From fb5a9ec741c4a2246624b694d88db2e5fabc9118 Mon Sep 17 00:00:00 2001
From: Angel Velazquez <angelcar@amazon.com>
Date: Thu, 20 May 2021 16:59:30 -0700
Subject: [PATCH] Limit the rate at which logger errors are logged into daemon
 logs

Logging to daemon logs every time there's an error with a log driver can be
problematic since daemon logs can grow rapidly, potentially exhausting disk
space.

Instead, it's preferable to limit the rate at which log driver errors are allowed
to be written. By default, this limit is 333 entries per second max.

Signed-off-by: Angel Velazquez <angelcar@amazon.com>
---
 daemon/logger/copier.go       |  6 ++----
 daemon/logger/logger_error.go | 24 ++++++++++++++++++++++++
 daemon/logger/ring.go         | 12 ++----------
 3 files changed, 28 insertions(+), 14 deletions(-)
 create mode 100644 daemon/logger/logger_error.go

diff --git a/daemon/logger/copier.go b/daemon/logger/copier.go
index e2ee36c0987e..30c68ea364d2 100644
--- a/daemon/logger/copier.go
+++ b/daemon/logger/copier.go
@@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
 					}
 
 					if logErr := c.dst.Log(msg); logErr != nil {
-						logWritesFailedCount.Inc(1)
-						logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
+						logDriverError(c.dst.Name(), string(msg.Line), logErr)
 					}
 				}
 				p += q + 1
@@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
 					hasMorePartial = true
 
 					if logErr := c.dst.Log(msg); logErr != nil {
-						logWritesFailedCount.Inc(1)
-						logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
+						logDriverError(c.dst.Name(), string(msg.Line), logErr)
 					}
 					p = 0
 					n = 0
diff --git a/daemon/logger/logger_error.go b/daemon/logger/logger_error.go
new file mode 100644
index 000000000000..70f4311979c4
--- /dev/null
+++ b/daemon/logger/logger_error.go
@@ -0,0 +1,24 @@
+package logger
+
+import (
+	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
+)
+
+// Rates based on journald defaults of 10,000 messages in 30s.
+// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=
+var logErrorLimiter = rate.NewLimiter(333, 333)
+
+// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount
+// metric.
+// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the
+// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation.
+func logDriverError(loggerName, msgLine string, logErr error) {
+	logWritesFailedCount.Inc(1)
+	if logErrorLimiter.Allow() {
+		logrus.WithError(logErr).
+			WithField("driver", loggerName).
+			WithField("message", msgLine).
+			Errorf("Error writing log message")
+	}
+}
diff --git a/daemon/logger/ring.go b/daemon/logger/ring.go
index b6432aed36f7..3b2652af63c1 100644
--- a/daemon/logger/ring.go
+++ b/daemon/logger/ring.go
@@ -4,8 +4,6 @@ import (
 	"errors"
 	"sync"
 	"sync/atomic"
-
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -104,10 +102,7 @@ func (r *RingLogger) Close() error {
 		}
 
 		if err := r.l.Log(msg); err != nil {
-			logrus.WithField("driver", r.l.Name()).
-				WithField("container", r.logInfo.ContainerID).
-				WithError(err).
-				Errorf("Error writing log message")
+			logDriverError(r.l.Name(), string(msg.Line), err)
 			logErr = true
 		}
 	}
@@ -128,10 +123,7 @@ func (r *RingLogger) run() {
 			return
 		}
 		if err := r.l.Log(msg); err != nil {
-			logrus.WithField("driver", r.l.Name()).
-				WithField("container", r.logInfo.ContainerID).
-				WithError(err).
-				Errorf("Error writing log message")
+			logDriverError(r.l.Name(), string(msg.Line), err)
 		}
 	}
 }