113 lines
3.8 KiB
Diff
113 lines
3.8 KiB
Diff
From fb5a9ec741c4a2246624b694d88db2e5fabc9118 Mon Sep 17 00:00:00 2001
|
|
From: Angel Velazquez <angelcar@amazon.com>
|
|
Date: Thu, 20 May 2021 16:59:30 -0700
|
|
Subject: [PATCH] Limit the rate at which logger errors are logged into daemon
|
|
logs
|
|
|
|
Logging to daemon logs every time there's an error with a log driver can be
|
|
problematic since daemon logs can grow rapidly, potentially exhausting disk
|
|
space.
|
|
|
|
Instead, it's preferable to limit the rate at which log driver errors are allowed
|
|
to be written. By default, this limit is 333 entries per second max.
|
|
|
|
Signed-off-by: Angel Velazquez <angelcar@amazon.com>
|
|
---
|
|
daemon/logger/copier.go | 6 ++----
|
|
daemon/logger/logger_error.go | 24 ++++++++++++++++++++++++
|
|
daemon/logger/ring.go | 12 ++----------
|
|
3 files changed, 28 insertions(+), 14 deletions(-)
|
|
create mode 100644 daemon/logger/logger_error.go
|
|
|
|
diff --git a/daemon/logger/copier.go b/daemon/logger/copier.go
|
|
index e2ee36c0987e..30c68ea364d2 100644
|
|
--- a/daemon/logger/copier.go
|
|
+++ b/daemon/logger/copier.go
|
|
@@ -126,8 +126,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
|
|
}
|
|
|
|
if logErr := c.dst.Log(msg); logErr != nil {
|
|
- logWritesFailedCount.Inc(1)
|
|
- logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
|
|
+ logDriverError(c.dst.Name(), string(msg.Line), logErr)
|
|
}
|
|
}
|
|
p += q + 1
|
|
@@ -159,8 +158,7 @@ func (c *Copier) copySrc(name string, src io.Reader) {
|
|
hasMorePartial = true
|
|
|
|
if logErr := c.dst.Log(msg); logErr != nil {
|
|
- logWritesFailedCount.Inc(1)
|
|
- logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr)
|
|
+ logDriverError(c.dst.Name(), string(msg.Line), logErr)
|
|
}
|
|
p = 0
|
|
n = 0
|
|
diff --git a/daemon/logger/logger_error.go b/daemon/logger/logger_error.go
|
|
new file mode 100644
|
|
index 000000000000..70f4311979c4
|
|
--- /dev/null
|
|
+++ b/daemon/logger/logger_error.go
|
|
@@ -0,0 +1,24 @@
|
|
+package logger
|
|
+
|
|
+import (
|
|
+ "github.com/sirupsen/logrus"
|
|
+ "golang.org/x/time/rate"
|
|
+)
|
|
+
|
|
+// Rates based on journald defaults of 10,000 messages in 30s.
|
|
+// reference: https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=
|
|
+var logErrorLimiter = rate.NewLimiter(333, 333)
|
|
+
|
|
+// logDriverError logs errors produced by log drivers to the daemon logs. It also increments the logWritesFailedCount
|
|
+// metric.
|
|
+// Logging to the daemon logs is limited to 333 operations per second at most. If this limit is exceeded, the
|
|
+// logWritesFailedCount is still counted, but logging to the daemon logs is omitted in order to prevent disk saturation.
|
|
+func logDriverError(loggerName, msgLine string, logErr error) {
|
|
+ logWritesFailedCount.Inc(1)
|
|
+ if logErrorLimiter.Allow() {
|
|
+ logrus.WithError(logErr).
|
|
+ WithField("driver", loggerName).
|
|
+ WithField("message", msgLine).
|
|
+ Errorf("Error writing log message")
|
|
+ }
|
|
+}
|
|
diff --git a/daemon/logger/ring.go b/daemon/logger/ring.go
|
|
index b6432aed36f7..3b2652af63c1 100644
|
|
--- a/daemon/logger/ring.go
|
|
+++ b/daemon/logger/ring.go
|
|
@@ -4,8 +4,6 @@ import (
|
|
"errors"
|
|
"sync"
|
|
"sync/atomic"
|
|
-
|
|
- "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
const (
|
|
@@ -104,10 +102,7 @@ func (r *RingLogger) Close() error {
|
|
}
|
|
|
|
if err := r.l.Log(msg); err != nil {
|
|
- logrus.WithField("driver", r.l.Name()).
|
|
- WithField("container", r.logInfo.ContainerID).
|
|
- WithError(err).
|
|
- Errorf("Error writing log message")
|
|
+ logDriverError(r.l.Name(), string(msg.Line), err)
|
|
logErr = true
|
|
}
|
|
}
|
|
@@ -128,10 +123,7 @@ func (r *RingLogger) run() {
|
|
return
|
|
}
|
|
if err := r.l.Log(msg); err != nil {
|
|
- logrus.WithField("driver", r.l.Name()).
|
|
- WithField("container", r.logInfo.ContainerID).
|
|
- WithError(err).
|
|
- Errorf("Error writing log message")
|
|
+ logDriverError(r.l.Name(), string(msg.Line), err)
|
|
}
|
|
}
|
|
}
|
