From 6822c916e68aef2f7d9cfecf2388619ee760998a Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:23:06 +0900 Subject: [PATCH 1/8] Ignore "Calculated write I/O size" message. --- cookbooks/vector/templates/etc/promtail/syslog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/vector/templates/etc/promtail/syslog.yaml b/cookbooks/vector/templates/etc/promtail/syslog.yaml index 83a95d7..558282d 100644 --- a/cookbooks/vector/templates/etc/promtail/syslog.yaml +++ b/cookbooks/vector/templates/etc/promtail/syslog.yaml @@ -39,7 +39,7 @@ scrape_configs: action: drop - match: - selector: '{job="syslog", hostname="esxi-new", appname="Hostd"} |~ "(->|IpmiIfcOpenIpmiOpen|LikewiseGetDomainJoinInfo|AddVirtualMachine: VM|Solo.HttpSvc.HTTPService|VigorCallback received fault|vim.fault.InvalidPowerState|Unable to get resource settings for a powered on VM|VigorOnlineStatusCb|N7Vmacore16TimeoutExceptionE)"' + selector: '{job="syslog", hostname="esxi-new", appname="Hostd"} |~ "(->|IpmiIfcOpenIpmiOpen|LikewiseGetDomainJoinInfo|AddVirtualMachine: VM|Solo.HttpSvc.HTTPService|VigorCallback received fault|vim.fault.InvalidPowerState|Unable to get resource settings for a powered on VM|VigorOnlineStatusCb|N7Vmacore16TimeoutExceptionE|Calculated write I/O size)"' action: drop - match: From 8497937786363c02f2e3beb49108f1929808cb59 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:23:39 +0900 Subject: [PATCH 2/8] Ignore "Last log rotation time" message. --- cookbooks/vector/templates/etc/promtail/syslog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/vector/templates/etc/promtail/syslog.yaml b/cookbooks/vector/templates/etc/promtail/syslog.yaml index 558282d..523bb82 100644 --- a/cookbooks/vector/templates/etc/promtail/syslog.yaml +++ b/cookbooks/vector/templates/etc/promtail/syslog.yaml @@ -51,7 +51,7 @@ scrape_configs: action: drop - match: - selector: '{job="syslog", hostname="esxi-new", appname="Rhttpproxy"} |~ "(warning rhttpproxy|->)"' + selector: '{job="syslog", hostname="esxi-new", appname="Rhttpproxy"} |~ "(warning rhttpproxy|->|last log rotation time)"' action: drop - match: From 935b2e1732822724ab81ef9f8374c100281212d0 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:25:05 +0900 Subject: [PATCH 3/8] Consolidate the `drop` stanzas. --- .../promtail/templates/etc/promtail/base.yaml | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/cookbooks/promtail/templates/etc/promtail/base.yaml b/cookbooks/promtail/templates/etc/promtail/base.yaml index c098384..eba6bfc 100644 --- a/cookbooks/promtail/templates/etc/promtail/base.yaml +++ b/cookbooks/promtail/templates/etc/promtail/base.yaml @@ -29,37 +29,12 @@ scrape_configs: pipeline_stages: - match: - selector: '{job="sudo"} |~ "/bin/sh"' - stages: - - drop: - expression: (CRON|sshd|session|securetty|systemd-logind) - - regex: - expression: '^(?P\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^ ]+ sudo: +(?P[^ ]+) : TTY=(?P[^ ]+) ; PWD=(?P[^ ]+) ; USER=(?P[^ ]+) ; COMMAND=(?P.+)$' - - - timestamp: - source: timestamp - format: Jan 2 15:04:05 - location: Asia/Tokyo - - - template: - source: message - template: 'USER={{ .user }} PWD={{ .pwd }} CMD={{ .cmd }}' - - - template: - source: level - template: 'info' - - - labels: - level: - - - output: - source: message + selector: '{job="sudo"} |~ "(CRON|sshd|session|securetty|systemd-logind|/bin/sh)"' + action: drop - match: selector: '{job="sudo"} !~ "/bin/sh"' stages: - - drop: - expression: (CRON|sshd|session|securetty|systemd-logind) - regex: expression: '^(?P\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^ ]+ sudo: +(?P[^ ]+) : TTY=(?P[^ ]+) ; PWD=(?P[^ ]+) ; USER=(?P[^ ]+) ; COMMAND=(?P.+)$' @@ -93,13 +68,14 @@ scrape_configs: __path__: /var/log/auth.log pipeline_stages: + - match: + selector: '{job="sshd"} |~ "(CRON|sudo|session)"' + action: drop + - match: selector: '{job="sshd"}' stages: - - drop: - expression: (CRON|sudo|session) - - regex: expression: '^(?P\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^:]+: (?P.+)$' From 90bfae99c911d5a9d7e50058e53ae4fd811e09e5 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:25:42 +0900 Subject: [PATCH 4/8] Ignore "libcontainer container" message. --- cookbooks/promtail/templates/etc/promtail/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/promtail/templates/etc/promtail/base.yaml b/cookbooks/promtail/templates/etc/promtail/base.yaml index eba6bfc..173ecf2 100644 --- a/cookbooks/promtail/templates/etc/promtail/base.yaml +++ b/cookbooks/promtail/templates/etc/promtail/base.yaml @@ -238,7 +238,7 @@ scrape_configs: pipeline_stages: - match: - selector: '{job="init"} |~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker)"' + selector: '{job="init"} |~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker|tmp-sanity|libcontainer container)"' stages: - template: source: level From 7c9e1ed48c58543ef0bf2d5732c16f88a2e2c513 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:26:04 +0900 Subject: [PATCH 5/8] Ignore "tmp-sanity" message. --- cookbooks/promtail/templates/etc/promtail/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/promtail/templates/etc/promtail/base.yaml b/cookbooks/promtail/templates/etc/promtail/base.yaml index 173ecf2..1c13f4a 100644 --- a/cookbooks/promtail/templates/etc/promtail/base.yaml +++ b/cookbooks/promtail/templates/etc/promtail/base.yaml @@ -248,7 +248,7 @@ scrape_configs: level: - match: - selector: '{job="init"} !~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker)"' + selector: '{job="init"} !~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker|tmp-sanity)"' stages: - template: source: level From 660420b10be69d5af69f5d937028d2c4eb33bf99 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:26:34 +0900 Subject: [PATCH 6/8] Ignore `consul` message from `syslog`. --- cookbooks/promtail/templates/etc/promtail/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/promtail/templates/etc/promtail/base.yaml b/cookbooks/promtail/templates/etc/promtail/base.yaml index 1c13f4a..b84b0c5 100644 --- a/cookbooks/promtail/templates/etc/promtail/base.yaml +++ b/cookbooks/promtail/templates/etc/promtail/base.yaml @@ -320,7 +320,7 @@ scrape_configs: target_label: 'unit' - action: drop - regex: '.*(cron|supervisor|ssh|promtail|local|grafana|motd|dnsmasq|snapd|logind|init|session|loki|monit).*' + regex: '.*(cron|supervisor|ssh|promtail|local|grafana|motd|dnsmasq|snapd|logind|init|session|loki|monit|consul).*' source_labels: - __journal__systemd_unit From 6d19ebf31bae8f32d3b5bfda33f8b521d5f8e94c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:28:23 +0900 Subject: [PATCH 7/8] Ignore `rclone` error messages. --- cookbooks/digdag/templates/etc/promtail/digdag.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cookbooks/digdag/templates/etc/promtail/digdag.yaml b/cookbooks/digdag/templates/etc/promtail/digdag.yaml index b78a19d..b201922 100644 --- a/cookbooks/digdag/templates/etc/promtail/digdag.yaml +++ b/cookbooks/digdag/templates/etc/promtail/digdag.yaml @@ -24,7 +24,11 @@ scrape_configs: action: drop - match: - selector: '{job="digdag"} |~ "^[0-9]+-[0-9]+-[0-9]+"' + selector: '{job="digdag"} |~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"' + action: drop + + - match: + selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"' stages: - regex: From 529d9adb7c7abbfc9e14ae11a684b020c05bd521 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 6 Dec 2020 12:28:48 +0900 Subject: [PATCH 8/8] Specify the YYYY-MM-DD more explicitly. --- cookbooks/digdag/templates/etc/promtail/digdag.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/digdag/templates/etc/promtail/digdag.yaml b/cookbooks/digdag/templates/etc/promtail/digdag.yaml index b201922..dcdb06c 100644 --- a/cookbooks/digdag/templates/etc/promtail/digdag.yaml +++ b/cookbooks/digdag/templates/etc/promtail/digdag.yaml @@ -20,7 +20,7 @@ scrape_configs: pipeline_stages: - match: - selector: '{job="digdag"} !~ "^[0-9]+-[0-9]+-[0-9]+"' + selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2}"' action: drop - match: