Merge pull request 'Promtail設定の累積的な変更' (#69) from change-promtail-configs into master

Reviewed-on: #69

cookbooks/digdag/templates/etc/promtail/digdag.yaml

@@ -20,11 +20,15 @@ scrape_configs:
 
     pipeline_stages:
     - match:
-        selector: '{job="digdag"} !~ "^[0-9]+-[0-9]+-[0-9]+"'
+        selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2}"'
         action: drop
 
     - match:
-        selector: '{job="digdag"} |~ "^[0-9]+-[0-9]+-[0-9]+"'
+        selector: '{job="digdag"} |~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"'
+        action: drop
+
+    - match:
+        selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"'
 
         stages:
         - regex:

cookbooks/promtail/templates/etc/promtail/base.yaml

@@ -29,37 +29,12 @@ scrape_configs:
 
     pipeline_stages:
     - match:
-        selector: '{job="sudo"} |~ "/bin/sh"'
+        selector: '{job="sudo"} |~ "(CRON|sshd|session|securetty|systemd-logind|/bin/sh)"'
-        stages:
+        action: drop
-        - drop:
-            expression: (CRON|sshd|session|securetty|systemd-logind)
-        - regex:
-            expression: '^(?P<timestamp>\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^ ]+ sudo: +(?P<user>[^ ]+) : TTY=(?P<tty>[^ ]+) ; PWD=(?P<pwd>[^ ]+) ; USER=(?P<foo>[^ ]+) ; COMMAND=(?P<cmd>.+)$'
-
-        - timestamp:
-            source: timestamp
-            format: Jan 2 15:04:05
-            location: Asia/Tokyo
-
-        - template:
-            source: message
-            template: 'USER={{ .user }} PWD={{ .pwd }} CMD={{ .cmd }}'
-
-        - template:
-            source: level
-            template: 'info'
-
-        - labels:
-            level:
-
-        - output:
-            source: message
 
     - match:
         selector: '{job="sudo"} !~ "/bin/sh"'
         stages:
-        - drop:
-            expression: (CRON|sshd|session|securetty|systemd-logind)
         - regex:
             expression: '^(?P<timestamp>\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^ ]+ sudo: +(?P<user>[^ ]+) : TTY=(?P<tty>[^ ]+) ; PWD=(?P<pwd>[^ ]+) ; USER=(?P<foo>[^ ]+) ; COMMAND=(?P<cmd>.+)$'
 
@@ -93,13 +68,14 @@ scrape_configs:
         __path__: /var/log/auth.log
 
     pipeline_stages:
+    - match:
+        selector: '{job="sshd"} |~ "(CRON|sudo|session)"'
+        action: drop
+
     - match:
         selector: '{job="sshd"}'
 
         stages:
-        - drop:
-            expression: (CRON|sudo|session)
-
         - regex:
             expression: '^(?P<timestamp>\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^:]+: (?P<message>.+)$'
 
@@ -262,7 +238,7 @@ scrape_configs:
 
     pipeline_stages:
     - match:
-        selector: '{job="init"} |~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker)"'
+        selector: '{job="init"} |~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker|tmp-sanity|libcontainer container)"'
         stages:
         - template:
             source: level
@@ -272,7 +248,7 @@ scrape_configs:
             level:
 
     - match:
-        selector: '{job="init"} !~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker)"'
+        selector: '{job="init"} !~ "(apt|Message of the Day|motd-news|Temporary Directories|man-db|fwupd|Firmware update daemon|systemd-tmpfiles-clean.service|Rotate log files|logrotate.service|[Pp]ackage[Kk]it|/run/dbus/system_bus_socket|[Ss]nap|lxd|Reloading|Mount unit|ext4 Metadata|e2scrub_all.service|docker|tmp-sanity)"'
         stages:
         - template:
             source: level
@@ -344,7 +320,7 @@ scrape_configs:
         target_label: 'unit'
 
       - action: drop
-        regex: '.*(cron|supervisor|ssh|promtail|local|grafana|motd|dnsmasq|snapd|logind|init|session|loki|monit).*'
+        regex: '.*(cron|supervisor|ssh|promtail|local|grafana|motd|dnsmasq|snapd|logind|init|session|loki|monit|consul).*'
         source_labels:
         - __journal__systemd_unit
 

cookbooks/vector/templates/etc/promtail/syslog.yaml

@@ -39,7 +39,7 @@ scrape_configs:
         action: drop
 
     - match:
-        selector: '{job="syslog", hostname="esxi-new", appname="Hostd"} |~ "(->|IpmiIfcOpenIpmiOpen|LikewiseGetDomainJoinInfo|AddVirtualMachine: VM|Solo.HttpSvc.HTTPService|VigorCallback received fault|vim.fault.InvalidPowerState|Unable to get resource settings for a powered on VM|VigorOnlineStatusCb|N7Vmacore16TimeoutExceptionE)"'
+        selector: '{job="syslog", hostname="esxi-new", appname="Hostd"} |~ "(->|IpmiIfcOpenIpmiOpen|LikewiseGetDomainJoinInfo|AddVirtualMachine: VM|Solo.HttpSvc.HTTPService|VigorCallback received fault|vim.fault.InvalidPowerState|Unable to get resource settings for a powered on VM|VigorOnlineStatusCb|N7Vmacore16TimeoutExceptionE|Calculated write I/O size)"'
         action: drop
 
     - match:
@@ -51,7 +51,7 @@ scrape_configs:
         action: drop
 
     - match:
-        selector: '{job="syslog", hostname="esxi-new", appname="Rhttpproxy"} |~ "(warning rhttpproxy|->)"'
+        selector: '{job="syslog", hostname="esxi-new", appname="Rhttpproxy"} |~ "(warning rhttpproxy|->|last log rotation time)"'
         action: drop
 
     - match: