Monitor `digdag` log files.

cookbooks/digdag/files/etc/vector/digdag.toml

@@ -0,0 +1,57 @@
+data_dir = "/var/lib/vector/"
+
+[sources.digdag]
+  type = "file"
+  include = [ "/var/log/digdag.log" ]
+  ignore_older_secs = 600
+  read_from = "beginning"
+
+[transforms.digdag_transform]
+  type = "remap"
+  inputs = ["digdag"]
+  source = '''
+    . |= parse_syslog!(.message)
+
+    .message = replace(.message, r'^\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} \+\d{4} ', "")
+
+    if starts_with(.message, "[") {
+      l = parse_regex!(.message, r'\[(?P<level>[^\]]+)\]')
+      . = merge(., l)
+
+      .level = downcase(.level)
+    } else {
+      .level = "debug"
+    }
+  '''
+
+[sinks.digdag_output]
+type = "file"
+inputs = [ "digdag_transform" ]
+compression = "none"
+path = "/tmp/digdag-%Y-%m-%d.log"
+
+  [sinks.digdag_output.encoding]
+  codec = "json"
+
+  [sinks.digdag_output.buffer]
+  max_size = 268435488
+  type = "disk"
+
+[sinks.digdag_loki]
+type = "loki"
+inputs = [ "digdag_transform" ]
+endpoint = "http://loki.service.consul:3100"
+compression = "snappy"
+
+  [sinks.digdag_loki.labels]
+  level = "{{ level }}"
+  hostname = "{{ hostname }}"
+  job = "digdag"
+  filename = "/var/log/digdag.log"
+
+  [sinks.digdag_loki.encoding]
+  codec = "json"
+
+  [sinks.digdag_loki.buffer]
+  max_size = 268435488
+  type = "disk"

cookbooks/digdag/files/lib/systemd/system/vector-digdag.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Vector
+Documentation=https://vector.dev
+After=network-online.target
+Requires=network-online.target
+
+[Service]
+ExecStart=/usr/bin/vector --config /etc/vector/digdag.toml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=vector
+
+[Install]
+WantedBy=multi-user.target

cookbooks/digdag/setup.rb

@@ -47,25 +47,34 @@ service 'digdag' do
   action [ :enable, :restart ]
 end
 
-# Deploy /etc/hosts file:
-HOSTNAME = run_command('uname -n').stdout.chomp
-
-template '/etc/promtail/digdag.yaml' do
+# Deploy `rsyslog` config file for `digdag`:
+remote_file '/etc/rsyslog.d/30-digdag.conf' do
   owner 'root'
   group 'root'
   mode '644'
 
-  variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['promtail']['lokiendpoint'])
-  end
+  notifies :restart, 'service[rsyslog]', :immediately
+end
+
+# Deploy the config file for `vector`:
+remote_file '/etc/vector/digdag.toml' do
+  owner 'root'
+  group 'root'
+  mode  '644'
+end
 
 # Deploy the `systemd` configuration:
-remote_file '/lib/systemd/system/promtail-digdag.service' do
+remote_file '/lib/systemd/system/vector-digdag.service' do
   owner 'root'
   group 'root'
   mode '644'
 end
 
 # Service setting:
-service 'promtail-digdag' do
+service 'vector-digdag' do
   action [ :enable, :restart ]
 end
+
+service 'rsyslog' do
+  action [ :nothing ]
+end

cookbooks/digdag/templates/etc/promtail/digdag.yaml

@@ -1,61 +0,0 @@
-server:
-  disable: true
-
-positions:
-  filename: /var/opt/promtail/promtail_digdag_position.yaml
-
-
-clients:
-  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
-
-scrape_configs:
-  - job_name: digdag
-    static_configs:
-    - targets:
-        - localhost
-      labels:
-        job: digdag
-        hostname: <%= @HOSTNAME %>
-        __path__: /var/log/supervisor/digdag.log
-
-    pipeline_stages:
-    - match:
-        selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2}"'
-        action: drop
-
-    - match:
-        selector: '{job="digdag"} |~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"'
-        action: drop
-
-    - match:
-        selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"'
-
-        stages:
-        - regex:
-            expression: '^(?P<datetime>\d+\-\d+\-\d+ \d+:\d+:\d+)([\.\d]+)? (?P<timezone>[\+\d]+) \[(?P<level>[^\]]+)\] (?P<message>.+)$'
-
-        - template:
-            source: timestamp
-            template: '{{ .datetime }} {{ .timezone }}'
-
-        - timestamp:
-            source: timestamp
-            format: 2006-01-02 15:04:05 -0700
-
-        - template:
-            source: level
-            template: '{{ if .level }}{{ .level }}{{ else }}notice{{ end }}'
-
-        - template:
-            source: level
-            template: '{{ ToLower .level }}'
-
-        - template:
-            source: level
-            template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}'
-
-        - labels:
-            level:
-
-        - output:
-            source: message