From 4c849d22c58084379c42e5ff4a1931cb619e24aa Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 15 Jan 2023 20:23:15 +0900 Subject: [PATCH] Monitor `digdag` log files. --- cookbooks/digdag/files/etc/vector/digdag.toml | 57 +++++++++++++++++ .../lib/systemd/system/vector-digdag.service | 16 +++++ cookbooks/digdag/setup.rb | 29 ++++++--- .../digdag/templates/etc/promtail/digdag.yaml | 61 ------------------- 4 files changed, 92 insertions(+), 71 deletions(-) create mode 100644 cookbooks/digdag/files/etc/vector/digdag.toml create mode 100644 cookbooks/digdag/files/lib/systemd/system/vector-digdag.service delete mode 100644 cookbooks/digdag/templates/etc/promtail/digdag.yaml diff --git a/cookbooks/digdag/files/etc/vector/digdag.toml b/cookbooks/digdag/files/etc/vector/digdag.toml new file mode 100644 index 0000000..42bb544 --- /dev/null +++ b/cookbooks/digdag/files/etc/vector/digdag.toml @@ -0,0 +1,57 @@ +data_dir = "/var/lib/vector/" + +[sources.digdag] + type = "file" + include = [ "/var/log/digdag.log" ] + ignore_older_secs = 600 + read_from = "beginning" + +[transforms.digdag_transform] + type = "remap" + inputs = ["digdag"] + source = ''' + . |= parse_syslog!(.message) + + .message = replace(.message, r'^\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2} \+\d{4} ', "") + + if starts_with(.message, "[") { + l = parse_regex!(.message, r'\[(?P[^\]]+)\]') + . = merge(., l) + + .level = downcase(.level) + } else { + .level = "debug" + } + ''' + +[sinks.digdag_output] +type = "file" +inputs = [ "digdag_transform" ] +compression = "none" +path = "/tmp/digdag-%Y-%m-%d.log" + + [sinks.digdag_output.encoding] + codec = "json" + + [sinks.digdag_output.buffer] + max_size = 268435488 + type = "disk" + +[sinks.digdag_loki] +type = "loki" +inputs = [ "digdag_transform" ] +endpoint = "http://loki.service.consul:3100" +compression = "snappy" + + [sinks.digdag_loki.labels] + level = "{{ level }}" + hostname = "{{ hostname }}" + job = "digdag" + filename = "/var/log/digdag.log" + + [sinks.digdag_loki.encoding] + codec = "json" + + [sinks.digdag_loki.buffer] + max_size = 268435488 + type = "disk" diff --git a/cookbooks/digdag/files/lib/systemd/system/vector-digdag.service b/cookbooks/digdag/files/lib/systemd/system/vector-digdag.service new file mode 100644 index 0000000..ff7bea5 --- /dev/null +++ b/cookbooks/digdag/files/lib/systemd/system/vector-digdag.service @@ -0,0 +1,16 @@ +[Unit] +Description=Vector +Documentation=https://vector.dev +After=network-online.target +Requires=network-online.target + +[Service] +ExecStart=/usr/bin/vector --config /etc/vector/digdag.toml +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=vector + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/digdag/setup.rb b/cookbooks/digdag/setup.rb index 53abb9b..cb0003f 100644 --- a/cookbooks/digdag/setup.rb +++ b/cookbooks/digdag/setup.rb @@ -47,25 +47,34 @@ service 'digdag' do action [ :enable, :restart ] end -# Deploy /etc/hosts file: -HOSTNAME = run_command('uname -n').stdout.chomp - -template '/etc/promtail/digdag.yaml' do +# Deploy `rsyslog` config file for `digdag`: +remote_file '/etc/rsyslog.d/30-digdag.conf' do owner 'root' - group 'root' - mode '644' + group 'root' + mode '644' - variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['promtail']['lokiendpoint']) - end + notifies :restart, 'service[rsyslog]', :immediately +end + +# Deploy the config file for `vector`: +remote_file '/etc/vector/digdag.toml' do + owner 'root' + group 'root' + mode '644' +end # Deploy the `systemd` configuration: -remote_file '/lib/systemd/system/promtail-digdag.service' do +remote_file '/lib/systemd/system/vector-digdag.service' do owner 'root' group 'root' mode '644' end # Service setting: -service 'promtail-digdag' do +service 'vector-digdag' do action [ :enable, :restart ] end + +service 'rsyslog' do + action [ :nothing ] +end diff --git a/cookbooks/digdag/templates/etc/promtail/digdag.yaml b/cookbooks/digdag/templates/etc/promtail/digdag.yaml deleted file mode 100644 index dcdb06c..0000000 --- a/cookbooks/digdag/templates/etc/promtail/digdag.yaml +++ /dev/null @@ -1,61 +0,0 @@ -server: - disable: true - -positions: - filename: /var/opt/promtail/promtail_digdag_position.yaml - - -clients: - - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push - -scrape_configs: - - job_name: digdag - static_configs: - - targets: - - localhost - labels: - job: digdag - hostname: <%= @HOSTNAME %> - __path__: /var/log/supervisor/digdag.log - - pipeline_stages: - - match: - selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2}"' - action: drop - - - match: - selector: '{job="digdag"} |~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"' - action: drop - - - match: - selector: '{job="digdag"} !~ "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} ERROR"' - - stages: - - regex: - expression: '^(?P\d+\-\d+\-\d+ \d+:\d+:\d+)([\.\d]+)? (?P[\+\d]+) \[(?P[^\]]+)\] (?P.+)$' - - - template: - source: timestamp - template: '{{ .datetime }} {{ .timezone }}' - - - timestamp: - source: timestamp - format: 2006-01-02 15:04:05 -0700 - - - template: - source: level - template: '{{ if .level }}{{ .level }}{{ else }}notice{{ end }}' - - - template: - source: level - template: '{{ ToLower .level }}' - - - template: - source: level - template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}' - - - labels: - level: - - - output: - source: message