From 173f4682fd4f27f2b9c6138a28ec3fc39f1da6fb Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Wed, 3 Apr 2024 18:07:56 +0900 Subject: [PATCH 1/6] Change the word `promtail` to `vector`. --- cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf b/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf index 9c26961..3c0d2ef 100644 --- a/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf +++ b/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf @@ -1,4 +1,4 @@ -# Log kernel generated promtail log messages to file +# Log kernel generated nomad log messages to file :syslogtag,contains,"nomad" /var/log/nomad.log # Uncomment the following to stop logging anything that matches the last rule. -- 2.40.1 From 217c3f0dd64f246c101e64b70f17af0c509a0bfb Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Wed, 3 Apr 2024 18:08:33 +0900 Subject: [PATCH 2/6] Delete `/lib/systemd/system/promtail-nomad.service`. --- .../files/lib/systemd/system/promtail-nomad.service | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service diff --git a/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service b/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service deleted file mode 100644 index a093d6f..0000000 --- a/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Grafana Promtail -Documentation=https://github.com/grafana/loki -After=network-online.target - -[Service] -User=root -Restart=always -ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/nomad.yaml - -[Install] -WantedBy=multi-user.target -- 2.40.1 From 6fb0e4919811cc51e0b47d2360ef8715c15575af Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Wed, 3 Apr 2024 18:09:31 +0900 Subject: [PATCH 3/6] Delete `promtail` config for `nomad`. --- .../nomad/templates/etc/promtail/nomad.yaml | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 cookbooks/nomad/templates/etc/promtail/nomad.yaml diff --git a/cookbooks/nomad/templates/etc/promtail/nomad.yaml b/cookbooks/nomad/templates/etc/promtail/nomad.yaml deleted file mode 100644 index b43a0e9..0000000 --- a/cookbooks/nomad/templates/etc/promtail/nomad.yaml +++ /dev/null @@ -1,88 +0,0 @@ -server: - disable: true - -positions: - filename: /var/opt/promtail/promtail_nomad_position.yaml - -clients: - - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push - -scrape_configs: - - job_name: nomad - static_configs: - - targets: - - localhost - labels: - job: nomad - hostname: <%= @HOSTNAME %> - __path__: /var/log/nomad.log - - pipeline_stages: - - match: - selector: '{job="nomad"} != "task="' - stages: - - regex: - expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4}) \[(?P[^\]]+)\] +(?P.*)$' - - - timestamp: - source: timestamp - format: 2006-01-02T15:04:05.000-0700 - location: Asia/Tokyo - - - template: - source: level - template: '{{ ToLower .level }}' - - - labels: - level: - - - output: - source: message - - - match: - selector: '{job="nomad"} |~ "task=.*timestamp="' - stages: - - regex: - expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4} \[(?P[^\]]+)\] +(?P.*) task=(?P[^ ]+) .* timestamp=(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4})$' - - - timestamp: - source: timestamp - format: 2006-01-02T15:04:05.000-0700 - location: Asia/Tokyo - - - template: - source: level - template: '{{ ToLower .level }}' - - - labels: - level: - task: - - - output: - source: message - - - match: - selector: '{job="nomad"} |~ "task=.*reason="' - stages: - - regex: - expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P[^\]]+)\] +(?P.*) task=(?P.*) reason="(?P.*)"$' - - - timestamp: - source: timestamp - format: 2006-01-02T15:04:05.000-0700 - location: Asia/Tokyo - - - template: - source: msg - template: 'Message={{ .message }} Reason={{ .reason }}' - - - template: - source: level - template: '{{ ToLower .level }}' - - - labels: - level: - task: - - - output: - source: msg -- 2.40.1 From b16ed0980359becb92416c720b75262b47c8cc6a Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Wed, 3 Apr 2024 18:11:41 +0900 Subject: [PATCH 4/6] Deploy `vector` config for `nomad`. --- cookbooks/nomad/files/etc/vector/nomad.toml | 53 +++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 cookbooks/nomad/files/etc/vector/nomad.toml diff --git a/cookbooks/nomad/files/etc/vector/nomad.toml b/cookbooks/nomad/files/etc/vector/nomad.toml new file mode 100644 index 0000000..2a858b5 --- /dev/null +++ b/cookbooks/nomad/files/etc/vector/nomad.toml @@ -0,0 +1,53 @@ +data_dir = "/var/lib/vector/" + +[sources.nomad] + type = "file" + include = [ "/var/log/nomad.log" ] + ignore_older_secs = 600 + read_from = "beginning" + + +[transforms.nomad_transform] + type = "remap" + inputs = ["nomad"] + source = ''' + . |= parse_syslog!(.message) + + lv, err = parse_regex(.message, r'(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P[^\]]+)\] +(?P.*)$') + . = merge(., lv) + + del(.host) + .level = downcase!(.level) + ''' + +[sinks.nomad_output] +type = "file" +inputs = [ "nomad_transform"] +compression = "none" +path = "/tmp/nomad-%Y-%m-%d.log" + + [sinks.nomad_output.encoding] + codec = "json" + + [sinks.nomad_output.buffer] + max_size = 268435488 + type = "disk" + +[sinks.nomad_loki] +type = "loki" +inputs = [ "nomad_transform" ] +endpoint = "http://loki.service.consul:3100" +compression = "snappy" + + [sinks.nomad_loki.labels] + level = "{{ level }}" + hostname = "{{ hostname }}" + job = "nomad" + filename = "{{ file }}" + + [sinks.nomad_loki.encoding] + codec = "json" + + [sinks.nomad_loki.buffer] + max_size = 268435488 + type = "disk" -- 2.40.1 From bef9a01bc1dfc05b39beeca042d4bcbe4757be62 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 6 Apr 2024 16:24:49 +0900 Subject: [PATCH 5/6] Deploy `/lib/systemd/system/vector-nomad.service`. --- .../lib/systemd/system/vector-nomad.service | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 cookbooks/nomad/files/lib/systemd/system/vector-nomad.service diff --git a/cookbooks/nomad/files/lib/systemd/system/vector-nomad.service b/cookbooks/nomad/files/lib/systemd/system/vector-nomad.service new file mode 100644 index 0000000..36e0b37 --- /dev/null +++ b/cookbooks/nomad/files/lib/systemd/system/vector-nomad.service @@ -0,0 +1,16 @@ +[Unit] +Description=Vector +Documentation=https://vector.dev +After=network-online.target +Requires=network-online.target + +[Service] +ExecStart=/usr/bin/vector --config /etc/vector/nomad.toml +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +StandardOutput=journal +StandardError=journal +SyslogIdentifier=vector + +[Install] +WantedBy=multi-user.target -- 2.40.1 From 4a1574a7875b97204c8dc9d953602154ad21589c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 6 Apr 2024 16:24:47 +0900 Subject: [PATCH 6/6] Deploy `vector`-related config files. --- cookbooks/nomad/setup.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cookbooks/nomad/setup.rb b/cookbooks/nomad/setup.rb index 3661691..3a26a5f 100644 --- a/cookbooks/nomad/setup.rb +++ b/cookbooks/nomad/setup.rb @@ -85,28 +85,24 @@ service 'nomad' do action [:enable, :start] end -# Deploy `promtail` config: -HOSTNAME = run_command('uname -n').stdout.chomp - -template '/etc/promtail/nomad.yaml' do +# Deploy `vector` config: +remote_file '/etc/vector/nomad.toml' do owner 'root' group 'root' mode '644' - variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['nomad']['lokiendpoint']) - - notifies :restart, 'service[promtail-nomad]' + notifies :restart, 'service[vector-nomad]' end # Deploy the `systemd` configuration: -remote_file '/lib/systemd/system/promtail-nomad.service' do +remote_file '/lib/systemd/system/vector-nomad.service' do owner 'root' group 'root' mode '644' end # Service setting: -service 'promtail-nomad' do +service 'vector-nomad' do action [ :enable, :restart ] end -- 2.40.1