From a7363ef7b1f73d156987c01a3bd5a23058b08b9c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 10 Jan 2022 15:09:59 +0900 Subject: [PATCH 1/5] Add `Promtail` setting for `nomad`. --- cookbooks/nomad/setup.rb | 12 +++ .../nomad/templates/etc/promtail/nomad.yaml | 88 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 cookbooks/nomad/templates/etc/promtail/nomad.yaml diff --git a/cookbooks/nomad/setup.rb b/cookbooks/nomad/setup.rb index 3738843..d18026f 100644 --- a/cookbooks/nomad/setup.rb +++ b/cookbooks/nomad/setup.rb @@ -67,3 +67,15 @@ service 'nomad' do action [:enable, :start] end +# Deploy `promtail` config: +HOSTNAME = run_command('uname -n').stdout.chomp + +template '/etc/promtail/nomad.yaml' do + owner 'root' + group 'root' + mode '644' + + variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['nomad']['lokiendpoint']) + + notifies :restart, 'service[promtail-nomad]' +end diff --git a/cookbooks/nomad/templates/etc/promtail/nomad.yaml b/cookbooks/nomad/templates/etc/promtail/nomad.yaml new file mode 100644 index 0000000..b43a0e9 --- /dev/null +++ b/cookbooks/nomad/templates/etc/promtail/nomad.yaml @@ -0,0 +1,88 @@ +server: + disable: true + +positions: + filename: /var/opt/promtail/promtail_nomad_position.yaml + +clients: + - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push + +scrape_configs: + - job_name: nomad + static_configs: + - targets: + - localhost + labels: + job: nomad + hostname: <%= @HOSTNAME %> + __path__: /var/log/nomad.log + + pipeline_stages: + - match: + selector: '{job="nomad"} != "task="' + stages: + - regex: + expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4}) \[(?P[^\]]+)\] +(?P.*)$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000-0700 + location: Asia/Tokyo + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + + - output: + source: message + + - match: + selector: '{job="nomad"} |~ "task=.*timestamp="' + stages: + - regex: + expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4} \[(?P[^\]]+)\] +(?P.*) task=(?P[^ ]+) .* timestamp=(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4})$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000-0700 + location: Asia/Tokyo + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + task: + + - output: + source: message + + - match: + selector: '{job="nomad"} |~ "task=.*reason="' + stages: + - regex: + expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P[^\]]+)\] +(?P.*) task=(?P.*) reason="(?P.*)"$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000-0700 + location: Asia/Tokyo + + - template: + source: msg + template: 'Message={{ .message }} Reason={{ .reason }}' + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + task: + + - output: + source: msg From d824d6afc8dfcbc79b9a4994659c40d5894fb11c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 10 Jan 2022 15:11:33 +0900 Subject: [PATCH 2/5] Add `systemd` config for `Promtail` monitoring `nomad`. --- .../files/lib/systemd/system/promtail-nomad.service | 12 ++++++++++++ cookbooks/nomad/setup.rb | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service diff --git a/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service b/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service new file mode 100644 index 0000000..a093d6f --- /dev/null +++ b/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Promtail +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/nomad.yaml + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/nomad/setup.rb b/cookbooks/nomad/setup.rb index d18026f..a2a5bcd 100644 --- a/cookbooks/nomad/setup.rb +++ b/cookbooks/nomad/setup.rb @@ -79,3 +79,15 @@ template '/etc/promtail/nomad.yaml' do notifies :restart, 'service[promtail-nomad]' end + +# Deploy the `systemd` configuration: +remote_file '/lib/systemd/system/promtail-nomad.service' do + owner 'root' + group 'root' + mode '644' +end + +# Service setting: +service 'promtail-nomad' do + action [ :enable, :restart ] +end From 9e6b05fbabd394f20a3b4e960c300198fe06048a Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 10 Jan 2022 15:12:10 +0900 Subject: [PATCH 3/5] Specify `Loki` endpoint. ``` --- a/cookbooks/nomad/attributes.rb +++ b/cookbooks/nomad/attributes.rb @@ -4,6 +4,7 @@ node.reverse_merge!({ 'nomad' => { 'manager' => false, - 'client' => false + 'client' => false, + 'lokiendpoint' => 'loki.service.consul:3100' } }) ``` --- cookbooks/nomad/attributes.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cookbooks/nomad/attributes.rb b/cookbooks/nomad/attributes.rb index 367b2d7..d0fb934 100644 --- a/cookbooks/nomad/attributes.rb +++ b/cookbooks/nomad/attributes.rb @@ -4,6 +4,7 @@ node.reverse_merge!({ 'nomad' => { 'manager' => false, - 'client' => false + 'client' => false, + 'lokiendpoint' => 'loki.service.consul:3100' } }) From dffb57e2fcfc36da1db394b068767838d148d738 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 10 Jan 2022 15:27:03 +0900 Subject: [PATCH 4/5] Add `rsyslog` config for `nomad`. --- cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf | 7 +++++++ cookbooks/nomad/setup.rb | 12 ++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf diff --git a/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf b/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf new file mode 100644 index 0000000..9c26961 --- /dev/null +++ b/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf @@ -0,0 +1,7 @@ +# Log kernel generated promtail log messages to file +:syslogtag,contains,"nomad" /var/log/nomad.log + +# Uncomment the following to stop logging anything that matches the last rule. +# Doing this will stop logging kernel generated UFW log messages to the file +# normally containing kern.* messages (eg, /var/log/kern.log) +& stop diff --git a/cookbooks/nomad/setup.rb b/cookbooks/nomad/setup.rb index a2a5bcd..2f2ddea 100644 --- a/cookbooks/nomad/setup.rb +++ b/cookbooks/nomad/setup.rb @@ -91,3 +91,15 @@ end service 'promtail-nomad' do action [ :enable, :restart ] end + +remote_file '/etc/rsyslog.d/30-nomad.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[rsyslog]' +end + +service 'rsyslog' do + action [ :nothing ] +end From de06f5575ce8fed164e65a3f5c8d83675fd419fd Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 10 Jan 2022 15:27:20 +0900 Subject: [PATCH 5/5] Add `logrotated` config for `nomad` log files. ``` --- /dev/null +++ b/cookbooks/nomad/files/etc/logrotate.d/nomad @@ -0,0 +1,13 @@ +/var/log/nomad.log +{ + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} ``` --- cookbooks/nomad/files/etc/logrotate.d/nomad | 13 +++++++++++++ cookbooks/nomad/setup.rb | 7 +++++++ 2 files changed, 20 insertions(+) create mode 100644 cookbooks/nomad/files/etc/logrotate.d/nomad diff --git a/cookbooks/nomad/files/etc/logrotate.d/nomad b/cookbooks/nomad/files/etc/logrotate.d/nomad new file mode 100644 index 0000000..adf7b5d --- /dev/null +++ b/cookbooks/nomad/files/etc/logrotate.d/nomad @@ -0,0 +1,13 @@ +/var/log/nomad.log +{ + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} diff --git a/cookbooks/nomad/setup.rb b/cookbooks/nomad/setup.rb index 2f2ddea..c9e7cb9 100644 --- a/cookbooks/nomad/setup.rb +++ b/cookbooks/nomad/setup.rb @@ -103,3 +103,10 @@ end service 'rsyslog' do action [ :nothing ] end + +# Deploy the `logrotated` configuration: +remote_file '/etc/logrotate.d/nomad' do + owner 'root' + group 'root' + mode '644' +end