diff --git a/cookbooks/nomad/attributes.rb b/cookbooks/nomad/attributes.rb index 367b2d7..d0fb934 100644 --- a/cookbooks/nomad/attributes.rb +++ b/cookbooks/nomad/attributes.rb @@ -4,6 +4,7 @@ node.reverse_merge!({ 'nomad' => { 'manager' => false, - 'client' => false + 'client' => false, + 'lokiendpoint' => 'loki.service.consul:3100' } }) diff --git a/cookbooks/nomad/files/etc/logrotate.d/nomad b/cookbooks/nomad/files/etc/logrotate.d/nomad new file mode 100644 index 0000000..adf7b5d --- /dev/null +++ b/cookbooks/nomad/files/etc/logrotate.d/nomad @@ -0,0 +1,13 @@ +/var/log/nomad.log +{ + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} diff --git a/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf b/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf new file mode 100644 index 0000000..9c26961 --- /dev/null +++ b/cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf @@ -0,0 +1,7 @@ +# Log kernel generated promtail log messages to file +:syslogtag,contains,"nomad" /var/log/nomad.log + +# Uncomment the following to stop logging anything that matches the last rule. +# Doing this will stop logging kernel generated UFW log messages to the file +# normally containing kern.* messages (eg, /var/log/kern.log) +& stop diff --git a/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service b/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service new file mode 100644 index 0000000..a093d6f --- /dev/null +++ b/cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Promtail +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/nomad.yaml + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/nomad/setup.rb b/cookbooks/nomad/setup.rb index 3738843..c9e7cb9 100644 --- a/cookbooks/nomad/setup.rb +++ b/cookbooks/nomad/setup.rb @@ -67,3 +67,46 @@ service 'nomad' do action [:enable, :start] end +# Deploy `promtail` config: +HOSTNAME = run_command('uname -n').stdout.chomp + +template '/etc/promtail/nomad.yaml' do + owner 'root' + group 'root' + mode '644' + + variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['nomad']['lokiendpoint']) + + notifies :restart, 'service[promtail-nomad]' +end + +# Deploy the `systemd` configuration: +remote_file '/lib/systemd/system/promtail-nomad.service' do + owner 'root' + group 'root' + mode '644' +end + +# Service setting: +service 'promtail-nomad' do + action [ :enable, :restart ] +end + +remote_file '/etc/rsyslog.d/30-nomad.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[rsyslog]' +end + +service 'rsyslog' do + action [ :nothing ] +end + +# Deploy the `logrotated` configuration: +remote_file '/etc/logrotate.d/nomad' do + owner 'root' + group 'root' + mode '644' +end diff --git a/cookbooks/nomad/templates/etc/promtail/nomad.yaml b/cookbooks/nomad/templates/etc/promtail/nomad.yaml new file mode 100644 index 0000000..b43a0e9 --- /dev/null +++ b/cookbooks/nomad/templates/etc/promtail/nomad.yaml @@ -0,0 +1,88 @@ +server: + disable: true + +positions: + filename: /var/opt/promtail/promtail_nomad_position.yaml + +clients: + - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push + +scrape_configs: + - job_name: nomad + static_configs: + - targets: + - localhost + labels: + job: nomad + hostname: <%= @HOSTNAME %> + __path__: /var/log/nomad.log + + pipeline_stages: + - match: + selector: '{job="nomad"} != "task="' + stages: + - regex: + expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4}) \[(?P[^\]]+)\] +(?P.*)$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000-0700 + location: Asia/Tokyo + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + + - output: + source: message + + - match: + selector: '{job="nomad"} |~ "task=.*timestamp="' + stages: + - regex: + expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4} \[(?P[^\]]+)\] +(?P.*) task=(?P[^ ]+) .* timestamp=(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4})$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000-0700 + location: Asia/Tokyo + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + task: + + - output: + source: message + + - match: + selector: '{job="nomad"} |~ "task=.*reason="' + stages: + - regex: + expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P[^\]]+)\] +(?P.*) task=(?P.*) reason="(?P.*)"$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000-0700 + location: Asia/Tokyo + + - template: + source: msg + template: 'Message={{ .message }} Reason={{ .reason }}' + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + task: + + - output: + source: msg