Merge pull request 'nomad用の`Promtail`設定ファイルのデプロイ' (#94) from promtail-setting-for-nomad into master

Reviewed-on: #94

cookbooks/nomad/attributes.rb

@@ -4,6 +4,7 @@
 node.reverse_merge!({
   'nomad' => {
     'manager' => false,
-    'client' => false
+    'client' => false,
+    'lokiendpoint' => 'loki.service.consul:3100'
   }
 })

cookbooks/nomad/files/etc/logrotate.d/nomad

@@ -0,0 +1,13 @@
+/var/log/nomad.log
+{
+        rotate 4
+        weekly
+        missingok
+        notifempty
+        compress
+        delaycompress
+        sharedscripts
+        postrotate
+                /usr/lib/rsyslog/rsyslog-rotate
+        endscript
+}

cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf

@@ -0,0 +1,7 @@
+# Log kernel generated promtail log messages to file
+:syslogtag,contains,"nomad" /var/log/nomad.log
+
+# Uncomment the following to stop logging anything that matches the last rule.
+# Doing this will stop logging kernel generated UFW log messages to the file
+# normally containing kern.* messages (eg, /var/log/kern.log)
+& stop

cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Grafana Promtail
+Documentation=https://github.com/grafana/loki
+After=network-online.target
+
+[Service]
+User=root
+Restart=always
+ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/nomad.yaml
+
+[Install]
+WantedBy=multi-user.target

cookbooks/nomad/setup.rb

@@ -67,3 +67,46 @@ service 'nomad' do
   action [:enable, :start]
 end
 
+# Deploy `promtail` config:
+HOSTNAME = run_command('uname -n').stdout.chomp
+
+template '/etc/promtail/nomad.yaml' do
+  owner 'root'
+  group 'root'
+  mode '644'
+
+  variables(HOSTNAME: HOSTNAME,  LOKIENDPOINT: node['nomad']['lokiendpoint'])
+
+  notifies :restart, 'service[promtail-nomad]'
+end
+
+# Deploy the `systemd` configuration:
+remote_file '/lib/systemd/system/promtail-nomad.service' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end
+
+# Service setting:
+service 'promtail-nomad' do
+  action [ :enable, :restart ]
+end
+
+remote_file '/etc/rsyslog.d/30-nomad.conf' do
+  owner 'root'
+  group 'root'
+  mode '644'
+
+  notifies :restart, 'service[rsyslog]'
+end
+
+service 'rsyslog' do
+  action [ :nothing ]
+end
+
+# Deploy the `logrotated` configuration:
+remote_file '/etc/logrotate.d/nomad' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end

cookbooks/nomad/templates/etc/promtail/nomad.yaml

@@ -0,0 +1,88 @@
+server:
+  disable: true
+
+positions:
+  filename: /var/opt/promtail/promtail_nomad_position.yaml
+
+clients:
+  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
+
+scrape_configs:
+  - job_name: nomad
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nomad
+        hostname: <%= @HOSTNAME %>
+        __path__: /var/log/nomad.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="nomad"} != "task="'
+        stages:
+        - regex:
+            expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4}) \[(?P<level>[^\]]+)\] +(?P<message>.*)$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.000-0700
+            location: Asia/Tokyo
+
+        - template:
+            source: level
+            template: '{{ ToLower .level }}'
+
+        - labels:
+            level:
+
+        - output:
+            source: message
+
+    - match:
+        selector: '{job="nomad"} |~ "task=.*timestamp="'
+        stages:
+        - regex:
+            expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4} \[(?P<level>[^\]]+)\] +(?P<message>.*) task=(?P<task>[^ ]+) .* timestamp=(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4})$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.000-0700
+            location: Asia/Tokyo
+
+        - template:
+            source: level
+            template: '{{ ToLower .level }}'
+
+        - labels:
+            level:
+            task:
+
+        - output:
+            source: message
+
+    - match:
+        selector: '{job="nomad"} |~ "task=.*reason="'
+        stages:
+        - regex:
+            expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P<level>[^\]]+)\] +(?P<message>.*) task=(?P<task>.*) reason="(?P<reason>.*)"$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.000-0700
+            location: Asia/Tokyo
+
+        - template:
+            source: msg
+            template: 'Message={{ .message }} Reason={{ .reason }}'
+
+        - template:
+            source: level
+            template: '{{ ToLower .level }}'
+
+        - labels:
+            level:
+            task:
+
+        - output:
+            source: msg