Merge pull request 'housekeep-promtail-config-for-nomad' (#18) from housekeep-promtail-config-for-nomad into main

Reviewed-on: #18

cookbooks/nomad/files/etc/rsyslog.d/30-nomad.conf

@@ -1,4 +1,4 @@
-# Log kernel generated promtail log messages to file
+# Log kernel generated nomad log messages to file
 :syslogtag,contains,"nomad" /var/log/nomad.log
 
 # Uncomment the following to stop logging anything that matches the last rule.

cookbooks/nomad/files/etc/vector/nomad.toml

@@ -0,0 +1,53 @@
+data_dir = "/var/lib/vector/"
+
+[sources.nomad]
+  type = "file"
+  include = [ "/var/log/nomad.log" ]
+  ignore_older_secs = 600
+  read_from = "beginning"
+
+
+[transforms.nomad_transform]
+  type = "remap"
+  inputs = ["nomad"]
+  source = '''
+    . |= parse_syslog!(.message)
+
+    lv, err = parse_regex(.message, r'(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P<level>[^\]]+)\] +(?P<message>.*)$')
+    . = merge(., lv)
+
+    del(.host)
+    .level = downcase!(.level)
+  '''
+
+[sinks.nomad_output]
+type = "file"
+inputs = [ "nomad_transform"]
+compression = "none"
+path = "/tmp/nomad-%Y-%m-%d.log"
+
+  [sinks.nomad_output.encoding]
+  codec = "json"
+
+  [sinks.nomad_output.buffer]
+  max_size = 268435488
+  type = "disk"
+
+[sinks.nomad_loki]
+type = "loki"
+inputs = [ "nomad_transform" ]
+endpoint = "http://loki.service.consul:3100"
+compression = "snappy"
+
+  [sinks.nomad_loki.labels]
+  level = "{{ level }}"
+  hostname = "{{ hostname }}"
+  job = "nomad"
+  filename = "{{ file }}"
+
+  [sinks.nomad_loki.encoding]
+  codec = "json"
+
+  [sinks.nomad_loki.buffer]
+  max_size = 268435488
+  type = "disk"

cookbooks/nomad/files/lib/systemd/system/promtail-nomad.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Grafana Promtail
-Documentation=https://github.com/grafana/loki
-After=network-online.target
-
-[Service]
-User=root
-Restart=always
-ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/nomad.yaml
-
-[Install]
-WantedBy=multi-user.target

cookbooks/nomad/files/lib/systemd/system/vector-nomad.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Vector
+Documentation=https://vector.dev
+After=network-online.target
+Requires=network-online.target
+
+[Service]
+ExecStart=/usr/bin/vector --config /etc/vector/nomad.toml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=vector
+
+[Install]
+WantedBy=multi-user.target

cookbooks/nomad/setup.rb

@@ -85,28 +85,24 @@ service 'nomad' do
   action [:enable, :start]
 end
 
-# Deploy `promtail` config:
-HOSTNAME = run_command('uname -n').stdout.chomp
-
-template '/etc/promtail/nomad.yaml' do
+# Deploy `vector` config:
+remote_file '/etc/vector/nomad.toml' do
   owner 'root'
   group 'root'
   mode '644'
 
-  variables(HOSTNAME: HOSTNAME,  LOKIENDPOINT: node['nomad']['lokiendpoint'])
-
-  notifies :restart, 'service[promtail-nomad]'
+  notifies :restart, 'service[vector-nomad]'
 end
 
 # Deploy the `systemd` configuration:
-remote_file '/lib/systemd/system/promtail-nomad.service' do
+remote_file '/lib/systemd/system/vector-nomad.service' do
   owner 'root'
   group 'root'
   mode '644'
 end
 
 # Service setting:
-service 'promtail-nomad' do
+service 'vector-nomad' do
   action [ :enable, :restart ]
 end
 

cookbooks/nomad/templates/etc/promtail/nomad.yaml

@@ -1,88 +0,0 @@
-server:
-  disable: true
-
-positions:
-  filename: /var/opt/promtail/promtail_nomad_position.yaml
-
-clients:
-  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
-
-scrape_configs:
-  - job_name: nomad
-    static_configs:
-    - targets:
-        - localhost
-      labels:
-        job: nomad
-        hostname: <%= @HOSTNAME %>
-        __path__: /var/log/nomad.log
-
-    pipeline_stages:
-    - match:
-        selector: '{job="nomad"} != "task="'
-        stages:
-        - regex:
-            expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4}) \[(?P<level>[^\]]+)\] +(?P<message>.*)$'
-
-        - timestamp:
-            source: timestamp
-            format: 2006-01-02T15:04:05.000-0700
-            location: Asia/Tokyo
-
-        - template:
-            source: level
-            template: '{{ ToLower .level }}'
-
-        - labels:
-            level:
-
-        - output:
-            source: message
-
-    - match:
-        selector: '{job="nomad"} |~ "task=.*timestamp="'
-        stages:
-        - regex:
-            expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4} \[(?P<level>[^\]]+)\] +(?P<message>.*) task=(?P<task>[^ ]+) .* timestamp=(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{4})$'
-
-        - timestamp:
-            source: timestamp
-            format: 2006-01-02T15:04:05.000-0700
-            location: Asia/Tokyo
-
-        - template:
-            source: level
-            template: '{{ ToLower .level }}'
-
-        - labels:
-            level:
-            task:
-
-        - output:
-            source: message
-
-    - match:
-        selector: '{job="nomad"} |~ "task=.*reason="'
-        stages:
-        - regex:
-            expression: '^\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ nomad[^:]+: +(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}) \[(?P<level>[^\]]+)\] +(?P<message>.*) task=(?P<task>.*) reason="(?P<reason>.*)"$'
-
-        - timestamp:
-            source: timestamp
-            format: 2006-01-02T15:04:05.000-0700
-            location: Asia/Tokyo
-
-        - template:
-            source: msg
-            template: 'Message={{ .message }} Reason={{ .reason }}'
-
-        - template:
-            source: level
-            template: '{{ ToLower .level }}'
-
-        - labels:
-            level:
-            task:
-
-        - output:
-            source: msg