From 44cede10da8161e1a6e7450034d1c8b10fb272d7 Mon Sep 17 00:00:00 2001
From: Kazuhiro MUSASHI <simoom634@yahoo.co.jp>
Date: Sun, 5 Feb 2023 19:49:14 +0900
Subject: [PATCH] Deploy `systemd` config for `vector-node_exporter`.

---
 .../system/vector-node_exporter.service       | 17 ++++++
 .../files/etc/vector/node_exporter.toml       | 54 +++++++++++++++++++
 .../node_exporter_setup.rb                    | 19 ++++++-
 .../etc/prometheus.d/alerts/services.yml      |  8 +++
 4 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 cookbooks/prometheus-exporters/files/etc/systemd/system/vector-node_exporter.service
 create mode 100644 cookbooks/prometheus-exporters/files/etc/vector/node_exporter.toml

diff --git a/cookbooks/prometheus-exporters/files/etc/systemd/system/vector-node_exporter.service b/cookbooks/prometheus-exporters/files/etc/systemd/system/vector-node_exporter.service
new file mode 100644
index 0000000..5474332
--- /dev/null
+++ b/cookbooks/prometheus-exporters/files/etc/systemd/system/vector-node_exporter.service
@@ -0,0 +1,17 @@
+[Unit]
+Description=Vector
+Documentation=https://vector.dev
+After=network-online.target
+Requires=network-online.target
+
+[Service]
+ExecStart=/usr/bin/vector --config /etc/vector/node_exporter.toml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=vector
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/cookbooks/prometheus-exporters/files/etc/vector/node_exporter.toml b/cookbooks/prometheus-exporters/files/etc/vector/node_exporter.toml
new file mode 100644
index 0000000..539dd46
--- /dev/null
+++ b/cookbooks/prometheus-exporters/files/etc/vector/node_exporter.toml
@@ -0,0 +1,54 @@
+data_dir = "/var/lib/vector/"
+
+[sources.node_exporter]
+  type = "file"
+  include = [ "/var/log/node_exporter.log" ]
+  ignore_older_secs = 600
+  read_from = "beginning"
+
+[transforms.node_exporter_transform]
+  type = "remap"
+  inputs = ["node_exporter"]
+  source = '''
+    . |= parse_syslog!(.message)
+
+    . |= parse_key_value!(.message)
+
+    del(.message)
+    del(.host)
+
+    .message = .msg
+    del(.msg)
+  '''
+
+[sinks.node_exporter_output]
+type = "file"
+inputs = [ "node_exporter_transform" ]
+compression = "none"
+path = "/tmp/node_exporter-%Y-%m-%d.log"
+
+  [sinks.node_exporter_output.encoding]
+  codec = "json"
+
+  [sinks.node_exporter_output.buffer]
+  max_size = 268435488
+  type = "disk"
+
+[sinks.node_exporter_loki]
+type = "loki"
+inputs = [ "node_exporter_transform" ]
+endpoint = "http://loki.service.consul:3100"
+compression = "snappy"
+
+  [sinks.node_exporter_loki.labels]
+  level = "{{ level }}"
+  hostname = "{{ hostname }}"
+  job = "node_exporter"
+  filename = "/var/log/node_exporter.log"
+
+  [sinks.node_exporter_loki.encoding]
+  codec = "json"
+
+  [sinks.node_exporter_loki.buffer]
+  max_size = 268435488
+  type = "disk"
diff --git a/cookbooks/prometheus-exporters/node_exporter_setup.rb b/cookbooks/prometheus-exporters/node_exporter_setup.rb
index 3f4d4f0..fa575f4 100644
--- a/cookbooks/prometheus-exporters/node_exporter_setup.rb
+++ b/cookbooks/prometheus-exporters/node_exporter_setup.rb
@@ -1,4 +1,4 @@
-# Deploy the `supervisord` configuration:
+# Deploy the `systemd` configuration:
 remote_file '/etc/systemd/system/node_exporter.service' do
   owner 'root'
   group 'root'
@@ -35,6 +35,23 @@ remote_file '/etc/logrotate.d/node_exporter' do
   mode  '0644'
 end
 
+# Deploy the `systemd` config for `vector`:
+remote_file '/etc/vector/node_exporter.toml' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end
+
+remote_file '/etc/systemd/system/vector-node_exporter.service' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end
+
+service 'vector-node_exporter' do
+  action [ :enable, :start]
+end
+
 # Deploy `consul` config for `node_exporter`:
 remote_file '/etc/consul.d/service-node_exporter.json' do
   owner 'consul'
diff --git a/cookbooks/prometheus/files/etc/prometheus.d/alerts/services.yml b/cookbooks/prometheus/files/etc/prometheus.d/alerts/services.yml
index 8048d00..a6d2fb1 100644
--- a/cookbooks/prometheus/files/etc/prometheus.d/alerts/services.yml
+++ b/cookbooks/prometheus/files/etc/prometheus.d/alerts/services.yml
@@ -19,3 +19,11 @@ groups:
       summary: "node_exporter is not running: {{ $labels.instance }}."
       description: "node_exporter is not running: {{ $labels.instance }}."
 
+  - alert: vector-node_exporter
+    expr: node_systemd_unit_state{name="vector-node_exporter.service", state="active"} != 1
+    for: 5m
+    labels:
+      severity: error
+    annotations:
+      summary: "vector-node_exporter is not running: {{ $labels.instance }}."
+      description: "vector-node_exporter is not running: {{ $labels.instance }}."