Add alert rules for `systemd` services.

cookbooks/prometheus/alertmanager_setup.rb

@@ -20,7 +20,7 @@ encrypted_remote_file '/etc/prometheus.d/alertmanager.yml' do
 end
 
 # Deploy alert setting file:
-%w(node_exporter prometheus filestat).each do |conf|
+%w(node_exporter prometheus filestat services snmp).each do |conf|
   remote_file "/etc/prometheus.d/alerts/#{conf}.yml" do
     owner  'root'
     group  'root'

cookbooks/prometheus/files/etc/prometheus.d/alerts/services.yml

@@ -0,0 +1,12 @@
+groups:
+- name: services
+  rules:
+  - alert: Digdag
+    expr: node_systemd_unit_state{name="digdag.service", state="active"} != 1
+    for: 5m
+    labels:
+      severity: error
+    annotations:
+      summary: "Digdag is not running: {{ $labels.instance }}."
+      description: "Digdag is not running: {{ $labels.instance }}."
+