Merge branch 'install-snmp-exporter' of kazu634/itamae into master

cookbooks/prometheus/attributes.rb

@@ -45,4 +45,11 @@ node.reverse_merge!({
     'storage' => '/opt/filestat_exporter/',
     'location' => '/usr/local/bin/'
   },
+  'snmp_exporter' => {
+    'url' => 'https://github.com/prometheus/snmp_exporter/releases/download/',
+    'prefix' => 'snmp_exporter-',
+    'postfix' => '.linux-amd64.tar.gz',
+    'storage' => '/opt/snmp_exporter/',
+    'location' => '/usr/local/bin/'
+  },
 })

cookbooks/prometheus/default.rb

@@ -10,6 +10,9 @@ include_recipe './alertmanager_setup.rb'
 include_recipe './alertmanager_webhook_install.rb'
 include_recipe './alertmanager_webhook_setup.rb'
 
+include_recipe './snmp_exporter_install.rb'
+include_recipe './snmp_exporter_setup.rb'
+
 # Deploy /etc/hosts file:
 HOSTNAME = run_command('uname -n').stdout.chomp
 

cookbooks/prometheus/files/etc/consul.d/service-snmp_exporter.json

@@ -0,0 +1,12 @@
+{
+  "service": {
+    "name": "snmp_exporter",
+    "port": 9116,
+    "check":{
+      "tcp": "127.0.0.1:9116",
+      "interval": "60s",
+      "timeout": "1s",
+      "success_before_passing": 3
+    }
+  }
+}

cookbooks/prometheus/files/etc/prometheus.d/alerts/snmp.yml

@@ -0,0 +1,21 @@
+groups:
+- name: snmp
+  rules:
+  - alert: NetworkInboundBandwidth
+    expr: ((delta(ifHCInOctets{ifName="bond0"}[10m]) * 8 * 100) / (600 * ifSpeed{ifName="bond0"})) * 100 > 80
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Network Bandwidth (inbound) exceeds 80%"
+      description: "Network Bandwidth (inbound) exceeds 80%"
+
+  - alert: NetworkOutboundBandwidth
+    expr: ((delta(ifHCOutOctets{ifName="bond0"}[10m]) * 8 * 100) / (600 * ifSpeed{ifName="bond0"})) * 100 > 80
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Network Bandwidth (outbound) exceeds 80%"
+      description: "Network Bandwidth (outbound) exceeds 80%"
+

cookbooks/prometheus/files/etc/prometheus.d/prometheus.yml

@@ -13,9 +13,7 @@ global:
 rule_files:
   # - "first.rules"
   # - "second.rules"
-  - '/etc/prometheus.d/alerts/node_exporter.yml'
+  - '/etc/prometheus.d/alerts/*.yml'
-  - '/etc/prometheus.d/alerts/filestat.yml'
-  - '/etc/prometheus.d/alerts/prometheus.yml'
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
@@ -66,6 +64,36 @@ scrape_configs:
       services:
         - 'grafana'
 
+  - job_name: 'synology'
+    static_configs:
+      - targets:
+        - 192.168.10.200  # SNMP device.
+    metrics_path: /snmp
+    params:
+      module: [synology]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9116  # The SNMP exporter's real hostname:port.
+
+  - job_name: 'vyos'
+    static_configs:
+      - targets:
+        - 192.168.10.1  # SNMP device.
+    metrics_path: /snmp
+    params:
+      module: [if_mib]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9116  # The SNMP exporter's real hostname:port.
+
 alerting:
   alertmanagers:
     - static_configs:

cookbooks/prometheus/files/etc/supervisor/conf.d/snmp_exporter.conf

@@ -0,0 +1,8 @@
+[program:snmp_exporter]
+command=/usr/local/bin/snmp_exporter --config.file /etc/prometheus_exporters.d/snmp.yml
+stdout_logfile=/var/log/supervisor/snmp_exporter.log
+redirect_stderr=true
+stdout_logfile_maxbytes=1MB
+stdout_logfile_backups=5
+autorestart=true
+stopsignal=HUP

cookbooks/prometheus/files/lib/systemd/system/promtail-snmp_exporter.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Grafana Promtail for snmp_exporter
+Documentation=https://github.com/grafana/loki
+After=network-online.target
+
+[Service]
+User=root
+Restart=always
+ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/snmp_exporter.yaml
+
+[Install]
+WantedBy=multi-user.target

cookbooks/prometheus/snmp_exporter_install.rb

@@ -0,0 +1,52 @@
+snmp_url = ''
+snmp_bin = ''
+
+vtag     = ''
+tag      = ''
+
+# Calculate the Download URL:
+begin
+  require 'net/http'
+
+  uri = URI.parse('https://github.com/prometheus/snmp_exporter/releases/latest')
+
+  Timeout.timeout(3) do
+    response = Net::HTTP.get_response(uri)
+
+    vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)}
+    tag = vtag.sub(/^v/, '')
+
+    snmp_bin = "#{node['snmp_exporter']['prefix']}#{tag}#{node['snmp_exporter']['postfix']}"
+
+    snmp_url = "#{node['snmp_exporter']['url']}/#{vtag}/#{snmp_bin}"
+  end
+rescue
+  # Abort the chef client process:
+  raise 'Cannot connect to http://github.com.'
+end
+
+# Download:
+TMP = "/tmp/#{snmp_bin}"
+
+execute "wget #{snmp_url} -O #{TMP}"
+
+# Install:
+directory node['snmp_exporter']['storage'] do
+  owner 'root'
+  group 'root'
+  mode '755'
+end
+
+execute "tar zxf #{TMP} -C #{node['snmp_exporter']['storage']} --strip-components 1"
+
+# Change Owner and Permissions:
+file "#{node['snmp_exporter']['storage']}snmp_exporter" do
+  owner 'root'
+  group 'root'
+  mode  '755'
+end
+
+# Create Link
+link "#{node['snmp_exporter']['location']}snmp_exporter" do
+  to "#{node['snmp_exporter']['storage']}snmp_exporter"
+end

cookbooks/prometheus/snmp_exporter_setup.rb

@@ -0,0 +1,50 @@
+# Create Link
+link '/etc/prometheus_exporters.d/snmp.yml' do
+  to "#{node['snmp_exporter']['storage']}snmp.yml"
+end
+
+# Deploy `supervisord` config:
+remote_file '/etc/supervisor/conf.d/snmp_exporter.conf' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  notifies :restart, 'service[supervisor]'
+end
+
+# Deploy `consul` config:
+remote_file '/etc/consul.d/service-snmp_exporter.json' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  notifies :restart, 'service[supervisor]'
+end
+
+# Restart the `supervisor`:
+service 'supervisor' do
+  action :nothing
+end
+
+# Deploy /etc/hosts file:
+template '/etc/promtail/snmp_exporter.yaml' do
+  owner 'root'
+  group 'root'
+  mode '644'
+
+  variables(HOSTNAME: node[:hostname], LOKIENDPOINT: node['promtail']['lokiendpoint'])
+
+  notifies :restart, 'service[promtail-prometheus]'
+end
+
+# Deploy the `systemd` configuration:
+remote_file '/lib/systemd/system/promtail-snmp_exporter.service' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end
+
+# Service setting:
+service 'promtail-snmp_exporter' do
+  action [ :enable, :restart ]
+end

cookbooks/prometheus/templates/etc/promtail/snmp_exporter.yaml

@@ -0,0 +1,40 @@
+server:
+  disable: true
+
+positions:
+  filename: /var/opt/promtail/promtail_snmp_exporter_position.yaml
+
+clients:
+  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
+
+scrape_configs:
+  - job_name: snmp_exporter
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: snmp_exporter
+        hostname: <%= @HOSTNAME %>
+        __path__: /var/log/supervisor/snmp_exporter.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="snmp_exporter"}'
+        stages:
+        - regex:
+            expression: '^level=(?P<level>[^ ]+) ts=(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z) (?P<message>.+)$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.000Z
+            location: Etc/GMT
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}'
+
+        - labels:
+            level:
+
+        - output:
+            source: message