diff --git a/cookbooks/prometheus/attributes.rb b/cookbooks/prometheus/attributes.rb index 221f13f..3c90455 100644 --- a/cookbooks/prometheus/attributes.rb +++ b/cookbooks/prometheus/attributes.rb @@ -45,4 +45,11 @@ node.reverse_merge!({ 'storage' => '/opt/filestat_exporter/', 'location' => '/usr/local/bin/' }, + 'snmp_exporter' => { + 'url' => 'https://github.com/prometheus/snmp_exporter/releases/download/', + 'prefix' => 'snmp_exporter-', + 'postfix' => '.linux-amd64.tar.gz', + 'storage' => '/opt/snmp_exporter/', + 'location' => '/usr/local/bin/' + }, }) diff --git a/cookbooks/prometheus/default.rb b/cookbooks/prometheus/default.rb index 32c478e..3ee9c9d 100644 --- a/cookbooks/prometheus/default.rb +++ b/cookbooks/prometheus/default.rb @@ -10,6 +10,9 @@ include_recipe './alertmanager_setup.rb' include_recipe './alertmanager_webhook_install.rb' include_recipe './alertmanager_webhook_setup.rb' +include_recipe './snmp_exporter_install.rb' +include_recipe './snmp_exporter_setup.rb' + # Deploy /etc/hosts file: HOSTNAME = run_command('uname -n').stdout.chomp diff --git a/cookbooks/prometheus/files/etc/consul.d/service-snmp_exporter.json b/cookbooks/prometheus/files/etc/consul.d/service-snmp_exporter.json new file mode 100644 index 0000000..30553f7 --- /dev/null +++ b/cookbooks/prometheus/files/etc/consul.d/service-snmp_exporter.json @@ -0,0 +1,12 @@ +{ + "service": { + "name": "snmp_exporter", + "port": 9116, + "check":{ + "tcp": "127.0.0.1:9116", + "interval": "60s", + "timeout": "1s", + "success_before_passing": 3 + } + } +} diff --git a/cookbooks/prometheus/files/etc/prometheus.d/alerts/snmp.yml b/cookbooks/prometheus/files/etc/prometheus.d/alerts/snmp.yml new file mode 100644 index 0000000..ffe98ec --- /dev/null +++ b/cookbooks/prometheus/files/etc/prometheus.d/alerts/snmp.yml @@ -0,0 +1,21 @@ +groups: +- name: snmp + rules: + - alert: NetworkInboundBandwidth + expr: ((delta(ifHCInOctets{ifName="bond0"}[10m]) * 8 * 100) / (600 * ifSpeed{ifName="bond0"})) * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "Network Bandwidth (inbound) exceeds 80%" + description: "Network Bandwidth (inbound) exceeds 80%" + + - alert: NetworkOutboundBandwidth + expr: ((delta(ifHCOutOctets{ifName="bond0"}[10m]) * 8 * 100) / (600 * ifSpeed{ifName="bond0"})) * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "Network Bandwidth (outbound) exceeds 80%" + description: "Network Bandwidth (outbound) exceeds 80%" + diff --git a/cookbooks/prometheus/files/etc/prometheus.d/prometheus.yml b/cookbooks/prometheus/files/etc/prometheus.d/prometheus.yml index b690880..aed8def 100644 --- a/cookbooks/prometheus/files/etc/prometheus.d/prometheus.yml +++ b/cookbooks/prometheus/files/etc/prometheus.d/prometheus.yml @@ -13,9 +13,7 @@ global: rule_files: # - "first.rules" # - "second.rules" - - '/etc/prometheus.d/alerts/node_exporter.yml' - - '/etc/prometheus.d/alerts/filestat.yml' - - '/etc/prometheus.d/alerts/prometheus.yml' + - '/etc/prometheus.d/alerts/*.yml' # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. @@ -66,6 +64,36 @@ scrape_configs: services: - 'grafana' + - job_name: 'synology' + static_configs: + - targets: + - 192.168.10.200 # SNMP device. + metrics_path: /snmp + params: + module: [synology] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9116 # The SNMP exporter's real hostname:port. + + - job_name: 'vyos' + static_configs: + - targets: + - 192.168.10.1 # SNMP device. + metrics_path: /snmp + params: + module: [if_mib] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9116 # The SNMP exporter's real hostname:port. + alerting: alertmanagers: - static_configs: diff --git a/cookbooks/prometheus/files/etc/supervisor/conf.d/snmp_exporter.conf b/cookbooks/prometheus/files/etc/supervisor/conf.d/snmp_exporter.conf new file mode 100644 index 0000000..38e00ac --- /dev/null +++ b/cookbooks/prometheus/files/etc/supervisor/conf.d/snmp_exporter.conf @@ -0,0 +1,8 @@ +[program:snmp_exporter] +command=/usr/local/bin/snmp_exporter --config.file /etc/prometheus_exporters.d/snmp.yml +stdout_logfile=/var/log/supervisor/snmp_exporter.log +redirect_stderr=true +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=5 +autorestart=true +stopsignal=HUP diff --git a/cookbooks/prometheus/files/lib/systemd/system/promtail-snmp_exporter.service b/cookbooks/prometheus/files/lib/systemd/system/promtail-snmp_exporter.service new file mode 100644 index 0000000..0fbf1ab --- /dev/null +++ b/cookbooks/prometheus/files/lib/systemd/system/promtail-snmp_exporter.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Promtail for snmp_exporter +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/snmp_exporter.yaml + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/prometheus/snmp_exporter_install.rb b/cookbooks/prometheus/snmp_exporter_install.rb new file mode 100644 index 0000000..fb75001 --- /dev/null +++ b/cookbooks/prometheus/snmp_exporter_install.rb @@ -0,0 +1,52 @@ +snmp_url = '' +snmp_bin = '' + +vtag = '' +tag = '' + +# Calculate the Download URL: +begin + require 'net/http' + + uri = URI.parse('https://github.com/prometheus/snmp_exporter/releases/latest') + + Timeout.timeout(3) do + response = Net::HTTP.get_response(uri) + + vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)} + tag = vtag.sub(/^v/, '') + + snmp_bin = "#{node['snmp_exporter']['prefix']}#{tag}#{node['snmp_exporter']['postfix']}" + + snmp_url = "#{node['snmp_exporter']['url']}/#{vtag}/#{snmp_bin}" + end +rescue + # Abort the chef client process: + raise 'Cannot connect to http://github.com.' +end + +# Download: +TMP = "/tmp/#{snmp_bin}" + +execute "wget #{snmp_url} -O #{TMP}" + +# Install: +directory node['snmp_exporter']['storage'] do + owner 'root' + group 'root' + mode '755' +end + +execute "tar zxf #{TMP} -C #{node['snmp_exporter']['storage']} --strip-components 1" + +# Change Owner and Permissions: +file "#{node['snmp_exporter']['storage']}snmp_exporter" do + owner 'root' + group 'root' + mode '755' +end + +# Create Link +link "#{node['snmp_exporter']['location']}snmp_exporter" do + to "#{node['snmp_exporter']['storage']}snmp_exporter" +end diff --git a/cookbooks/prometheus/snmp_exporter_setup.rb b/cookbooks/prometheus/snmp_exporter_setup.rb new file mode 100644 index 0000000..f20aea8 --- /dev/null +++ b/cookbooks/prometheus/snmp_exporter_setup.rb @@ -0,0 +1,50 @@ +# Create Link +link '/etc/prometheus_exporters.d/snmp.yml' do + to "#{node['snmp_exporter']['storage']}snmp.yml" +end + +# Deploy `supervisord` config: +remote_file '/etc/supervisor/conf.d/snmp_exporter.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Deploy `consul` config: +remote_file '/etc/consul.d/service-snmp_exporter.json' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Restart the `supervisor`: +service 'supervisor' do + action :nothing +end + +# Deploy /etc/hosts file: +template '/etc/promtail/snmp_exporter.yaml' do + owner 'root' + group 'root' + mode '644' + + variables(HOSTNAME: node[:hostname], LOKIENDPOINT: node['promtail']['lokiendpoint']) + + notifies :restart, 'service[promtail-prometheus]' +end + +# Deploy the `systemd` configuration: +remote_file '/lib/systemd/system/promtail-snmp_exporter.service' do + owner 'root' + group 'root' + mode '644' +end + +# Service setting: +service 'promtail-snmp_exporter' do + action [ :enable, :restart ] +end diff --git a/cookbooks/prometheus/templates/etc/promtail/snmp_exporter.yaml b/cookbooks/prometheus/templates/etc/promtail/snmp_exporter.yaml new file mode 100644 index 0000000..6009a02 --- /dev/null +++ b/cookbooks/prometheus/templates/etc/promtail/snmp_exporter.yaml @@ -0,0 +1,40 @@ +server: + disable: true + +positions: + filename: /var/opt/promtail/promtail_snmp_exporter_position.yaml + +clients: + - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push + +scrape_configs: + - job_name: snmp_exporter + static_configs: + - targets: + - localhost + labels: + job: snmp_exporter + hostname: <%= @HOSTNAME %> + __path__: /var/log/supervisor/snmp_exporter.log + + pipeline_stages: + - match: + selector: '{job="snmp_exporter"}' + stages: + - regex: + expression: '^level=(?P[^ ]+) ts=(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z) (?P.+)$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.000Z + location: Etc/GMT + + - template: + source: level + template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}' + + - labels: + level: + + - output: + source: message