From d463a2691d6eb0b6aeea6dede9c9f7ab7365f470 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 12 Sep 2020 15:04:44 +0900 Subject: [PATCH 1/4] Install `alertmanager-webhook-logger`. --- .../alertmanager_webhook_install.rb | 50 +++++++++++++++++++ cookbooks/prometheus/default.rb | 2 + 2 files changed, 52 insertions(+) create mode 100644 cookbooks/prometheus/alertmanager_webhook_install.rb diff --git a/cookbooks/prometheus/alertmanager_webhook_install.rb b/cookbooks/prometheus/alertmanager_webhook_install.rb new file mode 100644 index 0000000..23f48bc --- /dev/null +++ b/cookbooks/prometheus/alertmanager_webhook_install.rb @@ -0,0 +1,50 @@ +alertmanager_webhook_url = '' +alertmanager_webhook_bin = '' + +tag = '' + +# Calculate the Download URL: +begin + require 'net/http' + + uri = URI.parse('https://github.com/tomtom-international/alertmanager-webhook-logger/releases/latest') + + Timeout.timeout(3) do + response = Net::HTTP.get_response(uri) + + tag = $1 if response.body =~ %r{tag\/(\d+\.\d+)} + + alertmanager_webhook_bin = "#{node['alertmanager_webhook']['prefix']}#{tag}#{node['alertmanager_webhook']['postfix']}" + + alertmanager_webhook_url = "#{node['alertmanager_webhook']['url']}/#{tag}/#{alertmanager_webhook_bin}" + end +rescue + # Abort the chef client process: + raise 'Cannot connect to http://github.com.' +end + +# Download: +TMP = "/tmp/#{alertmanager_webhook_bin}" + +execute "wget #{alertmanager_webhook_url} -O #{TMP}" + +# Install: +directory node['alertmanager_webhook']['storage'] do + owner 'root' + group 'root' + mode '755' +end + +execute "tar zxf #{TMP} -C #{node['alertmanager_webhook']['storage']}" + +# Change Owner and Permissions: +file "#{node['alertmanager_webhook']['storage']}alertmanager-webhook-logger" do + owner 'root' + group 'root' + mode '755' +end + +# Create Link +link "#{node['alertmanager_webhook']['location']}alertmanager-webhook-logger" do + to "#{node['alertmanager_webhook']['storage']}alertmanager-webhook-logger" +end diff --git a/cookbooks/prometheus/default.rb b/cookbooks/prometheus/default.rb index 6cfdbbc..a2f0ee4 100644 --- a/cookbooks/prometheus/default.rb +++ b/cookbooks/prometheus/default.rb @@ -8,6 +8,8 @@ if node['prometheus']['manager'] include_recipe './alertmanager_install.rb' include_recipe './alertmanager_setup.rb' + + include_recipe './alertmanager_webhook_install.rb' end # Install the node_exporter here: From 1f08c9a784881084991dd3de3ff81750e8105f31 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 12 Sep 2020 15:14:37 +0900 Subject: [PATCH 2/4] Specify attributes for `alertmanager-webhook-logger` --- cookbooks/prometheus/attributes.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cookbooks/prometheus/attributes.rb b/cookbooks/prometheus/attributes.rb index 5676d8d..21c31b7 100644 --- a/cookbooks/prometheus/attributes.rb +++ b/cookbooks/prometheus/attributes.rb @@ -31,4 +31,11 @@ node.reverse_merge!({ 'storage' => '/opt/prometheus/', 'location' => '/usr/local/bin/' }, + 'alertmanager_webhook' => { + 'url' => 'https://github.com/tomtom-international/alertmanager-webhook-logger/releases/download/', + 'prefix' => 'alertmanager-webhook-logger-', + 'postfix' => '.tar.gz', + 'storage' => '/opt/prometheus/', + 'location' => '/usr/local/bin/' + }, }) From d27b979ab096d590513bd347b38e55dd9549368a Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sun, 13 Sep 2020 02:02:19 +0900 Subject: [PATCH 3/4] Deploy `supervisord` config for `alertmanager-webhook-logger`. --- cookbooks/prometheus/alertmanager_webhook_setup.rb | 14 ++++++++++++++ cookbooks/prometheus/default.rb | 1 + .../conf.d/alertmanager_webhook_logger.conf | 7 +++++++ 3 files changed, 22 insertions(+) create mode 100644 cookbooks/prometheus/alertmanager_webhook_setup.rb create mode 100644 cookbooks/prometheus/files/etc/supervisor/conf.d/alertmanager_webhook_logger.conf diff --git a/cookbooks/prometheus/alertmanager_webhook_setup.rb b/cookbooks/prometheus/alertmanager_webhook_setup.rb new file mode 100644 index 0000000..8dc8042 --- /dev/null +++ b/cookbooks/prometheus/alertmanager_webhook_setup.rb @@ -0,0 +1,14 @@ +# Deploy `supervisor` config for `Alert Manager Webhook Logger` +remote_file '/etc/supervisor/conf.d/alertmanager_webhook_logger.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Restart the `supervisor`: +service 'supervisor' do + action :nothing +end + diff --git a/cookbooks/prometheus/default.rb b/cookbooks/prometheus/default.rb index a2f0ee4..f40634d 100644 --- a/cookbooks/prometheus/default.rb +++ b/cookbooks/prometheus/default.rb @@ -10,6 +10,7 @@ if node['prometheus']['manager'] include_recipe './alertmanager_setup.rb' include_recipe './alertmanager_webhook_install.rb' + include_recipe './alertmanager_webhook_setup.rb' end # Install the node_exporter here: diff --git a/cookbooks/prometheus/files/etc/supervisor/conf.d/alertmanager_webhook_logger.conf b/cookbooks/prometheus/files/etc/supervisor/conf.d/alertmanager_webhook_logger.conf new file mode 100644 index 0000000..16d99ac --- /dev/null +++ b/cookbooks/prometheus/files/etc/supervisor/conf.d/alertmanager_webhook_logger.conf @@ -0,0 +1,7 @@ +[program:alertmanager-webhook-logger] +command=/usr/local/bin/alertmanager-webhook-logger +stdout_logfile=/var/log/supervisor/alertmanager-webhook-logger.log +redirect_stderr=true +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=5 +autorestart=true From 90c954a1a26e427b3fad50fda9674fc27e49437c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 12 Sep 2020 15:29:30 +0900 Subject: [PATCH 4/4] Modify `alertmanager` config to send alerts to webhook. --- cookbooks/prometheus/files/etc/prometheus.d/alertmanager.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cookbooks/prometheus/files/etc/prometheus.d/alertmanager.yml b/cookbooks/prometheus/files/etc/prometheus.d/alertmanager.yml index d95b147..c3e0395 100644 --- a/cookbooks/prometheus/files/etc/prometheus.d/alertmanager.yml +++ b/cookbooks/prometheus/files/etc/prometheus.d/alertmanager.yml @@ -15,3 +15,7 @@ receivers: title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}" text: "{{ range .Alerts }}{{ .Annotations.description }}\n{{ end }}" send_resolved: true + + webhook_configs: + - send_resolved: true + url: 'http://localhost:6725'