From 057b7fa979cb2d3601ef7dd20fcff1368b8d5700 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 23:56:41 +0900 Subject: [PATCH] Create dedicated cookbook for `Prometheus` exporters. --- cookbooks/prometheus-exporters/attributes.rb | 48 +++++++++++++++ cookbooks/prometheus-exporters/default.rb | 21 +++++++ .../prometheus-exporters/exporter_proxy.rb | 60 +++++++++++++++++++ .../consul.d/service-filestat_exporter.json | 6 ++ .../etc/consul.d/service-node_exporter.json | 6 ++ .../exporter_proxy/config.yml | 21 +++++++ .../etc/prometheus_exporters.d/filestat.yml | 16 +++++ .../etc/supervisor/conf.d/exporter_proxy.conf | 8 +++ .../supervisor/conf.d/filestat_exporter.conf | 8 +++ .../etc/supervisor/conf.d/node_exporter.conf | 8 +++ .../system/promtail-prometheus.service | 12 ++++ .../filestat_exporter_install.rb | 53 ++++++++++++++++ .../filestat_exporter_setup.rb | 31 ++++++++++ .../node_exporter_install.rb | 55 +++++++++++++++++ .../node_exporter_setup.rb | 39 ++++++++++++ 15 files changed, 392 insertions(+) create mode 100644 cookbooks/prometheus-exporters/attributes.rb create mode 100644 cookbooks/prometheus-exporters/default.rb create mode 100644 cookbooks/prometheus-exporters/exporter_proxy.rb create mode 100644 cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json create mode 100644 cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json create mode 100644 cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/exporter_proxy/config.yml create mode 100644 cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/filestat.yml create mode 100644 cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/exporter_proxy.conf create mode 100644 cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/filestat_exporter.conf create mode 100644 cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/node_exporter.conf create mode 100644 cookbooks/prometheus-exporters/files/lib/systemd/system/promtail-prometheus.service create mode 100644 cookbooks/prometheus-exporters/filestat_exporter_install.rb create mode 100644 cookbooks/prometheus-exporters/filestat_exporter_setup.rb create mode 100644 cookbooks/prometheus-exporters/node_exporter_install.rb create mode 100644 cookbooks/prometheus-exporters/node_exporter_setup.rb diff --git a/cookbooks/prometheus-exporters/attributes.rb b/cookbooks/prometheus-exporters/attributes.rb new file mode 100644 index 0000000..221f13f --- /dev/null +++ b/cookbooks/prometheus-exporters/attributes.rb @@ -0,0 +1,48 @@ +# ------------------------------------------- +# Specifying the default settings: +# ------------------------------------------- +node.reverse_merge!({ + 'prometheus' => { + 'manager' => false, + 'url' => 'https://github.com/prometheus/prometheus/releases/download/', + 'prefix' => 'prometheus-', + 'postfix' => '.linux-amd64.tar.gz', + 'storage' => '/opt/prometheus/', + 'location' => '/usr/local/bin/' + }, + 'node_exporter' => { + 'url' => 'https://github.com/prometheus/node_exporter/releases/download/', + 'prefix' => 'node_exporter-', + 'postfix' => '.linux-amd64.tar.gz', + 'storage' => '/opt/node_exporter/bin/', + 'location' => '/usr/local/bin/' + }, + 'blackbox_exporter' => { + 'url' => 'https://github.com/prometheus/blackbox_exporter/releases/download/', + 'prefix' => 'blackbox_exporter-', + 'postfix' => '.linux-amd64.tar.gz', + 'storage' => '/opt/blackbox_exporter/bin/', + 'location' => '/usr/local/bin/' + }, + 'alertmanager' => { + 'url' => 'https://github.com/prometheus/alertmanager/releases/download/', + 'prefix' => 'alertmanager-', + 'postfix' => '.linux-amd64.tar.gz', + 'storage' => '/opt/prometheus/', + 'location' => '/usr/local/bin/' + }, + 'alertmanager_webhook' => { + 'url' => 'https://github.com/tomtom-international/alertmanager-webhook-logger/releases/download/', + 'prefix' => 'alertmanager-webhook-logger-', + 'postfix' => '.tar.gz', + 'storage' => '/opt/prometheus/', + 'location' => '/usr/local/bin/' + }, + 'filestat_exporter' => { + 'url' => 'https://github.com/michael-doubez/filestat_exporter/releases/download/', + 'prefix' => 'filestat_exporter-', + 'postfix' => '.linux-amd64.tar.gz', + 'storage' => '/opt/filestat_exporter/', + 'location' => '/usr/local/bin/' + }, +}) diff --git a/cookbooks/prometheus-exporters/default.rb b/cookbooks/prometheus-exporters/default.rb new file mode 100644 index 0000000..81ca436 --- /dev/null +++ b/cookbooks/prometheus-exporters/default.rb @@ -0,0 +1,21 @@ +# Loading the attributes: +include_recipe './attributes.rb' + +# Create `/etc/prometheus.d/`: +%w(/etc/prometheus_exporters.d).each do |d| + directory d do + owner 'root' + group 'root' + mode '0755' + end +end + +# Install the node_exporter here: +include_recipe './node_exporter_install.rb' +include_recipe './node_exporter_setup.rb' + +# Install the filestat_exporter here: +include_recipe './filestat_exporter_install.rb' +include_recipe './filestat_exporter_setup.rb' + +include_recipe './exporter_proxy.rb' diff --git a/cookbooks/prometheus-exporters/exporter_proxy.rb b/cookbooks/prometheus-exporters/exporter_proxy.rb new file mode 100644 index 0000000..55ffca2 --- /dev/null +++ b/cookbooks/prometheus-exporters/exporter_proxy.rb @@ -0,0 +1,60 @@ +URL = 'https://github.com/rrreeeyyy/exporter_proxy/releases/download/v0.1.0/exporter_proxy_linux_amd64' +BIN = '/usr/local/bin/exporter_proxy' +CONFDIR = '/etc/prometheus_exporters.d/exporter_proxy/' +CONF = 'config.yml' + +execute "wget #{URL} -O #{BIN}" do + not_if "test -e #{BIN}" +end + +file BIN do + user 'root' + group 'root' + + mode '755' +end + +directory CONFDIR do + user 'root' + group 'root' + + mode '755' +end + +remote_file "#{CONFDIR}#{CONF}" do + user 'root' + group 'root' + + mode '644' +end + +remote_file '/etc/supervisor/conf.d/exporter_proxy.conf' do + user 'root' + group 'root' + + mode '644' + + notifies :restart, 'service[supervisor]' +end + +service 'supervisor' do + action :nothing +end + +# Firewall settings here: +%w( 60000/tcp ).each do |p| + execute "ufw allow #{p}" do + user 'root' + + not_if "LANG=c ufw status | grep #{p}" + + notifies :run, 'execute[ufw reload-or-enable]' + end +end + +execute 'ufw reload-or-enable' do + user 'root' + command 'LANG=C ufw reload | grep skipping && ufw --force enable || exit 0' + + action :nothing +end diff --git a/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json b/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json new file mode 100644 index 0000000..592d642 --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json @@ -0,0 +1,6 @@ +{ + "service": { + "name": "filestat-exporter", + "port": 60000 + } +} diff --git a/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json b/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json new file mode 100644 index 0000000..e3de53e --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json @@ -0,0 +1,6 @@ +{ + "service": { + "name": "node-exporter", + "port": 60000 + } +} diff --git a/cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/exporter_proxy/config.yml b/cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/exporter_proxy/config.yml new file mode 100644 index 0000000..a6d0d27 --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/exporter_proxy/config.yml @@ -0,0 +1,21 @@ +# listen: Addr and port to listen (required) +listen: "0.0.0.0:60000" + +# access_log (optional) +access_log: + path: "/dev/stdout" + format: "ltsv" + fields: ['time', 'time_nsec', 'status', 'size', 'reqtime_nsec', 'backend', 'path', 'query', 'method'] + +# error_log (required) +error_log: + path: "/dev/stderr" + +# exporters: The path of exporter_proxy and the URL of the destination exporter +exporters: + node_exporter: + path: "/node_exporter/metrics" + url: "http://127.0.0.1:9100/metrics" + filestat_exporter: + path: "/filestat_exporter/metrics" + url: "http://127.0.0.1:9943/metrics" diff --git a/cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/filestat.yml b/cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/filestat.yml new file mode 100644 index 0000000..2b11ade --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/prometheus_exporters.d/filestat.yml @@ -0,0 +1,16 @@ +exporter: + # Optional network parameters + listen_address: ':9943' + metrics_path: /metrics + + # Optional working directory - overridden by parameter '-path.cwd' + # working_directory: "/path/to/my/project" + + # Default enable/disable of metrics - overridden if not set by parameter '-metric.*' + enable_crc32_metric: true + enable_nb_line_metric: false + # list of patterns to apply - metrics can be enable/disabled for each group + files: + - patterns: ["/var/run/reboot-required"] + enable_crc32_metric: false + enable_nb_line_metric: false diff --git a/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/exporter_proxy.conf b/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/exporter_proxy.conf new file mode 100644 index 0000000..60b6944 --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/exporter_proxy.conf @@ -0,0 +1,8 @@ +[program:exporter_proxy] +command=/usr/local/bin/exporter_proxy -config /etc/prometheus_exporters.d/exporter_proxy/config.yml +stdout_logfile=/var/log/supervisor/exporter_proxy.log +redirect_stderr=true +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=5 +autorestart=true +stopsignal=HUP diff --git a/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/filestat_exporter.conf b/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/filestat_exporter.conf new file mode 100644 index 0000000..c61c76c --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/filestat_exporter.conf @@ -0,0 +1,8 @@ +[program:filestat_exporter] +command=/usr/local/bin/filestat_exporter --config.file=/etc/prometheus_exporters.d/filestat.yml +stdout_logfile=/var/log/supervisor/filestat_exporter.log +redirect_stderr=true +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=5 +autorestart=true +stopsignal=HUP diff --git a/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/node_exporter.conf b/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/node_exporter.conf new file mode 100644 index 0000000..b5e5c04 --- /dev/null +++ b/cookbooks/prometheus-exporters/files/etc/supervisor/conf.d/node_exporter.conf @@ -0,0 +1,8 @@ +[program:node_exporter] +command=/usr/local/bin/node_exporter --web.listen-address="127.0.0.1:9100" +stdout_logfile=/var/log/supervisor/node_exporter.log +redirect_stderr=true +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=5 +autorestart=true +stopsignal=HUP diff --git a/cookbooks/prometheus-exporters/files/lib/systemd/system/promtail-prometheus.service b/cookbooks/prometheus-exporters/files/lib/systemd/system/promtail-prometheus.service new file mode 100644 index 0000000..a52887f --- /dev/null +++ b/cookbooks/prometheus-exporters/files/lib/systemd/system/promtail-prometheus.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Promtail for Prometheus +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/prometheus.yaml + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/prometheus-exporters/filestat_exporter_install.rb b/cookbooks/prometheus-exporters/filestat_exporter_install.rb new file mode 100644 index 0000000..a1fa6db --- /dev/null +++ b/cookbooks/prometheus-exporters/filestat_exporter_install.rb @@ -0,0 +1,53 @@ +filestat_exporter_url = '' +filestat_exporter_bin = '' + +vtag = '' + +# Calculate the Download URL: +begin + require 'net/http' + + uri = URI.parse('https://github.com/michael-doubez/filestat_exporter/releases/latest') + + Timeout.timeout(3) do + response = Net::HTTP.get_response(uri) + + vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)} + + filestat_exporter_bin = "#{node['filestat_exporter']['prefix']}#{vtag}#{node['filestat_exporter']['postfix']}" + filestat_exporter_url = "#{node['filestat_exporter']['url']}/#{vtag}/#{filestat_exporter_bin}" + end +rescue + # Abort the chef client process: + raise 'Cannot connect to http://github.com.' +end + +# バージョン確認して、アップデート必要かどうか確認 +result = run_command("filestat_exporter --version 2>&1 | grep #{vtag}", error: false) +if result.exit_status != 0 + # Download: + TMP = "/tmp/#{filestat_exporter_bin}" + + execute "wget #{filestat_exporter_url} -O #{TMP}" + + # Install: + directory node['filestat_exporter']['storage'] do + owner 'root' + group 'root' + mode '755' + end + + execute "tar zxf #{TMP} -C #{node['filestat_exporter']['storage']} --strip-components 1" + + # Change Owner and Permissions: + file "#{node['filestat_exporter']['storage']}filestat_exporter" do + owner 'root' + group 'root' + mode '755' + end + + # Create Link + link "#{node['filestat_exporter']['location']}filestat_exporter" do + to "#{node['filestat_exporter']['storage']}filestat_exporter" + end +end diff --git a/cookbooks/prometheus-exporters/filestat_exporter_setup.rb b/cookbooks/prometheus-exporters/filestat_exporter_setup.rb new file mode 100644 index 0000000..693bfe2 --- /dev/null +++ b/cookbooks/prometheus-exporters/filestat_exporter_setup.rb @@ -0,0 +1,31 @@ +# Deploy the `supervisord` configuration: +remote_file '/etc/prometheus_exporters.d/filestat.yml' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Deploy the `supervisord` configuration: +remote_file '/etc/supervisor/conf.d/filestat_exporter.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Deploy `consul` config for `node_exporter`: +remote_file '/etc/consul.d/service-filestat_exporter.json' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +service 'supervisor' do + action :nothing +end + diff --git a/cookbooks/prometheus-exporters/node_exporter_install.rb b/cookbooks/prometheus-exporters/node_exporter_install.rb new file mode 100644 index 0000000..0eeab37 --- /dev/null +++ b/cookbooks/prometheus-exporters/node_exporter_install.rb @@ -0,0 +1,55 @@ +node_exporter_url = '' +node_exporter_bin = '' + +tag = '' +vtag = '' + +# Calculate the Download URL: +begin + require 'net/http' + + uri = URI.parse('https://github.com/prometheus/node_exporter/releases/latest') + + Timeout.timeout(3) do + response = Net::HTTP.get_response(uri) + + vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)} + tag = vtag.sub(/^v/, '') + + node_exporter_bin = "#{node['node_exporter']['prefix']}#{tag}#{node['node_exporter']['postfix']}" + node_exporter_url = "#{node['node_exporter']['url']}/#{vtag}/#{node_exporter_bin}" + end +rescue + # Abort the chef client process: + raise 'Cannot connect to http://github.com.' +end + +# バージョン確認して、アップデート必要かどうか確認 +result = run_command("node_exporter --version 2>&1 | grep #{tag}", error: false) +if result.exit_status != 0 + # Download: + TMP = "/tmp/#{node_exporter_bin}" + + execute "wget #{node_exporter_url} -O #{TMP}" + + # Install: + directory node['node_exporter']['storage'] do + owner 'root' + group 'root' + mode '755' + end + + execute "tar zxf #{TMP} -C #{node['node_exporter']['storage']} --strip-components 1" + + # Change Owner and Permissions: + file "#{node['node_exporter']['storage']}node_exporter" do + owner 'root' + group 'root' + mode '755' + end + + # Create Link + link "#{node['node_exporter']['location']}node_exporter" do + to "#{node['node_exporter']['storage']}node_exporter" + end +end diff --git a/cookbooks/prometheus-exporters/node_exporter_setup.rb b/cookbooks/prometheus-exporters/node_exporter_setup.rb new file mode 100644 index 0000000..9512cd4 --- /dev/null +++ b/cookbooks/prometheus-exporters/node_exporter_setup.rb @@ -0,0 +1,39 @@ +# Deploy the `supervisord` configuration: +remote_file '/etc/supervisor/conf.d/node_exporter.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Deploy `consul` config for `node_exporter`: +remote_file '/etc/consul.d/service-node_exporter.json' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +service 'supervisor' do + action :nothing +end + +# Firewall settings here: +%w( 9100/tcp ).each do |p| + execute "ufw allow #{p}" do + user 'root' + + not_if "LANG=c ufw status | grep #{p}" + + notifies :run, 'execute[ufw reload-or-enable]' + end +end + +execute 'ufw reload-or-enable' do + user 'root' + command 'LANG=C ufw reload | grep skipping && ufw --force enable || exit 0' + + action :nothing +end