From 147e1bc643d9f3556fe762313d7b86b6657098c1 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 17 Oct 2020 13:59:00 +0900 Subject: [PATCH 1/6] Healthcheck for `filestat_exporter`. --- .../files/etc/consul.d/service-filestat_exporter.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json b/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json index 592d642..2262de1 100644 --- a/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json +++ b/cookbooks/prometheus-exporters/files/etc/consul.d/service-filestat_exporter.json @@ -1,6 +1,14 @@ { "service": { "name": "filestat-exporter", - "port": 60000 + "port": 60000, + "check":{ + "http": "http://localhost:60000/filestat_exporter/metrics", + "tls_skip_verify": false, + "interval": "60s", + "timeout": "1s", + "status": "passing", + "success_before_passing": 3 + } } } From ccc4f90bac83ce17ecaba677d4bf944e89effa7e Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 17 Oct 2020 13:59:53 +0900 Subject: [PATCH 2/6] Healthcheck for `node_exporter`. --- .../files/etc/consul.d/service-node_exporter.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json b/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json index e3de53e..8798d9a 100644 --- a/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json +++ b/cookbooks/prometheus-exporters/files/etc/consul.d/service-node_exporter.json @@ -1,6 +1,13 @@ { "service": { "name": "node-exporter", - "port": 60000 + "port": 60000, + "check":{ + "http": "http://localhost:60000/node_exporter/metrics", + "tls_skip_verify": false, + "interval": "60s", + "timeout": "1s", + "success_before_passing": 3 + } } } From 6315dc759e21d29df82d2009a297b8dade7c1305 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 17 Oct 2020 14:16:44 +0900 Subject: [PATCH 3/6] Healthcheck for `consul`. --- cookbooks/consul/files/etc/consul.d/service-consul.json | 8 +++++++- cookbooks/consul/setup.rb | 2 -- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cookbooks/consul/files/etc/consul.d/service-consul.json b/cookbooks/consul/files/etc/consul.d/service-consul.json index 10e7840..2ef9cf2 100644 --- a/cookbooks/consul/files/etc/consul.d/service-consul.json +++ b/cookbooks/consul/files/etc/consul.d/service-consul.json @@ -1,6 +1,12 @@ { "service": { "name": "consul", - "port": 8600 + "port": 8600, + "check":{ + "tcp": "localhost:8600", + "interval": "60s", + "timeout": "1s", + "success_before_passing": 3 + } } } diff --git a/cookbooks/consul/setup.rb b/cookbooks/consul/setup.rb index 9654250..94ed8af 100644 --- a/cookbooks/consul/setup.rb +++ b/cookbooks/consul/setup.rb @@ -21,8 +21,6 @@ remote_file '/etc/consul.d/service-consul.json' do owner 'root' group 'root' mode '644' - - only_if '{ node["consul"]["manager"]}' end execute 'Reload supervisor' do From 654a98b3ce826fc437bf6754cabc318f54bfc70e Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 17 Oct 2020 14:25:45 +0900 Subject: [PATCH 4/6] Healthcheck for `prometheus`. --- .../files/etc/consul.d/service-prometheus.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cookbooks/prometheus/files/etc/consul.d/service-prometheus.json b/cookbooks/prometheus/files/etc/consul.d/service-prometheus.json index 5bd41af..f80f227 100644 --- a/cookbooks/prometheus/files/etc/consul.d/service-prometheus.json +++ b/cookbooks/prometheus/files/etc/consul.d/service-prometheus.json @@ -1,6 +1,13 @@ { "service": { "name": "prometheus", - "port": 9100 + "port": 9090, + "check":{ + "http": "http://localhost:9090/", + "tls_skip_verify": false, + "interval": "60s", + "timeout": "1s", + "success_before_passing": 3 + } } } From 25a17917f5da3f188536c5168b0e70babd51a0fe Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 17 Oct 2020 14:41:02 +0900 Subject: [PATCH 5/6] Healthcheck for `Loki`. --- .../loki/files/etc/consul.d/service-loki.json | 6 --- cookbooks/loki/setup.rb | 10 ++++- .../etc/consul.d/service-loki.json.erb | 12 +++++ .../loki/templates/etc/loki/loki-config.yml | 45 +++++++++++++++++++ 4 files changed, 65 insertions(+), 8 deletions(-) delete mode 100644 cookbooks/loki/files/etc/consul.d/service-loki.json create mode 100644 cookbooks/loki/templates/etc/consul.d/service-loki.json.erb create mode 100644 cookbooks/loki/templates/etc/loki/loki-config.yml diff --git a/cookbooks/loki/files/etc/consul.d/service-loki.json b/cookbooks/loki/files/etc/consul.d/service-loki.json deleted file mode 100644 index 7f64580..0000000 --- a/cookbooks/loki/files/etc/consul.d/service-loki.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "service": { - "name": "loki", - "port": 3100 - } -} diff --git a/cookbooks/loki/setup.rb b/cookbooks/loki/setup.rb index 297fefb..575701f 100644 --- a/cookbooks/loki/setup.rb +++ b/cookbooks/loki/setup.rb @@ -8,10 +8,14 @@ end # Deploy `prometheus` files: -remote_file '/etc/loki/loki-config.yml' do +template '/etc/loki/loki-config.yml' do owner 'root' group 'root' mode '644' + + variables(ipaddr: node['consul']['ipaddr']) + + notifies :restart, 'service[loki]' end # Deploy `systemd` configuration for `prometheus`: @@ -27,11 +31,13 @@ service 'loki' do end # Depoy `consul` service configuration for `loki`: -remote_file '/etc/consul.d/service-loki.json' do +template '/etc/consul.d/service-loki.json' do owner 'root' group 'root' mode '644' + variables(ipaddr: node['consul']['ipaddr']) + notifies :restart, 'service[supervisor]' end diff --git a/cookbooks/loki/templates/etc/consul.d/service-loki.json.erb b/cookbooks/loki/templates/etc/consul.d/service-loki.json.erb new file mode 100644 index 0000000..d645fae --- /dev/null +++ b/cookbooks/loki/templates/etc/consul.d/service-loki.json.erb @@ -0,0 +1,12 @@ +{ + "service": { + "name": "loki", + "port": 3100, + "check":{ + "tcp": "<%= @ipaddr %>:3100", + "interval": "60s", + "timeout": "1s", + "success_before_passing": 3 + } + } +} diff --git a/cookbooks/loki/templates/etc/loki/loki-config.yml b/cookbooks/loki/templates/etc/loki/loki-config.yml new file mode 100644 index 0000000..9dc3aba --- /dev/null +++ b/cookbooks/loki/templates/etc/loki/loki-config.yml @@ -0,0 +1,45 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + max_transfer_retries: 0 + +schema_config: + configs: + - from: 2018-04-15 + store: boltdb + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 168h + +storage_config: + boltdb: + directory: /var/opt/loki/index + + filesystem: + directory: /var/opt/loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s From 06f5f13ca59e48c7ef2edb9af655c3bbfa5f94bf Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Sat, 17 Oct 2020 15:33:16 +0900 Subject: [PATCH 6/6] Healthcheck for `Grafana`. --- cookbooks/grafana/files/etc/consul.d/service-grafana.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cookbooks/grafana/files/etc/consul.d/service-grafana.json b/cookbooks/grafana/files/etc/consul.d/service-grafana.json index b535c68..aaed5dc 100644 --- a/cookbooks/grafana/files/etc/consul.d/service-grafana.json +++ b/cookbooks/grafana/files/etc/consul.d/service-grafana.json @@ -1,6 +1,12 @@ { "service": { "name": "grafana", - "port": 3000 + "port": 3000, + "check":{ + "tcp": "localhost:3000", + "tls_skip_verify": false, + "interval": "60s", + "timeout": "1s" + } } }