From e0c58066eca29529756551f1299ba1006176275c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 23:20:52 +0900 Subject: [PATCH 1/9] Install & Setup `Loki`. --- cookbooks/loki/attributes.rb | 11 ++++ cookbooks/loki/default.rb | 7 +++ cookbooks/loki/install.rb | 56 +++++++++++++++++++ cookbooks/loki/setup.rb | 103 +++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+) create mode 100644 cookbooks/loki/attributes.rb create mode 100644 cookbooks/loki/default.rb create mode 100644 cookbooks/loki/install.rb create mode 100644 cookbooks/loki/setup.rb diff --git a/cookbooks/loki/attributes.rb b/cookbooks/loki/attributes.rb new file mode 100644 index 0000000..bffdee8 --- /dev/null +++ b/cookbooks/loki/attributes.rb @@ -0,0 +1,11 @@ +# ------------------------------------------- +# Specifying the default settings: +# ------------------------------------------- +node.reverse_merge!({ + 'loki' => { + 'url' => 'https://github.com/grafana/loki/releases/download/', + 'zip' => 'loki-linux-amd64.zip', + 'storage' => '/opt/loki/', + 'location' => '/usr/local/bin/' + }, +}) diff --git a/cookbooks/loki/default.rb b/cookbooks/loki/default.rb new file mode 100644 index 0000000..f0eb864 --- /dev/null +++ b/cookbooks/loki/default.rb @@ -0,0 +1,7 @@ +# Loading the attributes: +include_recipe './attributes.rb' + +# Install loki here: +include_recipe './install.rb' +include_recipe './setup.rb' + diff --git a/cookbooks/loki/install.rb b/cookbooks/loki/install.rb new file mode 100644 index 0000000..b950d01 --- /dev/null +++ b/cookbooks/loki/install.rb @@ -0,0 +1,56 @@ +loki_url = '' +loki_bin = '' + +vtag = '' +tag = '' + +# Calculate the Download URL: +begin + require 'net/http' + + uri = URI.parse('https://github.com/grafana/loki/releases/latest') + + Timeout.timeout(3) do + response = Net::HTTP.get_response(uri) + + vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)} + tag = vtag.sub(/^v/, '') + + loki_bin = "#{node['loki']['zip']}" + loki_url = "#{node['loki']['url']}/#{vtag}/#{loki_bin}" + end +rescue + # Abort the chef client process: + raise 'Cannot connect to http://github.com.' +end + + +# バージョン確認して、アップデート必要かどうか確認 +result = run_command("loki --version 2>&1 | grep #{tag}", error: false) +if result.exit_status != 0 + # Download: + TMP = "/tmp/#{loki_bin}" + + execute "wget #{loki_url} -O #{TMP}" + + # Install: + directory node['loki']['storage'] do + owner 'root' + group 'root' + mode '755' + end + + execute "unzip -d #{node['loki']['storage']} #{TMP}" + + # Change Owner and Permissions: + file "#{node['loki']['storage']}loki-linux-amd64" do + owner 'root' + group 'root' + mode '755' + end + + # Create Link + link "#{node['loki']['location']}loki" do + to "#{node['loki']['storage']}loki-linux-amd64" + end +end diff --git a/cookbooks/loki/setup.rb b/cookbooks/loki/setup.rb new file mode 100644 index 0000000..297fefb --- /dev/null +++ b/cookbooks/loki/setup.rb @@ -0,0 +1,103 @@ +# Create `/etc/loki/`: +%w(/etc/loki).each do |d| + directory d do + owner 'root' + group 'root' + mode '0755' + end +end + +# Deploy `prometheus` files: +remote_file '/etc/loki/loki-config.yml' do + owner 'root' + group 'root' + mode '644' +end + +# Deploy `systemd` configuration for `prometheus`: +remote_file '/etc/systemd/system/loki.service' do + owner 'root' + group 'root' + mode '644' +end + +# Service setting: +service 'loki' do + action [ :enable, :restart ] +end + +# Depoy `consul` service configuration for `loki`: +remote_file '/etc/consul.d/service-loki.json' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[supervisor]' +end + +# Depoy `promtail` configuration for `loki`: +HOSTNAME = run_command('uname -n').stdout.chomp + +template '/etc/promtail/loki.yaml' do + owner 'root' + group 'root' + mode '644' + + variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['promtail']['lokiendpoint']) + + notifies :restart, 'service[promtail-loki]' +end + +# Deploy `systemd` configuration for `promtail-loki`: +remote_file '/etc/systemd/system/promtail-loki.service' do + owner 'root' + group 'root' + mode '644' +end + +# Service setting: +service 'promtail-loki' do + action [ :enable, :restart ] +end + +remote_file '/etc/rsyslog.d/30-loki.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[rsyslog]' +end + +service 'rsyslog' do + action [ :nothing ] +end + +# Deploy the `logrotated` configuration: +remote_file '/etc/logrotate.d/loki' do + owner 'root' + group 'root' + mode '644' +end + +# Restart the `supervisor`: +service 'supervisor' do + action :nothing +end + +# Firewall settings here: +%w( 3100/tcp ).each do |p| + execute "ufw allow #{p}" do + user 'root' + + not_if "LANG=c ufw status | grep #{p}" + + notifies :run, 'execute[ufw reload-or-enable]' + end +end + +execute 'ufw reload-or-enable' do + user 'root' + command 'LANG=C ufw reload | grep skipping && ufw --force enable || exit 0' + + action :nothing +end From 49869c48fa07eb051ba88c9207a1b6bac97aa0f9 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 17:00:36 +0900 Subject: [PATCH 2/9] Deploy `Loki` config. --- cookbooks/loki/files/etc/loki/loki-config.yml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 cookbooks/loki/files/etc/loki/loki-config.yml diff --git a/cookbooks/loki/files/etc/loki/loki-config.yml b/cookbooks/loki/files/etc/loki/loki-config.yml new file mode 100644 index 0000000..9dc3aba --- /dev/null +++ b/cookbooks/loki/files/etc/loki/loki-config.yml @@ -0,0 +1,45 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + max_transfer_retries: 0 + +schema_config: + configs: + - from: 2018-04-15 + store: boltdb + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 168h + +storage_config: + boltdb: + directory: /var/opt/loki/index + + filesystem: + directory: /var/opt/loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s From d6c76791b6dbd98847e9b96d7b78bccf21281f69 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 17:00:38 +0900 Subject: [PATCH 3/9] Deploy `systemd` config for `Loki`. --- cookbooks/loki/files/etc/systemd/system/loki.service | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cookbooks/loki/files/etc/systemd/system/loki.service diff --git a/cookbooks/loki/files/etc/systemd/system/loki.service b/cookbooks/loki/files/etc/systemd/system/loki.service new file mode 100644 index 0000000..75b3373 --- /dev/null +++ b/cookbooks/loki/files/etc/systemd/system/loki.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Loki +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/loki --config.file=/etc/loki/loki-config.yml + +[Install] +WantedBy=multi-user.target From 9aa481fa56e53ecc5168e75fb6af1d91274409de Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 18:08:58 +0900 Subject: [PATCH 4/9] Deploy `rsyslog` config for `loki` logs. --- cookbooks/loki/files/etc/rsyslog.d/30-loki.conf | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cookbooks/loki/files/etc/rsyslog.d/30-loki.conf diff --git a/cookbooks/loki/files/etc/rsyslog.d/30-loki.conf b/cookbooks/loki/files/etc/rsyslog.d/30-loki.conf new file mode 100644 index 0000000..980eef9 --- /dev/null +++ b/cookbooks/loki/files/etc/rsyslog.d/30-loki.conf @@ -0,0 +1,7 @@ +# Log kernel generated promtail log messages to file +:syslogtag,contains,"loki" /var/log/loki.log + +# Uncomment the following to stop logging anything that matches the last rule. +# Doing this will stop logging kernel generated UFW log messages to the file +# normally containing kern.* messages (eg, /var/log/kern.log) +& stop From b95f7a0383d21b0388974fc889fcb36d6c221591 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 18:09:44 +0900 Subject: [PATCH 5/9] Deploy `logrotated` config for `Loki`. --- cookbooks/loki/files/etc/logrotate.d/loki | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 cookbooks/loki/files/etc/logrotate.d/loki diff --git a/cookbooks/loki/files/etc/logrotate.d/loki b/cookbooks/loki/files/etc/logrotate.d/loki new file mode 100644 index 0000000..b236c96 --- /dev/null +++ b/cookbooks/loki/files/etc/logrotate.d/loki @@ -0,0 +1,13 @@ +/var/log/loki.log +{ + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} From 0a9b4983d06d3614cca26a51190030ab17a531f4 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 17:20:18 +0900 Subject: [PATCH 6/9] Deploy `promtail` config for `Loki`. --- .../loki/templates/etc/promtail/loki.yaml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 cookbooks/loki/templates/etc/promtail/loki.yaml diff --git a/cookbooks/loki/templates/etc/promtail/loki.yaml b/cookbooks/loki/templates/etc/promtail/loki.yaml new file mode 100644 index 0000000..a684f40 --- /dev/null +++ b/cookbooks/loki/templates/etc/promtail/loki.yaml @@ -0,0 +1,41 @@ +server: + disable: true + +positions: + filename: /var/opt/promtail/promtail_loki_position.yaml + +clients: + - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push + +scrape_configs: + - job_name: loki + static_configs: + - targets: + - localhost + labels: + job: loki + hostname: <%= @HOSTNAME %> + __path__: /var/log/loki.log + + pipeline_stages: + - match: + selector: '{job="loki"}' + stages: + - regex: + + expression: '^[^ ]+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ loki[^ ]+ .*level=(?P[^ ]+) ts=(?P[^ ]+) (?P.+)$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.999999999Z + location: Etc/GMT + + - template: + source: level + template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}' + + - labels: + level: + + - output: + source: message From f2c3a5557d79e7e8d7a20960b13984545d31c6db Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 17:21:07 +0900 Subject: [PATCH 7/9] Deploy `systemd` config for `promtail-loki`. --- .../files/etc/systemd/system/promtail-loki.service | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cookbooks/loki/files/etc/systemd/system/promtail-loki.service diff --git a/cookbooks/loki/files/etc/systemd/system/promtail-loki.service b/cookbooks/loki/files/etc/systemd/system/promtail-loki.service new file mode 100644 index 0000000..48f8d37 --- /dev/null +++ b/cookbooks/loki/files/etc/systemd/system/promtail-loki.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Promtail +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/loki.yaml + +[Install] +WantedBy=multi-user.target From e8c01fc6ed0ade554405fe539d9283953159bf1d Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 16:57:41 +0900 Subject: [PATCH 8/9] Deploy `Consul` config for `Loki`. --- cookbooks/loki/files/etc/consul.d/service-loki.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 cookbooks/loki/files/etc/consul.d/service-loki.json diff --git a/cookbooks/loki/files/etc/consul.d/service-loki.json b/cookbooks/loki/files/etc/consul.d/service-loki.json new file mode 100644 index 0000000..7f64580 --- /dev/null +++ b/cookbooks/loki/files/etc/consul.d/service-loki.json @@ -0,0 +1,6 @@ +{ + "service": { + "name": "loki", + "port": 3100 + } +} From 80175d609c8cfcc8572f38c6d502ba7aedb5ff1f Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Mon, 12 Oct 2020 23:20:49 +0900 Subject: [PATCH 9/9] Change `Loki` endpoint to `loki.service.consul:3100`. --- .../prometheus/templates/etc/promtail/prometheus.yaml | 8 ++++---- cookbooks/promtail/attributes.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cookbooks/prometheus/templates/etc/promtail/prometheus.yaml b/cookbooks/prometheus/templates/etc/promtail/prometheus.yaml index 8b091c8..e48839b 100644 --- a/cookbooks/prometheus/templates/etc/promtail/prometheus.yaml +++ b/cookbooks/prometheus/templates/etc/promtail/prometheus.yaml @@ -5,7 +5,7 @@ positions: filename: /var/opt/promtail/promtail_prometheus_position.yaml clients: - - url: http://192.168.10.118:3100/loki/api/v1/push + - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push scrape_configs: - job_name: prometheus @@ -33,7 +33,7 @@ scrape_configs: location: Etc/UTC - labels: - level: + level: - output: source: message @@ -63,7 +63,7 @@ scrape_configs: location: Etc/UTC - labels: - level: + level: - output: source: message @@ -104,7 +104,7 @@ scrape_configs: template: '{{ regexReplaceAllLiteral "resolved" .Value "notice" }}' - labels: - level: + level: - output: source: message diff --git a/cookbooks/promtail/attributes.rb b/cookbooks/promtail/attributes.rb index 473421a..71adf26 100644 --- a/cookbooks/promtail/attributes.rb +++ b/cookbooks/promtail/attributes.rb @@ -8,7 +8,7 @@ node.reverse_merge!({ 'storage' => '/opt/promtail/bin/', 'location' => '/usr/local/bin/', 'data' => '/var/opt/promtail/', - 'lokiendpoint' => '192.168.10.118:3100' + 'lokiendpoint' => 'loki.service.consul:3100' }, })