From 8931d6709c03d97d40e8063dbcc0047404914291 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Tue, 1 Sep 2020 01:21:59 +0900 Subject: [PATCH 1/6] Create the entrypoint .rb. --- cookbooks/promtail/default.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 cookbooks/promtail/default.rb diff --git a/cookbooks/promtail/default.rb b/cookbooks/promtail/default.rb new file mode 100644 index 0000000..340c25a --- /dev/null +++ b/cookbooks/promtail/default.rb @@ -0,0 +1,6 @@ +# Loading the attributes: +include_recipe './attributes.rb' + +include_recipe './install.rb' + +include_recipe './setup.rb' From ee975266e0a2af76cd1c33e5254ab02a5e37ce7c Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Tue, 1 Sep 2020 01:32:27 +0900 Subject: [PATCH 2/6] Specify the cookbook-specific attiributes. --- cookbooks/promtail/attributes.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 cookbooks/promtail/attributes.rb diff --git a/cookbooks/promtail/attributes.rb b/cookbooks/promtail/attributes.rb new file mode 100644 index 0000000..473421a --- /dev/null +++ b/cookbooks/promtail/attributes.rb @@ -0,0 +1,14 @@ +# ------------------------------------------- +# Specifying the default settings: +# ------------------------------------------- +node.reverse_merge!({ + 'promtail' => { + 'url' => 'https://github.com/grafana/loki/releases/download/', + 'bin' => 'promtail-linux-amd64.zip', + 'storage' => '/opt/promtail/bin/', + 'location' => '/usr/local/bin/', + 'data' => '/var/opt/promtail/', + 'lokiendpoint' => '192.168.10.118:3100' + }, +}) + From 872382df1dc9ef1765cebc378a1186bb2dd8b8d3 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Tue, 1 Sep 2020 02:02:10 +0900 Subject: [PATCH 3/6] Install `promtail`. --- cookbooks/promtail/install.rb | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 cookbooks/promtail/install.rb diff --git a/cookbooks/promtail/install.rb b/cookbooks/promtail/install.rb new file mode 100644 index 0000000..bcb69af --- /dev/null +++ b/cookbooks/promtail/install.rb @@ -0,0 +1,55 @@ +promtail_url = '' +promtail_bin = '' + +tag = '' +vtag = '' + +# Calculate the Download URL: +begin + require 'net/http' + + uri = URI.parse('https://github.com/grafana/loki/releases/latest') + + Timeout.timeout(3) do + response = Net::HTTP.get_response(uri) + + vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)} + tag = vtag.sub(/^v/, '') + + promtail_url = "#{node['promtail']['url']}/#{vtag}/#{node['promtail']['bin']}" + end +rescue + # Abort the chef client process: + raise 'Cannot connect to http://github.com.' +end + +# バージョン確認して、アップデート必要かどうか確認 +result = run_command("promtail --version 2>&1 | grep #{tag}", error: false) +if result.exit_status != 0 + # Download: + TMP = "/tmp/#{node['promtail']['bin']}" + + execute "wget #{promtail_url} -O #{TMP}" + + # Install: + directory node['promtail']['storage'] do + owner 'root' + group 'root' + mode '755' + end + + execute "unzip #{TMP} -d #{node['promtail']['storage']}" + execute "mv #{node['promtail']['storage']}promtail-linux-amd64 #{node['promtail']['storage']}promtail" + + # Change Owner and Permissions: + file "#{node['promtail']['storage']}promtail" do + owner 'root' + group 'root' + mode '755' + end + + # Create Link + link "#{node['promtail']['location']}promtail" do + to "#{node['promtail']['storage']}promtail" + end +end From e750ac0ce92bb5b52b32dc657980b7a1e6203320 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Wed, 2 Sep 2020 00:21:29 +0900 Subject: [PATCH 4/6] Set up `promtail`: - Deploy `systemd` config file - Deploy `promtail` basic config file - Deploy `rsyslog` config file - Deploy `logrotated` config file --- .../promtail/files/etc/logrotate.d/promtail | 13 ++ .../files/etc/rsyslog.d/30-promtail.conf | 7 + .../lib/systemd/system/promtail-base.service | 12 ++ cookbooks/promtail/setup.rb | 51 +++++ .../promtail/templates/etc/promtail/base.yaml | 186 ++++++++++++++++++ 5 files changed, 269 insertions(+) create mode 100644 cookbooks/promtail/files/etc/logrotate.d/promtail create mode 100644 cookbooks/promtail/files/etc/rsyslog.d/30-promtail.conf create mode 100644 cookbooks/promtail/files/lib/systemd/system/promtail-base.service create mode 100644 cookbooks/promtail/setup.rb create mode 100644 cookbooks/promtail/templates/etc/promtail/base.yaml diff --git a/cookbooks/promtail/files/etc/logrotate.d/promtail b/cookbooks/promtail/files/etc/logrotate.d/promtail new file mode 100644 index 0000000..4ed9451 --- /dev/null +++ b/cookbooks/promtail/files/etc/logrotate.d/promtail @@ -0,0 +1,13 @@ +/var/log/promtail.log +{ + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} diff --git a/cookbooks/promtail/files/etc/rsyslog.d/30-promtail.conf b/cookbooks/promtail/files/etc/rsyslog.d/30-promtail.conf new file mode 100644 index 0000000..9f6d43f --- /dev/null +++ b/cookbooks/promtail/files/etc/rsyslog.d/30-promtail.conf @@ -0,0 +1,7 @@ +# Log kernel generated promtail log messages to file +:syslogtag,contains,"promtail" /var/log/promtail.log + +# Uncomment the following to stop logging anything that matches the last rule. +# Doing this will stop logging kernel generated UFW log messages to the file +# normally containing kern.* messages (eg, /var/log/kern.log) +& stop diff --git a/cookbooks/promtail/files/lib/systemd/system/promtail-base.service b/cookbooks/promtail/files/lib/systemd/system/promtail-base.service new file mode 100644 index 0000000..1766380 --- /dev/null +++ b/cookbooks/promtail/files/lib/systemd/system/promtail-base.service @@ -0,0 +1,12 @@ +[Unit] +Description=Grafana Promtail +Documentation=https://github.com/grafana/loki +After=network-online.target + +[Service] +User=root +Restart=always +ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/base.yaml + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/promtail/setup.rb b/cookbooks/promtail/setup.rb new file mode 100644 index 0000000..1ac3083 --- /dev/null +++ b/cookbooks/promtail/setup.rb @@ -0,0 +1,51 @@ +# Deploy the configuration file: +%w( /etc/promtail /var/opt/promtail ).each do |d| + directory d do + owner 'root' + group 'root' + mode '755' + end +end + +# Deploy /etc/hosts file: +HOSTNAME = run_command('uname -n').stdout.chomp + +template '/etc/promtail/base.yaml' do + owner 'root' + group 'root' + mode '644' + + variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['promtail']['lokiendpoint']) +end + +# Deploy the `systemd` configuration: +remote_file '/lib/systemd/system/promtail-base.service' do + owner 'root' + group 'root' + mode '644' +end + +# Service setting: +service 'promtail-base' do + action [ :enable, :restart ] +end + +# Deploy the `systemd` configuration: +remote_file '/etc/rsyslog.d/30-promtail.conf' do + owner 'root' + group 'root' + mode '644' + + notifies :restart, 'service[rsyslog]' +end + +service 'rsyslog' do + action [ :nothing ] +end + +# Deploy the `logrotated` configuration: +remote_file '/etc/logrotate.d/promtail' do + owner 'root' + group 'root' + mode '644' +end diff --git a/cookbooks/promtail/templates/etc/promtail/base.yaml b/cookbooks/promtail/templates/etc/promtail/base.yaml new file mode 100644 index 0000000..3c7307d --- /dev/null +++ b/cookbooks/promtail/templates/etc/promtail/base.yaml @@ -0,0 +1,186 @@ +server: + disable: true + +positions: + filename: /var/opt/promtail/promtail_base_position.yaml + +clients: + - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push + +scrape_configs: + - job_name: apt + static_configs: + - targets: + - localhost + labels: + job: apt + hostname: <%= @HOSTNAME %> + level: notice + __path__: /var/log/apt/history.log + + - job_name: sudo + static_configs: + - targets: + - localhost + labels: + job: sudo + hostname: <%= @HOSTNAME %> + level: notice + __path__: /var/log/auth.log + + pipeline_stages: + - match: + selector: '{job="sudo"}' + stages: + - drop: + expression: (CRON|sshd|session) + - regex: + expression: '^(?P\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^ ]+ sudo: +(?P[^ ]+) : TTY=(?P[^ ]+) ; PWD=(?P[^ ]+) ; USER=(?P[^ ]+) ; COMMAND=(?P.+)$' + + - timestamp: + source: timestamp + format: Jan 2 15:04:05 + location: Asia/Tokyo + + - template: + source: message + template: 'USER={{ .user }} PWD={{ .pwd }} CMD={{ .cmd }}' + + - output: + source: message + + - job_name: sshd + static_configs: + - targets: + - localhost + labels: + job: sshd + hostname: <%= @HOSTNAME %> + level: warning + __path__: /var/log/auth.log + + pipeline_stages: + - match: + selector: '{job="sshd"}' + + stages: + - drop: + expression: (CRON|sudo) + + - regex: + expression: '^(?P\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^:]+: (?P.+)$' + + - timestamp: + source: timestamp + format: Jan 2 15:04:05 + location: Asia/Tokyo + + - output: + source: message + + - job_name: supervisord + static_configs: + - targets: + - localhost + labels: + job: supervisord + hostname: <%= @HOSTNAME %> + level: notice + __path__: /var/log/supervisor/supervisord.log + + pipeline_stages: + - match: + selector: '{job="supervisord"}' + stages: + - regex: + expression: '^(?P[0-9]+\-[0-9]+\-[0-9]+ [0-9]+:[0-9]+:[0-9]+),[0-9]+ (?P[^ ]+) (?P.+)$' + + - timestamp: + source: timestamp + format: 2006-01-02 15:04:05 + location: Asia/Tokyo + + - template: + source: level + template: '{{ ToLower .level }}' + + - template: + source: level + template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}' + + - template: + source: level + template: '{{ regexReplaceAllLiteral "crit" .Value "critical" }}' + + - labels: + level: + + - output: + source: message + + - job_name: fail2ban + static_configs: + - targets: + - localhost + labels: + job: fail2ban + hostname: <%= @HOSTNAME %> + level: notice + __path__: /var/log/fail2ban.log + + pipeline_stages: + - match: + selector: '{job="fail2ban"}' + stages: + - regex: + expression: '^(?P[0-9]+\-[0-9]+\-[0-9]+ [0-9]+:[0-9]+:[0-9]+),[0-9]+ [^:]+: (?P[^ ]+)[^\[]+(?P.+)$' + + + - timestamp: + source: timestamp + format: 2006-01-02 15:04:05 + location: Asia/Tokyo + + - template: + source: level + template: '{{ ToLower .level }}' + + - labels: + level: + + - output: + source: message + + - job_name: promtail + static_configs: + - targets: + - localhost + labels: + job: promtail + hostname: <%= @HOSTNAME %> + __path__: /var/log/promtail.log + + pipeline_stages: + - match: + selector: '{job="promtail"}' + stages: + - regex: + expression: '^[^ ]+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ promtail[^ ]+ .*ts=(?P[^ ]+) (?P.+)$' + + - timestamp: + source: timestamp + format: 2006-01-02T15:04:05.999999999Z + location: Etc/GMT + + - regex: + expression: '^[^ ]+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ promtail[^ ]+ .*level=(?P[^\\" ]+).*$' + + - template: + source: level + template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}' + + - labels: + level: + + - output: + source: message From 211bbe79c725369822637503efb9f10214a352a7 Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Tue, 1 Sep 2020 01:45:00 +0900 Subject: [PATCH 5/6] Unregister `fluentd` cookbook. --- roles/base.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/base.rb b/roles/base.rb index 53bd52f..f323aee 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -5,5 +5,4 @@ include_recipe '../cookbooks/monit/default.rb' include_recipe '../cookbooks/consul/default.rb' include_recipe '../cookbooks/consul-template/default.rb' include_recipe '../cookbooks/fzf/default.rb' -include_recipe '../cookbooks/fluentd/default.rb' include_recipe '../cookbooks/prometheus/default.rb' From 9841ff26e8341eb32bc3c82dd20ac816b48d820b Mon Sep 17 00:00:00 2001 From: Kazuhiro MUSASHI Date: Tue, 1 Sep 2020 01:45:18 +0900 Subject: [PATCH 6/6] Instead, deploy `promtail` cookbook. --- roles/base.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/base.rb b/roles/base.rb index f323aee..c0fb200 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -5,4 +5,5 @@ include_recipe '../cookbooks/monit/default.rb' include_recipe '../cookbooks/consul/default.rb' include_recipe '../cookbooks/consul-template/default.rb' include_recipe '../cookbooks/fzf/default.rb' +include_recipe '../cookbooks/promtail/default.rb' include_recipe '../cookbooks/prometheus/default.rb'