Merge branch 'promtail' of kazu634/itamae into master

cookbooks/promtail/attributes.rb

@@ -0,0 +1,14 @@
+# -------------------------------------------
+# Specifying the default settings:
+# -------------------------------------------
+node.reverse_merge!({
+  'promtail' => {
+    'url' => 'https://github.com/grafana/loki/releases/download/',
+    'bin' => 'promtail-linux-amd64.zip',
+    'storage' => '/opt/promtail/bin/',
+    'location' => '/usr/local/bin/',
+    'data' => '/var/opt/promtail/',
+    'lokiendpoint' => '192.168.10.118:3100'
+  },
+})
+

cookbooks/promtail/default.rb

@@ -0,0 +1,6 @@
+# Loading the attributes:
+include_recipe './attributes.rb'
+
+include_recipe './install.rb'
+
+include_recipe './setup.rb'

cookbooks/promtail/files/etc/logrotate.d/promtail

@@ -0,0 +1,13 @@
+/var/log/promtail.log
+{
+        rotate 4
+        weekly
+        missingok
+        notifempty
+        compress
+        delaycompress
+        sharedscripts
+        postrotate
+                /usr/lib/rsyslog/rsyslog-rotate
+        endscript
+}

cookbooks/promtail/files/etc/rsyslog.d/30-promtail.conf

@@ -0,0 +1,7 @@
+# Log kernel generated promtail log messages to file
+:syslogtag,contains,"promtail" /var/log/promtail.log
+
+# Uncomment the following to stop logging anything that matches the last rule.
+# Doing this will stop logging kernel generated UFW log messages to the file
+# normally containing kern.* messages (eg, /var/log/kern.log)
+& stop

cookbooks/promtail/files/lib/systemd/system/promtail-base.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Grafana Promtail
+Documentation=https://github.com/grafana/loki
+After=network-online.target
+
+[Service]
+User=root
+Restart=always
+ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/base.yaml
+
+[Install]
+WantedBy=multi-user.target

cookbooks/promtail/install.rb

@@ -0,0 +1,55 @@
+promtail_url = ''
+promtail_bin = ''
+
+tag               = ''
+vtag              = ''
+
+# Calculate the Download URL:
+begin
+  require 'net/http'
+
+  uri = URI.parse('https://github.com/grafana/loki/releases/latest')
+
+  Timeout.timeout(3) do
+    response = Net::HTTP.get_response(uri)
+
+    vtag = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)}
+    tag = vtag.sub(/^v/, '')
+
+    promtail_url = "#{node['promtail']['url']}/#{vtag}/#{node['promtail']['bin']}"
+  end
+rescue
+  # Abort the chef client process:
+  raise 'Cannot connect to http://github.com.'
+end
+
+# バージョン確認して、アップデート必要かどうか確認
+result = run_command("promtail --version 2>&1 | grep #{tag}", error: false)
+if result.exit_status != 0
+  # Download:
+  TMP = "/tmp/#{node['promtail']['bin']}"
+
+  execute "wget #{promtail_url} -O #{TMP}"
+
+  # Install:
+  directory node['promtail']['storage'] do
+    owner 'root'
+    group 'root'
+    mode '755'
+  end
+
+  execute "unzip #{TMP} -d #{node['promtail']['storage']}"
+  execute "mv #{node['promtail']['storage']}promtail-linux-amd64 #{node['promtail']['storage']}promtail"
+
+  # Change Owner and Permissions:
+  file "#{node['promtail']['storage']}promtail" do
+    owner 'root'
+    group 'root'
+    mode  '755'
+  end
+
+  # Create Link
+  link "#{node['promtail']['location']}promtail" do
+    to "#{node['promtail']['storage']}promtail"
+  end
+end

cookbooks/promtail/setup.rb

@@ -0,0 +1,51 @@
+# Deploy the configuration file:
+%w( /etc/promtail /var/opt/promtail ).each do |d|
+  directory d do
+    owner 'root'
+    group 'root'
+    mode '755'
+  end
+end
+
+# Deploy /etc/hosts file:
+HOSTNAME = run_command('uname -n').stdout.chomp
+
+template '/etc/promtail/base.yaml' do
+  owner 'root'
+  group 'root'
+  mode '644'
+
+  variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['promtail']['lokiendpoint'])
+end
+
+# Deploy the `systemd` configuration:
+remote_file '/lib/systemd/system/promtail-base.service' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end
+
+# Service setting:
+service 'promtail-base' do
+  action [ :enable, :restart ]
+end
+
+# Deploy the `systemd` configuration:
+remote_file '/etc/rsyslog.d/30-promtail.conf' do
+  owner 'root'
+  group 'root'
+  mode '644'
+
+  notifies :restart, 'service[rsyslog]'
+end
+
+service 'rsyslog' do
+  action [ :nothing ]
+end
+
+# Deploy the `logrotated` configuration:
+remote_file '/etc/logrotate.d/promtail' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end

cookbooks/promtail/templates/etc/promtail/base.yaml

@@ -0,0 +1,186 @@
+server:
+  disable: true
+
+positions:
+  filename: /var/opt/promtail/promtail_base_position.yaml
+
+clients:
+  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
+
+scrape_configs:
+  - job_name: apt
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: apt
+        hostname: <%= @HOSTNAME %>
+        level: notice
+        __path__: /var/log/apt/history.log
+
+  - job_name: sudo
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: sudo
+        hostname: <%= @HOSTNAME %>
+        level: notice
+        __path__: /var/log/auth.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="sudo"}'
+        stages:
+        - drop:
+            expression: (CRON|sshd|session)
+        - regex:
+            expression: '^(?P<timestamp>\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^ ]+ sudo: +(?P<user>[^ ]+) : TTY=(?P<tty>[^ ]+) ; PWD=(?P<pwd>[^ ]+) ; USER=(?P<foo>[^ ]+) ; COMMAND=(?P<cmd>.+)$'
+
+        - timestamp:
+            source: timestamp
+            format: Jan 2 15:04:05
+            location: Asia/Tokyo
+
+        - template:
+            source: message
+            template: 'USER={{ .user }} PWD={{ .pwd }} CMD={{ .cmd }}'
+
+        - output:
+            source: message
+
+  - job_name: sshd
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: sshd
+        hostname: <%= @HOSTNAME %>
+        level: warning
+        __path__: /var/log/auth.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="sshd"}'
+
+        stages:
+        - drop:
+            expression: (CRON|sudo)
+
+        - regex:
+            expression: '^(?P<timestamp>\w+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+) [^:]+: (?P<message>.+)$'
+
+        - timestamp:
+            source: timestamp
+            format: Jan 2 15:04:05
+            location: Asia/Tokyo
+
+        - output:
+            source: message
+
+  - job_name: supervisord
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: supervisord
+        hostname: <%= @HOSTNAME %>
+        level: notice
+        __path__: /var/log/supervisor/supervisord.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="supervisord"}'
+        stages:
+        - regex:
+            expression: '^(?P<timestamp>[0-9]+\-[0-9]+\-[0-9]+ [0-9]+:[0-9]+:[0-9]+),[0-9]+ (?P<level>[^ ]+) (?P<message>.+)$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02 15:04:05
+            location: Asia/Tokyo
+
+        - template:
+            source: level
+            template: '{{ ToLower .level }}'
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}'
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "crit" .Value "critical" }}'
+
+        - labels:
+            level:
+
+        - output:
+            source: message
+
+  - job_name: fail2ban
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: fail2ban
+        hostname: <%= @HOSTNAME %>
+        level: notice
+        __path__: /var/log/fail2ban.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="fail2ban"}'
+        stages:
+        - regex:
+            expression: '^(?P<timestamp>[0-9]+\-[0-9]+\-[0-9]+ [0-9]+:[0-9]+:[0-9]+),[0-9]+ [^:]+: (?P<level>[^ ]+)[^\[]+(?P<message>.+)$'
+
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02 15:04:05
+            location: Asia/Tokyo
+
+        - template:
+            source: level
+            template: '{{ ToLower .level }}'
+
+        - labels:
+            level:
+
+        - output:
+            source: message
+
+  - job_name: promtail
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: promtail
+        hostname: <%= @HOSTNAME %>
+        __path__: /var/log/promtail.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="promtail"}'
+        stages:
+        - regex:
+            expression: '^[^ ]+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ promtail[^ ]+ .*ts=(?P<timestamp>[^ ]+) (?P<message>.+)$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.999999999Z
+            location: Etc/GMT
+
+        - regex:
+            expression: '^[^ ]+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ promtail[^ ]+ .*level=(?P<level>[^\\" ]+).*$'
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}'
+
+        - labels:
+            level:
+
+        - output:
+            source: message

roles/base.rb

@@ -5,5 +5,5 @@ include_recipe '../cookbooks/monit/default.rb'
 include_recipe '../cookbooks/consul/default.rb'
 include_recipe '../cookbooks/consul-template/default.rb'
 include_recipe '../cookbooks/fzf/default.rb'
-include_recipe '../cookbooks/fluentd/default.rb'
+include_recipe '../cookbooks/promtail/default.rb'
 include_recipe '../cookbooks/prometheus/default.rb'