Merge branch 'loki' of kazu634/itamae into master

cookbooks/loki/attributes.rb

@@ -0,0 +1,11 @@
+# -------------------------------------------
+# Specifying the default settings:
+# -------------------------------------------
+node.reverse_merge!({
+  'loki' => {
+    'url' => 'https://github.com/grafana/loki/releases/download/',
+    'zip' => 'loki-linux-amd64.zip',
+    'storage' => '/opt/loki/',
+    'location' => '/usr/local/bin/'
+  },
+})

cookbooks/loki/default.rb

@@ -0,0 +1,7 @@
+# Loading the attributes:
+include_recipe './attributes.rb'
+
+# Install loki here:
+include_recipe './install.rb'
+include_recipe './setup.rb'
+

cookbooks/loki/files/etc/consul.d/service-loki.json

@@ -0,0 +1,6 @@
+{
+  "service": {
+    "name": "loki",
+    "port": 3100
+  }
+}

cookbooks/loki/files/etc/logrotate.d/loki

@@ -0,0 +1,13 @@
+/var/log/loki.log
+{
+        rotate 4
+        weekly
+        missingok
+        notifempty
+        compress
+        delaycompress
+        sharedscripts
+        postrotate
+                /usr/lib/rsyslog/rsyslog-rotate
+        endscript
+}

cookbooks/loki/files/etc/loki/loki-config.yml

@@ -0,0 +1,45 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+    - from: 2018-04-15
+      store: boltdb
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 168h
+
+storage_config:
+  boltdb:
+    directory: /var/opt/loki/index
+
+  filesystem:
+    directory: /var/opt/loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s

cookbooks/loki/files/etc/rsyslog.d/30-loki.conf

@@ -0,0 +1,7 @@
+# Log kernel generated promtail log messages to file
+:syslogtag,contains,"loki" /var/log/loki.log
+
+# Uncomment the following to stop logging anything that matches the last rule.
+# Doing this will stop logging kernel generated UFW log messages to the file
+# normally containing kern.* messages (eg, /var/log/kern.log)
+& stop

cookbooks/loki/files/etc/systemd/system/loki.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Grafana Loki
+Documentation=https://github.com/grafana/loki
+After=network-online.target
+
+[Service]
+User=root
+Restart=always
+ExecStart=/usr/local/bin/loki --config.file=/etc/loki/loki-config.yml
+
+[Install]
+WantedBy=multi-user.target

cookbooks/loki/files/etc/systemd/system/promtail-loki.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Grafana Promtail
+Documentation=https://github.com/grafana/loki
+After=network-online.target
+
+[Service]
+User=root
+Restart=always
+ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/loki.yaml
+
+[Install]
+WantedBy=multi-user.target

cookbooks/loki/install.rb

@@ -0,0 +1,56 @@
+loki_url = ''
+loki_bin = ''
+
+vtag     = ''
+tag      = ''
+
+# Calculate the Download URL:
+begin
+  require 'net/http'
+
+  uri = URI.parse('https://github.com/grafana/loki/releases/latest')
+
+  Timeout.timeout(3) do
+    response = Net::HTTP.get_response(uri)
+
+    vtag     = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)}
+    tag      = vtag.sub(/^v/, '')
+
+    loki_bin = "#{node['loki']['zip']}"
+    loki_url = "#{node['loki']['url']}/#{vtag}/#{loki_bin}"
+  end
+rescue
+  # Abort the chef client process:
+  raise 'Cannot connect to http://github.com.'
+end
+
+
+# バージョン確認して、アップデート必要かどうか確認
+result = run_command("loki --version 2>&1 | grep #{tag}", error: false)
+if result.exit_status != 0
+  # Download:
+  TMP = "/tmp/#{loki_bin}"
+
+  execute "wget #{loki_url} -O #{TMP}"
+
+  # Install:
+  directory node['loki']['storage'] do
+    owner 'root'
+    group 'root'
+    mode '755'
+  end
+
+  execute "unzip -d #{node['loki']['storage']} #{TMP}"
+
+  # Change Owner and Permissions:
+  file "#{node['loki']['storage']}loki-linux-amd64" do
+    owner 'root'
+    group 'root'
+    mode  '755'
+  end
+
+  # Create Link
+  link "#{node['loki']['location']}loki" do
+    to "#{node['loki']['storage']}loki-linux-amd64"
+  end
+end

cookbooks/loki/setup.rb

@@ -0,0 +1,103 @@
+# Create `/etc/loki/`:
+%w(/etc/loki).each do |d|
+  directory d do
+    owner  'root'
+    group  'root'
+    mode   '0755'
+  end
+end
+
+# Deploy `prometheus` files:
+remote_file '/etc/loki/loki-config.yml' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+end
+
+# Deploy `systemd` configuration for `prometheus`:
+remote_file '/etc/systemd/system/loki.service' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+end
+
+# Service setting:
+service 'loki' do
+  action [ :enable, :restart ]
+end
+
+# Depoy `consul` service configuration for `loki`:
+remote_file '/etc/consul.d/service-loki.json' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  notifies :restart, 'service[supervisor]'
+end
+
+# Depoy `promtail` configuration for `loki`:
+HOSTNAME = run_command('uname -n').stdout.chomp
+
+template '/etc/promtail/loki.yaml' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  variables(HOSTNAME: HOSTNAME, LOKIENDPOINT: node['promtail']['lokiendpoint'])
+
+  notifies :restart, 'service[promtail-loki]'
+end
+
+# Deploy `systemd` configuration for `promtail-loki`:
+remote_file '/etc/systemd/system/promtail-loki.service' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+end
+
+# Service setting:
+service 'promtail-loki' do
+  action [ :enable, :restart ]
+end
+
+remote_file '/etc/rsyslog.d/30-loki.conf' do
+  owner 'root'
+  group 'root'
+  mode '644'
+
+  notifies :restart, 'service[rsyslog]'
+end
+
+service 'rsyslog' do
+  action [ :nothing ]
+end
+
+# Deploy the `logrotated` configuration:
+remote_file '/etc/logrotate.d/loki' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end
+
+# Restart the `supervisor`:
+service 'supervisor' do
+  action :nothing
+end
+
+# Firewall settings here:
+%w( 3100/tcp ).each do |p|
+  execute "ufw allow #{p}" do
+    user 'root'
+
+    not_if "LANG=c ufw status | grep #{p}"
+
+    notifies :run, 'execute[ufw reload-or-enable]'
+  end
+end
+
+execute 'ufw reload-or-enable' do
+  user 'root'
+  command 'LANG=C ufw reload | grep skipping && ufw --force enable || exit 0'
+
+  action :nothing
+end

cookbooks/loki/templates/etc/promtail/loki.yaml

@@ -0,0 +1,41 @@
+server:
+  disable: true
+
+positions:
+  filename: /var/opt/promtail/promtail_loki_position.yaml
+
+clients:
+  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
+
+scrape_configs:
+  - job_name: loki
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: loki
+        hostname: <%= @HOSTNAME %>
+        __path__: /var/log/loki.log
+
+    pipeline_stages:
+    - match:
+        selector: '{job="loki"}'
+        stages:
+        - regex:
+
+            expression: '^[^ ]+ +[0-9]+ [0-9]+:[0-9]+:[0-9]+ [^ ]+ loki[^ ]+ .*level=(?P<level>[^ ]+) ts=(?P<timestamp>[^ ]+) (?P<message>.+)$'
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.999999999Z
+            location: Etc/GMT
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "warn" .Value "warning" }}'
+
+        - labels:
+            level:
+
+        - output:
+            source: message

cookbooks/prometheus/templates/etc/promtail/prometheus.yaml

@@ -5,7 +5,7 @@ positions:
   filename: /var/opt/promtail/promtail_prometheus_position.yaml
 
 clients:
-  - url: http://192.168.10.118:3100/loki/api/v1/push
+  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
 
 scrape_configs:
   - job_name: prometheus
@@ -33,7 +33,7 @@ scrape_configs:
             location: Etc/UTC
 
         - labels:
-            level: 
+            level:
 
         - output:
             source: message
@@ -63,7 +63,7 @@ scrape_configs:
             location: Etc/UTC
 
         - labels:
-            level: 
+            level:
 
         - output:
             source: message
@@ -104,7 +104,7 @@ scrape_configs:
             template: '{{ regexReplaceAllLiteral "resolved" .Value "notice" }}'
 
         - labels:
-            level: 
+            level:
 
         - output:
             source: message

cookbooks/promtail/attributes.rb

@@ -8,7 +8,7 @@ node.reverse_merge!({
     'storage' => '/opt/promtail/bin/',
     'location' => '/usr/local/bin/',
     'data' => '/var/opt/promtail/',
-    'lokiendpoint' => '192.168.10.118:3100'
+    'lokiendpoint' => 'loki.service.consul:3100'
   },
 })