Merge branch 'install-vector' of kazu634/itamae into master

cookbooks/vector/attributes.rb

@@ -0,0 +1,23 @@
+# -------------------------------------------
+# Specifying the default settings:
+# -------------------------------------------
+case run_command('grep VERSION_ID /etc/os-release | awk -F\" \'{print $2}\'').stdout.chomp
+when "20.04"
+  cmd = 'LANG=C ip a | grep "inet " | grep -v -E "(127|172)" | cut -d" " -f6 | perl -pe "s/\/.+//g"'
+
+when "18.04"
+  cmd = 'LANG=C /sbin/ifconfig | grep "inet " | grep -v -E "(127|172)" | cut -d" " -f10'
+
+else
+  cmd = 'LANG=C /sbin/ifconfig | grep "inet addr" | grep -v -E "(127|172)" | awk "{print $2;}" | cut -d: -f2 | cut -f 1 -d " " | tail -1'
+end
+
+ipaddr = run_command(cmd).stdout.chomp
+
+node.reverse_merge!({
+  'vector' => {
+    'url' => 'https://github.com/timberio/vector/releases/download/',
+    'ipaddr' => ipaddr,
+    'deb' => 'vector-amd64.deb'
+  },
+})

cookbooks/vector/default.rb

@@ -0,0 +1,6 @@
+# Loading the attributes:
+include_recipe './attributes.rb'
+
+# Install loki here:
+include_recipe './install.rb'
+

cookbooks/vector/files/etc/logrotate.d/vector-syslog

@@ -0,0 +1,14 @@
+/var/log/vector/syslog.log {
+    ifempty
+    dateformat .%Y%m%d
+    missingok
+    compress
+    daily
+    rotate 10
+    prerotate
+        /bin/systemctl stop vector-syslog.service
+    endscript
+    postrotate
+        /bin/systemctl start vector-syslog.service
+    endscript
+}

cookbooks/vector/files/etc/systemd/system/promtail-vector-syslog.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Grafana Promtail
+Documentation=https://github.com/grafana/loki
+After=network-online.target
+
+[Service]
+User=root
+Restart=always
+ExecStart=/usr/local/bin/promtail --config.file=/etc/promtail/syslog.yaml
+
+[Install]
+WantedBy=multi-user.target

cookbooks/vector/files/etc/systemd/system/vector-syslog.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Vector
+Documentation=https://vector.dev
+After=network-online.target
+Requires=network-online.target
+
+[Service]
+ExecStart=/usr/bin/vector --config /etc/vector/syslog.toml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=vector
+
+[Install]
+WantedBy=multi-user.target

cookbooks/vector/files/etc/vector/syslog.toml

@@ -0,0 +1,16 @@
+data_dir = "/var/lib/vector"
+
+[sources.syslog]
+  address = "0.0.0.0:514" # required, required when mode = "tcp" or mode = "udp"
+  mode    = "tcp"         # required
+  type    = "syslog"      # required
+
+[sinks.syslog-file]
+  # General
+  type        = "file"                                # required
+  inputs      = ["syslog"]                            # required
+  healthcheck = true                                  # optional, default
+  path        = "/var/log/vector/syslog.log"          # required
+
+  # Encoding
+  encoding.codec = "ndjson" # required

cookbooks/vector/install.rb

@@ -0,0 +1,36 @@
+vector_url = ''
+vector_deb = ''
+
+tag        = ''
+vtag       = ''
+
+# Calculate the Download URL:
+begin
+  require 'net/http'
+
+  uri = URI.parse('https://github.com/timberio/vector/releases/latest')
+
+  Timeout.timeout(3) do
+    response = Net::HTTP.get_response(uri)
+
+    vtag     = $1 if response.body =~ %r{tag\/(v\d+\.\d+\.\d+)}
+    tag      = vtag.sub(/^v/, '')
+
+    vector_deb = "#{node['vector']['deb']}"
+    vector_url = "#{node['vector']['url']}/#{vtag}/#{vector_deb}"
+  end
+rescue
+  # Abort the chef client process:
+  raise 'Cannot connect to http://github.com.'
+end
+
+# バージョン確認して、アップデート必要かどうか確認
+result = run_command("vector --version 2>&1 | grep #{tag}", error: false)
+if result.exit_status != 0
+  # Download:
+  TMP = "/tmp/#{vector_deb}"
+
+  execute "wget #{vector_url} -O #{TMP}"
+
+  execute "dpkg -i #{TMP}"
+end

cookbooks/vector/syslog_setup.rb

@@ -0,0 +1,89 @@
+# Create `/var/log/vector`:
+%w( /var/log/vector ).each do |d|
+  directory d do
+    owner  'root'
+    group  'root'
+    mode   '0755'
+  end
+end
+
+# Deploy `vector` configuration for `syslog`:
+remote_file '/etc/vector/syslog.toml' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  notifies :restart, 'service[vector-syslog]'
+end
+
+# Deploy `systemd` configuration for `prometheus`:
+remote_file '/etc/systemd/system/vector-syslog.service' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  notifies :restart, 'service[vector-syslog]'
+end
+
+# Service setting:
+service 'vector-syslog' do
+  action [ :enable, :restart ]
+end
+
+# Firewall settings here:
+%w( 514/tcp ).each do |p|
+  execute "ufw allow #{p}" do
+    user 'root'
+
+    not_if "LANG=c ufw status | grep #{p}"
+
+    notifies :run, 'execute[ufw reload-or-enable]'
+  end
+end
+
+execute 'ufw reload-or-enable' do
+  user 'root'
+  command 'LANG=C ufw reload | grep skipping && ufw --force enable || exit 0'
+
+  action :nothing
+end
+
+# Depoy `consul` service configuration for `loki`:
+template '/etc/consul.d/service-vector-syslog.json' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  variables(ipaddr: node['vector']['ipaddr'])
+
+  notifies :restart, 'service[supervisor]'
+end
+
+template '/etc/promtail/syslog.yaml' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+
+  variables(LOKIENDPOINT: node['promtail']['lokiendpoint'])
+
+  notifies :restart, 'service[promtail-vector-syslog]'
+end
+
+# Deploy `systemd` configuration for `promtail-loki`:
+remote_file '/etc/systemd/system/promtail-vector-syslog.service' do
+  owner  'root'
+  group  'root'
+  mode   '644'
+end
+
+# Service setting:
+service 'promtail-vector-syslog' do
+  action [ :enable, :restart ]
+end
+
+# Deploy the `logrotated` configuration:
+remote_file '/etc/logrotate.d/vector-syslog' do
+  owner 'root'
+  group 'root'
+  mode '644'
+end

cookbooks/vector/templates/etc/consul.d/service-vector-syslog.json

@@ -0,0 +1,12 @@
+{
+  "service": {
+    "name": "vector-syslog",
+    "port": 514,
+    "check":{
+      "tcp": "<%= @ipaddr %>:514",
+      "interval": "60s",
+      "timeout": "1s",
+      "success_before_passing": 3
+    }
+  }
+}

cookbooks/vector/templates/etc/promtail/syslog.yaml

@@ -0,0 +1,104 @@
+server:
+  disable: true
+
+positions:
+  filename: /var/opt/promtail/promtail_syslog_position.yaml
+
+clients:
+  - url: http://<%= @LOKIENDPOINT %>/loki/api/v1/push
+
+scrape_configs:
+  - job_name: syslog
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: syslog
+        __path__: /var/log/vector/*.log
+
+    pipeline_stages:
+    - json:
+        expressions:
+          appname:
+          hostname:
+          level: severity
+          message:
+          timestamp:
+
+    - labels:
+        appname:
+        hostname:
+        level:
+
+    - match:
+        selector: '{job="syslog", level=~"(debug|DEBUG)"}'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="esxi-new", appname=~"(storageRM|sdrsInjector)"} |= "getting state for"'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="esxi-new", appname="Hostd"} |~ "(->|IpmiIfcOpenIpmiOpen|LikewiseGetDomainJoinInfo)"'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="esxi-new", appname="smartd"} |~ "(REALLOCATED SECTOR CT below threshold)"'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="esxi-new", appname="backup.sh"} |~ "(esx.conf|Creating archive)"'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="esxi-new", appname="Rhttpproxy"} |~ "(warning rhttpproxy)"'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="esxi-new"}'
+        stages:
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.999Z
+            location: Etc/GMT
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "err" .Value "error" }}'
+
+        - labeldrop:
+            - appname
+
+        - output:
+            source: message
+
+    - match:
+        selector: '{job="syslog", hostname="ubnt", appname="openvpn", level="notice"}'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="ubnt", appname="sudo", level="info"}'
+        action: drop
+
+    - match:
+        selector: '{job="syslog", hostname="ubnt"}'
+        stages:
+
+        - timestamp:
+            source: timestamp
+            format: 2006-01-02T15:04:05.999Z
+            location: Asia/Bangkok
+
+        - template:
+            source: level
+            template: '{{ regexReplaceAllLiteral "err" .Value "error" }}'
+
+        - labels:
+            level:
+            hostname:
+
+        - labeldrop:
+            - appname
+
+        - output:
+            source: message

roles/base.rb

@@ -4,4 +4,5 @@ include_recipe '../cookbooks/supervisor/default.rb'
 include_recipe '../cookbooks/consul/default.rb'
 include_recipe '../cookbooks/fzf/default.rb'
 include_recipe '../cookbooks/promtail/default.rb'
+include_recipe '../cookbooks/vector/default.rb'
 include_recipe '../cookbooks/prometheus-exporters/default.rb'

roles/prometheus.rb

@@ -1,3 +1,4 @@
 include_recipe '../cookbooks/prometheus/default.rb'
 include_recipe '../cookbooks/grafana/default.rb'
 include_recipe '../cookbooks/loki/default.rb'
+include_recipe '../cookbooks/vector/syslog_setup.rb'