Consul server setup

cookbooks/base/packages.rb

@@ -2,7 +2,7 @@
 execute 'apt update'
 
 # Install the necessary packages:
-%w[build-essential zsh vim-nox debian-keyring screen curl dstat direnv].each do |pkg|
+%w[build-essential zsh vim-nox debian-keyring screen curl dstat direnv jq avahi-daemon].each do |pkg|
   package pkg
 end
 
@@ -63,9 +63,6 @@ end
   end
 end
 
-# mDNS
-package 'avahi-daemon'
-
 execute 'ufw allow 5353/udp' do
   user 'root'
 

cookbooks/consul/attributes.rb

@@ -23,6 +23,7 @@ node.reverse_merge!({
     'ipaddr' => ipaddr,
     'dns' => dns,
     'encrypt' => 's2T3XUTb9MjHYOw8I820O5YkN2G6eJrjLjJRTnEAKoM=',
-    'token' => '39f8fc02-7ec7-ec7b-7bc6-e6e16bb8deca'
+    'token' => '39f8fc02-7ec7-ec7b-7bc6-e6e16bb8deca',
+    'server_token' => '655967fb-8a72-5ba9-b5ae-58c587c853b2'
   }
 })

cookbooks/consul/files/etc/consul.d/certs/consul-agent-ca-key.pem

@@ -0,0 +1,24 @@
+md5:4f683f562e56d8663a72584dfa67b247:salt:116-0-67-185-96-189-235-121:aes-256-cfb:/pbbA5VABbbCOrdOi/C+lfBXQ2v5kTXj7eid4Sh3CaL4fokVqN0txFlLJiqZ
+Hstgm8GIFK1+33JHrOdbPCszRjyTW18P0pDmZbXilh0kg5otpYrQ67f5E976
+mJz9Mr9MeBZ7At6H4nnWAAaSQu2MDgBgrkVLL87NYARn0t0qOz4PAU3mN8Ah
+e/fT1Awr8s4m/cupKAX6q0vZ4w1mfS+oY9tWeDDppvpr9AuYWLofvaB5axIu
+x54YETl7vvpu+rvkLs9iuzw1Xvws/ztUxS1QBMAlIxBd1ICHMTd0+7hZf/tU
+bH7gNqMDaTDAZzruDOisS7H6x47G9rOkGeuS/1RsEB8u3qV2IDXEgwfETfH4
+f1TUq1s3oaNGGTSnUBxjx0iUMZOk6+vy1xX4oa3bq4wa6PVQONdhrbqx4hvQ
+Y69Q36Q4gpTtI58YDx7U+Nc8TEnL6OCRewGOsBjUv5gjMZbn9wMUzFEvNyr3
+4gquHT9OARvvmqybm7L8Af7proWAo5Mxy+rB3o0eAjSZ5ND7hQtCMKlMqtkc
+Z5xBemLKXTBrqt5wZ+MACL51UJX0ngLAESW8vpy19y+kMOkM+kuqT4sqjsFw
+sHn4mOsg4PCio61zs7aylWLFWZZBfKcD/wuHseL22U3cabyRaUabotSgZrAf
+uqOJWVUihNSJ8ak6t/6Lrc8ilyLzULvwdiT+ZjavGXi1c6Ext/VmQIUHIjUy
+dd+ScLvgpKJVztW2sIpOXbUzYDgjMsYR4H6XJw6OFmneuL9z/0qaUs011y2z
+5kaakT+by7v+czTj0lflTdz55RLPNViYY3rZZiCn2X/UjXhRXFBuVOY4Rm7f
+hB8MmTyGPf9fQk0aKWEi6QTA+Lt3/4ak9rY5K3I+g1f66Hn8zZmJU+fETXuH
+RhbXgrhHdH/i44rkDVkJL4obf9JhsBWYAcMhQN/5z91gf45V0J8VWF46vGRR
+y4N04NifLXuZG7rA3Hw3xY9EyBOAPHII45qSzQM94Exo/Xik/QLfChnaLPQO
+XcUx32w8f1g0Vm+K4I4g3BEGmztLnoVodW9qqF//4/HmJqHry2znLhI0EAtN
+CUA270z6tGzuMUgZo7rJX06Pn76HRaBkyWMBcQzN72JaIbZvrsOJJHhVMLmu
+PBjXyJmF0tNwFtMEwGYJ2T4OfGpCCD+NPS5nEQ6bZ0sX3/qJwhGD8Vz3U4P6
+3fhRlnjM2KJFQ5CCl95lpTGwsbeP6NNaoCAmLZXcSMDiL6YQDy/rtN3Q4LfO
+pfgg3hf5a041qSo5FOY+7vjyc9ndYisLxnoAtB90cX8ORnKKZBugqlvarYMR
+x4tJ6S2h2177eRwIcm7uHCd96ne/g/s0ScEoqHZg0vXs645F+ovi2ymsKxxN
+CrRkuxP1P8bhK3Bn1ylK4aejktOzy4dcdEmUCwJlPElnJbuQtnjv

cookbooks/consul/files/etc/consul.d/certs/dc1-server-consul-1-key.pem

@@ -0,0 +1,6 @@
+md5:3fe62feb81f46bc71cf0e43936d98efa:salt:5-77-73-67-58-107-161-252:aes-256-cfb:6B67412gDFtS5lssg5QCREm5vrWJA2pfLbnqySZHkqEp9xd2rG4S+vak3DAk
+9ruQRqOZOs5vmzysY6AONeAK0ghGId9olkPDUqxuj0xZ466Yt4BDGnmTl2rU
+WoKONoX+GZz7zmVxqYfykHCgkHb0bfV7HRWJKiVBTvxlDQtceoTScu2qrlRj
+aRmsaKvpR2LvNeh+D9tVqmD3+T4924jjO1HqkyAXEu0RENRvwFJIb3Vd9WwC
+Mp/49miaRSdZ/OkXyu8hG00bqt40Edn9wJSWDHDw5XYYVoBlYYHCfzAuQw3K
+fL+BxPM=

cookbooks/consul/files/etc/consul.d/certs/dc1-server-consul-1.pem

@@ -0,0 +1,22 @@
+md5:3a2252bcb411d02014a3e1a14efc1cac:salt:4-79-66-221-55-44-210-100:aes-256-cfb:FoeqpBC4Wrwrmu/9a+Jo9ejvJMoLuqfxXLAi4sJHJowcallU4XWZzDWnJHqj
+a8gHLk1bx6ysJp/rTD7oEBp3WEbg3wGu2v+ywl+DrHbArS6lR0PQ0U6mu/EX
+RiqTPiv7HmvExIBVeMCUamkS7HnKLKdf/X2Mo2z/yWca0gORNeElQoCs/UaC
+AXnlgQzd6sYAzxYIYqOQSzz42f1CqA2yIy38R4j8O/VbQ7MBLuDa6b0nYsjp
+/gMH7KHdeP+5ZeE+JueJjbgChkmuq6soBfWuxuij1uJFhrnKsf9kX7n0uGOh
+IHHBMJbm+vORcha/En2KviNwmxQWF3RF+Ba0NazkiEdxvpaozYutSdwn6r2I
+XlGBg6OTTTXOZVXjB/gFZjBBXQruLw2vHKxEGtR3iNYJbfEA90wbaB8EP4bZ
+8E2ktbJ/HWD6V9ruQ3wRgh2YQ6WTjEi/a54uZJgWbxDsYIVdHrrwpQrPtZlg
+1FdpE2dzE8b1Lr+obub6U0Rn2Eo2Pc17leEsGmMxGyyJecCJxW49U4fCgsAM
+rJC3ky8/38sQh7W5NJdRhUkYW3sSWe8ZcTCFjONMea65DI6bt/pbj+sw+4a1
+NtJDOWb7yRSI4yqTNYntGZMygDW7PPLAh95lGkHWJgHXOCQcMK4Oyct7ArRl
+nwt2p5JvPEeJlgj0pbqqXMc3EenOVY2I2PNOb83YThXPEc5HYhciRee7W2TH
+VIacv30zV5iC/LA9ddHSe0SG00GujYdMLGDCqb4NBn5UUmEVKDQ+YRUcwolS
+J0triJjN7bDMDZihBrDkR/H+IhEzYE0CwHxwDO0jVzw6JPv72GN5dpabgoCC
+9Ul+3gM3k3uOUII6VZe85AHew7Ih5a7DJL3O9jlgCNqIla+LWDFsRsO+nXKK
+0X0l6mBivT1J0xs4WAFWJDMZzoXDhvrQgxPMeYiRVb42P9XgqvBaRzASa3I0
+VqP8bmgauBddgOPMf36R2U2KlomyDcS1qwncrtRX8Ix2jO7Ei+SBiO7g6jV3
+34tU9eeUpdY7Q2lc88oGJQ8lGRHppMJEm45U8XBzZBMn6kTtoEUTMSu3tz7s
+sslPifdBxp9k031DtpI0FRvgf9EDU9CSL3PWuIRkRyllbllpfuPABxe2n41e
+2X1q9yyMe88YByWy8P44dHT5ObsZWjtLspKFXme4jCRAwIljTBdeuBHPa5rX
+6awSzdPfwbrBeUNBcHBKxmGAODmkiv4akakYRl7kipcMIY1mx948Rc0XCwvx
+XGBNhzyQZogKZZUpTqANvqOf7e+RJpHJ

cookbooks/consul/files/etc/consul.d/howto/01-bootstrap.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export TOKEN_DIR=/etc/consul.d/tokens
+
+consul acl bootstrap --format json | tee -a "${TOKEN_DIR}/token-bootstrap.json"
+
+exit $?

cookbooks/consul/files/etc/consul.d/howto/02-policy.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+export POLICIES=`ls -1 /etc/consul.d/policies/*.hcl`
+export TOKEN_DIR=/etc/consul.d/tokens
+export ANONYMOUS_TOKEN="00000000-0000-0000-0000-000000000001"
+export CONSUL_HTTP_TOKEN=`cat ${TOKEN_DIR}/token-bootstrap.json | jq -r ".SecretID"`
+
+for conf in ${POLICIES}; do
+  policy=`basename ${conf} .hcl`
+
+  consul acl policy read -name "${policy}" &> /dev/null
+  if [ $? -ne 0 ]; then
+    consul acl policy create -name "${policy}" -rules @${conf}
+  fi
+
+  # anonymousは特別扱い
+  if [ ${policy} = "anonymous" ]; then
+    continue
+  fi
+
+  consul acl token list | grep ${policy} &> /dev/null
+  if [ $? -ne 0 ]; then
+    consul acl token create -description "${policy}" -policy-name ${policy} | tee ${TOKEN_DIR}/${policy}
+  fi
+done
+
+consul acl token update -id ${ANONYMOUS_TOKEN} -policy-name "anonymous" -description "Anonymous Token"
+
+exit 0

cookbooks/consul/files/etc/consul.d/policies/anonymous.hcl

@@ -0,0 +1,34 @@
+acl = "read"
+
+agent_prefix "" {
+  policy = "read"
+}
+
+event_prefix "" {
+  policy = "read"
+}
+
+key_prefix "" {
+  policy = "read"
+}
+
+keyring = "read"
+
+node_prefix "" {
+  policy = "read"
+}
+
+operator = "read"
+
+query_prefix "" {
+  policy = "read"
+}
+
+service_prefix "" {
+  policy = "read"
+  intentions = "read"
+}
+
+session_prefix "" {
+  policy = "read"
+}

cookbooks/consul/files/etc/consul.d/policies/nodes.hcl

@@ -0,0 +1,15 @@
+agent_prefix "" {
+  policy = "write"
+}
+
+node_prefix "" {
+  policy = "write"
+}
+
+service_prefix "" {
+  policy = "write"
+}
+
+session_prefix "" {
+  policy = "read"
+}

cookbooks/consul/files/etc/consul.d/policies/nomad-policy.hcl

@@ -0,0 +1,13 @@
+agent_prefix "" {
+  policy = "read"
+}
+
+node_prefix "" {
+  policy = "read"
+}
+
+service_prefix "" {
+  policy = "write"
+}
+
+acl = "write"

cookbooks/consul/files/etc/consul.d/policies/vault-service-registration.hcl

@@ -0,0 +1,3 @@
+service_prefix "" {
+  policy = "write"
+}

cookbooks/consul/setup.rb

@@ -9,17 +9,44 @@ end
 
 # deploy certificates
 if node['consul']['manager']
-else
+  encrypted_remote_file '/etc/consul.d/certs/consul-agent-ca-key.pem' do
-  encrypted_remote_file '/etc/consul.d/certs/consul-agent-ca.pem' do
     owner 'consul'
     group 'consul'
     mode '0444'
 
-    source   'files/etc/consul.d/certs/consul-agent-ca.pem'
+    source   'files/etc/consul.d/certs/consul-agent-ca-key.pem'
+    password ENV['ITAMAE_PASSWORD']
+  end
+
+  encrypted_remote_file '/etc/consul.d/certs/dc1-server-consul-1-key.pem' do
+    owner 'consul'
+    group 'consul'
+    mode '0444'
+
+    source   'files/etc/consul.d/certs/dc1-server-consul-1-key.pem'
+    password ENV['ITAMAE_PASSWORD']
+  end
+
+  encrypted_remote_file '/etc/consul.d/certs/dc1-server-consul-1.pem' do
+    owner 'consul'
+    group 'consul'
+    mode '0444'
+
+    source   'files/etc/consul.d/certs/dc1-server-consul-1.pem'
     password ENV['ITAMAE_PASSWORD']
   end
 end
 
+encrypted_remote_file '/etc/consul.d/certs/consul-agent-ca.pem' do
+  owner 'consul'
+  group 'consul'
+  mode '0444'
+
+  source   'files/etc/consul.d/certs/consul-agent-ca.pem'
+  password ENV['ITAMAE_PASSWORD']
+end
+
+# Deploy configs
 if node['consul']['manager']
   SRC = 'consul-server.hcl.erb'
 else
@@ -43,6 +70,34 @@ template '/etc/consul.d/consul.hcl' do
   notifies :restart, 'service[consul]'
 end
 
+# Deploy server specific config
+template '/etc/consul.d/server.hcl' do
+  owner 'consul'
+  group 'consul'
+  mode '644'
+
+  variables(server_token: node['consul']['server_token'])
+
+  notifies :restart, 'service[consul]'
+end
+
+%w( 01-bootstrap.sh 02-policy.sh ).each do |sh|
+  remote_file("/etc/consul.d/howto/#{sh}") {
+    owner 'root'
+    group 'root'
+    mode '0755'
+  }
+end
+
+%w( anonymous nodes nomad-policy vault-service-registration ).each do |policy|
+  remote_file("/etc/consul.d/policies/#{policy}.hcl") {
+    owner 'root'
+    group 'root'
+    mode '0644'
+  }
+end
+
+# misc
 directory '/var/log/consul/' do
   owner 'consul'
   group 'consul'

cookbooks/consul/templates/etc/consul.d/consul-server.hcl.erb

@@ -1,85 +1,43 @@
-# Full configuration options can be found at https://www.consul.io/docs/agent/options.html
+datacenter = "dc1"
+data_dir   = "/opt/consul"
 
-# datacenter
+encrypt                = "<%= @encrypt %>"
-# This flag controls the datacenter in which the agent is running. If not provided,
+verify_incoming        = true
-# it defaults to "dc1". Consul has first-class support for multiple datacenters, but
+verify_outgoing        = true
-# it relies on proper configuration. Nodes in the same datacenter should be on a
+verify_server_hostname = true
-# single LAN.
-#datacenter = "aws"
 
-# data_dir
+ca_file   = "/etc/consul.d/certs/consul-agent-ca.pem"
-# This flag provides a data directory for the agent to store state. This is required
-# for all agents. The directory should be durable across reboots. This is especially
-# critical for agents that are running in server mode as they must be able to persist
-# cluster state. Additionally, the directory must support the use of filesystem
-# locking, meaning some types of mounted folders (e.g. VirtualBox shared folders) may
-# not be suitable.
-data_dir = "/opt/consul"
-
-# client_addr
-# The address to which Consul will bind client interfaces, including the HTTP and DNS
-# servers. By default, this is "127.0.0.1", allowing only loopback connections. In
-# Consul 1.0 and later this can be set to a space-separated list of addresses to bind
-# to, or a go-sockaddr template that can potentially resolve to multiple addresses.
-client_addr = "0.0.0.0"
-
-# ui
-# Enables the built-in web UI server and the required HTTP routes. This eliminates
-# the need to maintain the Consul web UI files separately from the binary.
-ui = true
-
-# server
-# This flag is used to control if an agent is in server or client mode. When provided,
-# an agent will act as a Consul server. Each Consul cluster must have at least one
-# server and ideally no more than 5 per datacenter. All servers participate in the Raft
-# consensus algorithm to ensure that transactions occur in a consistent, linearizable
-# manner. Transactions modify cluster state, which is maintained on all server nodes to
-# ensure availability in the case of node failure. Server nodes also participate in a
-# WAN gossip pool with server nodes in other datacenters. Servers act as gateways to
-# other datacenters and forward traffic as appropriate.
-server = true
-
-# bootstrap_expect
-# This flag provides the number of expected servers in the datacenter. Either this value
-# should not be provided or the value must agree with other servers in the cluster. When
-# provided, Consul waits until the specified number of servers are available and then
-# bootstraps the cluster. This allows an initial leader to be elected automatically.
-# This cannot be used in conjunction with the legacy -bootstrap flag. This flag requires
-# -server mode.
-bootstrap_expect=3
-
-# encrypt
-# Specifies the secret key to use for encryption of Consul network traffic. This key must
-# be 32-bytes that are Base64-encoded. The easiest way to create an encryption key is to
-# use consul keygen. All nodes within a cluster must share the same encryption key to
-# communicate. The provided key is automatically persisted to the data directory and loaded
-# automatically whenever the agent is restarted. This means that to encrypt Consul's gossip
-# protocol, this option only needs to be provided once on each agent's initial startup
-# sequence. If it is provided after Consul has been initialized with an encryption key,
-# then the provided key is ignored and a warning will be displayed.
-encrypt = "LPKrNBQZnJIc8tJpViI4ug=="
-
-# retry_join
-# Similar to -join but allows retrying a join until it is successful. Once it joins
-# successfully to a member in a list of members it will never attempt to join again.
-# Agents will then solely maintain their membership via gossip. This is useful for
-# cases where you know the address will eventually be available. This option can be
-# specified multiple times to specify multiple agents to join. The value can contain
-# IPv4, IPv6, or DNS addresses. In Consul 1.1.0 and later this can be set to a go-sockaddr
-# template. If Consul is running on the non-default Serf LAN port, this must be specified
-# as well. IPv6 must use the "bracketed" syntax. If multiple values are given, they are
-# tried and retried in the order listed until the first succeeds. Here are some examples:
-retry_join = <%= @manager_hosts %>
 
 bind_addr = "<%= @ipaddr %>"
+client_addr    = "0.0.0.0"
+advertise_addr = "<%= @ipaddr %>"
+
+performance {
+  raft_multiplier = 1
+}
+
 disable_remote_exec = false
 disable_update_check = false
 enable_local_script_checks = true
 
-log_file = "/var/log/consul/"
+log_file             = "/var/log/consul/"
 log_rotate_max_files = -1
-log_level = "INFO"
+log_level            = "INFO"
-log_json = false
+log_json             = false
-log_rotate_bytes = 1000000
+log_rotate_bytes     = 1000000
 
+enable_central_service_config = true
+ports {
+  grpc_tls = 8502
+}
+connect {
+  enabled = true
+}
+
+telemetry {
+  prometheus_retention_time = "24h"
+  disable_hostname          = true
+}
+
+retry_join = [<%= @manager_hosts %>]
 rejoin_after_leave = true

cookbooks/consul/templates/etc/consul.d/server.hcl.erb

@@ -0,0 +1,50 @@
+server           = true
+bootstrap_expect = 3
+
+ca_file   = "/etc/consul.d/certs/consul-agent-ca.pem"
+cert_file = "/etc/consul.d/certs/dc1-server-consul-1.pem"
+key_file = "/etc/consul.d/certs/dc1-server-consul-1-key.pem"
+
+acl {
+  enabled                  = true
+  default_policy           = "deny"
+  enable_token_persistence = true
+
+  tokens {
+    master  = "<%= @manager_hosts %>"
+    agent   = "<%= @manager_hosts %>"
+    default = "<%= @manager_hosts %>"
+  }
+}
+
+ui_config {
+  enabled = true
+
+  metrics_provider = "prometheus"
+  metrics_proxy {
+    base_url = "http://192.168.10.101:9090"
+  }
+}
+
+config_entries {
+  bootstrap = [
+    {
+      kind = "proxy-defaults"
+      name = "global"
+      config {
+        envoy_prometheus_bind_addr = "0.0.0.0:9102"
+      }
+    }
+  ]
+}
+
+auto_config {
+  authorization {
+    enabled = true
+    static {
+      oidc_discovery_url = "http://vault.homelab:8200/v1/identity/oidc"
+      bound_issuer = "http://vault.homelab:8200/v1/identity/oidc"
+      bound_audiences = [ "dc1" ]
+    }
+  }
+}