auto unseal with cluster instructions

README.md

@@ -25,22 +25,25 @@ kubectl exec -ti vault-0 -- vault operator unseal
 
 ## Install (Cluster)
 
+This assumes you have an unseal vault available! See ha-test for manual unseal.
+
 ```bash
 kubectl apply -f k8s/certificate.yaml
 
 helm repo add hashicorp https://helm.releases.hashicorp.com
 helm search repo hashicorp/vault
-helm upgrade --install vault hashicorp/vault --values helm/values.yaml
+helm upgrade --install vault hashicorp/vault --values helm/ha.yaml
 
 mkdir ~/.vault-keys
-kubectl exec -ti vault-0 -- vault operator init -key-shares=5 -key-threshold=3 -format=json > ~/.vault-keys/cluster-keys.json
+kubectl exec -ti vault-0 -- vault operator init -format=json > ~/.vault-keys/vault-cluster-keys.json
-kubectl exec -ti vault-0 -- vault operator unseal
+# Only run the unseal if not autounsealing
+# kubectl exec -ti vault-0 -- vault operator unseal
 
 kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
-kubectl exec -ti vault-1 -- vault operator unseal
+# kubectl exec -ti vault-1 -- vault operator unseal
 
 kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
-kubectl exec -ti vault-2 -- vault operator unseal
+# kubectl exec -ti vault-2 -- vault operator unseal
 ```
 
 ## Add/Update policy
@@ -55,7 +58,7 @@ vault policy write ducoterra policies/ducoterra.hcl
 vault auth enable userpass
 vault write auth/userpass/users/ducoterra \
     policies=ducoterra \
-    password=password
+    password='<password>'
 ```
 
 ## Enable KV Secrets
@@ -73,7 +76,7 @@ vault kv get secret/okta
 vault secrets enable totp
 
 vault write totp/keys/okta \
-    url="otpauth://totp/Vault:test@test.com?secret=SECRET&issuer=Vault"
+    url='otpauth://totp/Vault:reese.wells@nimbisservices.com?secret=SECRET&issuer=Okta'
 
 vault read totp/code/okta
 ```
@@ -205,12 +208,13 @@ read_secrets:
 ### Create a CA
 
 ```bash
+# Note: 19800h is Apple's limit
 vault secrets enable -path=pki_dnet pki
-vault secrets tune -max-lease-ttl=87600h pki_dnet
+vault secrets tune -max-lease-ttl=19800h pki_dnet
 
 vault write pki_dnet/root/generate/internal \
     common_name=vault.ducoterra.net \
-    ttl=87600h
+    ttl=19800h
 
 vault write pki_dnet/config/urls \
     issuing_certificates="https://vault.ducoterra.net/v1/pki_dnet/ca" \
@@ -221,13 +225,13 @@ vault write pki_dnet/config/urls \
 
 ```bash
 vault secrets enable -path=pki_dnet_int pki
-vault secrets tune -max-lease-ttl=43800h pki_dnet_int
+vault secrets tune -max-lease-ttl=19800h pki_dnet_int
 vault write -format=json pki_dnet_int/intermediate/generate/internal \
     common_name="vault.ducoterra.net Intermediate Authority" \
     | jq -r '.data.csr' > certs/pki_dnet_intermediate.csr
 
 vault write -format=json pki_dnet/root/sign-intermediate \
-    csr=@certs/pki_dnet_intermediate.csr format=pem_bundle ttl=43800h \
+    csr=@certs/pki_dnet_intermediate.csr format=pem_bundle ttl=19800h \
     | jq -r '.data.certificate' > certs/pki_dnet_intermediate.cert.pem
 
 vault write pki_dnet_int/intermediate/set-signed certificate=@certs/pki_dnet_intermediate.cert.pem
@@ -238,11 +242,11 @@ vault write pki_dnet_int/config/urls \
 
 vault write pki_dnet_int/roles/dnet \
     allowed_domains=dnet \
-    allow_subdomains=true max_ttl=43800h
+    allow_subdomains=true max_ttl=19800h
 
 vault write pki_dnet_int/roles/pi_hole \
     allowed_domains=hole \
-    allow_subdomains=true max_ttl=43800h
+    allow_subdomains=true max_ttl=19800h
 ```
 
 Navigate to <https://vault.ducoterra.net/v1/pki_dnet_int/ca> and download the CA. Import to your devices.
@@ -258,6 +262,22 @@ vault write pki_dnet_int/issue/pi_hole \
     common_name=pi.hole > certs/pi.hole.cert
 ```
 
+#### Adding cert to pihole
+
+```bash
+# Paste the isser CA in here:
+vim /etc/ssl/certs/vault-ca.pem
+
+# Paste the certificate and private key in here:
+vim /etc/ssl/certs/pihole.pem
+
+# Make sure lighttpd is configured correctly
+vim /etc/lighttpd/external.conf
+
+# Restart the service
+service lighttpd restart
+```
+
 ### Revoke a certificate
 
 ```bash
@@ -268,7 +288,7 @@ vault write pki_dnet_int/tidy tidy_cert_store=true tidy_revoked_certs=true
 ### Use with cert-manager
 
 ```bash
-vault policy write pki_dnet - <<EOF
+vault policy write pki_dnet_int - <<EOF
 path "pki_dnet_int*"                        { capabilities = ["read", "list"] }
 path "pki_dnet_int/roles/*"   { capabilities = ["create", "update"] }
 path "pki_dnet_int/sign/*"    { capabilities = ["create", "update"] }
@@ -351,6 +371,7 @@ spec:
 ### Connect to external vault
 
 ```bash
+# On our other server
 helm install vault hashicorp/vault \
     --set "injector.externalVaultAddr=https://vault.ducoterra.net"
 
@@ -371,7 +392,7 @@ vault write auth/pikube/config \
 vault write auth/pikube/role/issuer \
     bound_service_account_names=issuer \
     bound_service_account_namespaces=cert-manager \
-    policies=pki_dnet \
+    policies=pki_dnet_int \
     ttl=20m
 ```
 
@@ -402,3 +423,48 @@ EOF
 
 kubectl apply -f cert-manager/pikube-vault-clusterissuer.yaml
 ```
+
+## Auto-unseal
+
+<https://learn.hashicorp.com/tutorials/vault/autounseal-transit>
+
+### Enable transit on vault that will do the unsealing
+
+```bash
+vault secrets enable transit
+vault write -f transit/keys/autounseal
+
+tee policies/autounseal.hcl <<EOF
+path "transit/encrypt/autounseal" {
+   capabilities = [ "update" ]
+}
+
+path "transit/decrypt/autounseal" {
+   capabilities = [ "update" ]
+}
+EOF
+
+vault policy write autounseal policies/autounseal.hcl
+vault token create -policy="autounseal" -wrap-ttl=120
+VAULT_TOKEN="wrapping token" vault unwrap
+```
+
+### Auto unseal
+
+#### Create a test vault in a new namespace
+
+```bash
+kubectl apply -f k8s/certificate.yaml
+
+# Get token from above output
+kubectl create secret generic auto-unseal-token --from-literal=VAULT_TOKEN="token from output"
+helm upgrade --install vault hashicorp/vault --values helm/ha.yaml
+```
+
+#### Test Auto Unseal
+
+```bash
+kubectl exec -it vault-0 -- vault operator init
+kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
+kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
+```

helm/ha-test.yaml

@@ -0,0 +1,50 @@
+global:
+  enabled: true
+
+injector:
+  enabled: false
+
+server:
+  authDelegator:
+    enabled: false
+
+  ha:
+    enabled: true
+    raft:
+      enabled: true
+      config: |
+        ui = true
+
+        listener "tcp" {
+          tls_disable = 1
+          address = "[::]:8200"
+          cluster_address = "[::]:8201"
+        }
+
+        storage "raft" {
+          path = "/vault/data"
+        }
+
+        service_registration "kubernetes" {}
+
+  dataStorage:
+    enabled: true
+    size: 32Gi
+    storageClass: null
+    accessMode: ReadWriteOnce
+
+  ingress:
+    enabled: true
+    hosts:
+      - host: vault-test.dnet
+        paths:
+          - /
+
+    tls:
+      - hosts:
+          - vault-test.dnet
+        secretName: vault-test-dnet-cert
+
+ui:
+  enabled: true
+  serviceType: ClusterIP

helm/ha.yaml

@@ -0,0 +1,61 @@
+global:
+  enabled: true
+
+server:
+
+  extraSecretEnvironmentVars:
+    - envName: VAULT_TOKEN
+      secretName: auto-unseal-token
+      secretKey: VAULT_TOKEN
+
+  ha:
+    enabled: true
+    raft:
+      enabled: true
+      config: |
+        ui = true
+
+        listener "tcp" {
+          tls_disable = 1
+          address = "[::]:8200"
+          cluster_address = "[::]:8201"
+        }
+
+        seal "transit" {
+          address = "https://pivault.dnet"
+          disable_renewal = "false"
+          key_name = "autounseal"
+          mount_path = "transit/"
+          tls_skip_verify = "true"
+        }
+
+        storage "raft" {
+          path = "/vault/data"
+        }
+
+        service_registration "kubernetes" {}
+
+  dataStorage:
+    enabled: true
+    size: 32Gi
+    storageClass: null
+    accessMode: ReadWriteOnce
+
+  ingress:
+    enabled: true
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt-prod
+      kubernetes.io/ingress.class: nginx
+    hosts:
+      - host: vault.ducoterra.net
+        paths:
+          - /
+
+    tls:
+    - hosts:
+      - vault.ducoterra.net
+      secretName: vault-tls-cert
+
+ui:
+  enabled: true
+  serviceType: ClusterIP

helm/pivault.yaml

@@ -31,8 +31,6 @@ server:
 
   ingress:
     enabled: true
-    annotations:
-      cert-manager.io/cluster-issuer: vault-issuer
     hosts:
       - host: pivault.dnet
         paths:

helm/standalone-test.yaml

@@ -0,0 +1,55 @@
+global:
+  enabled: true
+
+injector:
+  enabled: false
+
+server:
+  authDelegator:
+    enabled: false
+
+  standalone:
+    enabled: true
+    config: |
+      ui = true
+
+      listener "tcp" {
+        tls_disable = 1
+        address = "[::]:8200"
+        cluster_address = "[::]:8201"
+      }
+      storage "file" {
+        path = "/vault/data"
+      }
+
+  dataStorage:
+    enabled: true
+    size: 32Gi
+    storageClass: null
+    accessMode: ReadWriteOnce
+
+  # volumeMounts:
+  #   - mountPath: /vault/old_data
+  #     name: old-data
+  #     readOnly: true
+
+  # volumes:
+  #   - name: old-data
+  #     persistentVolumeClaim:
+  #       claimName: data-vault-transfer-0
+
+  ingress:
+    enabled: true
+    hosts:
+      - host: vault-test.dnet
+        paths:
+          - /
+
+    tls:
+      - hosts:
+          - vault-test.dnet
+        secretName: vault-test-dnet-cert
+
+ui:
+  enabled: true
+  serviceType: ClusterIP

helm/standalone.yaml

@@ -2,22 +2,6 @@ global:
   enabled: true
 
 server:
-  # ha:
-  #   enabled: true
-  #   config: |
-  #     ui = true
-
-  #     listener "tcp" {
-  #       address = "[::]:8200"
-  #       cluster_address = "[::]:8201"
-  #     }
-
-  #     storage "file" {
-  #       path = "/vault/data"
-  #     }
-
-  #   raft:
-  #     enabled: true
 
   standalone:
     enabled: true
@@ -39,7 +23,6 @@ server:
     storageClass: null
     accessMode: ReadWriteOnce
 
-
   ingress:
     enabled: true
     annotations:

k8s/test-certificate.yaml

@@ -0,0 +1,12 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: vault-test-dnet
+spec:
+  secretName: vault-test-dnet-cert
+  issuerRef:
+    name: vault-issuer
+    kind: ClusterIssuer
+  commonName: vault-test.dnet
+  dnsNames:
+  - vault-test.dnet

k8s/vault-dnet-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pivault-dnet
+spec:
+  ports:
+  - protocol: TCP
+    port: 443
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: external-vault
+subsets:
+  - addresses:
+      - ip: pivault.dnet
+    ports:
+      - port: 443

policies/autounseal.hcl

@@ -0,0 +1,7 @@
+path "transit/encrypt/autounseal" {
+   capabilities = [ "update" ]
+}
+
+path "transit/decrypt/autounseal" {
+   capabilities = [ "update" ]
+}

scripts/vault_unseal.py

@@ -8,7 +8,7 @@ import threading
 vaults = ["vault-0"]
 
 home = os.getenv("HOME")
-with open(os.path.join(home, ".vault-keys/cluster-keys.json")) as f:
+with open(os.path.join(home, ".vault-keys/pivault-cluster-keys.json")) as f:
     vault_secrets = json.load(f)
 
 procs = []
@@ -18,7 +18,7 @@ for vault in vaults:
         threading.Thread(
             target = subprocess.run,
             args = (
-                ["kubectl", "--context", "k3os-alpha.dnet-admin-vault",
+                ["kubectl", "--context", "pikube.dnet-admin-pivault",
                  "exec", "-ti", vault, "--",
                  "vault", "operator", "unseal",
                  vault_secrets.get("unseal_keys_b64")[key]],))