Vault

Prereqs

brew tap hashicorp/tap
brew install hashicorp/tap/vault

brew install jq

Install (Standalone)

kubectl apply -f k8s/certificate.yaml

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo hashicorp/vault
helm upgrade --install vault hashicorp/vault --values helm/values.yaml

mkdir ~/.vault-keys
kubectl exec -ti vault-0 -- vault operator init -key-shares=5 -key-threshold=3 -format=json > ~/.vault-keys/cluster-keys.json
kubectl exec -ti vault-0 -- vault operator unseal

Install (Cluster)

This assumes you have an unseal vault available! See ha-test for manual unseal.

kubectl apply -f k8s/certificate.yaml

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo hashicorp/vault
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml

mkdir ~/.vault-keys
kubectl exec -ti vault-0 -- vault operator init -format=json > ~/.vault-keys/vault-cluster-keys.json
# Only run the unseal if not autounsealing
# kubectl exec -ti vault-0 -- vault operator unseal

kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
# kubectl exec -ti vault-1 -- vault operator unseal

kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
# kubectl exec -ti vault-2 -- vault operator unseal

Add/Update policy

vault policy write ducoterra policies/ducoterra.hcl

Add user

vault auth enable userpass
vault write auth/userpass/users/ducoterra \
    policies=ducoterra \
    password='<password>'

Enable KV Secrets

vault secrets enable -path=secret kv-v2

vault kv put secret/okta username='static-user' password='static-password'
vault kv get secret/okta

TOTP

vault secrets enable totp

vault write totp/keys/okta \
    url='otpauth://totp/Vault:reese.wells@nimbisservices.com?secret=SECRET&issuer=Okta'

vault read totp/code/okta

Policy:

path "totp/keys/*" {
    capabilities = ["update"]
}

path "totp/code/*" {
    capabilities = ["read"] 
}

Kubernetes Secrets

kubectl exec -it vault-0 -- /bin/sh

vault login -method=token <token>
vault auth enable kubernetes

vault write auth/kubernetes/config \
    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

vault policy write internal-app - <<EOF
path "internal/data/database/config" {
  capabilities = ["read"]
}
EOF

vault write auth/kubernetes/role/internal-app \
    bound_service_account_names=internal-app \
    bound_service_account_namespaces=gitlab \
    policies=internal-app \
    ttl=24h

rm /home/vault/.vault-token

Set your kube config to the namespace you want to use vault with

kubectl apply -f k8s/service-account-internal-app.yaml

Add secrets to your pod

spec:
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: "internal-app"
        vault.hashicorp.com/agent-inject-secret-database-config.txt: "internal/data/database/config"

Gitlab Integration

Enable JWT auth for Gitlab

# Enable jwt auth
vault auth enable jwt

# Create a policy
vault policy write myproject-production - <<EOF
# Policy name: myproject-production
#
# Read-only permission on 'secret/data/myproject/production/*' path
path "secret/data/myproject/production/*" {
  capabilities = [ "read" ]
}
EOF

# Create a role
vault write auth/jwt/role/myproject-production - <<EOF
{
  "role_type": "jwt",
  "policies": ["myproject-production"],
  "token_explicit_max_ttl": 60,
  "user_claim": "user_email",
  "bound_claims_type": "glob",
  "bound_claims": {
    "project_id": "127",
    "ref_protected": "false",
    "ref_type": "branch",
    "ref": "main"
  }
}
EOF

# Write the auth method
vault write auth/jwt/config \
    jwks_url="https://gitlab.ducoterra.net/-/jwks" \
    bound_issuer="gitlab.ducoterra.net"

Pipeline syntax

read_secrets:
  script:
    # Check job's ref name
    - echo $CI_COMMIT_REF_NAME
    # and is this ref protected
    - echo $CI_COMMIT_REF_PROTECTED
    # Vault's address can be provided here or as CI/CD variable
    - export VAULT_ADDR=https://vault.ducoterra.net
    # Authenticate and get token. Token expiry time and other properties can be configured
    # when configuring JWT Auth - https://www.vaultproject.io/api/auth/jwt#parameters-1
    - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-production jwt=$CI_JOB_JWT)"
    # Now use the VAULT_TOKEN to read the secret and store it in environment variable
    - export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)"
    # Use the secret
    - echo $PASSWORD

Certificate Management

https://learn.hashicorp.com/tutorials/vault/pki-engine https://learn.hashicorp.com/tutorials/vault/kubernetes-cert-manager

Create a CA

export ROOT_PATH=dnet

# Note: 19800h is Apple's limit
vault secrets enable -path=$ROOT_PATH pki
vault secrets tune -max-lease-ttl=19800h $ROOT_PATH

# Create a root CA
vault write $ROOT_PATH/root/generate/internal \
    common_name="Ducoterra Root CA" \
    ttl=19800h

# Create a CA URL
vault write $ROOT_PATH/config/urls \
    issuing_certificates="https://vault.ducoterra.net/v1/$ROOT_PATH/ca" \
    crl_distribution_points="https://vault.ducoterra.net/v1/$ROOT_PATH/crl"

Navigate to https://vault.ducoterra.net/v1/$ROOT_PATH/ca and download the CA. Import to your devices.

Create an intermediate CA

export ROOT_PATH=dnet
export PKI_PATH=dnet_inter

vault secrets enable -path=$PKI_PATH pki
vault secrets tune -max-lease-ttl=8760h $PKI_PATH

# Create CSR to sign with root CA
vault write -format=json $PKI_PATH/intermediate/generate/internal \
    common_name="Ducoterra Intermediate CA" \
    | jq -r '.data.csr' > certs/$PKI_PATH.csr

# Sign the CSR with the root CA
vault write -format=json $ROOT_PATH/root/sign-intermediate \
    csr=@certs/$PKI_PATH.csr format=pem_bundle ttl=8760h \
    | jq -r '.data.certificate' > certs/$PKI_PATH.cert.pem

# Save the signed cert to vault
vault write $PKI_PATH/intermediate/set-signed certificate=@certs/$PKI_PATH.cert.pem

# Create a CA URL
vault write $PKI_PATH/config/urls \
    issuing_certificates="https://vault.ducoterra.net/v1/$PKI_PATH/ca" \
    crl_distribution_points="https://vault.ducoterra.net/v1/$PKI_PATH/crl"

Navigate to https://vault.ducoterra.net/v1/$PKI_PATH/ca and download the CA. Import to your devices.

Allow .dnet and .hole certificates

export PKI_PATH=dnet_inter

# Allow .dnet subdomain
vault write $PKI_PATH/roles/dnet \
    allowed_domains=dnet \
    allow_subdomains=true max_ttl=8760h

# Allow .hole subdomain
vault write $PKI_PATH/roles/pi_hole \
    allowed_domains=hole \
    allow_subdomains=true max_ttl=8760h

Issue a certificate

export PKI_PATH=dnet_inter
export CNAME=freenas.dnet

# Use -format=json to dump a json file
vault write $PKI_PATH/issue/dnet \
    common_name=$CNAME \
    max_ttl=8760h > certs/$CNAME.cert

# Pihole Example
vault write $PKI_PATH/issue/pi_hole \
    common_name=$CNAME \
    max_ttl=8760h > certs/$CNAME.cert

Adding cert to freenas

Only caveat here is to paste the certificate and then the full chain cert below in the "certificate" section. iOS will infintely refresh the page if the full chain isn't provided.

Adding cert to pihole

# Paste the isser CA in here:
vim /etc/ssl/certs/vault-ca.pem

# Paste the certificate and private key in here:
vim /etc/ssl/certs/pihole.pem

# Make sure lighttpd is configured correctly
vim /etc/lighttpd/external.conf

# Restart the service
service lighttpd restart

Adding cert to cloudkey

THIS DOESN'T WORK

service unifi stop

# Remove the UNIFI_SSL_KEYSTORE=/etc/ssl/private/unifi.keystore.jks line from config
cp /etc/default/unifi /etc/default/unifi.back
vim /etc/default/unifi

# Remove the keystore reference
mv /usr/lib/unifi/data/keystore /usr/lib/unifi/data/keystore.back

# Copy your primary SSL Certificate into /etc/ssl/private/cloudkey.crt
cp /etc/ssl/private/cloudkey.crt /etc/ssl/private/cloudkey.crt.back
vim /etc/ssl/private/cloudkey.crt

# Copy your private key into /etc/ssl/private/cloudkey.key
cp /etc/ssl/private/cloudkey.key /etc/ssl/private/cloudkey.key.back
vim /etc/ssl/private/cloudkey.key

# Add the CA to /etc/ssl/private/vault-ca.pem
vim /etc/ssl/private/vault-ca.pem

# Generate the key bundle
openssl pkcs12 -export -in /etc/ssl/private/cloudkey.crt -inkey \
/etc/ssl/private/cloudkey.key -out /etc/ssl/private/cloudkey.p12 \
-name unifi -CAfile /etc/ssl/private/vault-ca.pem -caname vault -password pass:temppass

# Import to keystore
# Default password is "aircontrolenterprise"
keytool -importkeystore -deststorepass aircontrolenterprise \
-destkeypass aircontrolenterprise -destkeystore /usr/lib/unifi/data/keystore \
-srckeystore /etc/ssl/private/cloudkey.p12 -srcstoretype PKCS12 -srcstorepass temppass -alias unifi -noprompt

service unifi start

Revoke a certificate

vault write pki_dnet_int/revoke serial_number=<serial_number>
vault write pki_dnet_int/tidy tidy_cert_store=true tidy_revoked_certs=true

Use with cert-manager

vault policy write pki_dnet_int - <<EOF
path "pki_dnet_int*"                        { capabilities = ["read", "list"] }
path "pki_dnet_int/roles/*"   { capabilities = ["create", "update"] }
path "pki_dnet_int/sign/*"    { capabilities = ["create", "update"] }
path "pki_dnet_int/issue/*"   { capabilities = ["create"] }
EOF

vault write auth/kubernetes/role/issuer \
    bound_service_account_names=issuer \
    bound_service_account_namespaces=cert-manager \
    policies=pki_dnet_int \
    ttl=20m

kubectl -n cert-manager create serviceaccount issuer
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")

cat > cert-manager/vault-clusterissuer.yaml <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: vault-issuer
spec:
  vault:
    server: https://vault.ducoterra.net
    path: pki_dnet_int/sign/dnet
    auth:
      kubernetes:
        mountPath: /v1/auth/kubernetes
        role: issuer
        secretRef:
          name: $ISSUER_SECRET_REF
          key: token
EOF

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    # cert-manager.io/cluster-issuer: letsencrypt-prod
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.org/client-max-body-size: "0"
  name: {{ .Release.Name }}
spec:
  rules:
  - host: test.dnet
    http:
      paths:
      - backend:
          service:
            name: {{ .Release.Name }}
            port:
              number: 80
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - test-dnet
    secretName: test-dnet-cert
---
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  name: test-dnet
spec:
  secretName: test-dnet-cert
  issuerRef:
    name: vault-issuer
    kind: ClusterIssuer
  commonName: test.dnet
  dnsNames:
  - test.dnet

Kubernetes External Vault Auth

https://learn.hashicorp.com/tutorials/vault/kubernetes-external-vault?in=vault/kubernetes

Connect to external vault

# On our other server
helm install vault hashicorp/vault \
    --set "injector.externalVaultAddr=https://vault.ducoterra.net"

VAULT_HELM_SECRET_NAME=$(kubectl get secrets --output=json | jq -r '.items[].metadata | select(.name|startswith("vault-token-")).name')
TOKEN_REVIEW_JWT=$(kubectl get secret $VAULT_HELM_SECRET_NAME --output='go-template={{ .data.token }}' | base64 --decode)
KUBE_CA_CERT=$(kubectl config view --raw --minify --flatten --output='jsonpath={.clusters[].cluster.certificate-authority-data}' | base64 --decode)
KUBE_HOST="https://3.14.3.104:6443"

vault auth enable -path=pikube kubernetes

vault write auth/pikube/config \
    token_reviewer_jwt="$TOKEN_REVIEW_JWT" \
    kubernetes_host="https://3.14.3.104:6443" \
    kubernetes_ca_cert="$KUBE_CA_CERT"

vault write auth/pikube/role/issuer \
    bound_service_account_names=issuer \
    bound_service_account_namespaces=cert-manager \
    policies=pki_dnet_int \
    ttl=20m

Install cert-manager

kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml
kubectl -n cert-manager create serviceaccount issuer
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")

cat > cert-manager/pikube-vault-clusterissuer.yaml <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: vault-issuer
spec:
  vault:
    server: https://vault.ducoterra.net
    path: pki_dnet_int/sign/dnet
    auth:
      kubernetes:
        mountPath: /v1/auth/pikube
        role: issuer
        secretRef:
          name: $ISSUER_SECRET_REF
          key: token
EOF

kubectl apply -f cert-manager/pikube-vault-clusterissuer.yaml

Auto-unseal

https://learn.hashicorp.com/tutorials/vault/autounseal-transit

Enable transit on vault that will do the unsealing

vault secrets enable transit
vault write -f transit/keys/autounseal

tee policies/autounseal.hcl <<EOF
path "transit/encrypt/autounseal" {
   capabilities = [ "update" ]
}

path "transit/decrypt/autounseal" {
   capabilities = [ "update" ]
}
EOF

vault policy write autounseal policies/autounseal.hcl
vault token create -policy="autounseal" -wrap-ttl=120
VAULT_TOKEN="wrapping token" vault unwrap

Auto unseal

Create a test vault in a new namespace

kubectl apply -f k8s/certificate.yaml

# Get token from above output
kubectl create secret generic auto-unseal-token --from-literal=VAULT_TOKEN="token from output"
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml

Test Auto Unseal

kubectl exec -it vault-0 -- vault operator init
kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200