Vault

Prereqs

brew tap hashicorp/tap
brew install hashicorp/tap/vault

brew install jq

Install (Standalone)

kubectl apply -f k8s/certificate.yaml

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo hashicorp/vault
helm upgrade --install vault hashicorp/vault --values helm/values.yaml

mkdir ~/.vault-keys
kubectl exec -ti vault-0 -- vault operator init -key-shares=5 -key-threshold=3 -format=json > ~/.vault-keys/cluster-keys.json
kubectl exec -ti vault-0 -- vault operator unseal

Install (Cluster)

This assumes you have an unseal vault available! See ha-test for manual unseal.

kubectl apply -f k8s/certificate.yaml

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo hashicorp/vault
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml

mkdir ~/.vault-keys
kubectl exec -ti vault-0 -- vault operator init -format=json > ~/.vault-keys/vault-cluster-keys.json
# Only run the unseal if not autounsealing
# kubectl exec -ti vault-0 -- vault operator unseal

kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
# kubectl exec -ti vault-1 -- vault operator unseal

kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
# kubectl exec -ti vault-2 -- vault operator unseal

Add/Update policy

vault policy write ducoterra policies/ducoterra.hcl

Add user

vault auth enable userpass
vault write auth/userpass/users/ducoterra \
    policies=ducoterra \
    password='<password>'

Enable KV Secrets

vault secrets enable -path=secret kv-v2

vault kv put secret/okta username='static-user' password='static-password'
vault kv get secret/okta

TOTP

vault secrets enable totp

vault write totp/keys/okta \
    url='otpauth://totp/Vault:reese.wells@nimbisservices.com?secret=SECRET&issuer=Okta'

vault read totp/code/okta

Policy:

path "totp/keys/*" {
    capabilities = ["update"]
}

path "totp/code/*" {
    capabilities = ["read"] 
}

Kubernetes Secrets

kubectl exec -it vault-0 -- /bin/sh

vault login -method=token <token>
vault auth enable kubernetes

vault write auth/kubernetes/config \
    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

vault policy write internal-app - <<EOF
path "internal/data/database/config" {
  capabilities = ["read"]
}
EOF

vault write auth/kubernetes/role/internal-app \
    bound_service_account_names=internal-app \
    bound_service_account_namespaces=gitlab \
    policies=internal-app \
    ttl=24h

rm /home/vault/.vault-token

Set your kube config to the namespace you want to use vault with

kubectl apply -f k8s/service-account-internal-app.yaml

Add secrets to your pod

spec:
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: "internal-app"
        vault.hashicorp.com/agent-inject-secret-database-config.txt: "internal/data/database/config"

Gitlab Integration

Enable JWT auth for Gitlab

# Enable jwt auth
vault auth enable jwt

# Create a policy
vault policy write myproject-production - <<EOF
# Policy name: myproject-production
#
# Read-only permission on 'secret/data/myproject/production/*' path
path "secret/data/myproject/production/*" {
  capabilities = [ "read" ]
}
EOF

# Create a role
vault write auth/jwt/role/myproject-production - <<EOF
{
  "role_type": "jwt",
  "policies": ["myproject-production"],
  "token_explicit_max_ttl": 60,
  "user_claim": "user_email",
  "bound_claims_type": "glob",
  "bound_claims": {
    "project_id": "127",
    "ref_protected": "false",
    "ref_type": "branch",
    "ref": "main"
  }
}
EOF

# Write the auth method
vault write auth/jwt/config \
    jwks_url="https://gitlab.ducoterra.net/-/jwks" \
    bound_issuer="gitlab.ducoterra.net"

Pipeline syntax

read_secrets:
  script:
    # Check job's ref name
    - echo $CI_COMMIT_REF_NAME
    # and is this ref protected
    - echo $CI_COMMIT_REF_PROTECTED
    # Vault's address can be provided here or as CI/CD variable
    - export VAULT_ADDR=https://vault.ducoterra.net
    # Authenticate and get token. Token expiry time and other properties can be configured
    # when configuring JWT Auth - https://www.vaultproject.io/api/auth/jwt#parameters-1
    - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-production jwt=$CI_JOB_JWT)"
    # Now use the VAULT_TOKEN to read the secret and store it in environment variable
    - export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)"
    # Use the secret
    - echo $PASSWORD

Certificate Management

https://learn.hashicorp.com/tutorials/vault/pki-engine https://learn.hashicorp.com/tutorials/vault/kubernetes-cert-manager

Create a CA

# Note: 19800h is Apple's limit
vault secrets enable -path=pki_dnet pki
vault secrets tune -max-lease-ttl=19800h pki_dnet

vault write pki_dnet/root/generate/internal \
    common_name=vault.ducoterra.net \
    ttl=19800h

vault write pki_dnet/config/urls \
    issuing_certificates="https://vault.ducoterra.net/v1/pki_dnet/ca" \
    crl_distribution_points="https://vault.ducoterra.net/v1/pki_dnet/crl"

Create an intermediate CA

vault secrets enable -path=pki_dnet_int pki
vault secrets tune -max-lease-ttl=19800h pki_dnet_int
vault write -format=json pki_dnet_int/intermediate/generate/internal \
    common_name="vault.ducoterra.net Intermediate Authority" \
    | jq -r '.data.csr' > certs/pki_dnet_intermediate.csr

vault write -format=json pki_dnet/root/sign-intermediate \
    csr=@certs/pki_dnet_intermediate.csr format=pem_bundle ttl=19800h \
    | jq -r '.data.certificate' > certs/pki_dnet_intermediate.cert.pem

vault write pki_dnet_int/intermediate/set-signed certificate=@certs/pki_dnet_intermediate.cert.pem

vault write pki_dnet_int/config/urls \
    issuing_certificates="https://vault.ducoterra.net/v1/pki_dnet_int/ca" \
    crl_distribution_points="https://vault.ducoterra.net/v1/pki_dnet_int/crl"

vault write pki_dnet_int/roles/dnet \
    allowed_domains=dnet \
    allow_subdomains=true max_ttl=19800h

vault write pki_dnet_int/roles/pi_hole \
    allowed_domains=hole \
    allow_subdomains=true max_ttl=19800h

Navigate to https://vault.ducoterra.net/v1/pki_dnet_int/ca and download the CA. Import to your devices.

Issue a certificate

# Use -format=json to dump a json file
vault write pki_dnet_int/issue/dnet \
    common_name=freenas.dnet > certs/freenas.dnet.cert

vault write pki_dnet_int/issue/pi_hole \
    common_name=pi.hole > certs/pi.hole.cert

Adding cert to pihole

# Paste the isser CA in here:
vim /etc/ssl/certs/vault-ca.pem

# Paste the certificate and private key in here:
vim /etc/ssl/certs/pihole.pem

# Make sure lighttpd is configured correctly
vim /etc/lighttpd/external.conf

# Restart the service
service lighttpd restart

Revoke a certificate

vault write pki_dnet_int/revoke serial_number=<serial_number>
vault write pki_dnet_int/tidy tidy_cert_store=true tidy_revoked_certs=true

Use with cert-manager

vault policy write pki_dnet_int - <<EOF
path "pki_dnet_int*"                        { capabilities = ["read", "list"] }
path "pki_dnet_int/roles/*"   { capabilities = ["create", "update"] }
path "pki_dnet_int/sign/*"    { capabilities = ["create", "update"] }
path "pki_dnet_int/issue/*"   { capabilities = ["create"] }
EOF

vault write auth/kubernetes/role/issuer \
    bound_service_account_names=issuer \
    bound_service_account_namespaces=cert-manager \
    policies=pki_dnet \
    ttl=20m

kubectl -n cert-manager create serviceaccount issuer
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")

cat > cert-manager/vault-issuer.yaml <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: vault-issuer
spec:
  vault:
    server: https://vault.ducoterra.net
    path: pki_dnet_int/sign/dnet
    auth:
      kubernetes:
        mountPath: /v1/auth/kubernetes
        role: issuer
        secretRef:
          name: $ISSUER_SECRET_REF
          key: token
EOF

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    # cert-manager.io/cluster-issuer: letsencrypt-prod
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.org/client-max-body-size: "0"
  name: {{ .Release.Name }}
spec:
  rules:
  - host: test.dnet
    http:
      paths:
      - backend:
          service:
            name: {{ .Release.Name }}
            port:
              number: 80
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - test-dnet
    secretName: test-dnet-cert
---
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  name: test-dnet
spec:
  secretName: test-dnet-cert
  issuerRef:
    name: vault-issuer
    kind: ClusterIssuer
  commonName: test.dnet
  dnsNames:
  - test.dnet

Kubernetes External Vault Auth

https://learn.hashicorp.com/tutorials/vault/kubernetes-external-vault?in=vault/kubernetes

Connect to external vault

# On our other server
helm install vault hashicorp/vault \
    --set "injector.externalVaultAddr=https://vault.ducoterra.net"

VAULT_HELM_SECRET_NAME=$(kubectl get secrets --output=json | jq -r '.items[].metadata | select(.name|startswith("vault-token-")).name')
TOKEN_REVIEW_JWT=$(kubectl get secret $VAULT_HELM_SECRET_NAME --output='go-template={{ .data.token }}' | base64 --decode)
KUBE_CA_CERT=$(kubectl config view --raw --minify --flatten --output='jsonpath={.clusters[].cluster.certificate-authority-data}' | base64 --decode)
KUBE_HOST="https://3.14.3.104:6443"

vault auth enable -path=pikube kubernetes

vault write auth/pikube/config \
    token_reviewer_jwt="$TOKEN_REVIEW_JWT" \
    kubernetes_host="https://3.14.3.104:6443" \
    kubernetes_ca_cert="$KUBE_CA_CERT"

vault write auth/pikube/role/issuer \
    bound_service_account_names=issuer \
    bound_service_account_namespaces=cert-manager \
    policies=pki_dnet_int \
    ttl=20m

Install cert-manager

kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml
kubectl -n cert-manager create serviceaccount issuer
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")

cat > cert-manager/pikube-vault-clusterissuer.yaml <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: vault-issuer
spec:
  vault:
    server: https://vault.ducoterra.net
    path: pki_dnet_int/sign/dnet
    auth:
      kubernetes:
        mountPath: /v1/auth/pikube
        role: issuer
        secretRef:
          name: $ISSUER_SECRET_REF
          key: token
EOF

kubectl apply -f cert-manager/pikube-vault-clusterissuer.yaml

Auto-unseal

https://learn.hashicorp.com/tutorials/vault/autounseal-transit

Enable transit on vault that will do the unsealing

vault secrets enable transit
vault write -f transit/keys/autounseal

tee policies/autounseal.hcl <<EOF
path "transit/encrypt/autounseal" {
   capabilities = [ "update" ]
}

path "transit/decrypt/autounseal" {
   capabilities = [ "update" ]
}
EOF

vault policy write autounseal policies/autounseal.hcl
vault token create -policy="autounseal" -wrap-ttl=120
VAULT_TOKEN="wrapping token" vault unwrap

Auto unseal

Create a test vault in a new namespace

kubectl apply -f k8s/certificate.yaml

# Get token from above output
kubectl create secret generic auto-unseal-token --from-literal=VAULT_TOKEN="token from output"
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml

Test Auto Unseal

kubectl exec -it vault-0 -- vault operator init
kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200