543 lines
14 KiB
Markdown
543 lines
14 KiB
Markdown
# Vault
|
|
|
|
## Prereqs
|
|
|
|
```bash
|
|
brew tap hashicorp/tap
|
|
brew install hashicorp/tap/vault
|
|
|
|
brew install jq
|
|
```
|
|
|
|
## Install (Standalone)
|
|
|
|
```bash
|
|
kubectl apply -f k8s/certificate.yaml
|
|
|
|
helm repo add hashicorp https://helm.releases.hashicorp.com
|
|
helm search repo hashicorp/vault
|
|
helm upgrade --install vault hashicorp/vault --values helm/values.yaml
|
|
|
|
mkdir ~/.vault-keys
|
|
kubectl exec -ti vault-0 -- vault operator init -key-shares=5 -key-threshold=3 -format=json > ~/.vault-keys/cluster-keys.json
|
|
kubectl exec -ti vault-0 -- vault operator unseal
|
|
```
|
|
|
|
## Install (Cluster)
|
|
|
|
This assumes you have an unseal vault available! See ha-test for manual unseal.
|
|
|
|
```bash
|
|
kubectl apply -f k8s/certificate.yaml
|
|
|
|
helm repo add hashicorp https://helm.releases.hashicorp.com
|
|
helm search repo hashicorp/vault
|
|
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml
|
|
|
|
mkdir ~/.vault-keys
|
|
kubectl exec -ti vault-0 -- vault operator init -format=json > ~/.vault-keys/vault-cluster-keys.json
|
|
# Only run the unseal if not autounsealing
|
|
# kubectl exec -ti vault-0 -- vault operator unseal
|
|
|
|
kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
|
|
# kubectl exec -ti vault-1 -- vault operator unseal
|
|
|
|
kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
|
|
# kubectl exec -ti vault-2 -- vault operator unseal
|
|
```
|
|
|
|
## Add/Update policy
|
|
|
|
```bash
|
|
vault policy write ducoterra policies/ducoterra.hcl
|
|
```
|
|
|
|
## Add user
|
|
|
|
```bash
|
|
vault auth enable userpass
|
|
vault write auth/userpass/users/ducoterra \
|
|
policies=ducoterra \
|
|
password='<password>'
|
|
```
|
|
|
|
## Enable KV Secrets
|
|
|
|
```bash
|
|
vault secrets enable -path=secret kv-v2
|
|
|
|
vault kv put secret/okta username='static-user' password='static-password'
|
|
vault kv get secret/okta
|
|
```
|
|
|
|
## TOTP
|
|
|
|
```bash
|
|
vault secrets enable totp
|
|
|
|
vault write totp/keys/okta \
|
|
url='otpauth://totp/Vault:reese.wells@nimbisservices.com?secret=SECRET&issuer=Okta'
|
|
|
|
vault read totp/code/okta
|
|
```
|
|
|
|
Policy:
|
|
|
|
```bash
|
|
path "totp/keys/*" {
|
|
capabilities = ["update"]
|
|
}
|
|
|
|
path "totp/code/*" {
|
|
capabilities = ["read"]
|
|
}
|
|
```
|
|
|
|
## Kubernetes Secrets
|
|
|
|
```bash
|
|
kubectl exec -it vault-0 -- /bin/sh
|
|
|
|
vault login -method=token <token>
|
|
vault auth enable kubernetes
|
|
|
|
vault write auth/kubernetes/config \
|
|
token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
|
|
kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
|
|
kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
|
|
vault policy write internal-app - <<EOF
|
|
path "internal/data/database/config" {
|
|
capabilities = ["read"]
|
|
}
|
|
EOF
|
|
|
|
vault write auth/kubernetes/role/internal-app \
|
|
bound_service_account_names=internal-app \
|
|
bound_service_account_namespaces=gitlab \
|
|
policies=internal-app \
|
|
ttl=24h
|
|
|
|
rm /home/vault/.vault-token
|
|
```
|
|
|
|
Set your kube config to the namespace you want to use vault with
|
|
|
|
```bash
|
|
kubectl apply -f k8s/service-account-internal-app.yaml
|
|
```
|
|
|
|
Add secrets to your pod
|
|
|
|
```bash
|
|
spec:
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
vault.hashicorp.com/agent-inject: "true"
|
|
vault.hashicorp.com/role: "internal-app"
|
|
vault.hashicorp.com/agent-inject-secret-database-config.txt: "internal/data/database/config"
|
|
```
|
|
|
|
## Gitlab Integration
|
|
|
|
### Enable JWT auth for Gitlab
|
|
|
|
```bash
|
|
# Enable jwt auth
|
|
vault auth enable jwt
|
|
|
|
# Create a policy
|
|
vault policy write myproject-production - <<EOF
|
|
# Policy name: myproject-production
|
|
#
|
|
# Read-only permission on 'secret/data/myproject/production/*' path
|
|
path "secret/data/myproject/production/*" {
|
|
capabilities = [ "read" ]
|
|
}
|
|
EOF
|
|
|
|
# Create a role
|
|
vault write auth/jwt/role/myproject-production - <<EOF
|
|
{
|
|
"role_type": "jwt",
|
|
"policies": ["myproject-production"],
|
|
"token_explicit_max_ttl": 60,
|
|
"user_claim": "user_email",
|
|
"bound_claims_type": "glob",
|
|
"bound_claims": {
|
|
"project_id": "127",
|
|
"ref_protected": "false",
|
|
"ref_type": "branch",
|
|
"ref": "main"
|
|
}
|
|
}
|
|
EOF
|
|
|
|
# Write the auth method
|
|
vault write auth/jwt/config \
|
|
jwks_url="https://gitlab.ducoterra.net/-/jwks" \
|
|
bound_issuer="gitlab.ducoterra.net"
|
|
```
|
|
|
|
### Pipeline syntax
|
|
|
|
```yaml
|
|
read_secrets:
|
|
script:
|
|
# Check job's ref name
|
|
- echo $CI_COMMIT_REF_NAME
|
|
# and is this ref protected
|
|
- echo $CI_COMMIT_REF_PROTECTED
|
|
# Vault's address can be provided here or as CI/CD variable
|
|
- export VAULT_ADDR=https://vault.ducoterra.net
|
|
# Authenticate and get token. Token expiry time and other properties can be configured
|
|
# when configuring JWT Auth - https://www.vaultproject.io/api/auth/jwt#parameters-1
|
|
- export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-production jwt=$CI_JOB_JWT)"
|
|
# Now use the VAULT_TOKEN to read the secret and store it in environment variable
|
|
- export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)"
|
|
# Use the secret
|
|
- echo $PASSWORD
|
|
```
|
|
|
|
## Certificate Management
|
|
|
|
<https://learn.hashicorp.com/tutorials/vault/pki-engine>
|
|
<https://learn.hashicorp.com/tutorials/vault/kubernetes-cert-manager>
|
|
|
|
### Create a CA
|
|
|
|
```bash
|
|
export ROOT_PATH=dnet
|
|
|
|
# Note: 19800h is Apple's limit
|
|
vault secrets enable -path=$ROOT_PATH pki
|
|
vault secrets tune -max-lease-ttl=19800h $ROOT_PATH
|
|
|
|
# Create a root CA
|
|
vault write $ROOT_PATH/root/generate/internal \
|
|
common_name="Ducoterra Root CA" \
|
|
ttl=19800h
|
|
|
|
# Create a CA URL
|
|
vault write $ROOT_PATH/config/urls \
|
|
issuing_certificates="https://vault.ducoterra.net/v1/$ROOT_PATH/ca" \
|
|
crl_distribution_points="https://vault.ducoterra.net/v1/$ROOT_PATH/crl"
|
|
```
|
|
|
|
Navigate to <https://vault.ducoterra.net/v1/$ROOT_PATH/ca> and download the CA. Import to your devices.
|
|
|
|
### Create an intermediate CA
|
|
|
|
```bash
|
|
export ROOT_PATH=dnet
|
|
export PKI_PATH=dnet_inter
|
|
|
|
vault secrets enable -path=$PKI_PATH pki
|
|
vault secrets tune -max-lease-ttl=8760h $PKI_PATH
|
|
|
|
# Create CSR to sign with root CA
|
|
vault write -format=json $PKI_PATH/intermediate/generate/internal \
|
|
common_name="Ducoterra Intermediate CA" \
|
|
| jq -r '.data.csr' > certs/$PKI_PATH.csr
|
|
|
|
# Sign the CSR with the root CA
|
|
vault write -format=json $ROOT_PATH/root/sign-intermediate \
|
|
csr=@certs/$PKI_PATH.csr format=pem_bundle ttl=8760h \
|
|
| jq -r '.data.certificate' > certs/$PKI_PATH.cert.pem
|
|
|
|
# Save the signed cert to vault
|
|
vault write $PKI_PATH/intermediate/set-signed certificate=@certs/$PKI_PATH.cert.pem
|
|
|
|
# Create a CA URL
|
|
vault write $PKI_PATH/config/urls \
|
|
issuing_certificates="https://vault.ducoterra.net/v1/$PKI_PATH/ca" \
|
|
crl_distribution_points="https://vault.ducoterra.net/v1/$PKI_PATH/crl"
|
|
```
|
|
|
|
Navigate to <https://vault.ducoterra.net/v1/$PKI_PATH/ca> and download the CA. Import to your devices.
|
|
|
|
### Allow .dnet and .hole certificates
|
|
|
|
```bash
|
|
export PKI_PATH=dnet_inter
|
|
|
|
# Allow .dnet subdomain
|
|
vault write $PKI_PATH/roles/dnet \
|
|
allowed_domains=dnet \
|
|
allow_subdomains=true max_ttl=8760h
|
|
|
|
# Allow .hole subdomain
|
|
vault write $PKI_PATH/roles/pi_hole \
|
|
allowed_domains=hole \
|
|
allow_subdomains=true max_ttl=8760h
|
|
```
|
|
|
|
### Issue a certificate
|
|
|
|
```bash
|
|
export PKI_PATH=dnet_inter
|
|
export CNAME=freenas.dnet
|
|
|
|
# Use -format=json to dump a json file
|
|
vault write $PKI_PATH/issue/dnet \
|
|
common_name=$CNAME \
|
|
max_ttl=8760h > certs/$CNAME.cert
|
|
|
|
# Pihole Example
|
|
vault write $PKI_PATH/issue/pi_hole \
|
|
common_name=$CNAME \
|
|
max_ttl=8760h > certs/$CNAME.cert
|
|
```
|
|
|
|
#### Adding cert to freenas
|
|
|
|
Only caveat here is to paste the certificate and then the full chain cert below in the
|
|
"certificate" section. iOS will infintely refresh the page if the full chain isn't provided.
|
|
|
|
#### Adding cert to pihole
|
|
|
|
```bash
|
|
# Paste the isser CA in here:
|
|
vim /etc/ssl/certs/vault-ca.pem
|
|
|
|
# Paste the certificate and private key in here:
|
|
vim /etc/ssl/certs/pihole.pem
|
|
|
|
# Make sure lighttpd is configured correctly
|
|
vim /etc/lighttpd/external.conf
|
|
|
|
# Restart the service
|
|
service lighttpd restart
|
|
```
|
|
|
|
#### Adding cert to cloudkey
|
|
|
|
**THIS DOESN'T WORK**
|
|
|
|
```bash
|
|
service unifi stop
|
|
|
|
# Remove the UNIFI_SSL_KEYSTORE=/etc/ssl/private/unifi.keystore.jks line from config
|
|
cp /etc/default/unifi /etc/default/unifi.back
|
|
vim /etc/default/unifi
|
|
|
|
# Remove the keystore reference
|
|
mv /usr/lib/unifi/data/keystore /usr/lib/unifi/data/keystore.back
|
|
|
|
# Copy your primary SSL Certificate into /etc/ssl/private/cloudkey.crt
|
|
cp /etc/ssl/private/cloudkey.crt /etc/ssl/private/cloudkey.crt.back
|
|
vim /etc/ssl/private/cloudkey.crt
|
|
|
|
# Copy your private key into /etc/ssl/private/cloudkey.key
|
|
cp /etc/ssl/private/cloudkey.key /etc/ssl/private/cloudkey.key.back
|
|
vim /etc/ssl/private/cloudkey.key
|
|
|
|
# Add the CA to /etc/ssl/private/vault-ca.pem
|
|
vim /etc/ssl/private/vault-ca.pem
|
|
|
|
# Generate the key bundle
|
|
openssl pkcs12 -export -in /etc/ssl/private/cloudkey.crt -inkey \
|
|
/etc/ssl/private/cloudkey.key -out /etc/ssl/private/cloudkey.p12 \
|
|
-name unifi -CAfile /etc/ssl/private/vault-ca.pem -caname vault -password pass:temppass
|
|
|
|
# Import to keystore
|
|
# Default password is "aircontrolenterprise"
|
|
keytool -importkeystore -deststorepass aircontrolenterprise \
|
|
-destkeypass aircontrolenterprise -destkeystore /usr/lib/unifi/data/keystore \
|
|
-srckeystore /etc/ssl/private/cloudkey.p12 -srcstoretype PKCS12 -srcstorepass temppass -alias unifi -noprompt
|
|
|
|
service unifi start
|
|
```
|
|
|
|
### Revoke a certificate
|
|
|
|
```bash
|
|
vault write pki_dnet_int/revoke serial_number=<serial_number>
|
|
vault write pki_dnet_int/tidy tidy_cert_store=true tidy_revoked_certs=true
|
|
```
|
|
|
|
### Use with cert-manager
|
|
|
|
```bash
|
|
vault policy write pki_dnet_int - <<EOF
|
|
path "pki_dnet_int*" { capabilities = ["read", "list"] }
|
|
path "pki_dnet_int/roles/*" { capabilities = ["create", "update"] }
|
|
path "pki_dnet_int/sign/*" { capabilities = ["create", "update"] }
|
|
path "pki_dnet_int/issue/*" { capabilities = ["create"] }
|
|
EOF
|
|
|
|
vault write auth/kubernetes/role/issuer \
|
|
bound_service_account_names=issuer \
|
|
bound_service_account_namespaces=cert-manager \
|
|
policies=pki_dnet_int \
|
|
ttl=20m
|
|
|
|
kubectl -n cert-manager create serviceaccount issuer
|
|
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")
|
|
|
|
cat > cert-manager/vault-clusterissuer.yaml <<EOF
|
|
apiVersion: cert-manager.io/v1
|
|
kind: ClusterIssuer
|
|
metadata:
|
|
name: vault-issuer
|
|
spec:
|
|
vault:
|
|
server: https://vault.ducoterra.net
|
|
path: pki_dnet_int/sign/dnet
|
|
auth:
|
|
kubernetes:
|
|
mountPath: /v1/auth/kubernetes
|
|
role: issuer
|
|
secretRef:
|
|
name: $ISSUER_SECRET_REF
|
|
key: token
|
|
EOF
|
|
```
|
|
|
|
```yaml
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
annotations:
|
|
# cert-manager.io/cluster-issuer: letsencrypt-prod
|
|
kubernetes.io/ingress.class: nginx
|
|
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
|
nginx.org/client-max-body-size: "0"
|
|
name: {{ .Release.Name }}
|
|
spec:
|
|
rules:
|
|
- host: test.dnet
|
|
http:
|
|
paths:
|
|
- backend:
|
|
service:
|
|
name: {{ .Release.Name }}
|
|
port:
|
|
number: 80
|
|
path: /
|
|
pathType: Prefix
|
|
tls:
|
|
- hosts:
|
|
- test-dnet
|
|
secretName: test-dnet-cert
|
|
---
|
|
apiVersion: cert-manager.io/v1alpha2
|
|
kind: Certificate
|
|
metadata:
|
|
name: test-dnet
|
|
spec:
|
|
secretName: test-dnet-cert
|
|
issuerRef:
|
|
name: vault-issuer
|
|
kind: ClusterIssuer
|
|
commonName: test.dnet
|
|
dnsNames:
|
|
- test.dnet
|
|
```
|
|
|
|
## Kubernetes External Vault Auth
|
|
|
|
<https://learn.hashicorp.com/tutorials/vault/kubernetes-external-vault?in=vault/kubernetes>
|
|
|
|
### Connect to external vault
|
|
|
|
```bash
|
|
# On our other server
|
|
helm install vault hashicorp/vault \
|
|
--set "injector.externalVaultAddr=https://vault.ducoterra.net"
|
|
|
|
VAULT_HELM_SECRET_NAME=$(kubectl get secrets --output=json | jq -r '.items[].metadata | select(.name|startswith("vault-token-")).name')
|
|
TOKEN_REVIEW_JWT=$(kubectl get secret $VAULT_HELM_SECRET_NAME --output='go-template={{ .data.token }}' | base64 --decode)
|
|
KUBE_CA_CERT=$(kubectl config view --raw --minify --flatten --output='jsonpath={.clusters[].cluster.certificate-authority-data}' | base64 --decode)
|
|
KUBE_HOST="https://3.14.3.104:6443"
|
|
```
|
|
|
|
```bash
|
|
vault auth enable -path=pikube kubernetes
|
|
|
|
vault write auth/pikube/config \
|
|
token_reviewer_jwt="$TOKEN_REVIEW_JWT" \
|
|
kubernetes_host="https://3.14.3.104:6443" \
|
|
kubernetes_ca_cert="$KUBE_CA_CERT"
|
|
|
|
vault write auth/pikube/role/issuer \
|
|
bound_service_account_names=issuer \
|
|
bound_service_account_namespaces=cert-manager \
|
|
policies=pki_dnet_int \
|
|
ttl=20m
|
|
```
|
|
|
|
### Install cert-manager
|
|
|
|
```bash
|
|
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml
|
|
kubectl -n cert-manager create serviceaccount issuer
|
|
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")
|
|
|
|
cat > cert-manager/pikube-vault-clusterissuer.yaml <<EOF
|
|
apiVersion: cert-manager.io/v1
|
|
kind: ClusterIssuer
|
|
metadata:
|
|
name: vault-issuer
|
|
spec:
|
|
vault:
|
|
server: https://vault.ducoterra.net
|
|
path: pki_dnet_int/sign/dnet
|
|
auth:
|
|
kubernetes:
|
|
mountPath: /v1/auth/pikube
|
|
role: issuer
|
|
secretRef:
|
|
name: $ISSUER_SECRET_REF
|
|
key: token
|
|
EOF
|
|
|
|
kubectl apply -f cert-manager/pikube-vault-clusterissuer.yaml
|
|
```
|
|
|
|
## Auto-unseal
|
|
|
|
<https://learn.hashicorp.com/tutorials/vault/autounseal-transit>
|
|
|
|
### Enable transit on vault that will do the unsealing
|
|
|
|
```bash
|
|
vault secrets enable transit
|
|
vault write -f transit/keys/autounseal
|
|
|
|
tee policies/autounseal.hcl <<EOF
|
|
path "transit/encrypt/autounseal" {
|
|
capabilities = [ "update" ]
|
|
}
|
|
|
|
path "transit/decrypt/autounseal" {
|
|
capabilities = [ "update" ]
|
|
}
|
|
EOF
|
|
|
|
vault policy write autounseal policies/autounseal.hcl
|
|
vault token create -policy="autounseal" -wrap-ttl=120
|
|
VAULT_TOKEN="wrapping token" vault unwrap
|
|
```
|
|
|
|
### Auto unseal
|
|
|
|
#### Create a test vault in a new namespace
|
|
|
|
```bash
|
|
kubectl apply -f k8s/certificate.yaml
|
|
|
|
# Get token from above output
|
|
kubectl create secret generic auto-unseal-token --from-literal=VAULT_TOKEN="token from output"
|
|
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml
|
|
```
|
|
|
|
#### Test Auto Unseal
|
|
|
|
```bash
|
|
kubectl exec -it vault-0 -- vault operator init
|
|
kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
|
|
kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
|
|
```
|