vault/README.md

# Vault

## Prereqs

```bash
brew tap hashicorp/tap
brew install hashicorp/tap/vault

brew install jq
```

## Install (Standalone)

```bash
kubectl apply -f k8s/certificate.yaml

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo hashicorp/vault
helm upgrade --install vault hashicorp/vault --values helm/values.yaml

mkdir ~/.vault-keys
kubectl exec -ti vault-0 -- vault operator init -key-shares=5 -key-threshold=3 -format=json > ~/.vault-keys/cluster-keys.json
kubectl exec -ti vault-0 -- vault operator unseal
```

## Install (Cluster)

This assumes you have an unseal vault available! See ha-test for manual unseal.

```bash
kubectl apply -f k8s/certificate.yaml

helm repo add hashicorp https://helm.releases.hashicorp.com
helm search repo hashicorp/vault
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml

mkdir ~/.vault-keys
kubectl exec -ti vault-0 -- vault operator init -format=json > ~/.vault-keys/vault-cluster-keys.json
# Only run the unseal if not autounsealing
# kubectl exec -ti vault-0 -- vault operator unseal

kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
# kubectl exec -ti vault-1 -- vault operator unseal

kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
# kubectl exec -ti vault-2 -- vault operator unseal
```

## Add/Update policy

```bash
vault policy write ducoterra policies/ducoterra.hcl
```

## Add user

```bash
vault auth enable userpass
vault write auth/userpass/users/ducoterra \
    policies=ducoterra \
    password='<password>'
```

## Enable KV Secrets

```bash
vault secrets enable -path=secret kv-v2

vault kv put secret/okta username='static-user' password='static-password'
vault kv get secret/okta
```

## TOTP

```bash
vault secrets enable totp

vault write totp/keys/okta \
    url='otpauth://totp/Vault:reese.wells@nimbisservices.com?secret=SECRET&issuer=Okta'

vault read totp/code/okta
```

Policy:

```bash
path "totp/keys/*" {
    capabilities = ["update"]
}

path "totp/code/*" {
    capabilities = ["read"] 
}
```

## Kubernetes Secrets

```bash
kubectl exec -it vault-0 -- /bin/sh

vault login -method=token <token>
vault auth enable kubernetes

vault write auth/kubernetes/config \
    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

vault policy write internal-app - <<EOF
path "internal/data/database/config" {
  capabilities = ["read"]
}
EOF

vault write auth/kubernetes/role/internal-app \
    bound_service_account_names=internal-app \
    bound_service_account_namespaces=gitlab \
    policies=internal-app \
    ttl=24h

rm /home/vault/.vault-token
```

Set your kube config to the namespace you want to use vault with

```bash
kubectl apply -f k8s/service-account-internal-app.yaml
```

Add secrets to your pod

```bash
spec:
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: "internal-app"
        vault.hashicorp.com/agent-inject-secret-database-config.txt: "internal/data/database/config"
```

## Gitlab Integration

### Enable JWT auth for Gitlab

```bash
# Enable jwt auth
vault auth enable jwt

# Create a policy
vault policy write myproject-production - <<EOF
# Policy name: myproject-production
#
# Read-only permission on 'secret/data/myproject/production/*' path
path "secret/data/myproject/production/*" {
  capabilities = [ "read" ]
}
EOF

# Create a role
vault write auth/jwt/role/myproject-production - <<EOF
{
  "role_type": "jwt",
  "policies": ["myproject-production"],
  "token_explicit_max_ttl": 60,
  "user_claim": "user_email",
  "bound_claims_type": "glob",
  "bound_claims": {
    "project_id": "127",
    "ref_protected": "false",
    "ref_type": "branch",
    "ref": "main"
  }
}
EOF

# Write the auth method
vault write auth/jwt/config \
    jwks_url="https://gitlab.ducoterra.net/-/jwks" \
    bound_issuer="gitlab.ducoterra.net"
```

### Pipeline syntax

```yaml
read_secrets:
  script:
    # Check job's ref name
    - echo $CI_COMMIT_REF_NAME
    # and is this ref protected
    - echo $CI_COMMIT_REF_PROTECTED
    # Vault's address can be provided here or as CI/CD variable
    - export VAULT_ADDR=https://vault.ducoterra.net
    # Authenticate and get token. Token expiry time and other properties can be configured
    # when configuring JWT Auth - https://www.vaultproject.io/api/auth/jwt#parameters-1
    - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-production jwt=$CI_JOB_JWT)"
    # Now use the VAULT_TOKEN to read the secret and store it in environment variable
    - export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)"
    # Use the secret
    - echo $PASSWORD
```

## Certificate Management

<https://learn.hashicorp.com/tutorials/vault/pki-engine>
<https://learn.hashicorp.com/tutorials/vault/kubernetes-cert-manager>

### Create a CA

```bash
export ROOT_PATH=dnet

# Note: 19800h is Apple's limit
vault secrets enable -path=$ROOT_PATH pki
vault secrets tune -max-lease-ttl=19800h $ROOT_PATH

# Create a root CA
vault write $ROOT_PATH/root/generate/internal \
    common_name="Ducoterra Root CA" \
    ttl=19800h

# Create a CA URL
vault write $ROOT_PATH/config/urls \
    issuing_certificates="https://vault.ducoterra.net/v1/$ROOT_PATH/ca" \
    crl_distribution_points="https://vault.ducoterra.net/v1/$ROOT_PATH/crl"
```

Navigate to <https://vault.ducoterra.net/v1/$ROOT_PATH/ca> and download the CA. Import to your devices.

### Create an intermediate CA

```bash
export ROOT_PATH=dnet
export PKI_PATH=dnet_inter

vault secrets enable -path=$PKI_PATH pki
vault secrets tune -max-lease-ttl=8760h $PKI_PATH

# Create CSR to sign with root CA
vault write -format=json $PKI_PATH/intermediate/generate/internal \
    common_name="Ducoterra Intermediate CA" \
    | jq -r '.data.csr' > certs/$PKI_PATH.csr

# Sign the CSR with the root CA
vault write -format=json $ROOT_PATH/root/sign-intermediate \
    csr=@certs/$PKI_PATH.csr format=pem_bundle ttl=8760h \
    | jq -r '.data.certificate' > certs/$PKI_PATH.cert.pem

# Save the signed cert to vault
vault write $PKI_PATH/intermediate/set-signed certificate=@certs/$PKI_PATH.cert.pem

# Create a CA URL
vault write $PKI_PATH/config/urls \
    issuing_certificates="https://vault.ducoterra.net/v1/$PKI_PATH/ca" \
    crl_distribution_points="https://vault.ducoterra.net/v1/$PKI_PATH/crl"
```

Navigate to <https://vault.ducoterra.net/v1/$PKI_PATH/ca> and download the CA. Import to your devices.

### Allow .dnet and .hole certificates

```bash
export PKI_PATH=dnet_inter

# Allow .dnet subdomain
vault write $PKI_PATH/roles/dnet \
    allowed_domains=dnet \
    allow_subdomains=true max_ttl=8760h

# Allow .hole subdomain
vault write $PKI_PATH/roles/pi_hole \
    allowed_domains=hole \
    allow_subdomains=true max_ttl=8760h
```

### Issue a certificate

```bash
export PKI_PATH=dnet_inter
export CNAME=freenas.dnet

# Use -format=json to dump a json file
vault write $PKI_PATH/issue/dnet \
    common_name=$CNAME \
    max_ttl=8760h > certs/$CNAME.cert

# Pihole Example
vault write $PKI_PATH/issue/pi_hole \
    common_name=$CNAME \
    max_ttl=8760h > certs/$CNAME.cert
```

#### Adding cert to freenas

Only caveat here is to paste the certificate and then the full chain cert below in the
"certificate" section. iOS will infintely refresh the page if the full chain isn't provided.

#### Adding cert to pihole

```bash
# Paste the isser CA in here:
vim /etc/ssl/certs/vault-ca.pem

# Paste the certificate and private key in here:
vim /etc/ssl/certs/pihole.pem

# Make sure lighttpd is configured correctly
vim /etc/lighttpd/external.conf

# Restart the service
service lighttpd restart
```

#### Adding cert to cloudkey

**THIS DOESN'T WORK**

```bash
service unifi stop

# Remove the UNIFI_SSL_KEYSTORE=/etc/ssl/private/unifi.keystore.jks line from config
cp /etc/default/unifi /etc/default/unifi.back
vim /etc/default/unifi

# Remove the keystore reference
mv /usr/lib/unifi/data/keystore /usr/lib/unifi/data/keystore.back

# Copy your primary SSL Certificate into /etc/ssl/private/cloudkey.crt
cp /etc/ssl/private/cloudkey.crt /etc/ssl/private/cloudkey.crt.back
vim /etc/ssl/private/cloudkey.crt

# Copy your private key into /etc/ssl/private/cloudkey.key
cp /etc/ssl/private/cloudkey.key /etc/ssl/private/cloudkey.key.back
vim /etc/ssl/private/cloudkey.key

# Add the CA to /etc/ssl/private/vault-ca.pem
vim /etc/ssl/private/vault-ca.pem

# Generate the key bundle
openssl pkcs12 -export -in /etc/ssl/private/cloudkey.crt -inkey \
/etc/ssl/private/cloudkey.key -out /etc/ssl/private/cloudkey.p12 \
-name unifi -CAfile /etc/ssl/private/vault-ca.pem -caname vault -password pass:temppass

# Import to keystore
# Default password is "aircontrolenterprise"
keytool -importkeystore -deststorepass aircontrolenterprise \
-destkeypass aircontrolenterprise -destkeystore /usr/lib/unifi/data/keystore \
-srckeystore /etc/ssl/private/cloudkey.p12 -srcstoretype PKCS12 -srcstorepass temppass -alias unifi -noprompt

service unifi start
```

### Revoke a certificate

```bash
export PKI_PATH=dnet_inter

vault write $PKI_PATH/revoke serial_number=<serial_number>
vault write $PKI_PATH/tidy tidy_cert_store=true tidy_revoked_certs=true
```

### Use with cert-manager

```bash
export PKI_PATH=dnet_inter

vault policy write $PKI_PATH - <<EOF
path "$PKI_PATH*"                        { capabilities = ["read", "list"] }
path "$PKI_PATH/roles/*"   { capabilities = ["create", "update"] }
path "$PKI_PATH/sign/*"    { capabilities = ["create", "update"] }
path "$PKI_PATH/issue/*"   { capabilities = ["create"] }
EOF

vault write auth/kubernetes/role/issuer \
    bound_service_account_names=issuer \
    bound_service_account_namespaces=cert-manager \
    policies=$PKI_PATH \
    ttl=20m

kubectl -n cert-manager create serviceaccount issuer
ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")

kubectl -n cert-manager apply -f - <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: vault-issuer
spec:
  vault:
    server: https://vault.ducoterra.net
    path: $PKI_PATH/sign/dnet
    auth:
      kubernetes:
        mountPath: /v1/auth/kubernetes
        role: issuer
        secretRef:
          name: $ISSUER_SECRET_REF
          key: token
EOF
```

```yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    # cert-manager.io/cluster-issuer: letsencrypt-prod
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.org/client-max-body-size: "0"
  name: {{ .Release.Name }}
spec:
  rules:
  - host: test.dnet
    http:
      paths:
      - backend:
          service:
            name: {{ .Release.Name }}
            port:
              number: 80
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - test-dnet
    secretName: test-dnet-cert
---
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  name: test-dnet
spec:
  secretName: test-dnet-cert
  issuerRef:
    name: vault-issuer
    kind: ClusterIssuer
  commonName: test.dnet
  dnsNames:
  - test.dnet
```

## Kubernetes External Vault Auth

<https://learn.hashicorp.com/tutorials/vault/kubernetes-external-vault?in=vault/kubernetes>

### Connect to external vault

```bash
# On the new server (pikube - vault)
helm install vault hashicorp/vault \
    --set "injector.externalVaultAddr=https://vault.ducoterra.net"

VAULT_HELM_SECRET_NAME=$(kubectl get secrets --output=json | jq -r '.items[].metadata | select(.name|startswith("vault-token-")).name')
TOKEN_REVIEW_JWT=$(kubectl get secret $VAULT_HELM_SECRET_NAME --output='go-template={{ .data.token }}' | base64 --decode)
KUBE_CA_CERT=$(kubectl config view --raw --minify --flatten --output='jsonpath={.clusters[].cluster.certificate-authority-data}' | base64 --decode)
KUBE_HOST="https://3.14.3.104:6443"
```

```bash
# On the existing vault server

export PKI_PATH=dnet_inter
vault auth enable -path=pikube kubernetes

vault write auth/pikube/config \
    token_reviewer_jwt="$TOKEN_REVIEW_JWT" \
    kubernetes_host="https://3.14.3.104:6443" \
    kubernetes_ca_cert="$KUBE_CA_CERT"

vault write auth/pikube/role/issuer \
    bound_service_account_names=issuer \
    bound_service_account_namespaces=cert-manager \
    policies=$PKI_PATH \
    ttl=20m
```

### Install cert-manager

```bash
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml
kubectl -n cert-manager create serviceaccount issuer
export ISSUER_SECRET_REF=$(kubectl -n cert-manager get serviceaccount issuer -o json | jq -r ".secrets[].name")
export PKI_PATH=dnet_inter

kubectl apply -f - <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: vault-issuer
spec:
  vault:
    server: https://vault.ducoterra.net
    path: $PKI_PATH/sign/dnet
    auth:
      kubernetes:
        mountPath: /v1/auth/pikube
        role: issuer
        secretRef:
          name: $ISSUER_SECRET_REF
          key: token
EOF
```

## Auto-unseal

<https://learn.hashicorp.com/tutorials/vault/autounseal-transit>

### Enable transit on vault that will do the unsealing

```bash
vault secrets enable transit
vault write -f transit/keys/autounseal

tee policies/autounseal.hcl <<EOF
path "transit/encrypt/autounseal" {
   capabilities = [ "update" ]
}

path "transit/decrypt/autounseal" {
   capabilities = [ "update" ]
}
EOF

vault policy write autounseal policies/autounseal.hcl
vault token create -policy="autounseal" -wrap-ttl=120
VAULT_TOKEN="wrapping token" vault unwrap
```

### Auto unseal

#### Create a test vault in a new namespace

```bash
kubectl apply -f k8s/certificate.yaml

# Get token from above output
kubectl create secret generic auto-unseal-token --from-literal=VAULT_TOKEN="token from output"
helm upgrade --install vault hashicorp/vault --values helm/ha.yaml
```

#### Test Auto Unseal

```bash
kubectl exec -it vault-0 -- vault operator init
kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200
```

### SSH

#### Client key signing

Enable secrets engine and generate a key

```bash
vault secrets enable -path=ssh-client-signer ssh
vault write ssh-client-signer/config/ca generate_signing_key=true
```

Retrieve the public CA (and add it to your /etc/ssh/trusted-user-ca-keys.pem) with

```bash
curl -o /etc/ssh/trusted-user-ca-keys.pem https://vault.ducoterra.net/v1/ssh-client-signer/public_key
```

Add it to your sshd_config

```bash
echo "TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem" >> /etc/ssh/sshd_config
```

Restart the SSH service

```bash
service ssh restart
```

Add signing role

```bash
vault write ssh-client-signer/roles/ducoterra -<<"EOH"
{
    "allow_user_certificates": true,
    "allowed_users": "*",
    "allowed_extensions": "permit-pty,permit-port-forwarding",
    "default_extensions": [
        { "permit-pty": "" }
    ],
    "key_type": "ca",
    "default_user": "ducoterra",
    "ttl": "30m0s"
}
EOH
```

```bash
vault write ssh-client-signer/roles/pi -<<"EOH"
{
    "allow_user_certificates": true,
    "allowed_users": "*",
    "allowed_extensions": "permit-pty,permit-port-forwarding",
    "default_extensions": [
        { "permit-pty": "" }
    ],
    "key_type": "ca",
    "default_user": "pi",
    "ttl": "30m0s"
}
EOH
```

```bash
vault write ssh-client-signer/roles/rancher -<<"EOH"
{
    "allow_user_certificates": true,
    "allowed_users": "*",
    "allowed_extensions": "permit-pty,permit-port-forwarding",
    "default_extensions": [
        { "permit-pty": "" }
    ],
    "key_type": "ca",
    "default_user": "rancher",
    "ttl": "30m0s"
}
EOH
```

Sign a key

```bash
export SSHUSER=pi; vault write -field=signed_key ssh-client-signer/sign/$SSHUSER public_key=@$HOME/.ssh/test_rsa.pub > ~/.ssh/test_rsa-cert.pub
```

SSH using the signed key

```bash
# If you saved the signed pub as key_name"-cert.pub" then you don't need to specify the signed-cert.pub part.
ssh -i signed-cert.pub -i ~/.ssh/test_rsa client

# or without the cert (using default client)
ssh -i ~/.ssh/test_rsa client
```

#### Server Host Signing

Enable secrets engine

```bash
vault secrets enable -path=ssh-host-signer ssh
```

Generate keys:

```bash
vault write ssh-host-signer/config/ca generate_signing_key=true
```

Extend host key's TTL

```bash
vault secrets tune -max-lease-ttl=87600h ssh-host-signer
```

Create host role

```bash
vault write ssh-host-signer/roles/hostrole \
    key_type=ca \
    ttl=87600h \
    allow_host_certificates=true \
    allowed_domains="localdomains,dnet,hole,ducoterra.net" \
    allow_subdomains=true
```

Sign the host's public key

```bash
vault write ssh-host-signer/sign/hostrole \
    cert_type=host \
    public_key=@$HOME/.ssh/id_rsa.pub
```

Write the signed certificate to the ssh config on the host

```bash
vault write -field=signed_key ssh-host-signer/sign/hostrole \
    cert_type=host \
    public_key=@$HOME/.ssh/id_rsa.pub > /etc/ssh/ssh_host_rsa_key-cert.pub
```

Assign correct permissions

```bash
chmod 0640 /etc/ssh/ssh_host_rsa_key-cert.pub
```

Add to sshd_config

```bash
echo HostKey /etc/ssh/ssh_host_rsa_key >> /etc/ssh/sshd_config
echo HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub >> /etc/ssh/sshd_config
```

Restart the ssh service

```bash
service ssh restart
```

Add certificate to client

```bash
echo '@cert-authority *.ducoterra.net '$(vault read -field=public_key ssh-host-signer/config/ca) >> ~/.ssh/known_hosts
```