homelab/active/kubernetes_userspace/userspace.md

Project Userspace

One provisioner to rule them all

Quickstart

./userspace/scripts/setup.sh 
./userspace/scripts/upsertuser.sh <server_fqdn> <username>
./userspace/scripts/removeuser.sh <server_fqdn> <username>

Update a user

export USER=user
helm template $USER ./namespace | kubectl --context admin apply -f -

Objectives

1. Provision a namespace with clusterroles, rolebindings, and a dedicated nfs-provisioner with one helm chart
2. Create an easy way for users to sign their certificates
3. Create a cleanup script without deleting user data
4. profit

Userspace

Namespace

apiVersion: v1
kind: Namespace
metadata:
  name: {{ .Release.Name }}

Roles

kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: namespace-manager
  namespace: {{ .Release.Name }}
rules:
- apiGroups:
    - ""
    - extensions
    - apps
    - batch
    - autoscaling
    - networking.k8s.io
    - traefik.containo.us
    - rbac.authorization.k8s.io
    - metrics.k8s.io
  resources: 
    - deployments
    - replicasets
    - pods
    - pods/exec
    - pods/log
    - pods/attach
    - daemonsets
    - statefulsets
    - replicationcontrollers
    - horizontalpodautoscalers
    - services
    - ingresses
    - persistentvolumeclaims
    - jobs
    - cronjobs
    - secrets
    - configmaps
    - serviceaccounts
    - rolebindings
    - ingressroutes
    - middlewares
    - endpoints
  verbs: 
    - "*"
- apiGroups:
    - ""
    - metrics.k8s.io
    - rbac.authorization.k8s.io
  resources:
    - resourcequotas
    - roles
  verbs:
    - list

Rolebinding

kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  namespace: {{ .Release.Name }}
  name: namespace-manager
subjects:
- kind: User
  name: {{ .Release.Name }}
  apiGroup: ""
roleRef:
  kind: ClusterRole
  name: namespace-manager
  apiGroup: ""

Create a kubernetes certsigner pod

This keeps the client-ca crt and key secret and allows the cert to be signed and stored on the pod

Create the certsigner secret

kubectl -n kube-system create secret generic certsigner --from-file /var/lib/rancher/k3s/server/tls/client-ca.crt --from-file /var/lib/rancher/k3s/server/tls/client-ca.key

Set up the certsigner pod

kubectl --context admin apply -f certsigner

Generate a cert

export USER=<user>
docker run -it -v $(pwd)/users/$USER:/$USER python:latest openssl genrsa -out /$USER/$USER.key 2048
docker run -it -v $(pwd)/users/$USER:/$USER python:latest openssl req -new -key /$USER/$USER.key -out /$USER/$USER.csr -subj "/CN=$USER/O=user"

Create a new Userspace

helm template $USER ./namespace | kubectl --context admin apply -f -

Sign the cert

export USER=<user>
kubectl --context admin cp $(pwd)/users/$USER/$USER.csr certsigner:/certs/$USER.csr
kubectl --context admin exec -it --context admin certsigner -- openssl x509 -in /certs/$USER.csr -req -CA /keys/client-ca.crt -CAkey /keys/client-ca.key -CAcreateserial -out /certs/$USER.crt -days 5000
kubectl --context admin cp certsigner:/certs/$USER.crt $(pwd)/users/$USER/$USER.crt

Add to the config

kubectl config set-credentials $USER --client-certificate=$USER.crt  --client-key=$USER.key
kubectl config set-context $USER --cluster=mainframe --namespace=$USER --user=$USER

Delete

kubectl config delete-context $USER
helm template $USER ./namespace | kubectl --context admin delete -f -

Signing a user cert - detailed notes

NOTE: ca.crt and ca.key are in /var/lib/rancher/k3s/server/tls/client-ca.*

# First we create the credentials
# /CN=<username> - the user
# /O=<group> - the group

# Navigate to the user directory
export USER=<username>
cd $USER

# Generate a private key
openssl genrsa -out $USER.key 2048
# Check the key
# openssl pkey -in ca.key -noout -text
# Generate and send me the CSR
# The "user" group is my default group
openssl req -new -key $USER.key -out $USER.csr -subj "/CN=$USER/O=user"

# Check the CSR
# openssl req -in $USER.csr -noout -text
# If satisfactory, sign the CSR
# Copy from /var/lib/rancher/k3s/server/tls/client-ca.crt and client-ca.key
openssl x509 -req -in $USER.csr -CA ../client-ca.crt -CAkey ../client-ca.key -CAcreateserial -out $USER.crt -days 5000
# Review the certificate
# openssl x509 -in $USER.crt -text -noout

# Send back the crt
# cp $USER.crt $USER.key ../server-ca.crt ~/.kube/
kubectl config set-credentials $USER --client-certificate=$USER.crt  --client-key=$USER.key
kubectl config set-context $USER --cluster=mainframe --namespace=$USER --user=$USER

# Now we create the namespace, rolebindings, and resource quotas
# kubectl apply -f k8s/

# Add the cluster
# CA file can be found at https://3.14.3.100:6443/cacerts
- cluster:
    certificate-authority: server-ca.crt
    server: https://3.14.3.100:6443
  name: mainframe

# Test if everything worked
kubectl --context=$USER-context get pods