ETCD Config and Testing

Config

/etc/hosts

cat <<EOF >> /etc/hosts
3.14.3.100  mainframe
3.14.3.101  freenas
3.14.3.102  red
3.14.3.103  grey
3.14.3.107  purple
EOF

Generate Certs

Pick one server to act as the CA.

Install make

apt install -y make

Install go

tar -C /usr/local -xzf go...
rm go...

cat <<EOF >> ~/.bashrc 
export PATH=$PATH:/usr/local/go/bin
EOF

source ~/.bashrc

Install cfssl

git clone https://github.com/cloudflare/cfssl
cd cfssl
make -j 4
cp bin/cfssl bin/cfssljson /usr/local/bin/

Create templates

mkdir ~/.cfssl
cd ~/.cfssl

cat <<EOF > ca-config.json
{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "876000h"
    }
  }
}
EOF

cat <<EOF > ca-csr.json
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "autogenerated",
      "OU": "etcd cluster",
      "L": "duconet"
    }
  ]
}
EOF

cat <<EOF > req-csr.json
{
  "CN": "etcd",
  "hosts": [
    "3.14.3.102",
    "3.14.3.103",
    "3.14.3.107",
    "127.0.0.1"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "autogenerated",
      "OU": "etcd cluster",
      "L": "duconet"
    }
  ]
}
EOF

Generate CA:

mkdir -p /certs
cfssl gencert -initca ca-csr.json | cfssljson -bare /certs/ca

Generate a Peer and Client Cert:

cfssl gencert \
  -ca /certs/ca.pem \
  -ca-key /certs/ca-key.pem \
  -config ca-config.json \
  req-csr.json | cfssljson -bare /certs/client

cfssl gencert \
  -ca /certs/ca.pem \
  -ca-key /certs/ca-key.pem \
  -config ca-config.json \
  req-csr.json | cfssljson -bare /certs/red

cfssl gencert \
  -ca /certs/ca.pem \
  -ca-key /certs/ca-key.pem \
  -config ca-config.json \
  req-csr.json | cfssljson -bare /certs/purple

cfssl gencert \
  -ca /certs/ca.pem \
  -ca-key /certs/ca-key.pem \
  -config ca-config.json \
  req-csr.json | cfssljson -bare /certs/grey

# Run this on every node
useradd etcd
usermod -aG etcd pi
chown -R etcd:etcd /certs
chmod 750 /certs

scp /certs/ca.pem /certs/client-key.pem /certs/client.pem /certs/grey-key.pem /certs/grey.pem grey:/certs/

scp /certs/ca.pem /certs/client-key.pem /certs/client.pem /certs/purple-key.pem /certs/purple.pem purple:/certs/

chown -R etcd:etcd /certs
chmod 600 /certs/*

Install ETCD

Download the latest version

tar xf $(find . -maxdepth 1 -name etcd*)
cp $(find . -maxdepth 1 -type d -name "etcd*")/etcd $(find . -maxdepth 1 -type d -name "etcd*")/etcdctl /usr/local/bin/
chmod +x /usr/local/bin/etcd /usr/local/bin/etcdctl
echo 'export ETCD_UNSUPPORTED_ARCH=arm64' >> ~/.bashrc
echo 'export ETCD_IP=3.14.3.102' >> ~/.bashrc
echo 'export ETCD_NAME=red' >> ~/.bashrc
source ~/.bashrc
mkdir -p /var/lib/etcd
chown -R etcd:etcd /var/lib/etcd
chmod -R 700 /var/lib/etcd

cat <<EOF > /etc/systemd/system/etcd.service
[Unit]
Description=etcd service
Documentation=https://github.com/etcd-io/etcd 
After=network.target

[Service]
User=etcd
Type=notify
Environment=ETCD_UNSUPPORTED_ARCH=arm64
Environment=ETCD_DATA_DIR=/var/lib/etcd/$ETCD_NAME
Environment=ETCD_NAME=$ETCD_NAME
Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=https://$ETCD_IP:2380
Environment=ETCD_LISTEN_PEER_URLS=https://$ETCD_IP:2380
Environment=ETCD_LISTEN_CLIENT_URLS=https://$ETCD_IP:2379,https://127.0.0.1:2379
Environment=ETCD_ADVERTISE_CLIENT_URLS=https://$ETCD_IP:2379
Environment=ETCD_INITIAL_CLUSTER_TOKEN=pi-cluster-1
Environment=ETCD_INITIAL_CLUSTER="red=https://3.14.3.102:2380,purple=https://3.14.3.107:2380,grey=https://3.14.3.103:2380"
Environment=ETCD_INITIAL_CLUSTER_STATE=new
Environment=ETCD_TRUSTED_CA_FILE=/certs/ca.pem
Environment=ETCD_CERT_FILE=/certs/client.pem
Environment=ETCD_KEY_FILE=/certs/client-key.pem
Environment=ETCD_PEER_TRUSTED_CA_FILE=/certs/ca.pem
Environment=ETCD_PEER_CERT_FILE=/certs/$ETCD_NAME.pem
Environment=ETCD_PEER_KEY_FILE=/certs/$ETCD_NAME-key.pem
ExecStart=/usr/local/bin/etcd --client-cert-auth --peer-client-cert-auth
Restart=on-failure
RestartSec=5
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target
EOF

systemctl start etcd
systemctl enable etcd
journalctl -u etcd -f

Use NFS

echo "freenas:/mnt/enc0/pi /var/lib/etcd nfs noexec,nosuid,nofail 0 0" >> /etc/fstab
service etcd stop
mount -t nfs freenas:/mnt/enc0/pi /media
rsync -a /var/lib/etcd/ /media/$SERVER/
mount -a
umount /media
service etcd start

Testing

scp red:/certs/client.pem red:/certs/client-key.pem red:/certs/ca.pem certs
export ETCDCTL_DIAL_TIMEOUT=3s;
export ETCDCTL_CACERT=./certs/ca.pem; # ca.pem
export ETCDCTL_CERT=./certs/client.pem; # 3.14.3.102.pem
export ETCDCTL_KEY=./certs/client-key.pem;  # 3.14.3.102-key.pem
export ETCDCTL_ENDPOINTS=https://3.14.3.102:2379,https://3.14.3.107:2379,https://3.14.3.103:2379;
etcdctl put foo bar
etcdctl get foo
while true; do etcdctl put foo $(( ( RANDOM % 1000 )  + 1 )) && etcdctl get foo; done;
etcdctl del "" --from-key=true