working nginx stream proxy

ansible.md

@@ -0,0 +1,21 @@
+# Ansible
+
+## Install
+
+```bash
+pacman -S ansible ansible-core python-kubernetes
+```
+
+## Setup
+
+```bash
+ansible kubernetes -m ping -i inventory.yaml
+```
+
+## Updates
+
+```bash
+ansible-playbook -i ansible/inventory.yaml ansible/upgrade-kubernetes-nodes.yaml
+ansible-playbook -i ansible/inventory.yaml ansible/upgrade-colors.yaml
+ansible-playbook -i ansible/inventory.yaml ansible/upgrade-apt.yaml
+```

ansible/inventory.yaml

@@ -0,0 +1,23 @@
+kubernetes:
+  hosts:
+    node1:
+      nodename: node1
+    node2:
+      nodename: node2
+    node3:
+      nodename: node3
+
+colors:
+  hosts:
+    orange:
+    yellow:
+
+apt:
+  hosts:
+    unifi-external:
+    nextcloud-aio:
+    replicator:
+
+hardware: 
+  hosts:
+    gamebox:

ansible/update-quadlets.yaml

@@ -0,0 +1,19 @@
+- name: Update quadlets
+  hosts: colors
+  become: true
+  become_user: root
+  become_method: sudo
+  tasks:
+  - name: Copy quadlets with owner and permissions
+    ansible.builtin.copy:
+      src: "{{ item }}"
+      dest: /usr/share/containers/systemd/
+      owner: root
+      group: root
+      mode: '0644'
+    with_items:
+    - ../quadlets/iperf3.container
+    - ../quadlets/pihole.container
+  - name: Daemon-reload to trigger re-read of quadlets
+    ansible.builtin.systemd_service:
+      daemon_reload: true

ansible/upgrade-apt.yaml

@@ -0,0 +1,17 @@
+- name: Upgrade
+  hosts: apt
+  tasks:
+  - name: Update all packages to their latest version
+    become: true
+    become_user: root
+    become_method: sudo
+    ansible.builtin.apt:
+      name: "*"
+      state: latest
+      update_cache: yes
+  - name: Reboot
+    become: true
+    become_user: root
+    become_method: sudo
+    ansible.builtin.reboot:
+      reboot_timeout: 600

ansible/upgrade-colors.yaml

@@ -0,0 +1,16 @@
+- name: Upgrade
+  hosts: colors
+  tasks:
+  - name: Upgrade all packages
+    become: true
+    become_user: root
+    become_method: sudo
+    ansible.builtin.dnf:
+      name: "*"
+      state: latest
+  - name: Reboot
+    become: true
+    become_user: root
+    become_method: sudo
+    ansible.builtin.reboot:
+      reboot_timeout: 600

ansible/upgrade-kubernetes-nodes.yaml

@@ -0,0 +1,34 @@
+- name: Upgrade
+  hosts: kubernetes
+  serial: 1
+  tasks:
+  - name: Drain node
+    delegate_to: localhost
+    kubernetes.core.k8s_drain:
+      kubeconfig: ~/.kube/admin-config
+      state: drain
+      name: "{{ nodename }}"
+      delete_options:
+        delete_emptydir_data: true
+        ignore_daemonsets: true
+  - name: Upgrade all packages
+    become: true
+    become_user: root
+    become_method: sudo
+    ansible.builtin.dnf:
+      name: "*"
+      state: latest
+  - name: Reboot
+    become: true
+    become_user: root
+    become_method: sudo
+    ansible.builtin.reboot:
+      reboot_timeout: 600
+  - name: Uncordon node
+    delegate_to: localhost
+    retries: 10
+    delay: 6
+    kubernetes.core.k8s_drain:
+      kubeconfig: ~/.kube/admin-config
+      state: uncordon
+      name: "{{ nodename }}"

compose/test-compose.yaml

@@ -1,33 +0,0 @@
-services:
-  db:
-    # We use a mariadb image which supports both amd64 & arm64 architecture
-    image: docker.io/library/mariadb:10.6.4-focal
-    # If you really want to use MySQL, uncomment the following line
-    #image: mysql:8.0.27
-    command: '--default-authentication-plugin=mysql_native_password'
-    volumes:
-      - db_data:/var/lib/mysql
-    restart: always
-    environment:
-      - MYSQL_ROOT_PASSWORD=somewordpress
-      - MYSQL_DATABASE=wordpress
-      - MYSQL_USER=wordpress
-      - MYSQL_PASSWORD=wordpress
-    expose:
-      - 3306
-      - 33060
-  wordpress:
-    image: docker.io/library/wordpress:latest
-    volumes:
-      - wp_data:/var/www/html
-    ports:
-      - 80:80
-    restart: always
-    environment:
-      - WORDPRESS_DB_HOST=db
-      - WORDPRESS_DB_USER=wordpress
-      - WORDPRESS_DB_PASSWORD=wordpress
-      - WORDPRESS_DB_NAME=wordpress
-volumes:
-  db_data:
-  wp_data:

nginx-stream/stream.conf

@@ -1,73 +0,0 @@
-load_module /usr/lib64/nginx/modules/ngx_stream_module.so;
-
-events {}
-
-http {
-    server {
-        listen 127.0.0.1:80 default_server;
-        return 301 https://$host$request_uri;
-    }
-    server {
-        listen 127.0.0.1:443;
-        server_name nextcloud-aio.reeseapps.com;
-
-        location / {
-            resolver 8.8.8.8;
-            proxy_pass  http://nextcloud-aio.reeselink.com:443;
-        }
-
-    }
-    server {
-        listen 127.0.0.1:8443;
-        server_name nextcloud-aio.reeseapps.com;
-
-        location / {
-            resolver 8.8.8.8;
-            proxy_pass  http://nextcloud-aio.reeselink.com:8443;
-        }
-
-    }
-}
-
-stream {
-
-    map $ssl_preread_server_name $name {
-        "" 127.0.0.1;
-        nextcloud-aio.reeseapps.com nextcloud-aio.reeselink.com;
-        driveripper.reeseapps.com driveripper.reeselink.com;
-        default nginx.reeselink.com;
-    }
-
-    map $ssl_preread_server_name $ssl_port {
-        "" 443;
-        nextcloud-aio.reeseapps.com 443;
-        driveripper.reeseapps.com 8443;
-        default 443;
-    }
-
-    server {
-        resolver 8.8.8.8;
-        listen 10.1.203.197:443;
-        proxy_pass $name:$ssl_port;
-        ssl_preread on;
-    }
-
-    server {
-        resolver 8.8.8.8;
-        listen 10.1.203.197:8443;
-        proxy_pass $name:8443;
-        ssl_preread on;
-    }
-
-    server {
-        resolver 8.8.8.8;
-        listen 10.1.203.197:3478;
-        proxy_pass $name:3478;
-    }
-
-    server {
-        resolver 8.8.8.8;
-        listen 10.1.203.197:3478 udp;
-        proxy_pass $name:3478;
-    }
-}

nginx/README.md

@@ -0,0 +1,10 @@
+# Nginx Ansible Configuration
+
+## Installation
+
+Check vars.yaml to edit your servers.
+
+```bash
+ansible-playbook -i ansible/inventory.yaml nginx/nginx.yaml
+ansible-playbook -i ansible/inventory.yaml nginx/certbot.yaml
+```

nginx/_generate_certs.yaml

@@ -0,0 +1,8 @@
+- name: Generate placeholder letsencrypt certs for domains if needed
+  block:
+  - name: check if fullchain already exists
+    stat: path=/etc/letsencrypt/live/{{ item.external }}/fullchain.pem
+    register: p
+  - name: Generate self signed cert
+    shell: openssl req -x509 -newkey rsa:4096 -keyout /etc/letsencrypt/live/{{ item.external }}/privkey.pem -out /etc/letsencrypt/live/{{ item.external }}/fullchain.pem -sha256 -days 3650 -nodes -subj "/C=US/ST=Ohio/L=Columbus/O=ducoterra/OU=ducoterra/CN={{ item.external }}"
+    when: not p.stat.exists

nginx/certbot.yaml

@@ -0,0 +1,17 @@
+
+- name: Update certbot certs
+  hosts: yellow
+  become: true
+  become_user: root
+  become_method: sudo
+  vars_files:
+    - vars.yaml
+  tasks:
+  - name: Ensure nginx, certbot, and nginx-mod-stream are installed
+    ansible.builtin.dnf:
+      name:
+        - certbot
+      state: present
+  - name: Get certs for all terminate domains
+    ansible.builtin.shell: /usr/bin/certbot certonly --standalone -d '{{ item.external_domain }}' -n
+    loop: "{{ terminate_ssl }}"

nginx/http.d/nextcloud-aio.conf

@@ -0,0 +1,10 @@
+server {
+    listen 127.0.0.1:80;
+    server_name nextcloud-aio.reeseapps.com;
+
+    location / {
+        access_log /var/log/nginx/nextcloud-http-443-access.log compression;
+        resolver 1.1.1.1;
+        proxy_pass  http://{{ nextcloud.domain }}:443;
+    }
+}

nginx/https.conf

@@ -0,0 +1,59 @@
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+server {
+    listen 127.0.0.1:80;
+    listen 127.0.0.1:443 ssl http2;
+    server_name {{ item.external_domain }};
+
+    access_log /var/log/nginx/{{ item.external_domain }}-access.log compression;
+
+    if ($scheme = "http") {
+        return 301 https://$host:443$request_uri;
+    }
+
+    # listen 443 ssl http2;      # for nginx versions below v1.25.1
+    # listen [::]:443 ssl http2; # for nginx versions below v1.25.1 - comment to disable IPv6
+	
+    # http2 on;                                 # uncomment to enable HTTP/2        - supported on nginx v1.25.1+
+    # http3 on;                                 # uncomment to enable HTTP/3 / QUIC - supported on nginx v1.25.0+
+    # quic_retry on;                            # uncomment to enable HTTP/3 / QUIC - supported on nginx v1.25.0+
+    # add_header Alt-Svc 'h3=":443"; ma=86400'; # uncomment to enable HTTP/3 / QUIC - supported on nginx v1.25.0+
+    # listen 8443 quic reuseport;       # uncomment to enable HTTP/3 / QUIC - supported on nginx v1.25.0+ - please remove "reuseport" if there is already another quic listener on port 443 with enabled reuseport
+    # listen [::]:8443 quic reuseport;  # uncomment to enable HTTP/3 / QUIC - supported on nginx v1.25.0+ - please remove "reuseport" if there is already another quic listener on port 443 with enabled reuseport - keep comment to disable IPv6
+
+    location / {
+        resolver 1.1.1.1;
+        proxy_pass {{ item.internal_protocol }}://{{ item.internal_domain }}:{{ item.internal_port }}$request_uri;
+
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header X-Forwarded-Scheme $scheme;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Accept-Encoding "";
+        proxy_set_header Host $host;
+    
+        client_body_buffer_size 512k;
+        proxy_read_timeout 86400s;
+        client_max_body_size 0;
+
+        # Websocket
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+    }
+
+    ssl_certificate /etc/letsencrypt/live/{{ item.external_domain }}/fullchain.pem;   # managed by certbot on host machine
+    ssl_certificate_key /etc/letsencrypt/live/{{ item.external_domain }}/privkey.pem;
+
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
+    ssl_session_tickets off;
+
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
+    ssl_prefer_server_ciphers on;
+}

nginx/nginx.conf

@@ -0,0 +1,62 @@
+load_module /usr/lib64/nginx/modules/ngx_stream_module.so;
+
+events {}
+
+stream {
+    log_format basic '$remote_addr $name [$time_local] '
+                 '$protocol $status $bytes_sent $bytes_received '
+                 '$session_time';
+
+    include /etc/nginx/stream.d/*.conf;
+
+    # Map all SSL parsed server names to hosts
+    map $ssl_preread_server_name $name {
+
+        "" 127.0.0.1;
+
+        # For each domain we need to stream to a remote server, forward to internal domain
+        {% for domain in stream_ssl %}
+        {{ domain.external_domain }} {{ domain.internal_domain }};
+        {% endfor %}
+
+        # For each domain we want to terminate, forward to internal http server
+        {% for domain in terminate_ssl %}
+        {{ domain.external_domain }} 127.0.0.1;
+        {% endfor %}
+
+        default {{ nginx.defaults.domain }};
+    }
+
+    # Forward 80 traffic
+    server {
+        access_log /var/log/nginx/stream-access-80.log basic;
+        listen {{ ansible_default_ipv4.address }}:80;
+        resolver 1.1.1.1;
+        proxy_pass $name:80;
+        ssl_preread on;
+    }
+
+    # Forward 443 traffic
+    server {
+        access_log /var/log/nginx/stream-access-443.log basic;
+        listen {{ ansible_default_ipv4.address }}:443;
+        resolver 1.1.1.1;
+        proxy_pass $name:443;
+        ssl_preread on;
+    }
+
+}
+
+http {
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                       '"$request" $status $bytes_sent '
+                       '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+    include /etc/nginx/http.d/*.conf;
+
+    server {
+        access_log /var/log/nginx/http-access.log compression;
+        listen 127.0.0.1:80 default_server;
+        return 301 https://$host$request_uri;
+    }
+}

nginx/nginx.yaml

@@ -0,0 +1,62 @@
+- name: Update nginx stream configuration
+  hosts: yellow
+  become: true
+  become_user: root
+  become_method: sudo
+  vars_files:
+    - vars.yaml
+  tasks:
+  - name: Ensure nginx, certbot, and nginx-mod-stream are installed
+    ansible.builtin.dnf:
+      name:
+        - nginx
+        - nginx-mod-stream
+      state: present
+  - name: Create stream.d dir
+    ansible.builtin.file:
+      path: /etc/nginx/stream.d
+      state: directory
+      mode: '0755'
+  - name: Create http.d dir
+    ansible.builtin.file:
+      path: /etc/nginx/http.d
+      state: directory
+      mode: '0755'
+  - name: Copy nginx.conf
+    template:
+      src: nginx.conf
+      dest: /etc/nginx/nginx.conf
+      owner: root
+      group: root
+      mode: '0644'
+  - name: Copy stream configurations
+    template:
+      src: "{{ item }}"
+      dest: /etc/nginx/stream.d/{{ item | basename }}
+      owner: root
+      group: root
+      mode: '0644'
+    with_fileglob:
+    - stream.d/*
+  - name: Copy http configurations
+    template:
+      src: "{{ item }}"
+      dest: /etc/nginx/http.d/{{ item | basename }}
+      owner: root
+      group: root
+      mode: '0644'
+    with_fileglob:
+    - http.d/*
+  - name: Template all http configurations
+    template:
+      src: https.conf
+      dest: /etc/nginx/http.d/{{ item.external_domain }}.conf
+      owner: root
+      group: root
+      mode: '0644'
+    with_items: "{{ terminate_ssl }}"
+  - name: Reload nginx service
+    ansible.builtin.systemd_service:
+      state: restarted
+      name: nginx
+      enabled: true

nginx/stream.d/nextcloud-aio-talk.conf

@@ -0,0 +1,7 @@
+server {
+    access_log /var/log/nginx/nextcloud-aio-talk-access.log basic;
+    resolver 1.1.1.1;
+    listen {{ ansible_default_ipv4.address }}:3478;
+    listen {{ ansible_default_ipv4.address }}:3478 udp;
+    proxy_pass {{ nextcloud.domain }}:3478;
+}

nginx/vars.yaml

@@ -0,0 +1,43 @@
+terminate_ssl:
+  - external_domain: octoprint.reeseapps.com
+    external_port: 443
+    internal_domain: replicator.reeselink.com
+    internal_port: 443
+    internal_protocol: https
+  - external_domain: truenas.reeseapps.com
+    external_port: 443
+    internal_domain: driveripper.reeselink.com
+    internal_port: 8443
+    internal_protocol: https
+  - external_domain: pihole-yellow.reeseapps.com
+    external_port: 443
+    internal_domain: yellow.reeselink.com
+    internal_port: 8080
+    internal_protocol: http
+  - external_domain: pihole-orange.reeseapps.com
+    external_port: 443
+    internal_domain: orange.reeselink.com
+    internal_port: 8080
+    internal_protocol: http
+  - external_domain: yellow.reeseapps.com
+    external_port: 443
+    internal_domain: yellow.reeselink.com
+    internal_port: 9090
+    internal_protocol: https
+  - external_domain: orange.reeseapps.com
+    external_port: 443
+    internal_domain: orange.reeselink.com
+    internal_port: 9090
+    internal_protocol: https
+stream_ssl:
+  - external_domain: nextcloud-aio.reeseapps.com
+    external_port: 443
+    internal_domain: nextcloud-aio.reeselink.com
+    internal_port: 443
+nextcloud:
+  domain: nextcloud-aio.reeseapps.com
+nginx:
+  defaults:
+    domain: nginx.reeselink.com
+iperf:
+  domain: lb.reeselink.com

quadlets/db.container

@@ -1,13 +0,0 @@
-[Container]
-Environment=MYSQL_ROOT_PASSWORD=somewordpress MYSQL_DATABASE=wordpress MYSQL_USER=wordpress MYSQL_PASSWORD=wordpress
-Exec="--default-authentication-plugin=mysql_native_password"
-ExposeHostPort=3306
-ExposeHostPort=33060
-Image=docker.io/library/mariadb:10.6.4-focal
-Volume=db_data:/var/lib/mysql
-
-[Service]
-Restart=always
-
-[Install]
-WantedBy=default.target

quadlets/wordpress.container

@@ -1,11 +0,0 @@
-[Container]
-Environment=WORDPRESS_DB_HOST=db WORDPRESS_DB_USER=wordpress WORDPRESS_DB_PASSWORD=wordpress WORDPRESS_DB_NAME=wordpress
-Image=docker.io/library/wordpress:latest
-PublishPort=80:80
-Volume=wp_data:/var/www/html
-
-[Service]
-Restart=always
-
-[Install]
-WantedBy=default.target