nginx load balancing internal/external working

nginx/README.md

@@ -14,3 +14,18 @@ ansible-playbook -i ansible/inventory.yaml nginx/nginx.yaml
 
 We can detect whether traffic is originating internally or externally by checking if
 it came in on port 443 or 444.
+
+External traffic always come in through 444.
+
+## Certbot
+
+Use `certbot delete` to remove unused certs.
+
+## vars.yaml
+
+`allowed_ips` restricts access to the endpoint (deny all) and then allows only the list
+of ips provided.
+
+## Logging
+
+You can tail all the nginx logs with `tail -f /var/log/nginx/*`

nginx/certbot.yaml

@@ -16,7 +16,3 @@
     ansible.builtin.shell: /usr/bin/certbot certonly --dns-route53 -d '{{ item.1 }}' -n
     # Loops over every external.domains sub list
     loop: "{{ http | subelements('external.domains') }}"
-  - name: Start nginx service
-    ansible.builtin.systemd_service:
-      state: reloaded
-      name: nginx

nginx/https.conf

@@ -4,26 +4,34 @@ map $http_upgrade $connection_upgrade {
 }
 
 server {
+    access_log /var/log/nginx/nginx_https_access.log basic;
+    error_log /var/log/nginx/nginx_https_error.log warn;
 
-    listen 127.0.0.1:443 ssl;
+    {%- for port in item.0.external.ports +%}
+    listen 127.0.0.1:{{ port }} ssl proxy_protocol;
+    {%- endfor +%}
+    listen 127.0.0.1:80;
+
+    if ($scheme = "http") {
+        return 301 https://$host:443$request_uri;
+    }
+
+    set_real_ip_from  127.0.0.1;
 
     server_name {{ item.1 }};
 
-    access_log /var/log/nginx/{{ item.1 }}-access.log compression;
-
-    http2 on;
-
     location / {
         proxy_pass {{ item.0.internal.protocol }}://{{ item.0.internal.ip }}:{{ item.0.internal.port }}$request_uri;
 
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Port $server_port;
         proxy_set_header X-Forwarded-Scheme $scheme;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header Accept-Encoding "";
-        proxy_set_header Host $host;
-    
+
+        proxy_set_header Host            $host;
+        proxy_set_header X-Real-IP       $proxy_protocol_addr;
+        proxy_set_header X-Forwarded-For $proxy_protocol_addr;
+
         client_body_buffer_size 512k;
         proxy_read_timeout 86400s;
         client_max_body_size 0;
@@ -45,13 +53,11 @@ server {
     ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
     ssl_prefer_server_ciphers on;
 
-    # Optional settings:
-
     # OCSP stapling
-    # ssl_stapling on;
-    # ssl_stapling_verify on;
-    # ssl_trusted_certificate /etc/letsencrypt/live/<your-nc-domain>/chain.pem;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ item.1 }}/fullchain.pem;
 
     # replace with the IP address of your resolver
-    # resolver 127.0.0.1; # needed for oscp stapling: e.g. use 94.140.15.15 for adguard / 1.1.1.1 for cloudflared or 8.8.8.8 for google - you can use the same nameserver as listed in your /etc/resolv.conf file
+    resolver 127.0.0.1;
 }

nginx/nginx.conf

@@ -1,3 +1,27 @@
+{%- set unique_ports = [] %}
+
+{%- for port in default_ports %}
+{{- unique_ports.append(port) }}
+{%- endfor %}
+
+# For each domain we want to terminate, forward to internal http server
+{%- set http_domains = [] %}
+{%- for item in (http | subelements('external.domains')) %}
+
+{#- Collect unique domains #}
+{%- if item.1 not in http_domains %}
+{{- http_domains.append(item.1) }}
+{%- endif %}
+
+{#- Collect unique ports #}
+{%- for port in item.0.external.ports %}
+{%- if port not in unique_ports %}
+{{- unique_ports.append(port) }}
+{%- endif %}
+{%- endfor %}
+
+{%- endfor %}
+
 load_module /usr/lib64/nginx/modules/ngx_stream_module.so;
 
 worker_processes 8;
@@ -5,53 +29,96 @@ worker_processes 8;
 events {}
 
 stream {
-    log_format basic '$remote_addr $domain [$time_local] '
-                 '$protocol $status $bytes_sent $bytes_received '
-                 '$session_time';
-
-    include /etc/nginx/stream.d/*.conf;
+    log_format basic '| Remote Addr: $remote_addr:$server_port | SSL Preread: $ssl_preread_server_name | Forward IP: $forward_ip:$upstream_port | Upstream Addr: $upstream_addr | $time_local | $protocol | $status | $bytes_sent | $bytes_received | $session_time |';
 
     # Map all SSL parsed server names to hosts
-    map $ssl_preread_server_name $domain {
+    map $ssl_preread_server_name $forward_ip {
 
-        "" 127.0.0.1:443;
+        # Empty ssl preread gets forwarded to internal
+        "" 127.0.0.1;
 
-        # For each domain we need to stream to a remote server, forward to internal ip
-        {% for item in (stream | subelements('external.domains')) %}
-        {{ item.1 }} {{ item.0.internal.ip }}:{{ item.0.internal.port }};
-        {% endfor %}
+{% for item in http_domains %}
+        {{ item }} 127.0.0.1;
+{% endfor %}
 
-        # For each domain we want to terminate, forward to internal http server
-        {% for item in (http | subelements('external.domains')) %}
-        {{ item.1 }} 127.0.0.1:443;
-        {% endfor %}
-
-        default {{ nginx.defaults.domain }}:443;
+        default {{ nginx.defaults.ip }};
+    }
+
+    # Since external traffic will be coming in on port 444, and we need to get some of that traffic
+    # to kubernetes ingress-nginx on port 443, we need to detect if the destination IP is kubernetes.
+    # If it is, forward that traffic to port 443. Otherwise, preserve the original port the traffic
+    # came in on.
+    map $forward_ip $upstream_port {
+        {{ nginx.defaults.ip }} 443;
+        default $server_port;
     }
 
-    # Forward 443 traffic
     server {
-        access_log /var/log/nginx/stream-access-443.log basic;
-        listen {{ ansible_default_ipv4.address }}:443;
-        resolver 1.1.1.1;
-        proxy_pass $domain;
+        access_log /var/log/nginx/nginx_stream_access.log basic;
+        error_log /var/log/nginx/nginx_stream_error.log warn;
+
+        proxy_protocol on;
+
+{% for port in unique_ports %}
+        listen {{ ansible_default_ipv4.address }}:{{ port }};
+{% endfor %}
+
+        proxy_pass $forward_ip:$upstream_port;
         ssl_preread on;
         proxy_socket_keepalive on;
     }
+
+    include /etc/nginx/stream.d/*.conf;
 }
 
 http {
-    log_format compression '$remote_addr - $remote_user [$time_local] '
-                       '"$request" $status $bytes_sent '
-                       '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+    log_format basic '| Proxy Proto Addr: $proxy_protocol_addr | Remote Addr: $remote_addr:$server_port | Host: $host | Forward IP: $forward_ip | Referer: $http_referer | $request | $time_local | $status |';
 
+    map $host $forward_ip {
+        "" "";
+
+{% for item in http_domains %}
+        {{ item }} "";
+{% endfor %}
+
+        default {{ nginx.defaults.ip }};
+    }
+
+    # Internal requests come through 80
     server {
-        access_log /var/log/nginx/http-access.log compression;
+        access_log /var/log/nginx/nginx_http_access.log basic;
+        error_log /var/log/nginx/nginx_http_error.log warn;
 
-        listen 80 default_server;
+        listen 127.0.0.1:80 default_server proxy_protocol;
 
-        if ($scheme = "http") {
-            return 301 https://$host:443$request_uri;
+        location / {
+            # If we have a foward IP, forward the traffic
+            if ($forward_ip) {
+                proxy_pass $forward_ip:80;
+            }
+            # Else redirect if the scheme is http
+            if ($scheme = "http") {
+                return 301 https://$host:443$request_uri;
+            }
+        }
+    }
+
+    # External requests come through 81
+    server {
+        access_log /var/log/nginx/nginx_http_access.log basic;
+        error_log /var/log/nginx/nginx_http_error.log warn;
+
+        listen 127.0.0.1:81 default_server proxy_protocol;
+
+        location / {
+            # If we have a foward IP, forward the traffic
+            if ($forward_ip) {
+                proxy_pass $forward_ip:81;
+            }
+            # Else redirect if the scheme is http
+            if ($scheme = "http") {
+                return 301 https://$host:443$request_uri;
+            }
         }
     }
 

nginx/nginx.yaml

@@ -58,13 +58,15 @@
   - name: Template all http configurations
     template:
       src: https.conf
-      dest: /etc/nginx/http.d/{{ item.1 }}.conf
+      dest: /etc/nginx/http.d/{{ item.1 }}.{{ item.0.internal.port }}.conf
       owner: root
       group: root
       mode: '0644'
     # item.0 == full dictionary
     # item.1 == external domain
     loop: "{{ http | subelements('external.domains') }}"
+  - name: Test nginx configuration
+    ansible.builtin.shell: /usr/sbin/nginx -t
   - name: Reload nginx service
     ansible.builtin.systemd_service:
       state: restarted

nginx/stream.d/gitea-ssh.conf

@@ -1,5 +1,7 @@
 server {
-    access_log /var/log/nginx/gitea-ssh.log basic;
+    access_log /var/log/nginx/nginx_stream_access.log basic;
+    error_log /var/log/nginx/nginx_stream_error.log warn;
+
     listen {{ ansible_default_ipv4.address }}:2222;
-    proxy_pass lb.reeselink.com:2222;
+    proxy_pass 10.1.2.100:2222;
 }

nginx/stream.d/iperf3.conf

@@ -1,5 +1,7 @@
 server {
-    access_log /var/log/nginx/iperf.log basic;
+    access_log /var/log/nginx/nginx_stream_access.log basic;
+    error_log /var/log/nginx/nginx_stream_error.log warn;
+
     listen {{ ansible_default_ipv4.address }}:5201;
     listen {{ ansible_default_ipv4.address }}:5201 udp;
     proxy_pass 127.0.0.1:5201;

nginx/stream.d/kube.conf

@@ -0,0 +1,13 @@
+upstream kube_backend {
+    server 10.1.2.13:6443 max_fails=2 fail_timeout=30s;
+    server 10.1.2.14:6443 max_fails=2 fail_timeout=30s;
+    server 10.1.2.15:6443 max_fails=2 fail_timeout=30s;
+}
+
+server {
+    access_log /var/log/nginx/nginx_stream_access.log basic;
+    error_log /var/log/nginx/nginx_stream_error.log warn;
+
+    listen {{ ansible_default_ipv4.address }}:6443;
+    proxy_pass kube_backend;
+}

nginx/stream.d/unifi-external.conf

@@ -1,6 +1,8 @@
 server {
-    access_log /var/log/nginx/unifi-external.log basic;
+    access_log /var/log/nginx/nginx_stream_access.log basic;
+    error_log /var/log/nginx/nginx_stream_error.log warn;
+
     resolver 1.1.1.1;
-    listen {{ ansible_default_ipv4.address }}:8080;
+    listen {{ ansible_default_ipv4.address }}:8082;
     proxy_pass {{ unifi_external.domain }}:8080;
 }

nginx/vars.yaml

@@ -1,35 +1,54 @@
 nginx:
   defaults:
-    domain: nginx.reeselink.com
+    ip: "10.1.2.101"
 iperf:
-  domain: lb.reeselink.com
+  domain: 10.1.2.100
 unifi_external:
   domain: unifi-server1.reeselink.com
 internal_ip: 10.1.0.0/16
-cr10se:
-  - external:
-      domains:
-      - cr10se.reeseseal.com
-      port: 443
-    internal:
-      ip: "10.3.165.70"
-      port: 80
-      protocol: http
+default_ports:
+  - 80
+  - 81
+  - 443
+  - 444
 
 http:
   - external:
       domains:
       - homeassistant.reeseapps.com
       - homeassistant.reeselink.com
-      port: 443
+      ports:
+      - 443
+      - 444
     internal:
       ip: "10.2.131.2"
       port: 8123
       protocol: https
+  - external:
+      domains:
+      - driveripper.reeseapps.com
+      - driveripper.reeselink.com
+      ports:
+      - 443
+      - 444
+    internal:
+      ip: "10.1.2.10"
+      port: 8443
+      protocol: https
+  - external:
+      domains:
+      - replicator.reeselink.com
+      ports:
+      - 443
+    internal:
+      ip: "10.2.224.77"
+      port: 80
+      protocol: http
   - external:
       domains:
       - yellow.reeselink.com
-      port: 443
+      ports:
+      - 443
     internal:
       ip: "10.1.203.197"
       port: 9090
@@ -37,7 +56,8 @@ http:
   - external:
       domains:
       - node1.reeselink.com
-      port: 443
+      ports:
+      - 443
     internal:
       ip: "10.1.2.13"
       port: 9090
@@ -45,7 +65,8 @@ http:
   - external:
       domains:
       - node2.reeselink.com
-      port: 443
+      ports:
+      - 443
     internal:
       ip: "10.1.2.14"
       port: 9090
@@ -53,17 +74,57 @@ http:
   - external:
       domains:
       - node3.reeselink.com
-      port: 443
+      ports:
+      - 443
     internal:
       ip: "10.1.2.15"
       port: 9090
       protocol: https
-
-stream:
+  # Printer
   - external:
       domains:
-      - containers.reeseapps.com
-      port: 443
+      - cr10se.reeselink.com
+      ports:
+      - 443
     internal:
-      ip: "10.1.2.13"
-      port: 6443
+      ip: "10.3.165.70"
+      port: 80
+      protocol: http
+  # Websocket
+  - external:
+      domains:
+      - cr10se.reeselink.com
+      ports:
+      - 9999
+    internal:
+      ip: "10.3.165.70"
+      port: 9999
+      protocol: http
+  # Camera
+  - external:
+      domains:
+      - cr10se.reeselink.com
+      ports:
+      - 8080
+    internal:
+      ip: "10.3.165.70"
+      port: 8080
+      protocol: http
+  - external:
+      domains:
+      - pihole.reeselink.com
+      ports:
+      - 443
+    internal:
+      ip: 10.1.203.197
+      port: 8081
+      protocol: http
+  - external:
+      domains:
+      - attmodem.reeselink.com
+      ports:
+      - 443
+    internal:
+      ip: 192.168.1.254
+      port: 80
+      protocol: http