standardize http/https ports in nginx config

2024-05-29 11:46:23 -04:00
parent a853df3cbc
commit 21d7cef56b
7 changed files with 197 additions and 144 deletions
--- a/nextcloud/README.md
+++ b/nextcloud/README.md
@@ -20,3 +20,11 @@ docker run \
 --volume /var/run/docker.sock:/var/run/docker.sock:ro \
 nextcloud/all-in-one:latest
 ```
+
+## Uninstall
+
+```bash
+docker stop $(docker ps -a -q)
+docker system prune
+docker volume prune -a -f
+```
--- a/nginx/certbot.yaml
+++ b/nginx/certbot.yaml
@@ -12,7 +12,12 @@
      name:
        - certbot
      state: present
-  - name: Get certs for all domains
-    ansible.builtin.shell: /usr/bin/certbot certonly --dns-route53 -d '{{ item.1 }}' -n
+  - name: Get certs for all internal domains
+    ansible.builtin.shell: /usr/bin/certbot certonly --dns-route53 -d '{{ item.external.domain }}{{ internal_tld }}' -n
    # Loops over every external.domains sub list
-    loop: "{{ http | subelements('external.domains') }}"
+    loop: "{{ http }}"
+  - name: Get certs for all external domains
+    ansible.builtin.shell: /usr/bin/certbot certonly --dns-route53 -d '{{ item.external.domain }}{{ expose_tld }}' -n
+    # Loops over every external.domains sub list
+    loop: "{{ http }}"
+    when: item.external.expose
--- a/nginx/https.conf
+++ b/nginx/https.conf
@@ -1,18 +1,16 @@
-map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-}
-
+# Internal Server
 server {
    access_log /var/log/nginx/nginx_https_access.log basic;
    error_log /var/log/nginx/nginx_https_error.log warn;

-    {%- for port in item.0.external.ports +%}
-    listen 127.0.0.1:{{ port }} ssl proxy_protocol;
-    {%- endfor +%}
+    # Listen for the default http internal ports
+    listen 127.0.0.1:{{ defaults.http.internal_http_port }} proxy_protocol;
+    listen 127.0.0.1:{{ defaults.http.internal_https_port }} ssl proxy_protocol;

-    listen 127.0.0.1:80 proxy_protocol;
-    listen 127.0.0.1:81 proxy_protocol;
+    # Listen for any extra ports specified by the user
+{% for port in item.external.extra_ports %}
+    listen 127.0.0.1:{{ port }} proxy_protocol ssl;
+{% endfor %}

    if ($scheme = "http") {
        return 301 https://$host:443$request_uri;
@@ -20,10 +18,17 @@ server {

    set_real_ip_from  127.0.0.1;

-    server_name {{ item.1 }};
+    server_name {{ item.external.domain }}{{ internal_tld }};

    location / {
-        proxy_pass {{ item.0.internal.protocol }}://{{ item.0.internal.ip }}:{{ item.0.internal.port }}$request_uri;
+{% for port in item.external.extra_ports %}
+        if ($server_port = "{{ port }}") {
+            proxy_pass {{ item.internal.protocol }}://{{ item.internal.ip }}:{{ port }}$request_uri;
+        }
+{% endfor %}
+        if ($server_port = "{{ defaults.http.internal_https_port }}"){
+            proxy_pass {{ item.internal.protocol }}://{{ item.internal.ip }}:{{ item.internal.port }}$request_uri;
+        }

        proxy_set_header X-Forwarded-Port $server_port;
        proxy_set_header X-Forwarded-Scheme $scheme;
@@ -44,8 +49,8 @@ server {
        proxy_set_header Connection $connection_upgrade;
    }

-    ssl_certificate /etc/letsencrypt/live/{{ item.1 }}/fullchain.pem;   # managed by certbot on host machine
-    ssl_certificate_key /etc/letsencrypt/live/{{ item.1 }}/privkey.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ item.external.domain }}{{ internal_tld }}/fullchain.pem;   # managed by certbot on host machine
+    ssl_certificate_key /etc/letsencrypt/live/{{ item.external.domain }}{{ internal_tld }}/privkey.pem;

    ssl_session_timeout 1d;
    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
@@ -58,8 +63,68 @@ server {
    # OCSP stapling
    ssl_stapling on;
    ssl_stapling_verify on;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ item.1 }}/fullchain.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ item.external.domain }}{{ internal_tld }}/fullchain.pem;

    # replace with the IP address of your resolver
    resolver 127.0.0.1;
 }
+
+# External Server
+{% if item.external.expose %}
+server {
+    access_log /var/log/nginx/nginx_https_access.log basic;
+    error_log /var/log/nginx/nginx_https_error.log warn;
+
+    listen 127.0.0.1:{{ defaults.http.external_http_port }} proxy_protocol;
+    listen 127.0.0.1:{{ defaults.http.external_https_port }} ssl proxy_protocol;
+
+    if ($scheme = "http") {
+        return 301 https://$host:443$request_uri;
+    }
+
+    set_real_ip_from  127.0.0.1;
+
+    server_name {{ item.external.domain }}{{ expose_tld }};
+
+    location / {
+        proxy_pass {{ item.internal.protocol }}://{{ item.internal.ip }}:{{ item.internal.port }}$request_uri;
+
+        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header X-Forwarded-Scheme $scheme;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Accept-Encoding "";
+
+        proxy_set_header Host            $host;
+        proxy_set_header X-Real-IP       $proxy_protocol_addr;
+        proxy_set_header X-Forwarded-For $proxy_protocol_addr;
+
+        client_body_buffer_size 512k;
+        proxy_read_timeout 86400s;
+        client_max_body_size 0;
+
+        # Websocket
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+    }
+
+    ssl_certificate /etc/letsencrypt/live/{{ item.external.domain }}{{ expose_tld }}/fullchain.pem;   # managed by certbot on host machine
+    ssl_certificate_key /etc/letsencrypt/live/{{ item.external.domain }}{{ expose_tld }}/privkey.pem;
+
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
+    ssl_session_tickets off;
+
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
+    ssl_prefer_server_ciphers on;
+
+    # OCSP stapling
+    ssl_stapling on;
+    ssl_stapling_verify on;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ item.external.domain }}{{ expose_tld }}/fullchain.pem;
+
+    # replace with the IP address of your resolver
+    resolver 127.0.0.1;
+}
+{%- endif %}
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -1,20 +1,7 @@
 {%- set unique_ports = [] %}

-{%- for port in default_ports %}
-{{- unique_ports.append(port) }}
-{%- endfor %}
-
-# For each domain we want to terminate, forward to internal http server
-{%- set http_domains = [] %}
-{%- for item in (http | subelements('external.domains')) %}
-
-{#- Collect unique domains #}
-{%- if item.1 not in http_domains %}
-{{- http_domains.append(item.1) }}
-{%- endif %}
-
-{#- Collect unique ports #}
-{%- for port in item.0.external.ports %}
+{%- for item in http %}
+{%- for port in item.external.extra_ports %}
 {%- if port not in unique_ports %}
 {{- unique_ports.append(port) }}
 {%- endif %}
@@ -29,27 +16,32 @@ worker_processes 8;
 events {}

 stream {
-    log_format basic '| Remote Addr: $remote_addr:$server_port | SSL Preread: $ssl_preread_server_name | Forward IP: $forward_ip:$upstream_port | Upstream Addr: $upstream_addr | $time_local | $protocol | $status | $bytes_sent | $bytes_received | $session_time |';
+    log_format basic '| Remote Addr: $remote_addr:$server_port | SSL Preread: $ssl_preread_server_name | Forward: $map_forward_ip:$upstream_port | Upstream Addr: $upstream_addr | $time_local | $protocol | $status | $bytes_sent | $bytes_received | $session_time |';

    # Map all SSL parsed server names to hosts
-    map $ssl_preread_server_name $forward_ip {
+    map $ssl_preread_server_name $map_forward_ip {

        # Empty ssl preread gets forwarded to internal
        "" 127.0.0.1;

-{% for item in http_domains %}
-        {{ item }} 127.0.0.1;
+        # These domains will get forwarded to the internal http server
+{% for item in http %}
+        {{ item.external.domain }}{{ internal_tld }} 127.0.0.1;
+{% if item.external.expose %}
+        {{ item.external.domain }}{{ expose_tld }} 127.0.0.1;
+{% endif %}
 {% endfor %}

-        default {{ nginx.defaults.ip }};
+        # By default forward to our internal nginx server (probably kubernetes)
+        default {{ defaults.forward_ip }};
    }

    # Since external traffic will be coming in on port 444, and we need to get some of that traffic
    # to kubernetes ingress-nginx on port 443, we need to detect if the destination IP is kubernetes.
    # If it is, forward that traffic to port 443. Otherwise, preserve the original port the traffic
    # came in on.
-    map $forward_ip $upstream_port {
-        {{ nginx.defaults.ip }} 443;
+    map $map_forward_ip $upstream_port {
+        {{ defaults.forward_ip }} 443;
        default $server_port;
    }

@@ -59,11 +51,17 @@ stream {

        proxy_protocol on;

+        # The default http ports
+{% for port in defaults.listen_ports %}
+        listen {{ ansible_default_ipv4.address }}:{{ port }};
+{% endfor %}
+
+        # Any unique ports listed in the extra_ports field
 {% for port in unique_ports %}
        listen {{ ansible_default_ipv4.address }}:{{ port }};
 {% endfor %}

-        proxy_pass $forward_ip:$upstream_port;
+        proxy_pass $map_forward_ip:$upstream_port;
        ssl_preread on;
        proxy_socket_keepalive on;
    }
@@ -72,16 +70,21 @@ stream {
 }

 http {
-    log_format basic '| Proxy Proto Addr: $proxy_protocol_addr | Remote Addr: $remote_addr:$server_port | Host: $host | Forward IP: $forward_ip | Referer: $http_referer | $request | $time_local | $status |';
+    log_format basic '| Proxy Proto Addr: $proxy_protocol_addr | Remote Addr: $remote_addr:$server_port | Host: $host | Forward: $map_forward_ip:$server_port | Referer: $http_referer | $request | $time_local | $status |';

-    map $host $forward_ip {
-        "" "";
+    map $host $map_forward_ip {
+        "" "127.0.0.1";

-{% for item in http_domains %}
-        {{ item }} "";
+        # We don't want to forward traffic we're terminating
+        # Rather we'll catch it here and redirect to 443.
+{% for item in http %}
+        {{ item.external.domain }}{{ internal_tld }} "127.0.0.1";
+{% if item.external.expose %}
+        {{ item.external.domain }}{{ expose_tld }} "127.0.0.1";
+{% endif %}
 {% endfor %}

-        default {{ nginx.defaults.ip }};
+        default {{ defaults.forward_ip }};
    }

    # Internal requests come through 80
@@ -92,9 +95,9 @@ http {
        listen 127.0.0.1:80 default_server proxy_protocol;

        location / {
-            # If we have a foward IP, forward the traffic
-            if ($forward_ip) {
-                proxy_pass $forward_ip:80;
+            # If we have an external forward IP, forward traffic
+            if ($map_forward_ip != "127.0.0.1") {
+                proxy_pass $map_forward_ip:80;
            }
            # Else redirect if the scheme is http
            if ($scheme = "http") {
@@ -112,8 +115,8 @@ http {

        location / {
            # If we have a foward IP, forward the traffic
-            if ($forward_ip) {
-                proxy_pass $forward_ip:81;
+            if ($map_forward_ip) {
+                proxy_pass $map_forward_ip:81;
            }
            # Else redirect if the scheme is http
            if ($scheme = "http") {
@@ -122,5 +125,10 @@ http {
        }
    }

+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
    include /etc/nginx/http.d/*.conf;
 }
--- a/nginx/nginx.yaml
+++ b/nginx/nginx.yaml
@@ -58,13 +58,11 @@
  - name: Template all http configurations
    template:
      src: https.conf
-      dest: /etc/nginx/http.d/{{ item.1 }}.{{ item.0.internal.port }}.conf
+      dest: /etc/nginx/http.d/{{ item.external.domain }}.conf
      owner: root
      group: root
      mode: '0644'
-    # item.0 == full dictionary
-    # item.1 == external domain
-    loop: "{{ http | subelements('external.domains') }}"
+    loop: "{{ http }}"
  - name: Test nginx configuration
    ansible.builtin.shell: /usr/sbin/nginx -t
  - name: Reload nginx service
--- a/nginx/stream.d/unifi-external.conf
+++ b/nginx/stream.d/unifi-external.conf
@@ -1,8 +1,8 @@
-server {
-    access_log /var/log/nginx/nginx_stream_access.log basic;
-    error_log /var/log/nginx/nginx_stream_error.log warn;
+# server {
+#     access_log /var/log/nginx/nginx_stream_access.log basic;
+#     error_log /var/log/nginx/nginx_stream_error.log warn;

-    resolver 1.1.1.1;
-    listen {{ ansible_default_ipv4.address }}:8082;
-    proxy_pass {{ unifi_external.domain }}:8080;
-}
+#     resolver 1.1.1.1;
+#     listen {{ ansible_default_ipv4.address }}:8082;
+#     proxy_pass {{ ansible_default_ipv4.address }}:8080;
+# }
--- a/nginx/vars.yaml
+++ b/nginx/vars.yaml
@@ -1,140 +1,109 @@
-nginx:
-  defaults:
-    ip: "10.1.2.101"
-iperf:
-  domain: 10.1.2.100
-unifi_external:
-  domain: unifi-server1.reeselink.com
-internal_ip: 10.1.0.0/16
-default_ports:
-  - 80
-  - 81
+defaults:
+  forward_ip: "10.1.2.101"
+  listen_ports:
  - 443
+  - 80
  - 444
+  - 81
+  http:
+    internal_http_port: 80
+    internal_https_port: 443
+    external_http_port: 81
+    external_https_port: 444
+internal_ip: 10.1.0.0/16
+expose_tld: .reeseapps.com
+internal_tld: .reeselink.com

 http:
  - external:
-      domains:
-      - homeassistant.reeseapps.com
-      - homeassistant.reeselink.com
-      ports:
-      - 443
-      - 444
+      domain: homeassistant
+      expose: true
+      extra_ports: []
    internal:
      ip: "10.2.131.2"
      port: 8123
      protocol: https
  - external:
-      domains:
-      - driveripper.reeseapps.com
-      - driveripper.reeselink.com
-      ports:
-      - 443
-      - 444
+      domain: driveripper
+      expose: true
+      extra_ports: []
    internal:
      ip: "10.1.2.10"
      port: 8443
      protocol: https
  - external:
-      domains:
-      - replicator.reeselink.com
-      ports:
-      - 443
+      domain: e3s1plus
+      expose: false
+      extra_ports: []
    internal:
      ip: "10.2.224.77"
      port: 80
      protocol: http
  - external:
-      domains:
-      - yellow.reeselink.com
-      ports:
-      - 443
+      domain: yellow
+      expose: false
+      extra_ports: []
    internal:
      ip: "10.1.203.197"
      port: 9090
      protocol: https
  - external:
-      domains:
-      - node1.reeselink.com
-      ports:
-      - 443
+      domain: node1
+      expose: false
+      extra_ports: []
    internal:
      ip: "10.1.2.13"
      port: 9090
      protocol: https
  - external:
-      domains:
-      - node2.reeselink.com
-      ports:
-      - 443
+      domain: node2
+      expose: false
+      extra_ports: []
    internal:
      ip: "10.1.2.14"
      port: 9090
      protocol: https
  - external:
-      domains:
-      - node3.reeselink.com
-      ports:
-      - 443
+      domain: node3
+      expose: false
+      extra_ports: []
    internal:
      ip: "10.1.2.15"
      port: 9090
      protocol: https
  # Printer
  - external:
-      domains:
-      - cr10se.reeselink.com
-      ports:
-      - 443
+      domain: cr10se
+      expose: false
+      extra_ports:
+      # websocket
+      - 9999
+      # camera
+      - 8080
    internal:
      ip: "10.3.165.70"
      port: 80
      protocol: http
-  # Websocket
  - external:
-      domains:
-      - cr10se.reeselink.com
-      ports:
-      - 9999
-    internal:
-      ip: "10.3.165.70"
-      port: 9999
-      protocol: http
-  # Camera
-  - external:
-      domains:
-      - cr10se.reeselink.com
-      ports:
-      - 8080
-    internal:
-      ip: "10.3.165.70"
-      port: 8080
-      protocol: http
-  - external:
-      domains:
-      - pihole.reeselink.com
-      ports:
-      - 443
+      domain: pihole
+      expose: false
+      extra_ports: []
    internal:
      ip: 10.1.203.197
      port: 8081
      protocol: http
  - external:
-      domains:
-      - attmodem.reeselink.com
-      ports:
-      - 443
+      domain: attmodem
+      expose: false
+      extra_ports: []
    internal:
      ip: 192.168.1.254
      port: 80
      protocol: http
  - external:
-      domains:
-      - nextcloud-aio.reeseapps.com
-      - nextcloud-aio.reeselink.com
-      ports:
-      - 443
-      - 444
+      domain: nextcloud-aio
+      expose: true
+      extra_ports: []
    internal:
      ip: 10.1.175.237
      port: 11000