nginx load balancing internal/external working

nginx/nginx.conf

@@ -1,3 +1,27 @@
+{%- set unique_ports = [] %}
+
+{%- for port in default_ports %}
+{{- unique_ports.append(port) }}
+{%- endfor %}
+
+# For each domain we want to terminate, forward to internal http server
+{%- set http_domains = [] %}
+{%- for item in (http | subelements('external.domains')) %}
+
+{#- Collect unique domains #}
+{%- if item.1 not in http_domains %}
+{{- http_domains.append(item.1) }}
+{%- endif %}
+
+{#- Collect unique ports #}
+{%- for port in item.0.external.ports %}
+{%- if port not in unique_ports %}
+{{- unique_ports.append(port) }}
+{%- endif %}
+{%- endfor %}
+
+{%- endfor %}
+
 load_module /usr/lib64/nginx/modules/ngx_stream_module.so;
 
 worker_processes 8;
@@ -5,53 +29,96 @@ worker_processes 8;
 events {}
 
 stream {
-    log_format basic '$remote_addr $domain [$time_local] '
-                 '$protocol $status $bytes_sent $bytes_received '
-                 '$session_time';
-
-    include /etc/nginx/stream.d/*.conf;
+    log_format basic '| Remote Addr: $remote_addr:$server_port | SSL Preread: $ssl_preread_server_name | Forward IP: $forward_ip:$upstream_port | Upstream Addr: $upstream_addr | $time_local | $protocol | $status | $bytes_sent | $bytes_received | $session_time |';
 
     # Map all SSL parsed server names to hosts
-    map $ssl_preread_server_name $domain {
+    map $ssl_preread_server_name $forward_ip {
 
-        "" 127.0.0.1:443;
+        # Empty ssl preread gets forwarded to internal
+        "" 127.0.0.1;
 
-        # For each domain we need to stream to a remote server, forward to internal ip
-        {% for item in (stream | subelements('external.domains')) %}
-        {{ item.1 }} {{ item.0.internal.ip }}:{{ item.0.internal.port }};
-        {% endfor %}
+{% for item in http_domains %}
+        {{ item }} 127.0.0.1;
+{% endfor %}
 
-        # For each domain we want to terminate, forward to internal http server
-        {% for item in (http | subelements('external.domains')) %}
-        {{ item.1 }} 127.0.0.1:443;
-        {% endfor %}
-
-        default {{ nginx.defaults.domain }}:443;
+        default {{ nginx.defaults.ip }};
+    }
+
+    # Since external traffic will be coming in on port 444, and we need to get some of that traffic
+    # to kubernetes ingress-nginx on port 443, we need to detect if the destination IP is kubernetes.
+    # If it is, forward that traffic to port 443. Otherwise, preserve the original port the traffic
+    # came in on.
+    map $forward_ip $upstream_port {
+        {{ nginx.defaults.ip }} 443;
+        default $server_port;
     }
 
-    # Forward 443 traffic
     server {
-        access_log /var/log/nginx/stream-access-443.log basic;
-        listen {{ ansible_default_ipv4.address }}:443;
-        resolver 1.1.1.1;
-        proxy_pass $domain;
+        access_log /var/log/nginx/nginx_stream_access.log basic;
+        error_log /var/log/nginx/nginx_stream_error.log warn;
+
+        proxy_protocol on;
+
+{% for port in unique_ports %}
+        listen {{ ansible_default_ipv4.address }}:{{ port }};
+{% endfor %}
+
+        proxy_pass $forward_ip:$upstream_port;
         ssl_preread on;
         proxy_socket_keepalive on;
     }
+
+    include /etc/nginx/stream.d/*.conf;
 }
 
 http {
-    log_format compression '$remote_addr - $remote_user [$time_local] '
-                       '"$request" $status $bytes_sent '
-                       '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+    log_format basic '| Proxy Proto Addr: $proxy_protocol_addr | Remote Addr: $remote_addr:$server_port | Host: $host | Forward IP: $forward_ip | Referer: $http_referer | $request | $time_local | $status |';
 
+    map $host $forward_ip {
+        "" "";
+
+{% for item in http_domains %}
+        {{ item }} "";
+{% endfor %}
+
+        default {{ nginx.defaults.ip }};
+    }
+
+    # Internal requests come through 80
     server {
-        access_log /var/log/nginx/http-access.log compression;
+        access_log /var/log/nginx/nginx_http_access.log basic;
+        error_log /var/log/nginx/nginx_http_error.log warn;
 
-        listen 80 default_server;
+        listen 127.0.0.1:80 default_server proxy_protocol;
 
-        if ($scheme = "http") {
-            return 301 https://$host:443$request_uri;
+        location / {
+            # If we have a foward IP, forward the traffic
+            if ($forward_ip) {
+                proxy_pass $forward_ip:80;
+            }
+            # Else redirect if the scheme is http
+            if ($scheme = "http") {
+                return 301 https://$host:443$request_uri;
+            }
+        }
+    }
+
+    # External requests come through 81
+    server {
+        access_log /var/log/nginx/nginx_http_access.log basic;
+        error_log /var/log/nginx/nginx_http_error.log warn;
+
+        listen 127.0.0.1:81 default_server proxy_protocol;
+
+        location / {
+            # If we have a foward IP, forward the traffic
+            if ($forward_ip) {
+                proxy_pass $forward_ip:81;
+            }
+            # Else redirect if the scheme is http
+            if ($scheme = "http") {
+                return 301 https://$host:443$request_uri;
+            }
         }
     }