fix: keep clustered nodes on the shared network

Dockerfile

@@ -93,6 +93,7 @@ RUN apt-get update \
   && apt-get install -y --no-install-recommends \
     ca-certificates=20250419 \
     curl=8.14.1-2+deb13u4 \
+    iproute2=6.15.0-1 \
     libncurses6=6.5+20250216-2 \
     libsctp1=1.0.21+dfsg-1 \
     libstdc++6=14.2.0-19 \

compose.yaml

@@ -11,7 +11,8 @@ x-app-environment: &app-environment
   # limit. OTP's normal default is 65,536; deployments can override it after
   # measuring their concurrent file/socket requirements.
   ERL_ZFLAGS: "+Q ${ERLANG_PORT_LIMIT:-65536}"
-  DNS_CLUSTER_QUERY: web
+  CLUSTER_INTERFACE: ${CLUSTER_INTERFACE:-eth0}
+  DNS_CLUSTER_QUERY: ${DNS_CLUSTER_QUERY:-cluster-web}
   PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env}
   PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env}
   PHX_URL_PORT: ${PHX_URL_PORT:?Set PHX_URL_PORT in .env}
@@ -115,7 +116,9 @@ services:
       retries: 20
     volumes:
       - postgres_data:/var/lib/postgresql
-    networks: [internal]
+    networks:
+      internal:
+        interface_name: eth0
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -181,7 +184,15 @@ services:
       retries: 10
     deploy:
       replicas: ${WEB_REPLICAS:-2}
-    networks: [ingress, internal, egress]
+    networks:
+      internal:
+        interface_name: eth0
+        aliases: [cluster-web]
+      ingress:
+        interface_name: eth1
+      egress:
+        interface_name: eth2
+        gw_priority: 1
     read_only: true
     tmpfs:
       - /tmp
@@ -207,7 +218,12 @@ services:
       retries: 10
     deploy:
       replicas: ${WORKER_REPLICAS:-2}
-    networks: [internal, egress]
+    networks:
+      internal:
+        interface_name: eth0
+      egress:
+        interface_name: eth1
+        gw_priority: 1
     read_only: true
     tmpfs:
       - /tmp

deploy/helm/who-need-help/templates/deployments.yaml

@@ -49,6 +49,8 @@ spec:
               value: migrate
             - name: ERL_ZFLAGS
               value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }}
+            - name: CLUSTER_INTERFACE
+              value: {{ $root.Values.app.clusterInterface | quote }}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
@@ -69,6 +71,8 @@ spec:
               value: {{ $component }}
             - name: ERL_ZFLAGS
               value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }}
+            - name: CLUSTER_INTERFACE
+              value: {{ $root.Values.app.clusterInterface | quote }}
             - name: PHX_SERVER
               value: {{ if eq $component "web" }}"true"{{ else }}"false"{{ end }}
             - name: PHX_HOST

deploy/helm/who-need-help/templates/migrate-job.yaml

@@ -35,6 +35,8 @@ spec:
               value: migrate
             - name: ERL_ZFLAGS
               value: {{ printf "+Q %d" (int .Values.app.erlangPortLimit) | quote }}
+            - name: CLUSTER_INTERFACE
+              value: {{ .Values.app.clusterInterface | quote }}
             - name: POOL_SIZE
               value: {{ .Values.app.migratePoolSize | quote }}
           securityContext:

deploy/helm/who-need-help/values.yaml

@@ -26,6 +26,7 @@ app:
   # Kubernetes runtimes expose a value so large that each BEAM instance
   # preallocates a multi-gigabyte port table.
   erlangPortLimit: 65536
+  clusterInterface: eth0
   codexSessionId: not-configured
   # Shared limits are opt-in; set only after product policy thresholds are approved.
   rateLimitPoliciesJson: "{}"

rel/env.sh.eex

@@ -3,8 +3,21 @@
 export RELEASE_DISTRIBUTION="${RELEASE_DISTRIBUTION:-name}"
 
 if [ -z "${RELEASE_NODE:-}" ]; then
+  if [ -n "${CLUSTER_INTERFACE:-}" ]; then
+    release_ip=$(
+      ip -o -4 address show dev "$CLUSTER_INTERFACE" scope global |
+        awk 'NR == 1 {sub(/\/.*/, "", $4); print $4}'
+    )
+
+    if [ -z "$release_ip" ]; then
+      echo "No global IPv4 address found on CLUSTER_INTERFACE=$CLUSTER_INTERFACE." >&2
+      exit 1
+    fi
+  else
     release_ip=$(hostname -i)
     release_ip=${release_ip%% *}
+  fi
+
   export RELEASE_NODE="who_need_help@$release_ip"
   unset release_ip
 fi

scripts/quality.sh

@@ -82,6 +82,7 @@ docker compose --env-file .env.example -f compose.yaml config --format json |
     | [$root.services.migrate, $root.services.web, $root.services.worker]
     | all(
         .environment.ERL_ZFLAGS == "+Q 65536" and
+        .environment.CLUSTER_INTERFACE == "eth0" and
         .read_only == true and
         .cap_drop == ["ALL"] and
         .security_opt == ["no-new-privileges:true"] and
@@ -95,6 +96,14 @@ docker compose --env-file .env.example -f compose.yaml config --format json |
     and ($root.services.proxy.networks | keys | sort) == ["docker-api", "edge", "ingress"]
     and ($root.services.web.networks | keys | sort) == ["egress", "ingress", "internal"]
     and ($root.services.worker.networks | keys | sort) == ["egress", "internal"]
+    and $root.services.web.networks.internal.interface_name == "eth0"
+    and $root.services.web.networks.internal.aliases == ["cluster-web"]
+    and $root.services.web.networks.ingress.interface_name == "eth1"
+    and $root.services.web.networks.egress.interface_name == "eth2"
+    and $root.services.web.networks.egress.gw_priority == 1
+    and $root.services.worker.networks.internal.interface_name == "eth0"
+    and $root.services.worker.networks.egress.interface_name == "eth1"
+    and $root.services.worker.networks.egress.gw_priority == 1
     and ($root.services.db.networks | keys) == ["internal"]
     and $root.networks.ingress.internal == true
     and $root.networks.internal.internal == true