diff --git a/Dockerfile b/Dockerfile index af97e42..6daa24b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,6 +93,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates=20250419 \ curl=8.14.1-2+deb13u4 \ + iproute2=6.15.0-1 \ libncurses6=6.5+20250216-2 \ libsctp1=1.0.21+dfsg-1 \ libstdc++6=14.2.0-19 \ diff --git a/compose.yaml b/compose.yaml index f3c6627..a3765fa 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,7 +11,8 @@ x-app-environment: &app-environment # limit. OTP's normal default is 65,536; deployments can override it after # measuring their concurrent file/socket requirements. ERL_ZFLAGS: "+Q ${ERLANG_PORT_LIMIT:-65536}" - DNS_CLUSTER_QUERY: web + CLUSTER_INTERFACE: ${CLUSTER_INTERFACE:-eth0} + DNS_CLUSTER_QUERY: ${DNS_CLUSTER_QUERY:-cluster-web} PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env} PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env} PHX_URL_PORT: ${PHX_URL_PORT:?Set PHX_URL_PORT in .env} @@ -115,7 +116,9 @@ services: retries: 20 volumes: - postgres_data:/var/lib/postgresql - networks: [internal] + networks: + internal: + interface_name: eth0 security_opt: - no-new-privileges:true restart: unless-stopped @@ -181,7 +184,15 @@ services: retries: 10 deploy: replicas: ${WEB_REPLICAS:-2} - networks: [ingress, internal, egress] + networks: + internal: + interface_name: eth0 + aliases: [cluster-web] + ingress: + interface_name: eth1 + egress: + interface_name: eth2 + gw_priority: 1 read_only: true tmpfs: - /tmp @@ -207,7 +218,12 @@ services: retries: 10 deploy: replicas: ${WORKER_REPLICAS:-2} - networks: [internal, egress] + networks: + internal: + interface_name: eth0 + egress: + interface_name: eth1 + gw_priority: 1 read_only: true tmpfs: - /tmp diff --git a/deploy/helm/who-need-help/templates/deployments.yaml b/deploy/helm/who-need-help/templates/deployments.yaml index 33b4ba3..51bb809 100644 --- a/deploy/helm/who-need-help/templates/deployments.yaml +++ b/deploy/helm/who-need-help/templates/deployments.yaml @@ -49,6 +49,8 @@ spec: value: migrate - name: ERL_ZFLAGS value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }} + - name: CLUSTER_INTERFACE + value: {{ $root.Values.app.clusterInterface | quote }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -69,6 +71,8 @@ spec: value: {{ $component }} - name: ERL_ZFLAGS value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }} + - name: CLUSTER_INTERFACE + value: {{ $root.Values.app.clusterInterface | quote }} - name: PHX_SERVER value: {{ if eq $component "web" }}"true"{{ else }}"false"{{ end }} - name: PHX_HOST diff --git a/deploy/helm/who-need-help/templates/migrate-job.yaml b/deploy/helm/who-need-help/templates/migrate-job.yaml index 80c4db3..0f6752b 100644 --- a/deploy/helm/who-need-help/templates/migrate-job.yaml +++ b/deploy/helm/who-need-help/templates/migrate-job.yaml @@ -35,6 +35,8 @@ spec: value: migrate - name: ERL_ZFLAGS value: {{ printf "+Q %d" (int .Values.app.erlangPortLimit) | quote }} + - name: CLUSTER_INTERFACE + value: {{ .Values.app.clusterInterface | quote }} - name: POOL_SIZE value: {{ .Values.app.migratePoolSize | quote }} securityContext: diff --git a/deploy/helm/who-need-help/values.yaml b/deploy/helm/who-need-help/values.yaml index 4b91478..a465a4f 100644 --- a/deploy/helm/who-need-help/values.yaml +++ b/deploy/helm/who-need-help/values.yaml @@ -26,6 +26,7 @@ app: # Kubernetes runtimes expose a value so large that each BEAM instance # preallocates a multi-gigabyte port table. erlangPortLimit: 65536 + clusterInterface: eth0 codexSessionId: not-configured # Shared limits are opt-in; set only after product policy thresholds are approved. rateLimitPoliciesJson: "{}" diff --git a/rel/env.sh.eex b/rel/env.sh.eex index 61e15be..7770781 100644 --- a/rel/env.sh.eex +++ b/rel/env.sh.eex @@ -3,8 +3,21 @@ export RELEASE_DISTRIBUTION="${RELEASE_DISTRIBUTION:-name}" if [ -z "${RELEASE_NODE:-}" ]; then - release_ip=$(hostname -i) - release_ip=${release_ip%% *} + if [ -n "${CLUSTER_INTERFACE:-}" ]; then + release_ip=$( + ip -o -4 address show dev "$CLUSTER_INTERFACE" scope global | + awk 'NR == 1 {sub(/\/.*/, "", $4); print $4}' + ) + + if [ -z "$release_ip" ]; then + echo "No global IPv4 address found on CLUSTER_INTERFACE=$CLUSTER_INTERFACE." >&2 + exit 1 + fi + else + release_ip=$(hostname -i) + release_ip=${release_ip%% *} + fi + export RELEASE_NODE="who_need_help@$release_ip" unset release_ip fi diff --git a/scripts/quality.sh b/scripts/quality.sh index 97a9da0..b392605 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -82,6 +82,7 @@ docker compose --env-file .env.example -f compose.yaml config --format json | | [$root.services.migrate, $root.services.web, $root.services.worker] | all( .environment.ERL_ZFLAGS == "+Q 65536" and + .environment.CLUSTER_INTERFACE == "eth0" and .read_only == true and .cap_drop == ["ALL"] and .security_opt == ["no-new-privileges:true"] and @@ -95,6 +96,14 @@ docker compose --env-file .env.example -f compose.yaml config --format json | and ($root.services.proxy.networks | keys | sort) == ["docker-api", "edge", "ingress"] and ($root.services.web.networks | keys | sort) == ["egress", "ingress", "internal"] and ($root.services.worker.networks | keys | sort) == ["egress", "internal"] + and $root.services.web.networks.internal.interface_name == "eth0" + and $root.services.web.networks.internal.aliases == ["cluster-web"] + and $root.services.web.networks.ingress.interface_name == "eth1" + and $root.services.web.networks.egress.interface_name == "eth2" + and $root.services.web.networks.egress.gw_priority == 1 + and $root.services.worker.networks.internal.interface_name == "eth0" + and $root.services.worker.networks.egress.interface_name == "eth1" + and $root.services.worker.networks.egress.gw_priority == 1 and ($root.services.db.networks | keys) == ["internal"] and $root.networks.ingress.internal == true and $root.networks.internal.internal == true