fix: prevent oversized BEAM port tables

.env.e2e.example

@@ -20,6 +20,7 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=GENERATE_INDEPENDENT_E2E_DATABASE_PASSWORD
 DATABASE_URL=ecto://postgres:GENERATE_URL_SAFE_PASSWORD@db/who_need_help_e2e
 POOL_SIZE=10
+ERLANG_PORT_LIMIT=65536
 
 SECRET_KEY_BASE=GENERATE_INDEPENDENT_E2E_SECRET_KEY_BASE
 HANDOVER_SECRET=GENERATE_INDEPENDENT_E2E_HANDOVER_SECRET

.env.example

@@ -63,6 +63,10 @@ POSTGRES_PASSWORD=replace-with-a-local-or-deployment-secret
 DATABASE_URL=ecto://postgres:replace-with-url-encoded-password@db/who_need_help
 
 POOL_SIZE=10
+# Maximum simultaneously existing Erlang ports (files, sockets and drivers).
+# Keeping this explicit prevents a host's very large nofile ulimit from making
+# every BEAM instance preallocate a multi-gigabyte port table.
+ERLANG_PORT_LIMIT=65536
 SECRET_KEY_BASE=generate-with-mix-phx-gen-secret
 HANDOVER_SECRET=generate-an-independent-random-secret
 RELEASE_COOKIE=generate-an-independent-beam-cluster-cookie

.env.load.example

@@ -21,6 +21,7 @@ POSTGRES_USER=wnh_load
 POSTGRES_PASSWORD=GENERATE_POSTGRES_PASSWORD
 DATABASE_URL=GENERATE_DATABASE_URL
 POOL_SIZE=10
+ERLANG_PORT_LIMIT=65536
 SECRET_KEY_BASE=GENERATE_SECRET_KEY_BASE
 HANDOVER_SECRET=GENERATE_HANDOVER_SECRET
 RELEASE_COOKIE=GENERATE_RELEASE_COOKIE

README.md

@@ -82,8 +82,16 @@ Inspect the exact state:
 docker compose -p who_need_help ps -a
 docker compose -p who_need_help logs -f web worker
 ./scripts/verify-realtime-cluster.sh compose
+./scripts/verify-beam-runtime.sh compose
 ```
 
+`ERLANG_PORT_LIMIT` defaults to the normal OTP port limit of `65536`. Keeping
+it explicit prevents a container runtime's unusually large `nofile` limit from
+making every BEAM instance preallocate a multi-gigabyte port table. The runtime
+check above reads every running web/worker VM and fails if its effective limit
+differs from the configured value. Increase it only from measured concurrent
+file, socket, and driver requirements.
+
 Create and restore-test a database backup without restoring over the source:
 
 ```bash
@@ -345,7 +353,7 @@ traffic. Supported actions are listed in `docs/trust-safety.md`.
 The kind script downloads checksum-verified kubectl, Helm, and kind binaries
 into `.tools/`, creates a project-owned cluster, loads the local image, and
 installs the Helm chart, waits for both Deployments, and verifies cross-node
-PubSub:
+PubSub and each BEAM VM's effective port limit:
 
 ```bash
 ./scripts/kind-up.sh

compose.yaml

@@ -7,6 +7,10 @@ x-app-environment: &app-environment
   HANDOVER_SECRET: ${HANDOVER_SECRET:?Set HANDOVER_SECRET in .env}
   RELEASE_COOKIE: ${RELEASE_COOKIE:?Set RELEASE_COOKIE in .env}
   METRICS_TOKEN: ${METRICS_TOKEN:?Set an independent METRICS_TOKEN in .env}
+  # Keep BEAM from deriving a massive port table from a host/container nofile
+  # limit. OTP's normal default is 65,536; deployments can override it after
+  # measuring their concurrent file/socket requirements.
+  ERL_ZFLAGS: "+Q ${ERLANG_PORT_LIMIT:-65536}"
   DNS_CLUSTER_QUERY: web
   PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env}
   PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env}

deploy/helm/who-need-help/templates/deployments.yaml

@@ -47,6 +47,8 @@ spec:
           env:
             - name: APP_ROLE
               value: migrate
+            - name: ERL_ZFLAGS
+              value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
@@ -65,6 +67,8 @@ spec:
           env:
             - name: APP_ROLE
               value: {{ $component }}
+            - name: ERL_ZFLAGS
+              value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }}
             - name: PHX_SERVER
               value: {{ if eq $component "web" }}"true"{{ else }}"false"{{ end }}
             - name: PHX_HOST

deploy/helm/who-need-help/templates/migrate-job.yaml

@@ -33,6 +33,8 @@ spec:
           env:
             - name: APP_ROLE
               value: migrate
+            - name: ERL_ZFLAGS
+              value: {{ printf "+Q %d" (int .Values.app.erlangPortLimit) | quote }}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

deploy/helm/who-need-help/values.schema.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "app": {
+      "type": "object",
+      "properties": {
+        "erlangPortLimit": {
+          "type": "integer",
+          "minimum": 1024,
+          "maximum": 134217727
+        }
+      },
+      "required": [
+        "erlangPortLimit"
+      ]
+    }
+  }
+}

deploy/helm/who-need-help/values.yaml

@@ -20,6 +20,10 @@ app:
   urlPort: "443"
   port: "4000"
   poolSize: "10"
+  # OTP otherwise derives this from the container nofile ulimit. Some
+  # Kubernetes runtimes expose a value so large that each BEAM instance
+  # preallocates a multi-gigabyte port table.
+  erlangPortLimit: 65536
   codexSessionId: not-configured
   # Shared limits are opt-in; set only after product policy thresholds are approved.
   rateLimitPoliciesJson: "{}"

docs/operations.md

@@ -284,6 +284,28 @@ The rollout timeout, probe interval/timeout/retry count, and cluster-join
 timeout are experiment inputs. They are not production SLOs or resource
 requirements.
 
+## BEAM runtime memory guard
+
+Compose sets `ERL_ZFLAGS="+Q ${ERLANG_PORT_LIMIT}"`; Helm renders the same flag
+from `app.erlangPortLimit`. Both default to `65536`, OTP's normal port-table
+limit. This makes the runtime independent of an unusually large host or nested
+container `nofile` limit. Validate all live replicas after deployment:
+
+```bash
+./scripts/verify-beam-runtime.sh compose
+./scripts/verify-beam-runtime.sh kind
+```
+
+The command records the effective port count/limit, allocated port-table bytes,
+BEAM memory, process count, cgroup memory, and RSS for each web and worker
+replica. It fails when a live VM does not use the configured limit; kind mode
+also requires the desired number of Ready application pods.
+
+`65536` is a concurrency ceiling for simultaneously existing Erlang ports
+(files, sockets, and drivers), not a container memory limit. Do not lower or
+raise it from a RAM estimate alone. A changed value must be validated against
+measured peak port usage and the target environment.
+
 ## Protected Prometheus metrics
 
 The web role exposes Prometheus text format at `/metrics`. It requires the

docs/performance.md

@@ -5,6 +5,38 @@ pool size, or autoscaling threshold is known yet. The repository therefore
 contains a reproducible measurement profile, not a capacity claim or blocking
 resource preflight.
 
+## Kind port-table memory investigation
+
+On 2026-07-20, the local kind control-plane container used 9.748 GiB while its
+four otherwise idle application `beam.smp` processes each had roughly
+2.3 GiB RSS. Read-only runtime probes found an effective Erlang port limit of
+134,217,727 and an allocated port table of 1,610,612,736 bytes in every kind
+BEAM VM. The nested runtime exposed `nofile=2,147,483,584`; OTP had derived the
+maximum port limit from that value.
+
+The same release image in ordinary Compose used the normal 65,536 port limit
+and a 786,432-byte port table. A separate VM probe with
+`ERL_ZFLAGS="+Q 65536"` reproduced those smaller values inside the kind pod
+before any rollout. Compose and Helm now set that value explicitly, and the
+startup and rolling-verification scripts check the effective value in every
+running web and worker VM.
+
+After the Helm rollout, the unchanged kind control-plane container measured
+1.796 GiB, a reduction of 7.952 GiB (81.6%). Its four application pod cgroups
+measured 188.5–215.6 MiB in the first post-change sample. A subsequent
+four-pod rolling replacement retained the 65,536 limit and 786,432-byte table;
+the replacement cgroups measured 194.3–211.4 MiB, all four replacement pods
+were Ready, cross-node PubSub passed, 184 readiness samples had no failed
+sample after retry handling, and the application-table count diff was empty.
+Evidence is retained under
+`output/memory/20260720-port-table-fix-005420` and
+`output/resilience/beam-port-limit-final-20260720`.
+
+In the ordinary 2-web/2-worker Compose profile, the immediate post-recreation
+sample measured 190.1–205.5 MiB per application container and 168.3 MiB for
+PostgreSQL. These local observations do not include a target server's OS and do
+not establish that a 1 GiB or 2 GiB machine is sufficient for production.
+
 The profile uses a separate Compose project, generated independent secrets, and
 a separate PostgreSQL volume. Its Traefik instance is constrained to that exact
 Compose project; its router, service, middleware, and Host rule are unique, so

docs/verification.md

@@ -268,6 +268,16 @@ fixture cleanup. Neither short run found a saturation point or represents
 production traffic, so the Helm chart does not invent resource limits or an HPA
 policy.
 
+The 2026-07-20 memory investigation did identify and remove a development
+runtime artifact: kind's nested container runtime exposed a huge `nofile`
+limit, causing each OTP VM to allocate a roughly 1.5 GiB port table. Explicit
+`+Q 65536` configuration reduced the entire kind container from 9.748 GiB to
+1.796 GiB. A later rolling replacement retained that value in all four pods,
+left the application-table count diff empty, and passed readiness and
+cross-node PubSub. This fixes that specific over-allocation; it is not a
+production minimum-RAM measurement. Detailed evidence and per-container values
+are recorded in `docs/performance.md`.
+
 ## Public staging observation
 
 On 2026-07-18, `whoneedhelp.imalto.site` was published through the existing

scripts/clean-deploy-verify.sh

@@ -159,6 +159,7 @@ POSTGRES_USER=$postgres_user
 POSTGRES_PASSWORD=$postgres_password
 DATABASE_URL=ecto://$postgres_user:$postgres_password@db/$postgres_db
 POOL_SIZE=10
+ERLANG_PORT_LIMIT=65536
 SECRET_KEY_BASE=$secret_key_base
 HANDOVER_SECRET=$handover_secret
 RELEASE_COOKIE=$release_cookie
@@ -315,6 +316,12 @@ curl --fail-with-body --silent --show-error \
   COMPOSE_PROJECT_NAME="$project" \
     ./scripts/verify-realtime-cluster.sh compose
 ) >"$output_dir/pubsub-probe.txt"
+(
+  cd "$workspace"
+  COMPOSE_PROJECT_NAME="$project" \
+    ./scripts/verify-beam-runtime.sh compose \
+    "$output_dir/beam-runtime.json"
+) >/dev/null
 
 "${compose[@]}" exec -T web /app/bin/who_need_help rpc '
   nodes = [node() | Node.list()] |> Enum.sort()

scripts/compose-up.sh

@@ -7,3 +7,4 @@ CODEX_SESSION_ID=${CODEX_SESSION_ID:-${CODEX_THREAD_ID:-not-configured}}
 export CODEX_SESSION_ID
 docker compose -p who_need_help up -d --build --wait
 "$ROOT/scripts/verify-realtime-cluster.sh" compose
+"$ROOT/scripts/verify-beam-runtime.sh" compose

scripts/ensure-local-e2e-env.sh

@@ -59,6 +59,7 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=$postgres_password
 DATABASE_URL=ecto://postgres:$postgres_password@db/who_need_help_e2e
 POOL_SIZE=10
+ERLANG_PORT_LIMIT=65536
 SECRET_KEY_BASE=$secret_key_base
 HANDOVER_SECRET=$handover_secret
 RELEASE_COOKIE=$release_cookie

scripts/kind-rolling-verify.sh

@@ -396,6 +396,8 @@ fi
   >"$output_dir/cluster-after.txt"
 "$ROOT/scripts/verify-realtime-cluster.sh" kind \
   >"$output_dir/pubsub-after.txt"
+"$ROOT/scripts/verify-beam-runtime.sh" kind \
+  "$output_dir/beam-runtime-after.json" >/dev/null
 
 stop_probe
 probe_pid=

scripts/kind-up.sh

@@ -160,6 +160,7 @@ kube --namespace "$NAMESPACE" rollout status deployment/who-need-help-who-need-h
 kube --namespace "$NAMESPACE" rollout status deployment/who-need-help-who-need-help-worker
 remove_legacy_inline_secrets
 "$ROOT/scripts/verify-realtime-cluster.sh" kind
+"$ROOT/scripts/verify-beam-runtime.sh" kind
 
 echo "Who Need Help: http://localhost:4011"
 echo "Mailpit:       http://localhost:8028"

scripts/load-stack-up.sh

@@ -46,6 +46,7 @@ docker compose \
   --scale "worker=$LOAD_WORKER_REPLICAS"
 
 COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-realtime-cluster.sh" compose
+COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-beam-runtime.sh" compose
 status=$(
   curl --silent --show-error \
     --header "Host: $LOAD_HOST" \

scripts/quality.sh

@@ -69,6 +69,11 @@ docker run --rm \
 
 echo "Rendering every Docker Compose profile"
 docker compose --env-file .env.example -f compose.yaml config --quiet
+docker compose --env-file .env.example -f compose.yaml config --format json |
+  jq --exit-status '
+    [.services.migrate, .services.web, .services.worker]
+    | all(.environment.ERL_ZFLAGS == "+Q 65536")
+  ' >/dev/null
 PORTABILITY_IMAGE=who-need-help:portability-render \
   docker compose --env-file .env.example \
   -f compose.yaml -f compose.portability.yaml config --quiet
@@ -157,6 +162,12 @@ tar --extract --file="$scan_tar" --directory "$scan_dir"
   --values "$ROOT/deploy/helm/who-need-help/values-kind.yaml" \
   "$ROOT/deploy/helm/who-need-help" \
   >"$scan_dir/rendered-helm.yaml"
+test "$(
+  grep -c 'name: ERL_ZFLAGS' "$scan_dir/rendered-helm.yaml"
+)" -eq 5
+test "$(
+  grep -c 'value: "+Q 65536"' "$scan_dir/rendered-helm.yaml"
+)" -eq 5
 mkdir -p "$ROOT/.tools/trivy-cache"
 docker run --rm \
   --volume "$scan_dir:/scan:ro" \

scripts/upgrade-rehearsal-compose.sh

@@ -410,6 +410,9 @@ docker exec "$web_container" /app/bin/who_need_help rpc '
 COMPOSE_PROJECT_NAME="$project" \
   "$ROOT/scripts/verify-realtime-cluster.sh" compose \
   >"$output_dir/pubsub.txt"
+COMPOSE_PROJECT_NAME="$project" \
+  "$ROOT/scripts/verify-beam-runtime.sh" compose \
+  "$output_dir/beam-runtime.json" >/dev/null
 
 image_id=$(docker image inspect --format '{{.Id}}' "$REHEARSAL_IMAGE")
 source_commit=$(git rev-parse HEAD)

scripts/verify-beam-runtime.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+MODE=${1:-compose}
+OUTPUT=${2:-}
+RPC_EXPRESSION='
+  IO.puts(
+    JSON.encode!(%{
+      node: to_string(node()),
+      port_count: :erlang.system_info(:port_count),
+      port_limit: :erlang.system_info(:port_limit),
+      process_count: :erlang.system_info(:process_count),
+      memory_total: :erlang.memory(:total),
+      memory_system: :erlang.memory(:system),
+      port_table:
+        :erlang.system_info(:allocated_areas)
+        |> Keyword.fetch!(:port_table)
+    })
+  )
+'
+
+temporary=$(mktemp "${TMPDIR:-/tmp}/wnh-beam-runtime.XXXXXX")
+trap 'rm -f "$temporary"' EXIT HUP INT TERM
+
+record_probe() {
+  local target=$1
+  local component=$2
+  local expected=$3
+  local runtime_json=$4
+  local cgroup_memory=$5
+  local rss_kb=$6
+
+  jq -cn \
+    --arg target "$target" \
+    --arg component "$component" \
+    --argjson expected_port_limit "$expected" \
+    --argjson cgroup_memory "$cgroup_memory" \
+    --argjson rss_kb "$rss_kb" \
+    --argjson runtime "$runtime_json" \
+    '{
+      target: $target,
+      component: $component,
+      expected_port_limit: $expected_port_limit,
+      cgroup_memory: $cgroup_memory,
+      rss_kb: $rss_kb
+    } + $runtime' >>"$temporary"
+}
+
+case "$MODE" in
+  compose)
+    mapfile -t targets < <(
+      docker compose -p "${COMPOSE_PROJECT_NAME:-who_need_help}" \
+        ps -q web worker |
+        sort -u
+    )
+
+    if [[ "${#targets[@]}" -eq 0 ]]; then
+      echo "No running Compose web or worker replicas were found." >&2
+      exit 1
+    fi
+
+    live_flags=$(
+      for target in "${targets[@]}"; do
+        docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
+          "$target" |
+          sed -n 's/^ERL_ZFLAGS=//p'
+      done |
+        sort -u
+    )
+
+    if [[ ! "$live_flags" =~ ^\+Q[[:space:]]+([0-9]+)$ ]]; then
+      echo "The live Compose replicas do not expose one unambiguous +Q limit." >&2
+      exit 1
+    fi
+
+    expected=${BASH_REMATCH[1]}
+
+    for target in "${targets[@]}"; do
+      component=$(
+        docker inspect --format \
+          '{{index .Config.Labels "com.docker.compose.service"}}' "$target"
+      )
+      runtime_json=$(
+        docker exec "$target" /app/bin/who_need_help rpc "$RPC_EXPRESSION" |
+          tail -n 1
+      )
+      cgroup_memory=$(docker exec "$target" cat /sys/fs/cgroup/memory.current)
+      rss_kb=$(
+        docker exec "$target" awk "/^VmRSS:/ {print \$2}" /proc/1/status
+      )
+      record_probe \
+        "$target" "$component" "$expected" "$runtime_json" \
+        "$cgroup_memory" "$rss_kb"
+    done
+    ;;
+
+  kind)
+    KUBECTL="$ROOT/.tools/bin/kubectl"
+    CONTEXT=kind-who-need-help
+    NAMESPACE=who-need-help
+
+    if [[ ! -x "$KUBECTL" ]]; then
+      echo "The project-owned kubectl is missing." >&2
+      exit 1
+    fi
+
+    live_flags=$(
+      "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
+        get deployment \
+        who-need-help-who-need-help-web \
+        who-need-help-who-need-help-worker \
+        -o json |
+        jq -r '
+          [
+            .items[].spec.template.spec.containers[].env[]
+            | select(.name == "ERL_ZFLAGS")
+            | .value
+          ]
+          | unique
+          | if length == 1 then .[0] else empty end
+        '
+    )
+
+    if [[ ! "$live_flags" =~ (^|[[:space:]])\+Q[[:space:]]+([0-9]+)($|[[:space:]]) ]]; then
+      echo "The live deployments do not expose one unambiguous +Q limit." >&2
+      exit 1
+    fi
+
+    expected=${BASH_REMATCH[2]}
+    mapfile -t pods < <(
+      "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
+        get pods \
+        -l app.kubernetes.io/name=who-need-help \
+        -o json |
+        jq -r '
+          .items[]
+          | .metadata.labels["app.kubernetes.io/component"] as $component
+          | select(
+              ($component == "web" or $component == "worker") and
+              .status.phase == "Running" and
+              ([.status.containerStatuses[]?.ready] | all)
+            )
+          | [.metadata.name, $component]
+          | @tsv
+        '
+    )
+
+    desired=$(
+      "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
+        get deployment \
+        who-need-help-who-need-help-web \
+        who-need-help-who-need-help-worker \
+        -o json |
+        jq '[.items[].spec.replicas] | add'
+    )
+
+    if [[ "${#pods[@]}" -ne "$desired" ]]; then
+      echo "Expected $desired Ready application pods; observed ${#pods[@]}." >&2
+      exit 1
+    fi
+
+    for entry in "${pods[@]}"; do
+      IFS=$'\t' read -r pod component <<<"$entry"
+      runtime_json=$(
+        "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
+          exec "$pod" --container "$component" -- \
+          /app/bin/who_need_help rpc "$RPC_EXPRESSION" |
+          tail -n 1
+      )
+      cgroup_memory=$(
+        "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
+          exec "$pod" --container "$component" -- \
+          cat /sys/fs/cgroup/memory.current
+      )
+      rss_kb=$(
+        "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
+          exec "$pod" --container "$component" -- \
+          awk "/^VmRSS:/ {print \$2}" /proc/1/status
+      )
+      record_probe \
+        "$pod" "$component" "$expected" "$runtime_json" \
+        "$cgroup_memory" "$rss_kb"
+    done
+    ;;
+
+  *)
+    echo "Usage: $0 [compose|kind] [output.json]" >&2
+    exit 1
+    ;;
+esac
+
+summary=$(jq -s 'sort_by(.component, .target)' "$temporary")
+
+if ! printf '%s\n' "$summary" |
+  jq -e \
+    --argjson expected "$expected" \
+    'length > 0 and all(.port_limit == $expected)' >/dev/null; then
+  echo "A running BEAM instance does not use the configured port limit." >&2
+  exit 1
+fi
+
+if [[ -n "$OUTPUT" ]]; then
+  printf '%s\n' "$summary" >"$OUTPUT"
+fi
+
+printf '%s\n' "$summary"