simpletest/who_need_help
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prevent oversized BEAM port tables

Browse Source
This commit is contained in:
SimpleTest 2026-07-20 04:01:31 +03:00
parent 673f94315c
commit fdaef2d685
21 changed files with 346 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env.e2e.example
View File

@ -20,6 +20,7 @@ POSTGRES_USER=postgres
POSTGRES_PASSWORD=GENERATE_INDEPENDENT_E2E_DATABASE_PASSWORD
DATABASE_URL=ecto://postgres:GENERATE_URL_SAFE_PASSWORD@db/who_need_help_e2e
POOL_SIZE=10
ERLANG_PORT_LIMIT=65536
SECRET_KEY_BASE=GENERATE_INDEPENDENT_E2E_SECRET_KEY_BASE
HANDOVER_SECRET=GENERATE_INDEPENDENT_E2E_HANDOVER_SECRET

4
.env.example
View File

@ -63,6 +63,10 @@ POSTGRES_PASSWORD=replace-with-a-local-or-deployment-secret
DATABASE_URL=ecto://postgres:replace-with-url-encoded-password@db/who_need_help
POOL_SIZE=10
# Maximum simultaneously existing Erlang ports (files, sockets and drivers).
# Keeping this explicit prevents a host's very large nofile ulimit from making
# every BEAM instance preallocate a multi-gigabyte port table.
ERLANG_PORT_LIMIT=65536
SECRET_KEY_BASE=generate-with-mix-phx-gen-secret
HANDOVER_SECRET=generate-an-independent-random-secret
RELEASE_COOKIE=generate-an-independent-beam-cluster-cookie

1
.env.load.example
View File

@ -21,6 +21,7 @@ POSTGRES_USER=wnh_load
POSTGRES_PASSWORD=GENERATE_POSTGRES_PASSWORD
DATABASE_URL=GENERATE_DATABASE_URL
POOL_SIZE=10
ERLANG_PORT_LIMIT=65536
SECRET_KEY_BASE=GENERATE_SECRET_KEY_BASE
HANDOVER_SECRET=GENERATE_HANDOVER_SECRET
RELEASE_COOKIE=GENERATE_RELEASE_COOKIE

10
README.md
View File

@ -82,8 +82,16 @@ Inspect the exact state:
docker compose -p who_need_help ps -a
docker compose -p who_need_help logs -f web worker
./scripts/verify-realtime-cluster.sh compose
./scripts/verify-beam-runtime.sh compose
```
`ERLANG_PORT_LIMIT` defaults to the normal OTP port limit of `65536`. Keeping
it explicit prevents a container runtime's unusually large `nofile` limit from
making every BEAM instance preallocate a multi-gigabyte port table. The runtime
check above reads every running web/worker VM and fails if its effective limit
differs from the configured value. Increase it only from measured concurrent
file, socket, and driver requirements.
Create and restore-test a database backup without restoring over the source:
```bash
@ -345,7 +353,7 @@ traffic. Supported actions are listed in `docs/trust-safety.md`.
The kind script downloads checksum-verified kubectl, Helm, and kind binaries
into `.tools/`, creates a project-owned cluster, loads the local image, and
installs the Helm chart, waits for both Deployments, and verifies cross-node
PubSub:
PubSub and each BEAM VM's effective port limit:
```bash
./scripts/kind-up.sh

4
compose.yaml
View File

@ -7,6 +7,10 @@ x-app-environment: &app-environment
HANDOVER_SECRET: ${HANDOVER_SECRET:?Set HANDOVER_SECRET in .env}
RELEASE_COOKIE: ${RELEASE_COOKIE:?Set RELEASE_COOKIE in .env}
METRICS_TOKEN: ${METRICS_TOKEN:?Set an independent METRICS_TOKEN in .env}
# Keep BEAM from deriving a massive port table from a host/container nofile
# limit. OTP's normal default is 65,536; deployments can override it after
# measuring their concurrent file/socket requirements.
ERL_ZFLAGS: "+Q ${ERLANG_PORT_LIMIT:-65536}"
DNS_CLUSTER_QUERY: web
PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env}
PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env}

4
deploy/helm/who-need-help/templates/deployments.yaml
View File

@ -47,6 +47,8 @@ spec:
env:
- name: APP_ROLE
value: migrate
- name: ERL_ZFLAGS
value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
@ -65,6 +67,8 @@ spec:
env:
- name: APP_ROLE
value: {{ $component }}
- name: ERL_ZFLAGS
value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }}
- name: PHX_SERVER
value: {{ if eq $component "web" }}"true"{{ else }}"false"{{ end }}
- name: PHX_HOST

2
deploy/helm/who-need-help/templates/migrate-job.yaml
View File

@ -33,6 +33,8 @@ spec:
env:
- name: APP_ROLE
value: migrate
- name: ERL_ZFLAGS
value: {{ printf "+Q %d" (int .Values.app.erlangPortLimit) | quote }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true

19
deploy/helm/who-need-help/values.schema.json Normal file
View File

@ -0,0 +1,19 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"app": {
"type": "object",
"properties": {
"erlangPortLimit": {
"type": "integer",
"minimum": 1024,
"maximum": 134217727
}
},
"required": [
"erlangPortLimit"
]
}
}
}

4
deploy/helm/who-need-help/values.yaml
View File

@ -20,6 +20,10 @@ app:
urlPort: "443"
port: "4000"
poolSize: "10"
# OTP otherwise derives this from the container nofile ulimit. Some
# Kubernetes runtimes expose a value so large that each BEAM instance
# preallocates a multi-gigabyte port table.
erlangPortLimit: 65536
codexSessionId: not-configured
# Shared limits are opt-in; set only after product policy thresholds are approved.
rateLimitPoliciesJson: "{}"

22
docs/operations.md
View File

@ -284,6 +284,28 @@ The rollout timeout, probe interval/timeout/retry count, and cluster-join
timeout are experiment inputs. They are not production SLOs or resource
requirements.
## BEAM runtime memory guard
Compose sets `ERL_ZFLAGS="+Q ${ERLANG_PORT_LIMIT}"`; Helm renders the same flag
from `app.erlangPortLimit`. Both default to `65536`, OTP's normal port-table
limit. This makes the runtime independent of an unusually large host or nested
container `nofile` limit. Validate all live replicas after deployment:
```bash
./scripts/verify-beam-runtime.sh compose
./scripts/verify-beam-runtime.sh kind
```
The command records the effective port count/limit, allocated port-table bytes,
BEAM memory, process count, cgroup memory, and RSS for each web and worker
replica. It fails when a live VM does not use the configured limit; kind mode
also requires the desired number of Ready application pods.
`65536` is a concurrency ceiling for simultaneously existing Erlang ports
(files, sockets, and drivers), not a container memory limit. Do not lower or
raise it from a RAM estimate alone. A changed value must be validated against
measured peak port usage and the target environment.
## Protected Prometheus metrics
The web role exposes Prometheus text format at `/metrics`. It requires the

32
docs/performance.md
View File

@ -5,6 +5,38 @@ pool size, or autoscaling threshold is known yet. The repository therefore
contains a reproducible measurement profile, not a capacity claim or blocking
resource preflight.
## Kind port-table memory investigation
On 2026-07-20, the local kind control-plane container used 9.748 GiB while its
four otherwise idle application `beam.smp` processes each had roughly
2.3 GiB RSS. Read-only runtime probes found an effective Erlang port limit of
134,217,727 and an allocated port table of 1,610,612,736 bytes in every kind
BEAM VM. The nested runtime exposed `nofile=2,147,483,584`; OTP had derived the
maximum port limit from that value.
The same release image in ordinary Compose used the normal 65,536 port limit
and a 786,432-byte port table. A separate VM probe with
`ERL_ZFLAGS="+Q 65536"` reproduced those smaller values inside the kind pod
before any rollout. Compose and Helm now set that value explicitly, and the
startup and rolling-verification scripts check the effective value in every
running web and worker VM.
After the Helm rollout, the unchanged kind control-plane container measured
1.796 GiB, a reduction of 7.952 GiB (81.6%). Its four application pod cgroups
measured 188.5–215.6 MiB in the first post-change sample. A subsequent
four-pod rolling replacement retained the 65,536 limit and 786,432-byte table;
the replacement cgroups measured 194.3–211.4 MiB, all four replacement pods
were Ready, cross-node PubSub passed, 184 readiness samples had no failed
sample after retry handling, and the application-table count diff was empty.
Evidence is retained under
`output/memory/20260720-port-table-fix-005420` and
`output/resilience/beam-port-limit-final-20260720`.
In the ordinary 2-web/2-worker Compose profile, the immediate post-recreation
sample measured 190.1–205.5 MiB per application container and 168.3 MiB for
PostgreSQL. These local observations do not include a target server's OS and do
not establish that a 1 GiB or 2 GiB machine is sufficient for production.
The profile uses a separate Compose project, generated independent secrets, and
a separate PostgreSQL volume. Its Traefik instance is constrained to that exact
Compose project; its router, service, middleware, and Host rule are unique, so

10
docs/verification.md
View File

@ -268,6 +268,16 @@ fixture cleanup. Neither short run found a saturation point or represents
production traffic, so the Helm chart does not invent resource limits or an HPA
policy.
The 2026-07-20 memory investigation did identify and remove a development
runtime artifact: kind's nested container runtime exposed a huge `nofile`
limit, causing each OTP VM to allocate a roughly 1.5 GiB port table. Explicit
`+Q 65536` configuration reduced the entire kind container from 9.748 GiB to
1.796 GiB. A later rolling replacement retained that value in all four pods,
left the application-table count diff empty, and passed readiness and
cross-node PubSub. This fixes that specific over-allocation; it is not a
production minimum-RAM measurement. Detailed evidence and per-container values
are recorded in `docs/performance.md`.
## Public staging observation
On 2026-07-18, `whoneedhelp.imalto.site` was published through the existing

7
scripts/clean-deploy-verify.sh
View File

@ -159,6 +159,7 @@ POSTGRES_USER=$postgres_user
POSTGRES_PASSWORD=$postgres_password
DATABASE_URL=ecto://$postgres_user:$postgres_password@db/$postgres_db
POOL_SIZE=10
ERLANG_PORT_LIMIT=65536
SECRET_KEY_BASE=$secret_key_base
HANDOVER_SECRET=$handover_secret
RELEASE_COOKIE=$release_cookie
@ -315,6 +316,12 @@ curl --fail-with-body --silent --show-error \
COMPOSE_PROJECT_NAME="$project" \
./scripts/verify-realtime-cluster.sh compose
) >"$output_dir/pubsub-probe.txt"
(
cd "$workspace"
COMPOSE_PROJECT_NAME="$project" \
./scripts/verify-beam-runtime.sh compose \
"$output_dir/beam-runtime.json"
) >/dev/null
"${compose[@]}" exec -T web /app/bin/who_need_help rpc '
nodes = [node() | Node.list()] |> Enum.sort()

1
scripts/compose-up.sh
View File

@ -7,3 +7,4 @@ CODEX_SESSION_ID=${CODEX_SESSION_ID:-${CODEX_THREAD_ID:-not-configured}}
export CODEX_SESSION_ID
docker compose -p who_need_help up -d --build --wait
"$ROOT/scripts/verify-realtime-cluster.sh" compose
"$ROOT/scripts/verify-beam-runtime.sh" compose

1
scripts/ensure-local-e2e-env.sh
View File

@ -59,6 +59,7 @@ POSTGRES_USER=postgres
POSTGRES_PASSWORD=$postgres_password
DATABASE_URL=ecto://postgres:$postgres_password@db/who_need_help_e2e
POOL_SIZE=10
ERLANG_PORT_LIMIT=65536
SECRET_KEY_BASE=$secret_key_base
HANDOVER_SECRET=$handover_secret
RELEASE_COOKIE=$release_cookie

2
scripts/kind-rolling-verify.sh
View File

@ -396,6 +396,8 @@ fi
>"$output_dir/cluster-after.txt"
"$ROOT/scripts/verify-realtime-cluster.sh" kind \
>"$output_dir/pubsub-after.txt"
"$ROOT/scripts/verify-beam-runtime.sh" kind \
"$output_dir/beam-runtime-after.json" >/dev/null
stop_probe
probe_pid=

1
scripts/kind-up.sh
View File

@ -160,6 +160,7 @@ kube --namespace "$NAMESPACE" rollout status deployment/who-need-help-who-need-h
kube --namespace "$NAMESPACE" rollout status deployment/who-need-help-who-need-help-worker
remove_legacy_inline_secrets
"$ROOT/scripts/verify-realtime-cluster.sh" kind
"$ROOT/scripts/verify-beam-runtime.sh" kind
echo "Who Need Help: http://localhost:4011"
echo "Mailpit: http://localhost:8028"

1
scripts/load-stack-up.sh
View File

@ -46,6 +46,7 @@ docker compose \
--scale "worker=$LOAD_WORKER_REPLICAS"
COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-realtime-cluster.sh" compose
COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-beam-runtime.sh" compose
status=$(
curl --silent --show-error \
--header "Host: $LOAD_HOST" \

11
scripts/quality.sh
View File

@ -69,6 +69,11 @@ docker run --rm \
echo "Rendering every Docker Compose profile"
docker compose --env-file .env.example -f compose.yaml config --quiet
docker compose --env-file .env.example -f compose.yaml config --format json |
jq --exit-status '
[.services.migrate, .services.web, .services.worker]
| all(.environment.ERL_ZFLAGS == "+Q 65536")
' >/dev/null
PORTABILITY_IMAGE=who-need-help:portability-render \
docker compose --env-file .env.example \
-f compose.yaml -f compose.portability.yaml config --quiet
@ -157,6 +162,12 @@ tar --extract --file="$scan_tar" --directory "$scan_dir"
--values "$ROOT/deploy/helm/who-need-help/values-kind.yaml" \
"$ROOT/deploy/helm/who-need-help" \
>"$scan_dir/rendered-helm.yaml"
test "$(
grep -c 'name: ERL_ZFLAGS' "$scan_dir/rendered-helm.yaml"
)" -eq 5
test "$(
grep -c 'value: "+Q 65536"' "$scan_dir/rendered-helm.yaml"
)" -eq 5
mkdir -p "$ROOT/.tools/trivy-cache"
docker run --rm \
--volume "$scan_dir:/scan:ro" \

3
scripts/upgrade-rehearsal-compose.sh
View File

@ -410,6 +410,9 @@ docker exec "$web_container" /app/bin/who_need_help rpc '
COMPOSE_PROJECT_NAME="$project" \
"$ROOT/scripts/verify-realtime-cluster.sh" compose \
>"$output_dir/pubsub.txt"
COMPOSE_PROJECT_NAME="$project" \
"$ROOT/scripts/verify-beam-runtime.sh" compose \
"$output_dir/beam-runtime.json" >/dev/null
image_id=$(docker image inspect --format '{{.Id}}' "$REHEARSAL_IMAGE")
source_commit=$(git rev-parse HEAD)

207
scripts/verify-beam-runtime.sh Executable file
View File

@ -0,0 +1,207 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
MODE=${1:-compose}
OUTPUT=${2:-}
RPC_EXPRESSION='
IO.puts(
JSON.encode!(%{
node: to_string(node()),
port_count: :erlang.system_info(:port_count),
port_limit: :erlang.system_info(:port_limit),
process_count: :erlang.system_info(:process_count),
memory_total: :erlang.memory(:total),
memory_system: :erlang.memory(:system),
port_table:
:erlang.system_info(:allocated_areas)
|> Keyword.fetch!(:port_table)
})
)
'
temporary=$(mktemp "${TMPDIR:-/tmp}/wnh-beam-runtime.XXXXXX")
trap 'rm -f "$temporary"' EXIT HUP INT TERM
record_probe() {
local target=$1
local component=$2
local expected=$3
local runtime_json=$4
local cgroup_memory=$5
local rss_kb=$6
jq -cn \
--arg target "$target" \
--arg component "$component" \
--argjson expected_port_limit "$expected" \
--argjson cgroup_memory "$cgroup_memory" \
--argjson rss_kb "$rss_kb" \
--argjson runtime "$runtime_json" \
'{
target: $target,
component: $component,
expected_port_limit: $expected_port_limit,
cgroup_memory: $cgroup_memory,
rss_kb: $rss_kb
} + $runtime' >>"$temporary"
}
case "$MODE" in
compose)
mapfile -t targets < <(
docker compose -p "${COMPOSE_PROJECT_NAME:-who_need_help}" \
ps -q web worker |
sort -u
)
if [[ "${#targets[@]}" -eq 0 ]]; then
echo "No running Compose web or worker replicas were found." >&2
exit 1
fi
live_flags=$(
for target in "${targets[@]}"; do
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
"$target" |
sed -n 's/^ERL_ZFLAGS=//p'
done |
sort -u
)
if [[ ! "$live_flags" =~ ^\+Q[[:space:]]+([0-9]+)$ ]]; then
echo "The live Compose replicas do not expose one unambiguous +Q limit." >&2
exit 1
fi
expected=${BASH_REMATCH[1]}
for target in "${targets[@]}"; do
component=$(
docker inspect --format \
'{{index .Config.Labels "com.docker.compose.service"}}' "$target"
)
runtime_json=$(
docker exec "$target" /app/bin/who_need_help rpc "$RPC_EXPRESSION" |
tail -n 1
)
cgroup_memory=$(docker exec "$target" cat /sys/fs/cgroup/memory.current)
rss_kb=$(
docker exec "$target" awk "/^VmRSS:/ {print \$2}" /proc/1/status
)
record_probe \
"$target" "$component" "$expected" "$runtime_json" \
"$cgroup_memory" "$rss_kb"
done
;;
kind)
KUBECTL="$ROOT/.tools/bin/kubectl"
CONTEXT=kind-who-need-help
NAMESPACE=who-need-help
if [[ ! -x "$KUBECTL" ]]; then
echo "The project-owned kubectl is missing." >&2
exit 1
fi
live_flags=$(
"$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
get deployment \
who-need-help-who-need-help-web \
who-need-help-who-need-help-worker \
-o json |
jq -r '
[
.items[].spec.template.spec.containers[].env[]
| select(.name == "ERL_ZFLAGS")
| .value
]
| unique
| if length == 1 then .[0] else empty end
'
)
if [[ ! "$live_flags" =~ (^|[[:space:]])\+Q[[:space:]]+([0-9]+)($|[[:space:]]) ]]; then
echo "The live deployments do not expose one unambiguous +Q limit." >&2
exit 1
fi
expected=${BASH_REMATCH[2]}
mapfile -t pods < <(
"$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
get pods \
-l app.kubernetes.io/name=who-need-help \
-o json |
jq -r '
.items[]
| .metadata.labels["app.kubernetes.io/component"] as $component
| select(
($component == "web" or $component == "worker") and
.status.phase == "Running" and
([.status.containerStatuses[]?.ready] | all)
)
| [.metadata.name, $component]
| @tsv
'
)
desired=$(
"$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
get deployment \
who-need-help-who-need-help-web \
who-need-help-who-need-help-worker \
-o json |
jq '[.items[].spec.replicas] | add'
)
if [[ "${#pods[@]}" -ne "$desired" ]]; then
echo "Expected $desired Ready application pods; observed ${#pods[@]}." >&2
exit 1
fi
for entry in "${pods[@]}"; do
IFS=$'\t' read -r pod component <<<"$entry"
runtime_json=$(
"$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
exec "$pod" --container "$component" -- \
/app/bin/who_need_help rpc "$RPC_EXPRESSION" |
tail -n 1
)
cgroup_memory=$(
"$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
exec "$pod" --container "$component" -- \
cat /sys/fs/cgroup/memory.current
)
rss_kb=$(
"$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \
exec "$pod" --container "$component" -- \
awk "/^VmRSS:/ {print \$2}" /proc/1/status
)
record_probe \
"$pod" "$component" "$expected" "$runtime_json" \
"$cgroup_memory" "$rss_kb"
done
;;
*)
echo "Usage: $0 [compose|kind] [output.json]" >&2
exit 1
;;
esac
summary=$(jq -s 'sort_by(.component, .target)' "$temporary")
if ! printf '%s\n' "$summary" |
jq -e \
--argjson expected "$expected" \
'length > 0 and all(.port_limit == $expected)' >/dev/null; then
echo "A running BEAM instance does not use the configured port limit." >&2
exit 1
fi
if [[ -n "$OUTPUT" ]]; then
printf '%s\n' "$summary" >"$OUTPUT"
fi
printf '%s\n' "$summary"