diff --git a/.env.e2e.example b/.env.e2e.example index 3266d40..b5de97b 100644 --- a/.env.e2e.example +++ b/.env.e2e.example @@ -20,6 +20,7 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=GENERATE_INDEPENDENT_E2E_DATABASE_PASSWORD DATABASE_URL=ecto://postgres:GENERATE_URL_SAFE_PASSWORD@db/who_need_help_e2e POOL_SIZE=10 +ERLANG_PORT_LIMIT=65536 SECRET_KEY_BASE=GENERATE_INDEPENDENT_E2E_SECRET_KEY_BASE HANDOVER_SECRET=GENERATE_INDEPENDENT_E2E_HANDOVER_SECRET diff --git a/.env.example b/.env.example index 0a55027..d7f4c7f 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,10 @@ POSTGRES_PASSWORD=replace-with-a-local-or-deployment-secret DATABASE_URL=ecto://postgres:replace-with-url-encoded-password@db/who_need_help POOL_SIZE=10 +# Maximum simultaneously existing Erlang ports (files, sockets and drivers). +# Keeping this explicit prevents a host's very large nofile ulimit from making +# every BEAM instance preallocate a multi-gigabyte port table. +ERLANG_PORT_LIMIT=65536 SECRET_KEY_BASE=generate-with-mix-phx-gen-secret HANDOVER_SECRET=generate-an-independent-random-secret RELEASE_COOKIE=generate-an-independent-beam-cluster-cookie diff --git a/.env.load.example b/.env.load.example index 73b78c9..1dd5f37 100644 --- a/.env.load.example +++ b/.env.load.example @@ -21,6 +21,7 @@ POSTGRES_USER=wnh_load POSTGRES_PASSWORD=GENERATE_POSTGRES_PASSWORD DATABASE_URL=GENERATE_DATABASE_URL POOL_SIZE=10 +ERLANG_PORT_LIMIT=65536 SECRET_KEY_BASE=GENERATE_SECRET_KEY_BASE HANDOVER_SECRET=GENERATE_HANDOVER_SECRET RELEASE_COOKIE=GENERATE_RELEASE_COOKIE diff --git a/README.md b/README.md index 6796780..895101d 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,16 @@ Inspect the exact state: docker compose -p who_need_help ps -a docker compose -p who_need_help logs -f web worker ./scripts/verify-realtime-cluster.sh compose +./scripts/verify-beam-runtime.sh compose ``` +`ERLANG_PORT_LIMIT` defaults to the normal OTP port limit of `65536`. Keeping +it explicit prevents a container runtime's unusually large `nofile` limit from +making every BEAM instance preallocate a multi-gigabyte port table. The runtime +check above reads every running web/worker VM and fails if its effective limit +differs from the configured value. Increase it only from measured concurrent +file, socket, and driver requirements. + Create and restore-test a database backup without restoring over the source: ```bash @@ -345,7 +353,7 @@ traffic. Supported actions are listed in `docs/trust-safety.md`. The kind script downloads checksum-verified kubectl, Helm, and kind binaries into `.tools/`, creates a project-owned cluster, loads the local image, and installs the Helm chart, waits for both Deployments, and verifies cross-node -PubSub: +PubSub and each BEAM VM's effective port limit: ```bash ./scripts/kind-up.sh diff --git a/compose.yaml b/compose.yaml index 0fd3a8f..1ab0648 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,6 +7,10 @@ x-app-environment: &app-environment HANDOVER_SECRET: ${HANDOVER_SECRET:?Set HANDOVER_SECRET in .env} RELEASE_COOKIE: ${RELEASE_COOKIE:?Set RELEASE_COOKIE in .env} METRICS_TOKEN: ${METRICS_TOKEN:?Set an independent METRICS_TOKEN in .env} + # Keep BEAM from deriving a massive port table from a host/container nofile + # limit. OTP's normal default is 65,536; deployments can override it after + # measuring their concurrent file/socket requirements. + ERL_ZFLAGS: "+Q ${ERLANG_PORT_LIMIT:-65536}" DNS_CLUSTER_QUERY: web PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env} PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env} diff --git a/deploy/helm/who-need-help/templates/deployments.yaml b/deploy/helm/who-need-help/templates/deployments.yaml index 9a4fc4b..d147fca 100644 --- a/deploy/helm/who-need-help/templates/deployments.yaml +++ b/deploy/helm/who-need-help/templates/deployments.yaml @@ -47,6 +47,8 @@ spec: env: - name: APP_ROLE value: migrate + - name: ERL_ZFLAGS + value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -65,6 +67,8 @@ spec: env: - name: APP_ROLE value: {{ $component }} + - name: ERL_ZFLAGS + value: {{ printf "+Q %d" (int $root.Values.app.erlangPortLimit) | quote }} - name: PHX_SERVER value: {{ if eq $component "web" }}"true"{{ else }}"false"{{ end }} - name: PHX_HOST diff --git a/deploy/helm/who-need-help/templates/migrate-job.yaml b/deploy/helm/who-need-help/templates/migrate-job.yaml index 210f131..9a5493c 100644 --- a/deploy/helm/who-need-help/templates/migrate-job.yaml +++ b/deploy/helm/who-need-help/templates/migrate-job.yaml @@ -33,6 +33,8 @@ spec: env: - name: APP_ROLE value: migrate + - name: ERL_ZFLAGS + value: {{ printf "+Q %d" (int .Values.app.erlangPortLimit) | quote }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/deploy/helm/who-need-help/values.schema.json b/deploy/helm/who-need-help/values.schema.json new file mode 100644 index 0000000..16637fd --- /dev/null +++ b/deploy/helm/who-need-help/values.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "erlangPortLimit": { + "type": "integer", + "minimum": 1024, + "maximum": 134217727 + } + }, + "required": [ + "erlangPortLimit" + ] + } + } +} diff --git a/deploy/helm/who-need-help/values.yaml b/deploy/helm/who-need-help/values.yaml index abe6a78..03f6e98 100644 --- a/deploy/helm/who-need-help/values.yaml +++ b/deploy/helm/who-need-help/values.yaml @@ -20,6 +20,10 @@ app: urlPort: "443" port: "4000" poolSize: "10" + # OTP otherwise derives this from the container nofile ulimit. Some + # Kubernetes runtimes expose a value so large that each BEAM instance + # preallocates a multi-gigabyte port table. + erlangPortLimit: 65536 codexSessionId: not-configured # Shared limits are opt-in; set only after product policy thresholds are approved. rateLimitPoliciesJson: "{}" diff --git a/docs/operations.md b/docs/operations.md index c29d141..21a6fc5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -284,6 +284,28 @@ The rollout timeout, probe interval/timeout/retry count, and cluster-join timeout are experiment inputs. They are not production SLOs or resource requirements. +## BEAM runtime memory guard + +Compose sets `ERL_ZFLAGS="+Q ${ERLANG_PORT_LIMIT}"`; Helm renders the same flag +from `app.erlangPortLimit`. Both default to `65536`, OTP's normal port-table +limit. This makes the runtime independent of an unusually large host or nested +container `nofile` limit. Validate all live replicas after deployment: + +```bash +./scripts/verify-beam-runtime.sh compose +./scripts/verify-beam-runtime.sh kind +``` + +The command records the effective port count/limit, allocated port-table bytes, +BEAM memory, process count, cgroup memory, and RSS for each web and worker +replica. It fails when a live VM does not use the configured limit; kind mode +also requires the desired number of Ready application pods. + +`65536` is a concurrency ceiling for simultaneously existing Erlang ports +(files, sockets, and drivers), not a container memory limit. Do not lower or +raise it from a RAM estimate alone. A changed value must be validated against +measured peak port usage and the target environment. + ## Protected Prometheus metrics The web role exposes Prometheus text format at `/metrics`. It requires the diff --git a/docs/performance.md b/docs/performance.md index f201ae4..7a71424 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -5,6 +5,38 @@ pool size, or autoscaling threshold is known yet. The repository therefore contains a reproducible measurement profile, not a capacity claim or blocking resource preflight. +## Kind port-table memory investigation + +On 2026-07-20, the local kind control-plane container used 9.748 GiB while its +four otherwise idle application `beam.smp` processes each had roughly +2.3 GiB RSS. Read-only runtime probes found an effective Erlang port limit of +134,217,727 and an allocated port table of 1,610,612,736 bytes in every kind +BEAM VM. The nested runtime exposed `nofile=2,147,483,584`; OTP had derived the +maximum port limit from that value. + +The same release image in ordinary Compose used the normal 65,536 port limit +and a 786,432-byte port table. A separate VM probe with +`ERL_ZFLAGS="+Q 65536"` reproduced those smaller values inside the kind pod +before any rollout. Compose and Helm now set that value explicitly, and the +startup and rolling-verification scripts check the effective value in every +running web and worker VM. + +After the Helm rollout, the unchanged kind control-plane container measured +1.796 GiB, a reduction of 7.952 GiB (81.6%). Its four application pod cgroups +measured 188.5–215.6 MiB in the first post-change sample. A subsequent +four-pod rolling replacement retained the 65,536 limit and 786,432-byte table; +the replacement cgroups measured 194.3–211.4 MiB, all four replacement pods +were Ready, cross-node PubSub passed, 184 readiness samples had no failed +sample after retry handling, and the application-table count diff was empty. +Evidence is retained under +`output/memory/20260720-port-table-fix-005420` and +`output/resilience/beam-port-limit-final-20260720`. + +In the ordinary 2-web/2-worker Compose profile, the immediate post-recreation +sample measured 190.1–205.5 MiB per application container and 168.3 MiB for +PostgreSQL. These local observations do not include a target server's OS and do +not establish that a 1 GiB or 2 GiB machine is sufficient for production. + The profile uses a separate Compose project, generated independent secrets, and a separate PostgreSQL volume. Its Traefik instance is constrained to that exact Compose project; its router, service, middleware, and Host rule are unique, so diff --git a/docs/verification.md b/docs/verification.md index 225fff4..5801576 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -268,6 +268,16 @@ fixture cleanup. Neither short run found a saturation point or represents production traffic, so the Helm chart does not invent resource limits or an HPA policy. +The 2026-07-20 memory investigation did identify and remove a development +runtime artifact: kind's nested container runtime exposed a huge `nofile` +limit, causing each OTP VM to allocate a roughly 1.5 GiB port table. Explicit +`+Q 65536` configuration reduced the entire kind container from 9.748 GiB to +1.796 GiB. A later rolling replacement retained that value in all four pods, +left the application-table count diff empty, and passed readiness and +cross-node PubSub. This fixes that specific over-allocation; it is not a +production minimum-RAM measurement. Detailed evidence and per-container values +are recorded in `docs/performance.md`. + ## Public staging observation On 2026-07-18, `whoneedhelp.imalto.site` was published through the existing diff --git a/scripts/clean-deploy-verify.sh b/scripts/clean-deploy-verify.sh index b342447..e1fe757 100755 --- a/scripts/clean-deploy-verify.sh +++ b/scripts/clean-deploy-verify.sh @@ -159,6 +159,7 @@ POSTGRES_USER=$postgres_user POSTGRES_PASSWORD=$postgres_password DATABASE_URL=ecto://$postgres_user:$postgres_password@db/$postgres_db POOL_SIZE=10 +ERLANG_PORT_LIMIT=65536 SECRET_KEY_BASE=$secret_key_base HANDOVER_SECRET=$handover_secret RELEASE_COOKIE=$release_cookie @@ -315,6 +316,12 @@ curl --fail-with-body --silent --show-error \ COMPOSE_PROJECT_NAME="$project" \ ./scripts/verify-realtime-cluster.sh compose ) >"$output_dir/pubsub-probe.txt" +( + cd "$workspace" + COMPOSE_PROJECT_NAME="$project" \ + ./scripts/verify-beam-runtime.sh compose \ + "$output_dir/beam-runtime.json" +) >/dev/null "${compose[@]}" exec -T web /app/bin/who_need_help rpc ' nodes = [node() | Node.list()] |> Enum.sort() diff --git a/scripts/compose-up.sh b/scripts/compose-up.sh index d6c6166..d617126 100755 --- a/scripts/compose-up.sh +++ b/scripts/compose-up.sh @@ -7,3 +7,4 @@ CODEX_SESSION_ID=${CODEX_SESSION_ID:-${CODEX_THREAD_ID:-not-configured}} export CODEX_SESSION_ID docker compose -p who_need_help up -d --build --wait "$ROOT/scripts/verify-realtime-cluster.sh" compose +"$ROOT/scripts/verify-beam-runtime.sh" compose diff --git a/scripts/ensure-local-e2e-env.sh b/scripts/ensure-local-e2e-env.sh index 1667f2b..5213a72 100755 --- a/scripts/ensure-local-e2e-env.sh +++ b/scripts/ensure-local-e2e-env.sh @@ -59,6 +59,7 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=$postgres_password DATABASE_URL=ecto://postgres:$postgres_password@db/who_need_help_e2e POOL_SIZE=10 +ERLANG_PORT_LIMIT=65536 SECRET_KEY_BASE=$secret_key_base HANDOVER_SECRET=$handover_secret RELEASE_COOKIE=$release_cookie diff --git a/scripts/kind-rolling-verify.sh b/scripts/kind-rolling-verify.sh index 2551953..a20eefd 100755 --- a/scripts/kind-rolling-verify.sh +++ b/scripts/kind-rolling-verify.sh @@ -396,6 +396,8 @@ fi >"$output_dir/cluster-after.txt" "$ROOT/scripts/verify-realtime-cluster.sh" kind \ >"$output_dir/pubsub-after.txt" +"$ROOT/scripts/verify-beam-runtime.sh" kind \ + "$output_dir/beam-runtime-after.json" >/dev/null stop_probe probe_pid= diff --git a/scripts/kind-up.sh b/scripts/kind-up.sh index 9c75ca9..af08bfd 100755 --- a/scripts/kind-up.sh +++ b/scripts/kind-up.sh @@ -160,6 +160,7 @@ kube --namespace "$NAMESPACE" rollout status deployment/who-need-help-who-need-h kube --namespace "$NAMESPACE" rollout status deployment/who-need-help-who-need-help-worker remove_legacy_inline_secrets "$ROOT/scripts/verify-realtime-cluster.sh" kind +"$ROOT/scripts/verify-beam-runtime.sh" kind echo "Who Need Help: http://localhost:4011" echo "Mailpit: http://localhost:8028" diff --git a/scripts/load-stack-up.sh b/scripts/load-stack-up.sh index 184b951..c9f6072 100755 --- a/scripts/load-stack-up.sh +++ b/scripts/load-stack-up.sh @@ -46,6 +46,7 @@ docker compose \ --scale "worker=$LOAD_WORKER_REPLICAS" COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-realtime-cluster.sh" compose +COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-beam-runtime.sh" compose status=$( curl --silent --show-error \ --header "Host: $LOAD_HOST" \ diff --git a/scripts/quality.sh b/scripts/quality.sh index 38daf48..564e3e6 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -69,6 +69,11 @@ docker run --rm \ echo "Rendering every Docker Compose profile" docker compose --env-file .env.example -f compose.yaml config --quiet +docker compose --env-file .env.example -f compose.yaml config --format json | + jq --exit-status ' + [.services.migrate, .services.web, .services.worker] + | all(.environment.ERL_ZFLAGS == "+Q 65536") + ' >/dev/null PORTABILITY_IMAGE=who-need-help:portability-render \ docker compose --env-file .env.example \ -f compose.yaml -f compose.portability.yaml config --quiet @@ -157,6 +162,12 @@ tar --extract --file="$scan_tar" --directory "$scan_dir" --values "$ROOT/deploy/helm/who-need-help/values-kind.yaml" \ "$ROOT/deploy/helm/who-need-help" \ >"$scan_dir/rendered-helm.yaml" +test "$( + grep -c 'name: ERL_ZFLAGS' "$scan_dir/rendered-helm.yaml" +)" -eq 5 +test "$( + grep -c 'value: "+Q 65536"' "$scan_dir/rendered-helm.yaml" +)" -eq 5 mkdir -p "$ROOT/.tools/trivy-cache" docker run --rm \ --volume "$scan_dir:/scan:ro" \ diff --git a/scripts/upgrade-rehearsal-compose.sh b/scripts/upgrade-rehearsal-compose.sh index c050684..69ed0b3 100755 --- a/scripts/upgrade-rehearsal-compose.sh +++ b/scripts/upgrade-rehearsal-compose.sh @@ -410,6 +410,9 @@ docker exec "$web_container" /app/bin/who_need_help rpc ' COMPOSE_PROJECT_NAME="$project" \ "$ROOT/scripts/verify-realtime-cluster.sh" compose \ >"$output_dir/pubsub.txt" +COMPOSE_PROJECT_NAME="$project" \ + "$ROOT/scripts/verify-beam-runtime.sh" compose \ + "$output_dir/beam-runtime.json" >/dev/null image_id=$(docker image inspect --format '{{.Id}}' "$REHEARSAL_IMAGE") source_commit=$(git rev-parse HEAD) diff --git a/scripts/verify-beam-runtime.sh b/scripts/verify-beam-runtime.sh new file mode 100755 index 0000000..0b707a7 --- /dev/null +++ b/scripts/verify-beam-runtime.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +MODE=${1:-compose} +OUTPUT=${2:-} +RPC_EXPRESSION=' + IO.puts( + JSON.encode!(%{ + node: to_string(node()), + port_count: :erlang.system_info(:port_count), + port_limit: :erlang.system_info(:port_limit), + process_count: :erlang.system_info(:process_count), + memory_total: :erlang.memory(:total), + memory_system: :erlang.memory(:system), + port_table: + :erlang.system_info(:allocated_areas) + |> Keyword.fetch!(:port_table) + }) + ) +' + +temporary=$(mktemp "${TMPDIR:-/tmp}/wnh-beam-runtime.XXXXXX") +trap 'rm -f "$temporary"' EXIT HUP INT TERM + +record_probe() { + local target=$1 + local component=$2 + local expected=$3 + local runtime_json=$4 + local cgroup_memory=$5 + local rss_kb=$6 + + jq -cn \ + --arg target "$target" \ + --arg component "$component" \ + --argjson expected_port_limit "$expected" \ + --argjson cgroup_memory "$cgroup_memory" \ + --argjson rss_kb "$rss_kb" \ + --argjson runtime "$runtime_json" \ + '{ + target: $target, + component: $component, + expected_port_limit: $expected_port_limit, + cgroup_memory: $cgroup_memory, + rss_kb: $rss_kb + } + $runtime' >>"$temporary" +} + +case "$MODE" in + compose) + mapfile -t targets < <( + docker compose -p "${COMPOSE_PROJECT_NAME:-who_need_help}" \ + ps -q web worker | + sort -u + ) + + if [[ "${#targets[@]}" -eq 0 ]]; then + echo "No running Compose web or worker replicas were found." >&2 + exit 1 + fi + + live_flags=$( + for target in "${targets[@]}"; do + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ + "$target" | + sed -n 's/^ERL_ZFLAGS=//p' + done | + sort -u + ) + + if [[ ! "$live_flags" =~ ^\+Q[[:space:]]+([0-9]+)$ ]]; then + echo "The live Compose replicas do not expose one unambiguous +Q limit." >&2 + exit 1 + fi + + expected=${BASH_REMATCH[1]} + + for target in "${targets[@]}"; do + component=$( + docker inspect --format \ + '{{index .Config.Labels "com.docker.compose.service"}}' "$target" + ) + runtime_json=$( + docker exec "$target" /app/bin/who_need_help rpc "$RPC_EXPRESSION" | + tail -n 1 + ) + cgroup_memory=$(docker exec "$target" cat /sys/fs/cgroup/memory.current) + rss_kb=$( + docker exec "$target" awk "/^VmRSS:/ {print \$2}" /proc/1/status + ) + record_probe \ + "$target" "$component" "$expected" "$runtime_json" \ + "$cgroup_memory" "$rss_kb" + done + ;; + + kind) + KUBECTL="$ROOT/.tools/bin/kubectl" + CONTEXT=kind-who-need-help + NAMESPACE=who-need-help + + if [[ ! -x "$KUBECTL" ]]; then + echo "The project-owned kubectl is missing." >&2 + exit 1 + fi + + live_flags=$( + "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \ + get deployment \ + who-need-help-who-need-help-web \ + who-need-help-who-need-help-worker \ + -o json | + jq -r ' + [ + .items[].spec.template.spec.containers[].env[] + | select(.name == "ERL_ZFLAGS") + | .value + ] + | unique + | if length == 1 then .[0] else empty end + ' + ) + + if [[ ! "$live_flags" =~ (^|[[:space:]])\+Q[[:space:]]+([0-9]+)($|[[:space:]]) ]]; then + echo "The live deployments do not expose one unambiguous +Q limit." >&2 + exit 1 + fi + + expected=${BASH_REMATCH[2]} + mapfile -t pods < <( + "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \ + get pods \ + -l app.kubernetes.io/name=who-need-help \ + -o json | + jq -r ' + .items[] + | .metadata.labels["app.kubernetes.io/component"] as $component + | select( + ($component == "web" or $component == "worker") and + .status.phase == "Running" and + ([.status.containerStatuses[]?.ready] | all) + ) + | [.metadata.name, $component] + | @tsv + ' + ) + + desired=$( + "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \ + get deployment \ + who-need-help-who-need-help-web \ + who-need-help-who-need-help-worker \ + -o json | + jq '[.items[].spec.replicas] | add' + ) + + if [[ "${#pods[@]}" -ne "$desired" ]]; then + echo "Expected $desired Ready application pods; observed ${#pods[@]}." >&2 + exit 1 + fi + + for entry in "${pods[@]}"; do + IFS=$'\t' read -r pod component <<<"$entry" + runtime_json=$( + "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \ + exec "$pod" --container "$component" -- \ + /app/bin/who_need_help rpc "$RPC_EXPRESSION" | + tail -n 1 + ) + cgroup_memory=$( + "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \ + exec "$pod" --container "$component" -- \ + cat /sys/fs/cgroup/memory.current + ) + rss_kb=$( + "$KUBECTL" --context "$CONTEXT" --namespace "$NAMESPACE" \ + exec "$pod" --container "$component" -- \ + awk "/^VmRSS:/ {print \$2}" /proc/1/status + ) + record_probe \ + "$pod" "$component" "$expected" "$runtime_json" \ + "$cgroup_memory" "$rss_kb" + done + ;; + + *) + echo "Usage: $0 [compose|kind] [output.json]" >&2 + exit 1 + ;; +esac + +summary=$(jq -s 'sort_by(.component, .target)' "$temporary") + +if ! printf '%s\n' "$summary" | + jq -e \ + --argjson expected "$expected" \ + 'length > 0 and all(.port_limit == $expected)' >/dev/null; then + echo "A running BEAM instance does not use the configured port limit." >&2 + exit 1 +fi + +if [[ -n "$OUTPUT" ]]; then + printf '%s\n' "$summary" >"$OUTPUT" +fi + +printf '%s\n' "$summary"