fix: drain replicas and clean isolated load cycles

2026-07-23 05:11:14 +03:00 · 2026-07-23 05:11:14 +03:00 · 0e81ee7f36
commit 0e81ee7f36
parent 4a1af1b658
16 changed files with 378 additions and 22 deletions
--- a/.env.example
+++ b/.env.example
@ -51,6 +51,10 @@ DOCKER_SOCKET_GID=REPLACE_WITH_DOCKER_SOCKET_NUMERIC_GID
 # accepts. Keep loopback locally; set the exact VPN proxy address for staging.
 TRAEFIK_TRUSTED_IPS=127.0.0.1/32
 TRAEFIK_RETRY_ATTEMPTS=3
+# Keep false for ordinary deployments. The isolated load profile enables the
+# unbound port-8080 API only inside its private Compose networks so its rolling
+# drill can observe when a drained backend leaves service.
+TRAEFIK_API_INSECURE=false
 # Docker-provider isolation and names. A second Compose project must use its
 # own project constraint, router/service name, Docker network, and Host rule.
 TRAEFIK_PROJECT_CONSTRAINT=who_need_help
--- a/.env.load.example
+++ b/.env.load.example
@ -21,6 +21,7 @@ PHX_SCHEME=https
 PHX_URL_PORT=443
 TRAEFIK_TRUSTED_IPS=127.0.0.1/32
 TRAEFIK_RETRY_ATTEMPTS=3
+TRAEFIK_API_INSECURE=true
 TRAEFIK_PROJECT_CONSTRAINT=who_need_help_load
 TRAEFIK_APP_NAME=who-need-help-load
 TRAEFIK_DOCKER_NETWORK=who_need_help_load_ingress
--- a/compose.load.yaml
+++ b/compose.load.yaml
@ -18,10 +18,10 @@ services:
      ingress: {}

  migrate:
-    image: who-need-help:load
+    image: ${APP_IMAGE:?Set APP_IMAGE in the isolated load environment}

  web:
-    image: who-need-help:load
+    image: ${APP_IMAGE:?Set APP_IMAGE in the isolated load environment}
    labels:
      - traefik.http.routers.${TRAEFIK_APP_NAME}.middlewares=${TRAEFIK_APP_NAME}-forwarded,${TRAEFIK_APP_NAME}-retry
      - traefik.http.middlewares.${TRAEFIK_APP_NAME}-forwarded.headers.customrequestheaders.X-Forwarded-Proto=https
@ -32,4 +32,4 @@ services:
      - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.tls=true

  worker:
-    image: who-need-help:load
+    image: ${APP_IMAGE:?Set APP_IMAGE in the isolated load environment}
--- a/compose.yaml
+++ b/compose.yaml
@ -94,7 +94,8 @@ services:
      context: .
      dockerfile: Dockerfile.traefik
    command:
-      - --api.dashboard=false
+      - --api.dashboard=${TRAEFIK_API_INSECURE:-false}
+      - --api.insecure=${TRAEFIK_API_INSECURE:-false}
      - --providers.docker=true
      - --providers.docker.endpoint=tcp://docker-api-proxy:2375
      - --providers.docker.exposedbydefault=false
@ -241,7 +242,11 @@ services:
      - traefik.http.routers.${TRAEFIK_APP_NAME:-who-need-help}.service=${TRAEFIK_APP_NAME:-who-need-help}
      - traefik.http.routers.${TRAEFIK_APP_NAME:-who-need-help}.middlewares=${TRAEFIK_APP_NAME:-who-need-help}-retry
      - traefik.http.middlewares.${TRAEFIK_APP_NAME:-who-need-help}-retry.retry.attempts=${TRAEFIK_RETRY_ATTEMPTS:-3}
+      - traefik.http.middlewares.${TRAEFIK_APP_NAME:-who-need-help}-retry.retry.status=500-599
      - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.server.port=4000
+      - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.healthcheck.path=/healthz/ready
+      - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.healthcheck.interval=10s
+      - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.healthcheck.timeout=3s
    healthcheck:
      test: ["CMD", "curl", "--fail", "--silent", "http://localhost:4000/healthz/ready"]
      interval: 10s
--- a/docs/operations.md
+++ b/docs/operations.md
@ -521,10 +521,15 @@ The isolated load project can exercise process crashes, sequential container
 replacement, and a real Oban retry without touching the normal Compose project:

 ```bash
-./scripts/load-stack-up.sh
-./scripts/load-resilience-run.sh local-resilience
+./scripts/load-cycle.sh local-resilience resilience
 ```

+The lifecycle wrapper assigns unique Compose and image names and removes that
+exact run's containers, networks, named volumes, temporary environment, and
+image tags on success, failure, or interruption. Use `load-stack-up.sh` plus
+`load-resilience-run.sh` only when the isolated stack must remain available for
+manual inspection.
+
 The resilience script refuses `LOAD_PROJECT=who_need_help` and verifies the
 Compose project/service labels of every container before stopping it. It:

--- a/docs/performance.md
+++ b/docs/performance.md
@ -84,6 +84,21 @@ counts, database state, and experiment inputs.

 ## Create the isolated profile

+For an ordinary one-run measurement, prefer the lifecycle wrapper. It creates
+unique Compose and image names, then removes only that run's containers,
+networks, named volumes, temporary environment, and image tags on success,
+failure, or interruption:
+
+```sh
+./scripts/load-cycle.sh local-load load
+./scripts/load-cycle.sh local-resilience resilience
+./scripts/load-cycle.sh local-complete both
+```
+
+Evidence remains under `output/performance/` and `output/resilience/`. The
+commands below are the manual diagnostic workflow; unlike the lifecycle
+wrapper, it deliberately keeps the isolated database volume between commands.
+
 ```sh
 ./scripts/ensure-local-load-env.sh
 ./scripts/load-stack-up.sh
--- a/lib/who_need_help/application.ex
+++ b/lib/who_need_help/application.ex
@ -9,6 +9,8 @@ defmodule WhoNeedHelp.Application do

  @impl true
  def start(_type, _args) do
+    WhoNeedHelpWeb.DrainState.reset()
+
    if @e2e_routes do
      boot_id = :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false)
      :persistent_term.put({WhoNeedHelp, :e2e_boot_id}, boot_id)
--- a/lib/who_need_help_web/controllers/health_controller.ex
+++ b/lib/who_need_help_web/controllers/health_controller.ex
@ -4,9 +4,13 @@ defmodule WhoNeedHelpWeb.HealthController do
  def live(conn, _params), do: json(conn, %{status: "ok"})

  def ready(conn, _params) do
+    if WhoNeedHelpWeb.DrainState.draining?() do
+      conn |> put_status(:service_unavailable) |> json(%{status: "draining"})
+    else
      case Ecto.Adapters.SQL.query(WhoNeedHelp.Repo, "SELECT 1", []) do
        {:ok, _} -> json(conn, %{status: "ready"})
        {:error, _} -> conn |> put_status(:service_unavailable) |> json(%{status: "not_ready"})
      end
    end
  end
+end
--- a/lib/who_need_help_web/drain_state.ex
+++ b/lib/who_need_help_web/drain_state.ex
@ -0,0 +1,26 @@
+defmodule WhoNeedHelpWeb.DrainState do
+  @moduledoc """
+  Tracks whether the current web node is draining before a planned shutdown.
+
+  The flag is node-local by design. Operators can mark one replica as draining
+  through a release RPC; readiness then fails on that replica while liveness
+  remains available.
+  """
+
+  @key {__MODULE__, :draining}
+
+  @spec reset() :: :ok
+  def reset do
+    :persistent_term.put(@key, false)
+    :ok
+  end
+
+  @spec begin_drain() :: :ok
+  def begin_drain do
+    :persistent_term.put(@key, true)
+    :ok
+  end
+
+  @spec draining?() :: boolean()
+  def draining?, do: :persistent_term.get(@key, false)
+end
--- a/scripts/ensure-local-load-env.sh
+++ b/scripts/ensure-local-load-env.sh
@ -59,6 +59,7 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS
  needs_resilience_interval=true
  needs_resilience_request_timeout=true
  needs_traefik_retry_attempts=true
+  needs_traefik_api_insecure=true
  needs_observability_prometheus_port=true
  needs_observability_alertmanager_port=true
  needs_observability_grafana_port=true
@ -90,6 +91,7 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS
  grep -q '^LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=' "$ENV_FILE" &&
    needs_resilience_request_timeout=false
  grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$ENV_FILE" && needs_traefik_retry_attempts=false
+  grep -q '^TRAEFIK_API_INSECURE=' "$ENV_FILE" && needs_traefik_api_insecure=false
  grep -q '^OBSERVABILITY_PROMETHEUS_PORT=' "$ENV_FILE" &&
    needs_observability_prometheus_port=false
  grep -q '^OBSERVABILITY_ALERTMANAGER_PORT=' "$ENV_FILE" &&
@ -135,6 +137,7 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS
    [ "$needs_resilience_interval" = false ] &&
    [ "$needs_resilience_request_timeout" = false ] &&
    [ "$needs_traefik_retry_attempts" = false ] &&
+    [ "$needs_traefik_api_insecure" = false ] &&
    [ "$needs_observability_prometheus_port" = false ] &&
    [ "$needs_observability_alertmanager_port" = false ] &&
    [ "$needs_observability_grafana_port" = false ] &&
@ -235,6 +238,10 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS
      printf 'TRAEFIK_RETRY_ATTEMPTS=3\n'
    fi

+    if [ "$needs_traefik_api_insecure" = true ]; then
+      printf 'TRAEFIK_API_INSECURE=true\n'
+    fi
+
    if [ "$needs_observability_prometheus_port" = true ] ||
      [ "$needs_observability_alertmanager_port" = true ] ||
      [ "$needs_observability_grafana_port" = true ] ||
@ -338,7 +345,8 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS
    needs_fixture_password needs_oban_maintenance_concurrency \
    needs_oban_push_concurrency needs_resilience_timeout \
    needs_resilience_interval needs_resilience_request_timeout \
-    needs_traefik_retry_attempts needs_observability_prometheus_port \
+    needs_traefik_retry_attempts needs_traefik_api_insecure \
+    needs_observability_prometheus_port \
    needs_observability_alertmanager_port needs_observability_grafana_port \
    needs_observability_scrape_interval needs_observability_evaluation_interval \
    needs_observability_timeout needs_observability_grafana_user \
--- a/scripts/load-cycle.sh
+++ b/scripts/load-cycle.sh
@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+BASE_ENV=${WNH_LOAD_BASE_ENV_FILE:-"$ROOT/.env.load"}
+LABEL=${1:-"cycle-$(date -u +%Y%m%dT%H%M%SZ)"}
+MODE=${2:-load}
+
+if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
+  echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
+  exit 1
+fi
+
+case "$MODE" in
+  load | resilience | both) ;;
+  *)
+    echo "Mode must be load, resilience, or both." >&2
+    exit 1
+    ;;
+esac
+
+WNH_LOAD_ENV_FILE="$BASE_ENV" "$ROOT/scripts/ensure-local-load-env.sh"
+
+if [[ ! -f "$BASE_ENV" ]]; then
+  echo "Missing base load environment: $BASE_ENV." >&2
+  exit 1
+fi
+
+run_id="$(date -u +%Y%m%d%H%M%S)-$$"
+safe_id=$(printf '%s' "$run_id" | tr -cd 'A-Za-z0-9')
+project="who_need_help_load_$safe_id"
+temporary_env=$(mktemp "${TMPDIR:-/tmp}/wnh-load-cycle.XXXXXX.env")
+export WNH_LOAD_ENV_FILE=$temporary_env
+export WNH_LOAD_TOOLS_IMAGE="who-need-help:load-tools-$safe_id"
+
+cp "$BASE_ENV" "$temporary_env"
+chmod 600 "$temporary_env"
+
+set_env_value() {
+  local name=$1
+  local value=$2
+  local replacement="$name=$value"
+
+  if grep -q "^${name}=" "$temporary_env"; then
+    sed -i "s|^${name}=.*|$replacement|" "$temporary_env"
+  else
+    printf '%s\n' "$replacement" >>"$temporary_env"
+  fi
+}
+
+set_env_value LOAD_PROJECT "$project"
+set_env_value COMPOSE_PROJECT_NAME "$project"
+set_env_value TRAEFIK_PROJECT_CONSTRAINT "$project"
+set_env_value TRAEFIK_DOCKER_NETWORK "${project}_ingress"
+set_env_value APP_IMAGE "who-need-help:load-$safe_id"
+set_env_value SOCKET_PROXY_IMAGE "who-need-help:socket-proxy-load-$safe_id"
+set_env_value POSTGIS_IMAGE "who-need-help:postgis-load-$safe_id"
+
+set -a
+# shellcheck source=/dev/null
+. "$temporary_env"
+set +a
+
+for name in APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE; do
+  if [[ -z "${!name:-}" ]]; then
+    echo "$name is missing from the generated cycle environment." >&2
+    exit 1
+  fi
+done
+
+if [[ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" ]] ||
+  [[ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" ]] ||
+  [[ -n "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" ]]; then
+  echo "Generated load-cycle project already owns Docker resources: $project." >&2
+  exit 1
+fi
+
+started=0
+
+cleanup() {
+  local status=$?
+  local cleanup_status=0
+  trap - EXIT HUP INT TERM
+
+  if [[ "$started" -eq 1 ]] ||
+    [[ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" ]] ||
+    [[ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" ]]; then
+    "$ROOT/scripts/load-stack-stop.sh" >/dev/null 2>&1 || cleanup_status=1
+  fi
+
+  while IFS= read -r volume; do
+    [[ -n "$volume" ]] || continue
+    observed_project=$(docker volume inspect --format \
+      '{{index .Labels "com.docker.compose.project"}}' "$volume")
+    references=$(docker ps -aq --filter "volume=$volume")
+
+    if [[ "$observed_project" != "$project" ]] || [[ -n "$references" ]]; then
+      echo "Refusing unexpected or referenced load-cycle volume: $volume" >&2
+      cleanup_status=1
+      continue
+    fi
+
+    docker volume rm "$volume" >/dev/null || cleanup_status=1
+  done < <(docker volume ls -q --filter "label=com.docker.compose.project=$project")
+
+  for image in "$APP_IMAGE" "$SOCKET_PROXY_IMAGE" "$POSTGIS_IMAGE" \
+    "$WNH_LOAD_TOOLS_IMAGE"; do
+    if image_id=$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null); then
+      if [[ -n "$(docker ps -aq --filter "ancestor=$image_id")" ]]; then
+        echo "Refusing referenced load-cycle image: $image" >&2
+        cleanup_status=1
+      else
+        docker image rm "$image" >/dev/null || cleanup_status=1
+      fi
+    fi
+  done
+
+  rm -f "$temporary_env"
+
+  if [[ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" ]] ||
+    [[ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" ]] ||
+    [[ -n "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" ]]; then
+    echo "Load-cycle cleanup left project resources behind: $project." >&2
+    cleanup_status=1
+  fi
+
+  if [[ "$status" -eq 0 && "$cleanup_status" -ne 0 ]]; then
+    status=$cleanup_status
+  fi
+
+  exit "$status"
+}
+
+trap cleanup EXIT HUP INT TERM
+
+"$ROOT/scripts/load-stack-up.sh"
+started=1
+
+case "$MODE" in
+  load)
+    "$ROOT/scripts/load-run.sh" "$LABEL"
+    ;;
+  resilience)
+    "$ROOT/scripts/load-resilience-run.sh" "$LABEL"
+    ;;
+  both)
+    "$ROOT/scripts/load-run.sh" "$LABEL-load"
+    "$ROOT/scripts/load-resilience-run.sh" "$LABEL-resilience"
+    ;;
+esac
+
+echo "Load cycle completed; its project resources will now be removed: $project"
--- a/scripts/load-resilience-run.sh
+++ b/scripts/load-resilience-run.sh
@ -2,7 +2,7 @@
 set -euo pipefail

 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
-ENV_FILE="$ROOT/.env.load"
+ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"}
 LABEL=${1:-"resilience-$(date -u +%Y%m%dT%H%M%SZ)"}

 if [[ ! -f "$ENV_FILE" ]]; then
@ -19,13 +19,18 @@ for name in LOAD_PROJECT LOAD_HOST LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS \
  LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS \
  LOAD_RESILIENCE_PROBE_INTERVAL_SECONDS \
  LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS TRAEFIK_RETRY_ATTEMPTS \
-  HTTP_PORT POSTGRES_DB METRICS_TOKEN; do
+  TRAEFIK_API_INSECURE HTTP_PORT POSTGRES_DB METRICS_TOKEN; do
  if [[ -z "${!name:-}" ]]; then
    echo "$name is missing from .env.load" >&2
    exit 1
  fi
 done

+if [[ "$TRAEFIK_API_INSECURE" != "true" ]]; then
+  echo "The isolated resilience profile requires TRAEFIK_API_INSECURE=true." >&2
+  exit 1
+fi
+
 if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
  echo "The resilience profile must not use the staging Compose project." >&2
  exit 1
@ -291,11 +296,109 @@ delete_retry_job() {

 restore_topology() {
  "${compose[@]}" up -d --no-deps \
+    --force-recreate \
    --scale "web=$LOAD_WEB_REPLICAS" \
    --scale "worker=$LOAD_WORKER_REPLICAS" \
    web worker >/dev/null 2>&1 || true
 }

+traefik_service_json() {
+  local proxy_id edge_network proxy_ip
+
+  proxy_id=$(service_ids proxy | head -n 1)
+  assert_scope "$proxy_id" proxy
+  edge_network="${LOAD_PROJECT}_edge"
+  proxy_ip=$(
+    docker inspect --format \
+      "{{(index .NetworkSettings.Networks \"$edge_network\").IPAddress}}" \
+      "$proxy_id"
+  )
+
+  if [[ -z "$proxy_ip" ]]; then
+    echo "The isolated Traefik proxy has no address on $edge_network." >&2
+    return 1
+  fi
+
+  curl --silent --show-error --fail \
+    --max-time "$LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS" \
+    "http://$proxy_ip:8080/api/http/services/${TRAEFIK_APP_NAME}@docker"
+}
+
+drain_web_from_traefik() {
+  local container_id=$1
+  local destination=$2
+  local ingress_network observed_project observed_role ingress_ip server_url
+  local deadline status
+
+  ingress_network=$(
+    docker inspect --format \
+      '{{index .Config.Labels "traefik.docker.network"}}' \
+      "$container_id"
+  )
+
+  if [[ -z "$ingress_network" ]]; then
+    echo "Container $container_id has no declared Traefik ingress network." >&2
+    return 1
+  fi
+
+  if ! docker inspect --format '{{json .NetworkSettings.Networks}}' \
+    "$container_id" | jq -e --arg network "$ingress_network" \
+      'has($network)' >/dev/null; then
+    echo "Container $container_id is not attached to $ingress_network." >&2
+    return 1
+  fi
+
+  observed_project=$(
+    docker network inspect --format \
+      '{{index .Labels "com.docker.compose.project"}}' \
+      "$ingress_network"
+  )
+  observed_role=$(
+    docker network inspect --format \
+      '{{index .Labels "com.docker.compose.network"}}' \
+      "$ingress_network"
+  )
+
+  if [[ "$observed_project" != "$LOAD_PROJECT" ||
+    "$observed_role" != "ingress" ]]; then
+    echo "Ingress network scope mismatch for $ingress_network." >&2
+    return 1
+  fi
+
+  ingress_ip=$(
+    docker inspect --format \
+      "{{(index .NetworkSettings.Networks \"$ingress_network\").IPAddress}}" \
+      "$container_id"
+  )
+  server_url="http://$ingress_ip:4000"
+
+  docker exec "$container_id" /app/bin/who_need_help rpc \
+    'IO.inspect(WhoNeedHelpWeb.DrainState.begin_drain())' >"$destination"
+
+  deadline=$((SECONDS + LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    status=$(
+      traefik_service_json |
+        jq -r --arg server_url "$server_url" \
+          '.serverStatus[$server_url] // "MISSING"'
+    )
+
+    printf 'server=%s status=%s observed_at=%s\n' \
+      "$server_url" "$status" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+      >>"$destination"
+
+    if [[ "$status" == "DOWN" ]]; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for Traefik to drain $server_url." >&2
+  return 1
+}
+
 cleanup() {
  local status=$?
  local cleanup_status=0
@ -376,8 +479,19 @@ docker inspect "$worker_target" |

 mapfile -t original_web_ids < <(service_ids web)

+traefik_service_json |
+  jq -e --argjson expected "$LOAD_WEB_REPLICAS" '
+    .status == "enabled" and
+      (.loadBalancer.healthCheck.path == "/healthz/ready") and
+      ([.serverStatus[]] | length) == $expected and
+      ([.serverStatus[]] | all(. == "UP"))
+  ' >"$output_dir/traefik-before-replacements.json"
+
 for container_id in "${original_web_ids[@]}"; do
  assert_scope "$container_id" web
+  drain_web_from_traefik \
+    "$container_id" \
+    "$output_dir/web-drain-${container_id:0:12}.txt"
  {
    docker stop --timeout 30 "$container_id"
    docker rm "$container_id"
--- a/scripts/load-run.sh
+++ b/scripts/load-run.sh
@ -2,7 +2,8 @@
 set -euo pipefail

 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
-ENV_FILE="$ROOT/.env.load"
+ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"}
+LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools}
 K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669"
 PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4"
 LABEL=${1:-"run-$(date -u +%Y%m%dT%H%M%SZ)"}
@ -106,7 +107,7 @@ if [[ -z "$internal_network_id" ]]; then
  exit 1
 fi

-if ! docker image inspect who-need-help:load-tools >/dev/null 2>&1; then
+if ! docker image inspect "$LOAD_TOOLS_IMAGE" >/dev/null 2>&1; then
  echo "The isolated load-tools image is missing. Run scripts/load-stack-up.sh first." >&2
  exit 1
 fi
@ -649,7 +650,7 @@ run_fixture_tool() {
    --env "WNH_LOAD_FIXTURE_COUNT=$fixture_count" \
    --env "WNH_LOAD_FIXTURE_PATH=/output/fixtures.json" \
    --volume "$output_dir:/output" \
-    who-need-help:load-tools \
+    "$LOAD_TOOLS_IMAGE" \
    mix wnh.load_fixtures "$action"
 }

--- a/scripts/load-stack-stop.sh
+++ b/scripts/load-stack-stop.sh
@ -2,7 +2,8 @@
 set -eu

 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
-ENV_FILE="$ROOT/.env.load"
+ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"}
+LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools}

 if [ ! -f "$ENV_FILE" ]; then
  echo "Missing $ENV_FILE; no load-profile project was selected." >&2
@ -46,9 +47,9 @@ for network_id in $(docker network ls -q \
  docker network rm "$network_id" >/dev/null
 done

-if image_id=$(docker image inspect --format '{{.Id}}' who-need-help:load-tools 2>/dev/null); then
+if image_id=$(docker image inspect --format '{{.Id}}' "$LOAD_TOOLS_IMAGE" 2>/dev/null); then
  if [ -z "$(docker ps -aq --filter "ancestor=$image_id")" ]; then
-    docker image rm who-need-help:load-tools >/dev/null
+    docker image rm "$LOAD_TOOLS_IMAGE" >/dev/null
  fi
 fi

--- a/scripts/load-stack-up.sh
+++ b/scripts/load-stack-up.sh
@ -2,7 +2,8 @@
 set -eu

 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
-ENV_FILE="$ROOT/.env.load"
+ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"}
+LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools}
 REPLICAS=${1:-}

 "$ROOT/scripts/ensure-local-load-env.sh"
@ -67,9 +68,9 @@ cleanup_failed_start() {
      sleep 1
    done

-    if image_id=$(docker image inspect --format '{{.Id}}' who-need-help:load-tools 2>/dev/null); then
+    if image_id=$(docker image inspect --format '{{.Id}}' "$LOAD_TOOLS_IMAGE" 2>/dev/null); then
      if [ -z "$(docker ps -aq --filter "ancestor=$image_id")" ]; then
-        docker image rm who-need-help:load-tools >/dev/null 2>&1 || true
+        docker image rm "$LOAD_TOOLS_IMAGE" >/dev/null 2>&1 || true
      fi
    fi

@ -85,7 +86,7 @@ cleanup_failed_start() {

 trap cleanup_failed_start EXIT HUP INT TERM

-docker build --target load_tools --tag who-need-help:load-tools .
+docker build --target load_tools --tag "$LOAD_TOOLS_IMAGE" .

 compose up -d --build --wait \
  --scale "web=$LOAD_WEB_REPLICAS" \
--- a/test/who_need_help_web/controllers/page_controller_test.exs
+++ b/test/who_need_help_web/controllers/page_controller_test.exs
@ -124,6 +124,22 @@ defmodule WhoNeedHelpWeb.PageControllerTest do
             |> json_response(200)
  end

+  test "a draining web node fails readiness but remains live", %{conn: conn} do
+    on_exit(&WhoNeedHelpWeb.DrainState.reset/0)
+    assert :ok = WhoNeedHelpWeb.DrainState.begin_drain()
+
+    assert %{"status" => "draining"} =
+             conn
+             |> get(~p"/healthz/ready")
+             |> json_response(503)
+
+    assert %{"status" => "ok"} =
+             conn
+             |> recycle()
+             |> get(~p"/healthz/live")
+             |> json_response(200)
+  end
+
  test "GET /safety selects Ukrainian locale and keeps it in the session", %{conn: conn} do
    conn = get(conn, ~p"/safety?locale=uk")
    assert html_response(conn, 200) =~ "Правила безпеки"