From 0e81ee7f360d0dc2e309471c401c56761702be86 Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Thu, 23 Jul 2026 05:11:14 +0300 Subject: [PATCH] fix: drain replicas and clean isolated load cycles --- .env.example | 4 + .env.load.example | 1 + compose.load.yaml | 6 +- compose.yaml | 7 +- docs/operations.md | 9 +- docs/performance.md | 15 ++ lib/who_need_help/application.ex | 2 + .../controllers/health_controller.ex | 10 +- lib/who_need_help_web/drain_state.ex | 26 +++ scripts/ensure-local-load-env.sh | 10 +- scripts/load-cycle.sh | 153 ++++++++++++++++++ scripts/load-resilience-run.sh | 118 +++++++++++++- scripts/load-run.sh | 7 +- scripts/load-stack-stop.sh | 7 +- scripts/load-stack-up.sh | 9 +- .../controllers/page_controller_test.exs | 16 ++ 16 files changed, 378 insertions(+), 22 deletions(-) create mode 100644 lib/who_need_help_web/drain_state.ex create mode 100755 scripts/load-cycle.sh diff --git a/.env.example b/.env.example index 6d9690b..c53c784 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,10 @@ DOCKER_SOCKET_GID=REPLACE_WITH_DOCKER_SOCKET_NUMERIC_GID # accepts. Keep loopback locally; set the exact VPN proxy address for staging. TRAEFIK_TRUSTED_IPS=127.0.0.1/32 TRAEFIK_RETRY_ATTEMPTS=3 +# Keep false for ordinary deployments. The isolated load profile enables the +# unbound port-8080 API only inside its private Compose networks so its rolling +# drill can observe when a drained backend leaves service. +TRAEFIK_API_INSECURE=false # Docker-provider isolation and names. A second Compose project must use its # own project constraint, router/service name, Docker network, and Host rule. TRAEFIK_PROJECT_CONSTRAINT=who_need_help diff --git a/.env.load.example b/.env.load.example index 28e154b..a5d03c0 100644 --- a/.env.load.example +++ b/.env.load.example @@ -21,6 +21,7 @@ PHX_SCHEME=https PHX_URL_PORT=443 TRAEFIK_TRUSTED_IPS=127.0.0.1/32 TRAEFIK_RETRY_ATTEMPTS=3 +TRAEFIK_API_INSECURE=true TRAEFIK_PROJECT_CONSTRAINT=who_need_help_load TRAEFIK_APP_NAME=who-need-help-load TRAEFIK_DOCKER_NETWORK=who_need_help_load_ingress diff --git a/compose.load.yaml b/compose.load.yaml index 1381d82..71b61ba 100644 --- a/compose.load.yaml +++ b/compose.load.yaml @@ -18,10 +18,10 @@ services: ingress: {} migrate: - image: who-need-help:load + image: ${APP_IMAGE:?Set APP_IMAGE in the isolated load environment} web: - image: who-need-help:load + image: ${APP_IMAGE:?Set APP_IMAGE in the isolated load environment} labels: - traefik.http.routers.${TRAEFIK_APP_NAME}.middlewares=${TRAEFIK_APP_NAME}-forwarded,${TRAEFIK_APP_NAME}-retry - traefik.http.middlewares.${TRAEFIK_APP_NAME}-forwarded.headers.customrequestheaders.X-Forwarded-Proto=https @@ -32,4 +32,4 @@ services: - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.tls=true worker: - image: who-need-help:load + image: ${APP_IMAGE:?Set APP_IMAGE in the isolated load environment} diff --git a/compose.yaml b/compose.yaml index 88146ca..06f369e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -94,7 +94,8 @@ services: context: . dockerfile: Dockerfile.traefik command: - - --api.dashboard=false + - --api.dashboard=${TRAEFIK_API_INSECURE:-false} + - --api.insecure=${TRAEFIK_API_INSECURE:-false} - --providers.docker=true - --providers.docker.endpoint=tcp://docker-api-proxy:2375 - --providers.docker.exposedbydefault=false @@ -241,7 +242,11 @@ services: - traefik.http.routers.${TRAEFIK_APP_NAME:-who-need-help}.service=${TRAEFIK_APP_NAME:-who-need-help} - traefik.http.routers.${TRAEFIK_APP_NAME:-who-need-help}.middlewares=${TRAEFIK_APP_NAME:-who-need-help}-retry - traefik.http.middlewares.${TRAEFIK_APP_NAME:-who-need-help}-retry.retry.attempts=${TRAEFIK_RETRY_ATTEMPTS:-3} + - traefik.http.middlewares.${TRAEFIK_APP_NAME:-who-need-help}-retry.retry.status=500-599 - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.server.port=4000 + - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.healthcheck.path=/healthz/ready + - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.healthcheck.interval=10s + - traefik.http.services.${TRAEFIK_APP_NAME:-who-need-help}.loadbalancer.healthcheck.timeout=3s healthcheck: test: ["CMD", "curl", "--fail", "--silent", "http://localhost:4000/healthz/ready"] interval: 10s diff --git a/docs/operations.md b/docs/operations.md index 20bdf3f..4336144 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -521,10 +521,15 @@ The isolated load project can exercise process crashes, sequential container replacement, and a real Oban retry without touching the normal Compose project: ```bash -./scripts/load-stack-up.sh -./scripts/load-resilience-run.sh local-resilience +./scripts/load-cycle.sh local-resilience resilience ``` +The lifecycle wrapper assigns unique Compose and image names and removes that +exact run's containers, networks, named volumes, temporary environment, and +image tags on success, failure, or interruption. Use `load-stack-up.sh` plus +`load-resilience-run.sh` only when the isolated stack must remain available for +manual inspection. + The resilience script refuses `LOAD_PROJECT=who_need_help` and verifies the Compose project/service labels of every container before stopping it. It: diff --git a/docs/performance.md b/docs/performance.md index 19512cc..ad760f4 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -84,6 +84,21 @@ counts, database state, and experiment inputs. ## Create the isolated profile +For an ordinary one-run measurement, prefer the lifecycle wrapper. It creates +unique Compose and image names, then removes only that run's containers, +networks, named volumes, temporary environment, and image tags on success, +failure, or interruption: + +```sh +./scripts/load-cycle.sh local-load load +./scripts/load-cycle.sh local-resilience resilience +./scripts/load-cycle.sh local-complete both +``` + +Evidence remains under `output/performance/` and `output/resilience/`. The +commands below are the manual diagnostic workflow; unlike the lifecycle +wrapper, it deliberately keeps the isolated database volume between commands. + ```sh ./scripts/ensure-local-load-env.sh ./scripts/load-stack-up.sh diff --git a/lib/who_need_help/application.ex b/lib/who_need_help/application.ex index 9179f42..8ddc03f 100644 --- a/lib/who_need_help/application.ex +++ b/lib/who_need_help/application.ex @@ -9,6 +9,8 @@ defmodule WhoNeedHelp.Application do @impl true def start(_type, _args) do + WhoNeedHelpWeb.DrainState.reset() + if @e2e_routes do boot_id = :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false) :persistent_term.put({WhoNeedHelp, :e2e_boot_id}, boot_id) diff --git a/lib/who_need_help_web/controllers/health_controller.ex b/lib/who_need_help_web/controllers/health_controller.ex index 9476487..8b29219 100644 --- a/lib/who_need_help_web/controllers/health_controller.ex +++ b/lib/who_need_help_web/controllers/health_controller.ex @@ -4,9 +4,13 @@ defmodule WhoNeedHelpWeb.HealthController do def live(conn, _params), do: json(conn, %{status: "ok"}) def ready(conn, _params) do - case Ecto.Adapters.SQL.query(WhoNeedHelp.Repo, "SELECT 1", []) do - {:ok, _} -> json(conn, %{status: "ready"}) - {:error, _} -> conn |> put_status(:service_unavailable) |> json(%{status: "not_ready"}) + if WhoNeedHelpWeb.DrainState.draining?() do + conn |> put_status(:service_unavailable) |> json(%{status: "draining"}) + else + case Ecto.Adapters.SQL.query(WhoNeedHelp.Repo, "SELECT 1", []) do + {:ok, _} -> json(conn, %{status: "ready"}) + {:error, _} -> conn |> put_status(:service_unavailable) |> json(%{status: "not_ready"}) + end end end end diff --git a/lib/who_need_help_web/drain_state.ex b/lib/who_need_help_web/drain_state.ex new file mode 100644 index 0000000..62a8be7 --- /dev/null +++ b/lib/who_need_help_web/drain_state.ex @@ -0,0 +1,26 @@ +defmodule WhoNeedHelpWeb.DrainState do + @moduledoc """ + Tracks whether the current web node is draining before a planned shutdown. + + The flag is node-local by design. Operators can mark one replica as draining + through a release RPC; readiness then fails on that replica while liveness + remains available. + """ + + @key {__MODULE__, :draining} + + @spec reset() :: :ok + def reset do + :persistent_term.put(@key, false) + :ok + end + + @spec begin_drain() :: :ok + def begin_drain do + :persistent_term.put(@key, true) + :ok + end + + @spec draining?() :: boolean() + def draining?, do: :persistent_term.get(@key, false) +end diff --git a/scripts/ensure-local-load-env.sh b/scripts/ensure-local-load-env.sh index 6014471..daf358f 100755 --- a/scripts/ensure-local-load-env.sh +++ b/scripts/ensure-local-load-env.sh @@ -59,6 +59,7 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS needs_resilience_interval=true needs_resilience_request_timeout=true needs_traefik_retry_attempts=true + needs_traefik_api_insecure=true needs_observability_prometheus_port=true needs_observability_alertmanager_port=true needs_observability_grafana_port=true @@ -90,6 +91,7 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS grep -q '^LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=' "$ENV_FILE" && needs_resilience_request_timeout=false grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$ENV_FILE" && needs_traefik_retry_attempts=false + grep -q '^TRAEFIK_API_INSECURE=' "$ENV_FILE" && needs_traefik_api_insecure=false grep -q '^OBSERVABILITY_PROMETHEUS_PORT=' "$ENV_FILE" && needs_observability_prometheus_port=false grep -q '^OBSERVABILITY_ALERTMANAGER_PORT=' "$ENV_FILE" && @@ -135,6 +137,7 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS [ "$needs_resilience_interval" = false ] && [ "$needs_resilience_request_timeout" = false ] && [ "$needs_traefik_retry_attempts" = false ] && + [ "$needs_traefik_api_insecure" = false ] && [ "$needs_observability_prometheus_port" = false ] && [ "$needs_observability_alertmanager_port" = false ] && [ "$needs_observability_grafana_port" = false ] && @@ -235,6 +238,10 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS printf 'TRAEFIK_RETRY_ATTEMPTS=3\n' fi + if [ "$needs_traefik_api_insecure" = true ]; then + printf 'TRAEFIK_API_INSECURE=true\n' + fi + if [ "$needs_observability_prometheus_port" = true ] || [ "$needs_observability_alertmanager_port" = true ] || [ "$needs_observability_grafana_port" = true ] || @@ -338,7 +345,8 @@ GOOGLE_OAUTH_HTTP_RECEIVE_TIMEOUT_MS needs_fixture_password needs_oban_maintenance_concurrency \ needs_oban_push_concurrency needs_resilience_timeout \ needs_resilience_interval needs_resilience_request_timeout \ - needs_traefik_retry_attempts needs_observability_prometheus_port \ + needs_traefik_retry_attempts needs_traefik_api_insecure \ + needs_observability_prometheus_port \ needs_observability_alertmanager_port needs_observability_grafana_port \ needs_observability_scrape_interval needs_observability_evaluation_interval \ needs_observability_timeout needs_observability_grafana_user \ diff --git a/scripts/load-cycle.sh b/scripts/load-cycle.sh new file mode 100755 index 0000000..f162a65 --- /dev/null +++ b/scripts/load-cycle.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +BASE_ENV=${WNH_LOAD_BASE_ENV_FILE:-"$ROOT/.env.load"} +LABEL=${1:-"cycle-$(date -u +%Y%m%dT%H%M%SZ)"} +MODE=${2:-load} + +if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2 + exit 1 +fi + +case "$MODE" in + load | resilience | both) ;; + *) + echo "Mode must be load, resilience, or both." >&2 + exit 1 + ;; +esac + +WNH_LOAD_ENV_FILE="$BASE_ENV" "$ROOT/scripts/ensure-local-load-env.sh" + +if [[ ! -f "$BASE_ENV" ]]; then + echo "Missing base load environment: $BASE_ENV." >&2 + exit 1 +fi + +run_id="$(date -u +%Y%m%d%H%M%S)-$$" +safe_id=$(printf '%s' "$run_id" | tr -cd 'A-Za-z0-9') +project="who_need_help_load_$safe_id" +temporary_env=$(mktemp "${TMPDIR:-/tmp}/wnh-load-cycle.XXXXXX.env") +export WNH_LOAD_ENV_FILE=$temporary_env +export WNH_LOAD_TOOLS_IMAGE="who-need-help:load-tools-$safe_id" + +cp "$BASE_ENV" "$temporary_env" +chmod 600 "$temporary_env" + +set_env_value() { + local name=$1 + local value=$2 + local replacement="$name=$value" + + if grep -q "^${name}=" "$temporary_env"; then + sed -i "s|^${name}=.*|$replacement|" "$temporary_env" + else + printf '%s\n' "$replacement" >>"$temporary_env" + fi +} + +set_env_value LOAD_PROJECT "$project" +set_env_value COMPOSE_PROJECT_NAME "$project" +set_env_value TRAEFIK_PROJECT_CONSTRAINT "$project" +set_env_value TRAEFIK_DOCKER_NETWORK "${project}_ingress" +set_env_value APP_IMAGE "who-need-help:load-$safe_id" +set_env_value SOCKET_PROXY_IMAGE "who-need-help:socket-proxy-load-$safe_id" +set_env_value POSTGIS_IMAGE "who-need-help:postgis-load-$safe_id" + +set -a +# shellcheck source=/dev/null +. "$temporary_env" +set +a + +for name in APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE; do + if [[ -z "${!name:-}" ]]; then + echo "$name is missing from the generated cycle environment." >&2 + exit 1 + fi +done + +if [[ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" ]] || + [[ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" ]] || + [[ -n "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" ]]; then + echo "Generated load-cycle project already owns Docker resources: $project." >&2 + exit 1 +fi + +started=0 + +cleanup() { + local status=$? + local cleanup_status=0 + trap - EXIT HUP INT TERM + + if [[ "$started" -eq 1 ]] || + [[ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" ]] || + [[ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" ]]; then + "$ROOT/scripts/load-stack-stop.sh" >/dev/null 2>&1 || cleanup_status=1 + fi + + while IFS= read -r volume; do + [[ -n "$volume" ]] || continue + observed_project=$(docker volume inspect --format \ + '{{index .Labels "com.docker.compose.project"}}' "$volume") + references=$(docker ps -aq --filter "volume=$volume") + + if [[ "$observed_project" != "$project" ]] || [[ -n "$references" ]]; then + echo "Refusing unexpected or referenced load-cycle volume: $volume" >&2 + cleanup_status=1 + continue + fi + + docker volume rm "$volume" >/dev/null || cleanup_status=1 + done < <(docker volume ls -q --filter "label=com.docker.compose.project=$project") + + for image in "$APP_IMAGE" "$SOCKET_PROXY_IMAGE" "$POSTGIS_IMAGE" \ + "$WNH_LOAD_TOOLS_IMAGE"; do + if image_id=$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null); then + if [[ -n "$(docker ps -aq --filter "ancestor=$image_id")" ]]; then + echo "Refusing referenced load-cycle image: $image" >&2 + cleanup_status=1 + else + docker image rm "$image" >/dev/null || cleanup_status=1 + fi + fi + done + + rm -f "$temporary_env" + + if [[ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" ]] || + [[ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" ]] || + [[ -n "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" ]]; then + echo "Load-cycle cleanup left project resources behind: $project." >&2 + cleanup_status=1 + fi + + if [[ "$status" -eq 0 && "$cleanup_status" -ne 0 ]]; then + status=$cleanup_status + fi + + exit "$status" +} + +trap cleanup EXIT HUP INT TERM + +"$ROOT/scripts/load-stack-up.sh" +started=1 + +case "$MODE" in + load) + "$ROOT/scripts/load-run.sh" "$LABEL" + ;; + resilience) + "$ROOT/scripts/load-resilience-run.sh" "$LABEL" + ;; + both) + "$ROOT/scripts/load-run.sh" "$LABEL-load" + "$ROOT/scripts/load-resilience-run.sh" "$LABEL-resilience" + ;; +esac + +echo "Load cycle completed; its project resources will now be removed: $project" diff --git a/scripts/load-resilience-run.sh b/scripts/load-resilience-run.sh index f500477..db17711 100755 --- a/scripts/load-resilience-run.sh +++ b/scripts/load-resilience-run.sh @@ -2,7 +2,7 @@ set -euo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -ENV_FILE="$ROOT/.env.load" +ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"} LABEL=${1:-"resilience-$(date -u +%Y%m%dT%H%M%SZ)"} if [[ ! -f "$ENV_FILE" ]]; then @@ -19,13 +19,18 @@ for name in LOAD_PROJECT LOAD_HOST LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS \ LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS \ LOAD_RESILIENCE_PROBE_INTERVAL_SECONDS \ LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS TRAEFIK_RETRY_ATTEMPTS \ - HTTP_PORT POSTGRES_DB METRICS_TOKEN; do + TRAEFIK_API_INSECURE HTTP_PORT POSTGRES_DB METRICS_TOKEN; do if [[ -z "${!name:-}" ]]; then echo "$name is missing from .env.load" >&2 exit 1 fi done +if [[ "$TRAEFIK_API_INSECURE" != "true" ]]; then + echo "The isolated resilience profile requires TRAEFIK_API_INSECURE=true." >&2 + exit 1 +fi + if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then echo "The resilience profile must not use the staging Compose project." >&2 exit 1 @@ -291,11 +296,109 @@ delete_retry_job() { restore_topology() { "${compose[@]}" up -d --no-deps \ + --force-recreate \ --scale "web=$LOAD_WEB_REPLICAS" \ --scale "worker=$LOAD_WORKER_REPLICAS" \ web worker >/dev/null 2>&1 || true } +traefik_service_json() { + local proxy_id edge_network proxy_ip + + proxy_id=$(service_ids proxy | head -n 1) + assert_scope "$proxy_id" proxy + edge_network="${LOAD_PROJECT}_edge" + proxy_ip=$( + docker inspect --format \ + "{{(index .NetworkSettings.Networks \"$edge_network\").IPAddress}}" \ + "$proxy_id" + ) + + if [[ -z "$proxy_ip" ]]; then + echo "The isolated Traefik proxy has no address on $edge_network." >&2 + return 1 + fi + + curl --silent --show-error --fail \ + --max-time "$LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS" \ + "http://$proxy_ip:8080/api/http/services/${TRAEFIK_APP_NAME}@docker" +} + +drain_web_from_traefik() { + local container_id=$1 + local destination=$2 + local ingress_network observed_project observed_role ingress_ip server_url + local deadline status + + ingress_network=$( + docker inspect --format \ + '{{index .Config.Labels "traefik.docker.network"}}' \ + "$container_id" + ) + + if [[ -z "$ingress_network" ]]; then + echo "Container $container_id has no declared Traefik ingress network." >&2 + return 1 + fi + + if ! docker inspect --format '{{json .NetworkSettings.Networks}}' \ + "$container_id" | jq -e --arg network "$ingress_network" \ + 'has($network)' >/dev/null; then + echo "Container $container_id is not attached to $ingress_network." >&2 + return 1 + fi + + observed_project=$( + docker network inspect --format \ + '{{index .Labels "com.docker.compose.project"}}' \ + "$ingress_network" + ) + observed_role=$( + docker network inspect --format \ + '{{index .Labels "com.docker.compose.network"}}' \ + "$ingress_network" + ) + + if [[ "$observed_project" != "$LOAD_PROJECT" || + "$observed_role" != "ingress" ]]; then + echo "Ingress network scope mismatch for $ingress_network." >&2 + return 1 + fi + + ingress_ip=$( + docker inspect --format \ + "{{(index .NetworkSettings.Networks \"$ingress_network\").IPAddress}}" \ + "$container_id" + ) + server_url="http://$ingress_ip:4000" + + docker exec "$container_id" /app/bin/who_need_help rpc \ + 'IO.inspect(WhoNeedHelpWeb.DrainState.begin_drain())' >"$destination" + + deadline=$((SECONDS + LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + status=$( + traefik_service_json | + jq -r --arg server_url "$server_url" \ + '.serverStatus[$server_url] // "MISSING"' + ) + + printf 'server=%s status=%s observed_at=%s\n' \ + "$server_url" "$status" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + >>"$destination" + + if [[ "$status" == "DOWN" ]]; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for Traefik to drain $server_url." >&2 + return 1 +} + cleanup() { local status=$? local cleanup_status=0 @@ -376,8 +479,19 @@ docker inspect "$worker_target" | mapfile -t original_web_ids < <(service_ids web) +traefik_service_json | + jq -e --argjson expected "$LOAD_WEB_REPLICAS" ' + .status == "enabled" and + (.loadBalancer.healthCheck.path == "/healthz/ready") and + ([.serverStatus[]] | length) == $expected and + ([.serverStatus[]] | all(. == "UP")) + ' >"$output_dir/traefik-before-replacements.json" + for container_id in "${original_web_ids[@]}"; do assert_scope "$container_id" web + drain_web_from_traefik \ + "$container_id" \ + "$output_dir/web-drain-${container_id:0:12}.txt" { docker stop --timeout 30 "$container_id" docker rm "$container_id" diff --git a/scripts/load-run.sh b/scripts/load-run.sh index 9558e93..e13a118 100755 --- a/scripts/load-run.sh +++ b/scripts/load-run.sh @@ -2,7 +2,8 @@ set -euo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -ENV_FILE="$ROOT/.env.load" +ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"} +LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools} K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669" PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4" LABEL=${1:-"run-$(date -u +%Y%m%dT%H%M%SZ)"} @@ -106,7 +107,7 @@ if [[ -z "$internal_network_id" ]]; then exit 1 fi -if ! docker image inspect who-need-help:load-tools >/dev/null 2>&1; then +if ! docker image inspect "$LOAD_TOOLS_IMAGE" >/dev/null 2>&1; then echo "The isolated load-tools image is missing. Run scripts/load-stack-up.sh first." >&2 exit 1 fi @@ -649,7 +650,7 @@ run_fixture_tool() { --env "WNH_LOAD_FIXTURE_COUNT=$fixture_count" \ --env "WNH_LOAD_FIXTURE_PATH=/output/fixtures.json" \ --volume "$output_dir:/output" \ - who-need-help:load-tools \ + "$LOAD_TOOLS_IMAGE" \ mix wnh.load_fixtures "$action" } diff --git a/scripts/load-stack-stop.sh b/scripts/load-stack-stop.sh index 111670b..b06819d 100755 --- a/scripts/load-stack-stop.sh +++ b/scripts/load-stack-stop.sh @@ -2,7 +2,8 @@ set -eu ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -ENV_FILE="$ROOT/.env.load" +ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"} +LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools} if [ ! -f "$ENV_FILE" ]; then echo "Missing $ENV_FILE; no load-profile project was selected." >&2 @@ -46,9 +47,9 @@ for network_id in $(docker network ls -q \ docker network rm "$network_id" >/dev/null done -if image_id=$(docker image inspect --format '{{.Id}}' who-need-help:load-tools 2>/dev/null); then +if image_id=$(docker image inspect --format '{{.Id}}' "$LOAD_TOOLS_IMAGE" 2>/dev/null); then if [ -z "$(docker ps -aq --filter "ancestor=$image_id")" ]; then - docker image rm who-need-help:load-tools >/dev/null + docker image rm "$LOAD_TOOLS_IMAGE" >/dev/null fi fi diff --git a/scripts/load-stack-up.sh b/scripts/load-stack-up.sh index f7af926..ef0b037 100755 --- a/scripts/load-stack-up.sh +++ b/scripts/load-stack-up.sh @@ -2,7 +2,8 @@ set -eu ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -ENV_FILE="$ROOT/.env.load" +ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/.env.load"} +LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools} REPLICAS=${1:-} "$ROOT/scripts/ensure-local-load-env.sh" @@ -67,9 +68,9 @@ cleanup_failed_start() { sleep 1 done - if image_id=$(docker image inspect --format '{{.Id}}' who-need-help:load-tools 2>/dev/null); then + if image_id=$(docker image inspect --format '{{.Id}}' "$LOAD_TOOLS_IMAGE" 2>/dev/null); then if [ -z "$(docker ps -aq --filter "ancestor=$image_id")" ]; then - docker image rm who-need-help:load-tools >/dev/null 2>&1 || true + docker image rm "$LOAD_TOOLS_IMAGE" >/dev/null 2>&1 || true fi fi @@ -85,7 +86,7 @@ cleanup_failed_start() { trap cleanup_failed_start EXIT HUP INT TERM -docker build --target load_tools --tag who-need-help:load-tools . +docker build --target load_tools --tag "$LOAD_TOOLS_IMAGE" . compose up -d --build --wait \ --scale "web=$LOAD_WEB_REPLICAS" \ diff --git a/test/who_need_help_web/controllers/page_controller_test.exs b/test/who_need_help_web/controllers/page_controller_test.exs index 2932e5e..b350048 100644 --- a/test/who_need_help_web/controllers/page_controller_test.exs +++ b/test/who_need_help_web/controllers/page_controller_test.exs @@ -124,6 +124,22 @@ defmodule WhoNeedHelpWeb.PageControllerTest do |> json_response(200) end + test "a draining web node fails readiness but remains live", %{conn: conn} do + on_exit(&WhoNeedHelpWeb.DrainState.reset/0) + assert :ok = WhoNeedHelpWeb.DrainState.begin_drain() + + assert %{"status" => "draining"} = + conn + |> get(~p"/healthz/ready") + |> json_response(503) + + assert %{"status" => "ok"} = + conn + |> recycle() + |> get(~p"/healthz/live") + |> json_response(200) + end + test "GET /safety selects Ukrainian locale and keeps it in the session", %{conn: conn} do conn = get(conn, ~p"/safety?locale=uk") assert html_response(conn, 200) =~ "Правила безпеки"