diff --git a/compose.observability.yaml b/compose.observability.yaml index cef4c6c..6225570 100644 --- a/compose.observability.yaml +++ b/compose.observability.yaml @@ -1,6 +1,9 @@ services: alert-receiver: - image: python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 + image: ${WNH_PYTHON_RUNTIME_IMAGE:-who-need-help:python-runtime} + build: + context: ops/external-boundaries + target: python_runtime command: ["python", "/opt/who-need-help/alert-receiver.py"] volumes: - ./scripts/alert-receiver.py:/opt/who-need-help/alert-receiver.py:ro diff --git a/ops/external-boundaries/Dockerfile b/ops/external-boundaries/Dockerfile index f964216..8e706cf 100644 --- a/ops/external-boundaries/Dockerfile +++ b/ops/external-boundaries/Dockerfile @@ -1,4 +1,9 @@ -FROM python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 +FROM python:3.14.7-alpine3.23@sha256:6b8f06d04d5305c1d1288435388df9165ab41e681fae6439d6349d8053cc3f83 AS python_runtime + +RUN apk add --no-cache sqlite-libs=3.53.4-r0 \ + && python -m pip uninstall --yes pip + +FROM python_runtime AS external_boundary WORKDIR /app diff --git a/scripts/load-cycle.sh b/scripts/load-cycle.sh index 1493486..f1b1dd4 100755 --- a/scripts/load-cycle.sh +++ b/scripts/load-cycle.sh @@ -33,6 +33,7 @@ project="who_need_help_load_$safe_id" temporary_env=$(mktemp "${TMPDIR:-/tmp}/wnh-load-cycle.XXXXXX.env") export WNH_LOAD_ENV_FILE=$temporary_env export WNH_LOAD_TOOLS_IMAGE="who-need-help:load-tools-$safe_id" +export WNH_PYTHON_RUNTIME_IMAGE="who-need-help:python-runtime-$safe_id" cp "$BASE_ENV" "$temporary_env" chmod 600 "$temporary_env" @@ -128,7 +129,7 @@ cleanup() { done < <(docker volume ls -q --filter "label=com.docker.compose.project=$project") for image in "$APP_IMAGE" "$SOCKET_PROXY_IMAGE" "$POSTGIS_IMAGE" \ - "$WNH_LOAD_TOOLS_IMAGE"; do + "$WNH_LOAD_TOOLS_IMAGE" "$WNH_PYTHON_RUNTIME_IMAGE"; do if image_id=$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null); then if [[ -n "$(docker ps -aq --filter "ancestor=$image_id")" ]]; then echo "Refusing referenced load-cycle image: $image" >&2 diff --git a/scripts/load-run.sh b/scripts/load-run.sh index c0a291a..e92ced1 100755 --- a/scripts/load-run.sh +++ b/scripts/load-run.sh @@ -5,7 +5,7 @@ ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/output/runtime/load.env"} LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools} K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669" -PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4" +PYTHON_RUNTIME_IMAGE=${WNH_PYTHON_RUNTIME_IMAGE:-who-need-help:python-runtime} LABEL=${1:-"run-$(date -u +%Y%m%dT%H%M%SZ)"} duration_override=${LOAD_DURATION_OVERRIDE:-} @@ -360,8 +360,9 @@ summarize_resources() { --user "$(id -u):$(id -g)" \ --volume "$ROOT/scripts/summarize-docker-stats.py:/scripts/summarize-docker-stats.py:ro" \ --volume "$output_dir:/output" \ - "$PYTHON_IMAGE" \ - python /scripts/summarize-docker-stats.py \ + --entrypoint python \ + "$PYTHON_RUNTIME_IMAGE" \ + /scripts/summarize-docker-stats.py \ /output/docker-stats.jsonl \ /output/resource-summary.json } @@ -750,7 +751,7 @@ trap cleanup_on_exit EXIT HUP INT TERM { printf 'observed_at=%s\n' "$run_started_at" printf 'k6_image=%s\n' "$K6_IMAGE" - printf 'resource_summarizer_image=%s\n' "$PYTHON_IMAGE" + printf 'resource_summarizer_image=%s\n' "$PYTHON_RUNTIME_IMAGE" printf 'load_project=%s\n' "$LOAD_PROJECT" printf 'web_replicas=%s\n' "$LOAD_WEB_REPLICAS" printf 'worker_replicas=%s\n' "$LOAD_WORKER_REPLICAS" diff --git a/scripts/load-stack-stop.sh b/scripts/load-stack-stop.sh index 8088803..3a42ca4 100755 --- a/scripts/load-stack-stop.sh +++ b/scripts/load-stack-stop.sh @@ -4,7 +4,7 @@ set -eu ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/output/runtime/load.env"} LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools} -RUNTIME_OWNER_IMAGE=python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 +PYTHON_RUNTIME_IMAGE=${WNH_PYTHON_RUNTIME_IMAGE:-who-need-help:python-runtime} if [ ! -f "$ENV_FILE" ]; then echo "Missing $ENV_FILE; no load-profile project was selected." >&2 @@ -78,10 +78,17 @@ if [ -d "$observability_runtime" ]; then --user 0:0 \ --volume "$observability_runtime:/runtime" \ --entrypoint /bin/sh \ - "$RUNTIME_OWNER_IMAGE" \ + "$PYTHON_RUNTIME_IMAGE" \ -euc "chown -R $(id -u):$(id -g) /runtime; chmod -R u+rwX /runtime" find "$observability_runtime" -xdev -depth -delete fi + +if image_id=$(docker image inspect --format '{{.Id}}' "$PYTHON_RUNTIME_IMAGE" 2>/dev/null); then + if [ -z "$(docker ps -aq --filter "ancestor=$image_id")" ]; then + docker image rm "$PYTHON_RUNTIME_IMAGE" >/dev/null + fi +fi + echo "Removed the isolated load-profile containers, networks, and generated observability runtime; its named volumes remain." diff --git a/scripts/load-stack-up.sh b/scripts/load-stack-up.sh index c6ca524..ca1e9a4 100755 --- a/scripts/load-stack-up.sh +++ b/scripts/load-stack-up.sh @@ -4,6 +4,7 @@ set -eu ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/output/runtime/load.env"} LOAD_TOOLS_IMAGE=${WNH_LOAD_TOOLS_IMAGE:-who-need-help:load-tools} +PYTHON_RUNTIME_IMAGE=${WNH_PYTHON_RUNTIME_IMAGE:-who-need-help:python-runtime} REPLICAS=${1:-} "$ROOT/scripts/ensure-local-load-env.sh" @@ -86,6 +87,12 @@ cleanup_failed_start() { fi fi + if image_id=$(docker image inspect --format '{{.Id}}' "$PYTHON_RUNTIME_IMAGE" 2>/dev/null); then + if [ -z "$(docker ps -aq --filter "ancestor=$image_id")" ]; then + docker image rm "$PYTHON_RUNTIME_IMAGE" >/dev/null 2>&1 || true + fi + fi + if [ -n "$(docker ps -aq --filter "label=com.docker.compose.project=$LOAD_PROJECT")" ] || \ [ -n "$(docker network ls -q --filter "label=com.docker.compose.project=$LOAD_PROJECT")" ]; then echo "Load-profile startup cleanup left project resources behind." >&2 @@ -99,6 +106,11 @@ cleanup_failed_start() { trap cleanup_failed_start EXIT HUP INT TERM docker build --target load_tools --tag "$LOAD_TOOLS_IMAGE" . +docker build \ + --target python_runtime \ + --tag "$PYTHON_RUNTIME_IMAGE" \ + --file ops/external-boundaries/Dockerfile \ + ops/external-boundaries compose up -d --build --wait \ --scale "web=$LOAD_WEB_REPLICAS" \ diff --git a/scripts/observability-run.sh b/scripts/observability-run.sh index f319188..be14993 100755 --- a/scripts/observability-run.sh +++ b/scripts/observability-run.sh @@ -4,6 +4,7 @@ set -euo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) ENV_FILE=${WNH_LOAD_ENV_FILE:-"$ROOT/output/runtime/load.env"} LABEL=${1:-"observability-$(date -u +%Y%m%dT%H%M%SZ)"} +PYTHON_RUNTIME_IMAGE=${WNH_PYTHON_RUNTIME_IMAGE:-who-need-help:python-runtime} if [[ ! -f "$ENV_FILE" ]]; then echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2 @@ -94,7 +95,7 @@ set_runtime_owner() { --user 0:0 \ --volume "$directory:/runtime" \ --entrypoint /bin/sh \ - python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 \ + "$PYTHON_RUNTIME_IMAGE" \ -euc "chown -R $owner /runtime; chmod 700 /runtime" } diff --git a/scripts/quality.sh b/scripts/quality.sh index 4f76f54..38790b2 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -10,8 +10,6 @@ ACTIONLINT_IMAGE="rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0 TRIVY_IMAGE="aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969" PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893" ALERTMANAGER_IMAGE="quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d" -PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4" - run_id="$(date -u +%Y%m%d%H%M%S)-$$" project="wnh_quality_$(printf '%s' "$run_id" | tr -d '-')" quality_image="who-need-help:quality-$run_id" @@ -22,6 +20,7 @@ backup_image="who-need-help:backup-audit-$run_id" minio_image="who-need-help:minio-audit-$run_id" mc_image="who-need-help:mc-audit-$run_id" boundary_mock_image="who-need-help:boundary-mock-audit-$run_id" +python_runtime_image="who-need-help:python-runtime-audit-$run_id" socket_proxy_image="who-need-help:socket-proxy-audit-$run_id" postgis_image="who-need-help:postgis-audit-$run_id" caddy_image="who-need-help:caddy-audit-$run_id" @@ -50,7 +49,8 @@ cleanup() { docker rm --force "$socket_proxy_container" >/dev/null 2>&1 || true docker image rm "$quality_image" "$assets_image" "$e2e_image" "$release_image" \ "$backup_image" "$minio_image" "$mc_image" \ - "$boundary_mock_image" "$socket_proxy_image" "$postgis_image" \ + "$boundary_mock_image" "$python_runtime_image" \ + "$socket_proxy_image" "$postgis_image" \ "$caddy_image" "$traefik_image" "$mailpit_image" \ >/dev/null 2>&1 || true rm -f "$android_fingerprint_probe" @@ -1547,6 +1547,16 @@ EXTERNAL_HANDOVER_SECRET=render-handover-secret \ docker compose -f compose.external-boundaries.yaml config --quiet echo "Validating local observability configuration" +docker build \ + --target python_runtime \ + --tag "$python_runtime_image" \ + --file ops/external-boundaries/Dockerfile \ + ops/external-boundaries +test -z "$(docker run --rm --entrypoint /bin/sh "$python_runtime_image" \ + -euc 'find /usr/local/lib/python3.14/site-packages -maxdepth 1 -name "pip*" -print')" +test "$(docker run --rm --entrypoint /bin/sh "$python_runtime_image" \ + -euc 'apk info -v | grep "^sqlite-libs-"')" = "sqlite-libs-3.53.4-r0" +scan_image "$python_runtime_image" sed \ -e 's/__SCRAPE_INTERVAL__/1s/g' \ -e 's/__EVALUATION_INTERVAL__/1s/g' \ @@ -1571,25 +1581,25 @@ docker run --rm \ "$ALERTMANAGER_IMAGE" check-config /etc/alertmanager/alertmanager.yml docker run --rm \ --volume "$ROOT/scripts/alert-receiver.py:/src/alert-receiver.py:ro" \ - "$PYTHON_IMAGE" python -c \ + "$python_runtime_image" python -c \ 'import py_compile; py_compile.compile("/src/alert-receiver.py", cfile="/tmp/alert-receiver.pyc", doraise=True)' docker run --rm \ --volume "$ROOT:/src:ro" \ --workdir /src \ - "$PYTHON_IMAGE" python test/scripts/production_external_monitor_test.py + "$python_runtime_image" python test/scripts/production_external_monitor_test.py docker run --rm \ --volume "$ROOT:/src:ro" \ --workdir /src \ - "$PYTHON_IMAGE" python test/scripts/override_production_monitor_smtp_test.py + "$python_runtime_image" python test/scripts/override_production_monitor_smtp_test.py docker run --rm \ --volume "$ROOT:/src:ro" \ --workdir /src \ - "$PYTHON_IMAGE" python test/scripts/install_production_external_monitor_test.py + "$python_runtime_image" python test/scripts/install_production_external_monitor_test.py python3 test/scripts/production_release_artifact_root_test.py python3 test/scripts/production_release_clean_test.py docker run --rm \ --volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \ - "$PYTHON_IMAGE" python -c \ + "$python_runtime_image" python -c \ 'import py_compile; py_compile.compile("/src/mock_server.py", cfile="/tmp/mock_server.pyc", doraise=True)' jq --exit-status \ 'type == "object" and .uid == "wnh-overview" and (.panels | length) == 12' \