diff --git a/.env.load.example b/.env.load.example index 385e82e..0070935 100644 --- a/.env.load.example +++ b/.env.load.example @@ -67,3 +67,16 @@ OBSERVABILITY_EVALUATION_INTERVAL=1s OBSERVABILITY_TIMEOUT_SECONDS=90 OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD + +# The encrypted backup drill keeps its MinIO API and console on dynamically +# allocated loopback ports. Secrets are generated only in ignored .env.load. +BACKUP_MINIO_API_PORT=0 +BACKUP_MINIO_CONSOLE_PORT=0 +BACKUP_MINIO_ROOT_USER=GENERATE_BACKUP_MINIO_ROOT_USER +BACKUP_MINIO_ROOT_PASSWORD=GENERATE_BACKUP_MINIO_ROOT_PASSWORD +BACKUP_RESTIC_PASSWORD=GENERATE_BACKUP_RESTIC_PASSWORD +BACKUP_BUCKET_PREFIX=wnh-backup +BACKUP_TIMEOUT_SECONDS=120 +BACKUP_INTERRUPTION_CHUNKS=300 +BACKUP_INTERRUPTION_CHUNK_BYTES=1048576 +BACKUP_INTERRUPTION_INTERVAL_SECONDS=0.1 diff --git a/Dockerfile.backup b/Dockerfile.backup new file mode 100644 index 0000000..1cb0019 --- /dev/null +++ b/Dockerfile.backup @@ -0,0 +1,20 @@ +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS restic + +RUN CGO_ENABLED=0 go install github.com/restic/restic/cmd/restic@v0.19.1 + +FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 + +RUN apk add --no-cache \ + ca-certificates=20260611-r0 \ + libcrypto3=3.5.7-r0 \ + libssl3=3.5.7-r0 \ + musl=1.2.5-r23 \ + musl-utils=1.2.5-r23 \ + postgresql18-client=18.4-r0 \ + zlib=1.3.2-r0 + +COPY --from=restic /go/bin/restic /usr/local/bin/restic + +ENTRYPOINT [] +USER 10001:10001 +CMD ["restic", "version"] diff --git a/Dockerfile.minio b/Dockerfile.minio new file mode 100644 index 0000000..ffe4827 --- /dev/null +++ b/Dockerfile.minio @@ -0,0 +1,81 @@ +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS minio_builder + +ADD --checksum=sha256:45521908307306e925c98d629e1c17d78c8b72b6ee242b1bfb1409f7d8ee5841 \ + https://github.com/minio/minio/archive/9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a.tar.gz \ + /tmp/minio.tar.gz + +WORKDIR /src + +RUN tar --extract --gzip --file /tmp/minio.tar.gz \ + --directory . --strip-components=1 \ + && go get \ + github.com/apache/thrift@v0.23.0 \ + github.com/buger/jsonparser@v1.1.2 \ + github.com/prometheus/prometheus@v0.311.3 \ + golang.org/x/crypto@v0.52.0 \ + golang.org/x/net@v0.55.0 \ + google.golang.org/grpc@v1.81.1 \ + && CGO_ENABLED=0 go build \ + -mod=mod \ + -trimpath \ + -tags kqueue \ + -ldflags="-s -w \ + -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z \ + -X github.com/minio/minio/cmd.CopyrightYear=2025 \ + -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z \ + -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a \ + -X github.com/minio/minio/cmd.ShortCommitID=9e49d5e7a648" \ + -o /out/minio . + +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS mc_builder + +ADD --checksum=sha256:95cd293c7119f16921a6dc515a1fb74a2227f19fd994b9c8b770a154e802ac44 \ + https://github.com/minio/mc/archive/7394ce0dd2a80935aded936b09fa12cbb3cb8096.tar.gz \ + /tmp/mc.tar.gz + +WORKDIR /src + +RUN tar --extract --gzip --file /tmp/mc.tar.gz \ + --directory . --strip-components=1 \ + && go get \ + github.com/prometheus/prometheus@v0.311.3 \ + golang.org/x/crypto@v0.52.0 \ + golang.org/x/net@v0.55.0 \ + google.golang.org/grpc@v1.81.1 \ + && go mod tidy \ + && CGO_ENABLED=0 go build \ + -trimpath \ + -tags kqueue \ + -ldflags="-s -w \ + -X github.com/minio/mc/cmd.Version=2025-08-13T08:35:41Z \ + -X github.com/minio/mc/cmd.CopyrightYear=2025 \ + -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-08-13T08-35-41Z \ + -X github.com/minio/mc/cmd.CommitID=7394ce0dd2a80935aded936b09fa12cbb3cb8096 \ + -X github.com/minio/mc/cmd.ShortCommitID=7394ce0dd2a8" \ + -o /out/mc . + +FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS runtime + +RUN apk add --no-cache \ + ca-certificates=20260611-r0 \ + libcrypto3=3.5.7-r0 \ + libssl3=3.5.7-r0 \ + musl=1.2.5-r23 \ + musl-utils=1.2.5-r23 \ + zlib=1.3.2-r0 + +USER 10001:10001 + +FROM runtime AS server + +COPY --from=minio_builder /out/minio /usr/local/bin/minio + +ENTRYPOINT ["minio"] +CMD ["--version"] + +FROM runtime AS client + +COPY --from=mc_builder /out/mc /usr/local/bin/mc + +ENTRYPOINT ["mc"] +CMD ["--version"] diff --git a/README.md b/README.md index 3e1e993..4cb5057 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,20 @@ backup_path=$(printf '%s\n' "$backup_output" | sed -n 's/^Backup: //p') ./scripts/restore-drill-compose.sh "$backup_path" ``` +For the reproducible encrypted S3-compatible drill, first start the isolated +load project and then run: + +```bash +./scripts/load-stack-up.sh +./scripts/backup-s3-drill.sh local-encrypted-backup +``` + +The command generates MinIO and Restic secrets only in ignored mode-`0600` +`.env.load`, streams `pg_dump` directly into an encrypted Restic repository, +restores it into a new temporary database, checks corruption and interruption +failure paths, removes those temporary buckets, and retains the successful +encrypted bucket in local MinIO. It never writes a plaintext dump to the host. + The commands, boundaries, and unclaimed production properties are documented in [the operations runbook](docs/operations.md). @@ -151,10 +165,11 @@ an isolated PostgreSQL volume: It checks shell scripts, Dockerfiles, the GitHub Actions workflow, every Compose profile, the rendered Helm chart, tracked-source secrets and infrastructure misconfigurations, Elixir formatting/compilation/xref/Credo/Sobelow/Dialyzer, -retired Hex packages, locked npm dependencies, all Phoenix tests, and the -production release image. Its generated database credentials are random and -exist only for that run. The exact database volume, networks, temporary source -snapshot, and one-run images are removed automatically. +retired Hex packages, locked npm dependencies, all Phoenix tests, the pinned +backup-tool, MinIO server/client, and production release images. Its generated +database credentials are random and exist only for that run. The exact database +volume, networks, temporary source snapshot, and one-run images are removed +automatically. The cursor-pagination database benchmark also creates a one-run Compose project, random database credentials, and a separate PostgreSQL volume: diff --git a/compose.backup.yaml b/compose.backup.yaml new file mode 100644 index 0000000..f94e472 --- /dev/null +++ b/compose.backup.yaml @@ -0,0 +1,118 @@ +services: + minio-permissions: + image: who-need-help:minio-local + build: + context: . + dockerfile: Dockerfile.minio + target: server + entrypoint: ["/bin/sh", "-euc"] + command: + - chown -R 10001:10001 /data + volumes: + - minio_data:/data + user: "0:0" + read_only: true + cap_drop: [ALL] + cap_add: [CHOWN] + security_opt: + - no-new-privileges:true + networks: [internal] + restart: "no" + profiles: [backup] + + minio: + image: who-need-help:minio-local + build: + context: . + dockerfile: Dockerfile.minio + target: server + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${BACKUP_MINIO_ROOT_USER:?Set BACKUP_MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${BACKUP_MINIO_ROOT_PASSWORD:?Set BACKUP_MINIO_ROOT_PASSWORD} + volumes: + - minio_data:/data + ports: + - target: 9000 + published: "${BACKUP_MINIO_API_PORT:-0}" + host_ip: 127.0.0.1 + protocol: tcp + - target: 9001 + published: "${BACKUP_MINIO_CONSOLE_PORT:-0}" + host_ip: 127.0.0.1 + protocol: tcp + healthcheck: + test: ["CMD", "wget", "--quiet", "--spider", "http://127.0.0.1:9000/minio/health/ready"] + interval: 2s + timeout: 3s + retries: 30 + depends_on: + minio-permissions: + condition: service_completed_successfully + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal, backup_host] + restart: unless-stopped + profiles: [backup] + + minio-client: + image: who-need-help:mc-local + build: + context: . + dockerfile: Dockerfile.minio + target: client + entrypoint: ["mc"] + command: ["--version"] + environment: + MC_CONFIG_DIR: /tmp/mc + MC_HOST_local: http://${BACKUP_MINIO_ROOT_USER:?Set BACKUP_MINIO_ROOT_USER}:${BACKUP_MINIO_ROOT_PASSWORD:?Set BACKUP_MINIO_ROOT_PASSWORD}@minio:9000 + volumes: + - ${BACKUP_RUNTIME_DIR:?Set BACKUP_RUNTIME_DIR for the backup profile}:/work + user: "${BACKUP_HOST_UID:?Set BACKUP_HOST_UID}:${BACKUP_HOST_GID:?Set BACKUP_HOST_GID}" + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal] + profiles: [backup] + + backup-tools: + image: who-need-help:backup-tools + build: + context: . + dockerfile: Dockerfile.backup + environment: + PGHOST: db + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:?Set POSTGRES_USER} + PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + PGDATABASE: ${POSTGRES_DB:?Set POSTGRES_DB} + AWS_ACCESS_KEY_ID: ${BACKUP_MINIO_ROOT_USER:?Set BACKUP_MINIO_ROOT_USER} + AWS_SECRET_ACCESS_KEY: ${BACKUP_MINIO_ROOT_PASSWORD:?Set BACKUP_MINIO_ROOT_PASSWORD} + RESTIC_PASSWORD: ${BACKUP_RESTIC_PASSWORD:?Set BACKUP_RESTIC_PASSWORD} + RESTIC_CACHE_DIR: /tmp/restic-cache + BACKUP_INTERRUPTION_CHUNKS: ${BACKUP_INTERRUPTION_CHUNKS:?Set BACKUP_INTERRUPTION_CHUNKS} + BACKUP_INTERRUPTION_CHUNK_BYTES: ${BACKUP_INTERRUPTION_CHUNK_BYTES:?Set BACKUP_INTERRUPTION_CHUNK_BYTES} + BACKUP_INTERRUPTION_INTERVAL_SECONDS: ${BACKUP_INTERRUPTION_INTERVAL_SECONDS:?Set BACKUP_INTERRUPTION_INTERVAL_SECONDS} + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal] + profiles: [backup] + +networks: + backup_host: + +volumes: + minio_data: diff --git a/docs/dependency-baseline.md b/docs/dependency-baseline.md index 383318a..a7e4b2f 100644 --- a/docs/dependency-baseline.md +++ b/docs/dependency-baseline.md @@ -42,6 +42,10 @@ package checksums are in `mix.lock` and `assets/package-lock.json`. | Alertmanager | 0.33.1 | | Grafana | 13.1.0 | | Python alert-boundary runtime | 3.14.6 / Alpine 3.23 | +| Restic | 0.19.1, rebuilt with Go 1.26.5 | +| MinIO server | RELEASE.2025-10-15T17-29-55Z, rebuilt with Go 1.26.5 | +| MinIO client | RELEASE.2025-08-13T08-35-41Z, rebuilt with Go 1.26.5 | +| Backup runtime | Alpine 3.23.3 / PostgreSQL client 18.4-r0 | | Debian builder/runner snapshot | trixie-20260713-slim | Every external Compose/kind service image and every Dockerfile base image is @@ -49,6 +53,15 @@ locked to both an exact tag and an OCI digest. The observed local Docker tooling was Docker Engine/CLI 29.6.2, Compose 5.3.1, and Buildx 0.35.0; their official release feeds reported those same versions as current during verification. +`Dockerfile.minio` fetches checksum-pinned upstream source commits for MinIO +server and client and records every dependency override used to rebuild them. +The current overrides move Apache Thrift, jsonparser, Prometheus, Go crypto/net, +and gRPC dependencies to the versions checked by the image security gate. +Both projects are AGPLv3; anyone distributing or operating modified builds must +review and satisfy the applicable license obligations. This repository keeps +the exact upstream source identifiers and the complete modification/build +recipe; this statement is operational documentation, not legal advice. + ## Android and local Kubernetes | Component | Locked version | @@ -106,6 +119,10 @@ Credo, Dialyxir, and Sobelow are locked in `mix.lock`. - [Alertmanager releases](https://github.com/prometheus/alertmanager/releases) - [Grafana releases](https://github.com/grafana/grafana/releases) - [Python releases](https://www.python.org/downloads/) +- [Go release history](https://go.dev/doc/devel/release) +- [Restic releases](https://github.com/restic/restic/releases) +- [MinIO releases](https://github.com/minio/minio/releases) +- [MinIO client releases](https://github.com/minio/mc/releases) - [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes) - [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk) - [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests) @@ -136,4 +153,5 @@ docker run --rm who-need-help:node-deps npm outdated --json ./scripts/android-build.sh ./scripts/android-instrumentation-test.sh ./scripts/observability-run.sh local-observability +./scripts/backup-s3-drill.sh local-encrypted-backup ``` diff --git a/docs/local-hardening-plan.md b/docs/local-hardening-plan.md index 7991c34..60e3975 100644 --- a/docs/local-hardening-plan.md +++ b/docs/local-hardening-plan.md @@ -14,7 +14,7 @@ item below unless the evidence column explicitly describes a local mock. | Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset | | Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data | | Observability | Completed locally: protected per-process metrics feed a pinned Prometheus/Grafana/Alertmanager profile | All 3 direct web targets are up before/after the drill; a verified replica stop delivers firing and resolved webhooks; Grafana datasource/dashboard and an empty DB-count diff are retained | -| Backup | Validated local custom-format dump and restore drill exist | An encrypted artifact is uploaded to local S3-compatible MinIO and restored into a fresh database; corruption and interrupted-upload checks fail closed | +| Backup | Completed locally: a plaintext-free Restic stream is retained in pinned local MinIO | The encrypted repository passes full-data checking and fresh-database restore; corrupted configuration and an interrupted upload fail closed, leave no snapshot, and their temporary buckets are removed | | External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout | | Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits | @@ -94,5 +94,12 @@ The goal remains open while any row lacks reproducible local evidence. dashboard, and routes the `up == 0` rule through Alertmanager. The canonical drill observed both firing and resolved webhooks for the exact stopped replica, restored all three targets, and left database counts unchanged. +- The encrypted backup profile streams PostgreSQL custom format through Restic + directly into local MinIO without a host plaintext archive. The canonical + drill restored 23 public tables with all 10 migrations and PostGIS 3.6.4, + rejected an intentionally corrupted repository, stopped a scoped upload + after an encrypted object reached MinIO, published no interrupted snapshot, + pruned the unreferenced packs, removed both temporary buckets and the + temporary restore database, and left source table counts unchanged. - The remaining rows above are still pending; this document is not a completion claim for the entire hardening goal. diff --git a/docs/operations.md b/docs/operations.md index 2ce0a7b..73e1a64 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -26,6 +26,62 @@ globals such as roles and tablespaces are not part of `pg_dump`; deployment credentials and database roles must be provisioned separately from secrets. Local backup files on the same workstation are not an off-site backup. +## Encrypted local S3-compatible backup drill + +The isolated load project can run a complete encrypted Restic/MinIO drill: + +```bash +./scripts/load-stack-up.sh +./scripts/backup-s3-drill.sh local-encrypted-backup +``` + +The script refuses the staging Compose project and validates the project and +service labels of every pre-existing container in its scope. On first use, +`scripts/ensure-local-load-env.sh` generates independent random MinIO and +Restic credentials in ignored `.env.load` and restricts that file to mode +`0600`. MinIO publishes Docker-assigned ports only on `127.0.0.1`; the observed +API and console URLs are printed after a successful run. + +The backup tool combines the matching PostgreSQL 18 client with pinned Restic +rebuilt on Go 1.26.5. MinIO server and client are also rebuilt as non-root +Alpine images from checksum-pinned upstream source commits with the exact +dependency updates recorded in `Dockerfile.minio`. The quality gate verifies +their reported release, commit, Go runtime, configured user, and current +HIGH/CRITICAL vulnerability scan. `restic backup --stdin-from-command` runs a +custom-format `pg_dump`, checks the producer exit status, encrypts the data, +and uploads it directly to MinIO. No plaintext database dump is written to the +host. The drill then: + +1. runs `restic check --read-data`; +2. streams `restic dump` into `pg_restore --list`; +3. restores into a uniquely named database created from `template0`; +4. checks tables, current Ecto migrations, PostGIS, and release migration + readiness before removing that exact database; +5. clones and corrupts an isolated repository and requires both check and dump + to fail; +6. stops an exact scoped in-progress backup container only after encrypted + objects reach MinIO, requires zero published snapshots, prunes unreferenced + packs, and rechecks the repository; +7. removes and verifies removal of the corruption/interruption buckets and + requires source table counts to remain unchanged. + +The successful encrypted bucket is deliberately retained in the named local +MinIO volume. Non-secret evidence is written under ignored +`output/backups-s3//`; the runtime scratch directory is under +ignored `tmp/backup-s3/`. Use a unique lowercase run label of at most 32 +characters. The command refuses to replace an existing retained bucket. + +MinIO server and client are AGPLv3. `Dockerfile.minio` identifies the exact +upstream source commits and contains the dependency changes and complete build +commands used here. Before distributing or publicly operating modified images, +review the license and make the corresponding source available as required; +this runbook does not provide legal advice. + +This verifies encryption, local S3 protocol use, restore mechanics, and two +failure paths on the observed workstation. A MinIO volume on that same +workstation is not an off-site backup and does not establish production RPO, +RTO, retention, capacity, key custody, object locking, or database HA. + ## Isolated restore drill Run a real restore into a uniquely named temporary database: diff --git a/docs/verification.md b/docs/verification.md index 1316790..ff56118 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -21,6 +21,7 @@ results from product limits and unknown production properties. | Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. | | Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. | | Local observability | Implemented and protocol-verified | Pinned Prometheus scraped all 3 direct load web targets with a file Bearer credential; Grafana provisioned a healthy datasource and four-panel dashboard; Alertmanager delivered firing and resolved webhooks for an induced scoped replica stop. | Local delivery does not establish production retention, notification-provider reliability, on-call policy, or measured alert thresholds. | +| Encrypted local backup | Implemented and failure-verified | Pinned Restic streamed PostgreSQL custom format into pinned local MinIO with no host plaintext dump, passed full-data checking and a fresh-database restore, rejected a corrupted repository, and published no snapshot for an interrupted upload. | The retained MinIO volume is on the same workstation; this is not off-site storage, database HA, or a production RPO/RTO/retention claim. | ## Reproducible checks @@ -31,12 +32,15 @@ results from product limits and unknown production properties. - `mix compile --force --warnings-as-errors` and `mix format --check-formatted`: passed against the same final source. - `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning - threshold, actionlint 1.7.12, all five Compose renders, Helm lint, Trivy + threshold, actionlint 1.7.12, all six Compose renders, Helm lint, Trivy source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the - production-image vulnerability scan. The rendered Helm manifest and Debian - 13.6 release image each reported zero HIGH/CRITICAL findings under the - configured gates. + backup/MinIO/mc/release-image vulnerability scans. The rendered Helm manifest + reported zero HIGH/CRITICAL misconfigurations; the Alpine backup, MinIO, and + mc images and the Debian 13.6 release image each reported zero HIGH/CRITICAL + vulnerabilities under the configured gates. The backup binary reported + Restic 0.19.1 compiled with Go 1.26.5; MinIO and mc reported their pinned + commits and Go 1.26.5. - Dialyzer passed with three path- and warning-specific documented filters and zero unused filters. Two findings are opaque-type warnings at Ecto `Multi`/`MapSet` call sites; the third is the generated Gettext backend's @@ -90,6 +94,19 @@ results from product limits and unknown production properties. running, and the checked user/request/message counts were identical before and after. A subsequent full drill returned all monitoring services to healthy. +- The encrypted S3 canonical drill restored 23 public application tables, all + 10 current migrations, 14 categories, and PostGIS 3.6.4 into a fresh + temporary database. `restic check --read-data` passed for the retained + repository. Flipping the first byte of an isolated cloned repository made + both `check` and `dump` exit nonzero. Stopping the exact backup container + after uploaded encrypted objects produced exit 130 and zero snapshots; + Restic identified and pruned 32.809 MiB of unreferenced packs, then passed a + second full-data check. Only the retained canonical bucket remained, the + restore database and interruption container were absent, no configured + secret appeared in retained evidence, and the source database count diff was + empty. The obsolete successful-run bucket was then removed; only the final + canonical bucket remains. Evidence is retained at + `output/backups-s3/backup-canonical-20260719d`. - The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium specs against a fresh PostGIS volume with two web and two worker replicas on 2026-07-19. The retained successful-run artifact directory is diff --git a/scripts/backup-s3-drill.sh b/scripts/backup-s3-drill.sh new file mode 100755 index 0000000..249223a --- /dev/null +++ b/scripts/backup-s3-drill.sh @@ -0,0 +1,631 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +ENV_FILE="$ROOT/.env.load" +LABEL=${1:-"backup-$(date -u +%Y%m%d%H%M%S)"} + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2 + exit 1 +fi + +set -a +# shellcheck source=/dev/null +. "$ENV_FILE" +set +a + +for name in LOAD_PROJECT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD DATABASE_URL \ + BACKUP_MINIO_API_PORT BACKUP_MINIO_CONSOLE_PORT BACKUP_MINIO_ROOT_USER \ + BACKUP_MINIO_ROOT_PASSWORD BACKUP_RESTIC_PASSWORD BACKUP_BUCKET_PREFIX \ + BACKUP_TIMEOUT_SECONDS BACKUP_INTERRUPTION_CHUNKS \ + BACKUP_INTERRUPTION_CHUNK_BYTES BACKUP_INTERRUPTION_INTERVAL_SECONDS; do + if [[ -z "${!name:-}" ]]; then + echo "$name is missing from .env.load" >&2 + exit 1 + fi +done + +if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then + echo "The encrypted backup drill must not use the staging Compose project." >&2 + exit 1 +fi + +if [[ ! "$LABEL" =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]]; then + echo "Run label must be 1-32 lowercase letters, numbers, or dashes." >&2 + exit 1 +fi + +if [[ ! "$BACKUP_BUCKET_PREFIX" =~ ^[a-z0-9][a-z0-9-]{1,19}$ ]]; then + echo "BACKUP_BUCKET_PREFIX must be 2-20 lowercase letters, numbers, or dashes." >&2 + exit 1 +fi + +for name in BACKUP_MINIO_API_PORT BACKUP_MINIO_CONSOLE_PORT; do + if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then + echo "$name must be a TCP port number from 0 through 65535." >&2 + exit 1 + fi +done + +for name in BACKUP_TIMEOUT_SECONDS BACKUP_INTERRUPTION_CHUNKS \ + BACKUP_INTERRUPTION_CHUNK_BYTES; do + if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then + echo "$name must be a positive integer." >&2 + exit 1 + fi +done + +if ! awk -v value="$BACKUP_INTERRUPTION_INTERVAL_SECONDS" \ + 'BEGIN {exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value > 0)}'; then + echo "BACKUP_INTERRUPTION_INTERVAL_SECONDS must be greater than zero." >&2 + exit 1 +fi + +for command in awk dd diff docker grep head jq od sleep tr unlink; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "Required command is unavailable: $command" >&2 + exit 1 + fi +done + +bucket="$BACKUP_BUCKET_PREFIX-$LABEL" +corrupt_bucket="$bucket-corrupt" +interrupt_bucket="$bucket-interrupt" + +for candidate in "$bucket" "$corrupt_bucket" "$interrupt_bucket"; do + if ((${#candidate} > 63)); then + echo "Derived S3 bucket name exceeds 63 characters: $candidate" >&2 + exit 1 + fi +done + +runtime_dir="$ROOT/tmp/backup-s3/$LOAD_PROJECT/$LABEL" +output_dir="$ROOT/output/backups-s3/$LABEL" +mkdir -p "$runtime_dir" "$output_dir" +chmod 700 "$ROOT/tmp" "$ROOT/tmp/backup-s3" \ + "$ROOT/tmp/backup-s3/$LOAD_PROJECT" "$runtime_dir" \ + "$ROOT/output" "$ROOT/output/backups-s3" "$output_dir" + +export BACKUP_RUNTIME_DIR="$runtime_dir" +export BACKUP_HOST_UID +export BACKUP_HOST_GID +BACKUP_HOST_UID=$(id -u) +BACKUP_HOST_GID=$(id -g) + +compose=( + docker compose + --env-file "$ENV_FILE" + -p "$LOAD_PROJECT" + -f compose.yaml + -f compose.load.yaml + -f compose.backup.yaml + --profile backup +) + +service_ids() { + "${compose[@]}" ps --all -q "$1" +} + +assert_scope() { + local container_id=$1 + local expected_service=$2 + local observed_project observed_service + + observed_project=$( + docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \ + "$container_id" + ) + observed_service=$( + docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \ + "$container_id" + ) + + if [[ "$observed_project" != "$LOAD_PROJECT" || + "$observed_service" != "$expected_service" ]]; then + echo "Container scope mismatch for $container_id." >&2 + exit 1 + fi +} + +mc_run() { + "${compose[@]}" run --rm --no-deps -T minio-client "$@" +} + +restic_run_repo() { + local repository=$1 + shift + "${compose[@]}" run --rm --no-deps -T \ + --env "RESTIC_REPOSITORY=$repository" \ + backup-tools restic "$@" +} + +database_snapshot() { + # Variables are intentionally expanded inside the isolated PostGIS container. + # shellcheck disable=SC2016 + "${compose[@]}" exec -T db sh -c \ + 'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL' +BEGIN READ ONLY; +SELECT 'users' AS table_name, count(*) AS row_count FROM users +UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens +UNION ALL SELECT 'help_requests', count(*) FROM help_requests +UNION ALL SELECT 'messages', count(*) FROM messages +UNION ALL SELECT 'categories', count(*) FROM categories +UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments +UNION ALL SELECT 'activities', count(*) FROM activities +UNION ALL SELECT 'reports', count(*) FROM reports +UNION ALL SELECT 'social_identities', count(*) FROM social_identities +UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions +UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions +UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations +ORDER BY table_name; +COMMIT; +SQL +} + +wait_for_minio() { + local minio_id=$1 + local deadline=$((SECONDS + BACKUP_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + local state health + state=$(docker inspect --format '{{.State.Status}}' "$minio_id") + health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \ + "$minio_id") + + if [[ "$state" == "running" && "$health" == "healthy" ]]; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for the scoped MinIO container." >&2 + return 1 +} + +remove_bucket() { + local candidate=$1 + mc_run rb --force "local/$candidate" >/dev/null 2>&1 || true +} + +remove_bucket_and_verify() { + local candidate=$1 + + mc_run rb --force "local/$candidate" >/dev/null + + if mc_run stat "local/$candidate" >/dev/null 2>&1; then + echo "Temporary MinIO bucket was not removed: $candidate" >&2 + exit 1 + fi +} + +remove_interrupt_container() { + if [[ -n "${interrupt_container_id:-}" ]] && + docker inspect "$interrupt_container_id" >/dev/null 2>&1; then + assert_scope "$interrupt_container_id" backup-tools + docker stop --time 1 "$interrupt_container_id" >/dev/null 2>&1 || true + docker rm "$interrupt_container_id" >/dev/null 2>&1 || true + fi +} + +drop_drill_database() { + if [[ "${drill_database_created:-false}" == "true" ]]; then + "${compose[@]}" exec -T db \ + dropdb --username "$POSTGRES_USER" --if-exists "$drill_db" >/dev/null 2>&1 || true + fi +} + +cleanup() { + local status=$? + trap - EXIT HUP INT TERM + drop_drill_database + remove_interrupt_container + + if [[ "${corrupt_bucket_created:-false}" == "true" ]]; then + remove_bucket "$corrupt_bucket" + fi + + if [[ "${interrupt_bucket_created:-false}" == "true" ]]; then + remove_bucket "$interrupt_bucket" + fi + + exit "$status" +} + +trap cleanup EXIT HUP INT TERM + +for service in minio-permissions minio minio-client backup-tools; do + while IFS= read -r existing_id; do + [[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service" + done < <(service_ids "$service") +done + +database_snapshot "$output_dir/database-before.txt" + +"${compose[@]}" build backup-tools minio minio-client \ + >"$output_dir/backup-images-build.txt" +"${compose[@]}" up -d --wait minio >"$output_dir/minio-up.txt" +minio_id=$(service_ids minio | head -n 1) +assert_scope "$minio_id" minio +wait_for_minio "$minio_id" + +if mc_run stat "local/$bucket" >/dev/null 2>&1; then + echo "Refusing to overwrite retained canonical bucket: $bucket" >&2 + exit 1 +fi + +mc_run mb "local/$bucket" >"$output_dir/canonical-bucket-create.txt" +canonical_repository="s3:http://minio:9000/$bucket" +restic_run_repo "$canonical_repository" init >"$output_dir/restic-init.txt" + +restic_run_repo "$canonical_repository" \ + backup \ + --json \ + --host who-need-help-local \ + --tag "$LABEL" \ + --stdin-filename database.dump \ + --stdin-from-command \ + -- \ + pg_dump \ + --host db \ + --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" \ + --format custom \ + --no-owner \ + >"$output_dir/backup.jsonl" + +restic_run_repo "$canonical_repository" snapshots --json --tag "$LABEL" \ + >"$output_dir/snapshots.json" + +if ! jq -e --arg label "$LABEL" ' + length == 1 and + .[0].tags == [$label] and + (.[0].id | length) == 64 +' "$output_dir/snapshots.json" >/dev/null; then + echo "The canonical encrypted backup did not create exactly one tagged snapshot." >&2 + exit 1 +fi + +snapshot_id=$(jq -r '.[0].id' "$output_dir/snapshots.json") + +restic_run_repo "$canonical_repository" check --read-data \ + >"$output_dir/canonical-check.txt" 2>&1 +restic_run_repo "$canonical_repository" ls --json "$snapshot_id" \ + >"$output_dir/snapshot-files.jsonl" + +if ! jq -s -e ' + any( + .[]; + .struct_type == "node" and + .path == "/database.dump" and + .type == "file" and + .size > 0 + ) +' "$output_dir/snapshot-files.jsonl" >/dev/null; then + echo "The encrypted snapshot does not contain a non-empty database.dump." >&2 + exit 1 +fi + +mc_run ls --recursive --json "local/$bucket" \ + >"$output_dir/minio-objects.jsonl" + +if ! jq -s -e ' + length > 0 and + any(.[]; (.key // "") | startswith("data/")) +' "$output_dir/minio-objects.jsonl" >/dev/null; then + echo "MinIO did not retain encrypted Restic data objects." >&2 + exit 1 +fi + +pack_key=$( + jq -sr ' + [.[] | select((.key // "") | startswith("data/")) | .key] + | sort + | first + ' "$output_dir/minio-objects.jsonl" +) + +mc_run cat "local/$bucket/$pack_key" | + dd of="$runtime_dir/encrypted-prefix.bin" bs=1 count=16 status=none +od -An -tx1 -v "$runtime_dir/encrypted-prefix.bin" | + tr -d ' \n' >"$output_dir/encrypted-prefix.hex" + +if grep -aFq 'PGDMP' "$runtime_dir/encrypted-prefix.bin"; then + echo "A Restic data object exposed the PostgreSQL custom-format header." >&2 + exit 1 +fi + +restic_run_repo "$canonical_repository" dump "$snapshot_id" /database.dump | + "${compose[@]}" exec -T db pg_restore --list \ + >"$output_dir/pg-restore-list.txt" + +drill_db="wnh_s3_restore_${LABEL//-/_}_$$" +"${compose[@]}" exec -T db \ + createdb \ + --username "$POSTGRES_USER" \ + --template template0 \ + "$drill_db" +drill_database_created=true + +restic_run_repo "$canonical_repository" dump "$snapshot_id" /database.dump | + "${compose[@]}" exec -T db \ + pg_restore \ + --username "$POSTGRES_USER" \ + --dbname "$drill_db" \ + --exit-on-error \ + --no-owner \ + --no-privileges + +"${compose[@]}" exec -T db \ + psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" --dbname "$drill_db" \ + >"$output_dir/restored-database.json" <<'SQL' +SELECT json_build_object( + 'public_table_count', + ( + SELECT count(*) + FROM pg_tables + WHERE schemaname = 'public' + AND tablename <> 'spatial_ref_sys' + ), + 'migration_count', (SELECT count(*) FROM schema_migrations), + 'user_count', (SELECT count(*) FROM users), + 'request_count', (SELECT count(*) FROM help_requests), + 'message_count', (SELECT count(*) FROM messages), + 'category_count', (SELECT count(*) FROM categories), + 'postgis_version', PostGIS_Lib_Version() +); +SQL + +if ! jq -e ' + .public_table_count > 0 and + .migration_count > 0 and + .category_count > 0 and + (.postgis_version | type == "string" and length > 0) +' "$output_dir/restored-database.json" >/dev/null; then + echo "The fresh restore database failed structural validation." >&2 + exit 1 +fi + +case "$DATABASE_URL" in + *\?*) + database_base=${DATABASE_URL%%\?*} + database_query="?${DATABASE_URL#*\?}" + ;; + + *) + database_base=$DATABASE_URL + database_query= + ;; +esac + +database_prefix=${database_base%/*} + +if [[ "$database_prefix" == "$database_base" ]]; then + echo "Could not derive the temporary drill DATABASE_URL." >&2 + exit 1 +fi + +drill_database_url="$database_prefix/$drill_db$database_query" + +"${compose[@]}" run --rm --no-deps \ + --env APP_ROLE=migrate \ + --env "DATABASE_URL=$drill_database_url" \ + migrate /app/bin/migrate \ + >"$output_dir/restore-migrate.txt" + +"${compose[@]}" run --rm --no-deps \ + --env APP_ROLE=migrate \ + --env "DATABASE_URL=$drill_database_url" \ + migrate /app/bin/who_need_help eval 'WhoNeedHelp.Release.await_migrations()' \ + >"$output_dir/restore-migration-readiness.txt" + +"${compose[@]}" exec -T db \ + dropdb --username "$POSTGRES_USER" "$drill_db" +drill_database_created=false + +remaining=$( + "${compose[@]}" exec -T db \ + psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" --dbname postgres \ + --command "SELECT count(*) FROM pg_database WHERE datname = '$drill_db';" +) + +if [[ "$remaining" != "0" ]]; then + echo "Restore drill database was not removed: $drill_db" >&2 + exit 1 +fi + +mc_run mb "local/$corrupt_bucket" >"$output_dir/corrupt-bucket-create.txt" +corrupt_bucket_created=true +mc_run mirror "local/$bucket" "local/$corrupt_bucket" \ + >"$output_dir/corrupt-mirror.txt" +mc_run cp "local/$corrupt_bucket/config" /work/corrupt-config \ + >"$output_dir/corrupt-download.txt" + +first_byte=$(od -An -tu1 -N1 "$runtime_dir/corrupt-config" | tr -d ' ') + +if [[ "$first_byte" == "0" ]]; then + printf '\001' | dd of="$runtime_dir/corrupt-config" bs=1 count=1 conv=notrunc status=none +else + printf '\000' | dd of="$runtime_dir/corrupt-config" bs=1 count=1 conv=notrunc status=none +fi + +mc_run cp /work/corrupt-config "local/$corrupt_bucket/config" \ + >"$output_dir/corrupt-upload.txt" +corrupt_repository="s3:http://minio:9000/$corrupt_bucket" + +set +e +restic_run_repo "$corrupt_repository" check --read-data \ + >"$output_dir/corrupt-check.txt" 2>&1 +corrupt_check_status=$? +restic_run_repo "$corrupt_repository" dump "$snapshot_id" /database.dump \ + >"/dev/null" 2>"$output_dir/corrupt-restore.txt" +corrupt_restore_status=$? +set -e + +if [[ "$corrupt_check_status" -eq 0 || "$corrupt_restore_status" -eq 0 ]]; then + echo "The corrupted encrypted repository did not fail closed." >&2 + exit 1 +fi + +remove_bucket_and_verify "$corrupt_bucket" +corrupt_bucket_created=false +unlink "$runtime_dir/corrupt-config" + +mc_run mb "local/$interrupt_bucket" >"$output_dir/interrupt-bucket-create.txt" +interrupt_bucket_created=true +interrupt_repository="s3:http://minio:9000/$interrupt_bucket" +restic_run_repo "$interrupt_repository" init >"$output_dir/interrupt-init.txt" + +interrupt_container="$LOAD_PROJECT-backup-interrupt-$LABEL" + +if docker inspect "$interrupt_container" >/dev/null 2>&1; then + echo "Refusing to replace an existing interruption container: $interrupt_container" >&2 + exit 1 +fi + +# The producer variables must expand inside the isolated backup-tools container. +# shellcheck disable=SC2016 +interrupt_container_id=$( + "${compose[@]}" run -d \ + --name "$interrupt_container" \ + --no-deps \ + --env "RESTIC_REPOSITORY=$interrupt_repository" \ + backup-tools \ + restic backup \ + --json \ + --host who-need-help-local \ + --tag "$LABEL-interrupted" \ + --stdin-filename interrupted.bin \ + --stdin-from-command \ + -- \ + /bin/sh -euc \ + 'i=0 + while [ "$i" -lt "$BACKUP_INTERRUPTION_CHUNKS" ]; do + head -c "$BACKUP_INTERRUPTION_CHUNK_BYTES" /dev/urandom + sleep "$BACKUP_INTERRUPTION_INTERVAL_SECONDS" + i=$((i + 1)) + done' +) +assert_scope "$interrupt_container_id" backup-tools + +upload_deadline=$((SECONDS + BACKUP_TIMEOUT_SECONDS)) +uploaded_data_objects=0 + +while ((SECONDS < upload_deadline)); do + mc_run ls --recursive --json "local/$interrupt_bucket/data" \ + >"$output_dir/interrupt-objects-current.jsonl" 2>/dev/null || true + uploaded_data_objects=$( + jq -s '[.[] | select(.type == "file")] | length' \ + "$output_dir/interrupt-objects-current.jsonl" + ) + + if ((uploaded_data_objects > 0)); then + break + fi + + if [[ "$(docker inspect --format '{{.State.Running}}' "$interrupt_container_id")" != "true" ]]; then + echo "Interruption producer exited before an uploaded data object was observed." >&2 + exit 1 + fi + + sleep 1 +done + +if ((uploaded_data_objects == 0)); then + echo "Timed out waiting for the interruption upload to reach MinIO." >&2 + exit 1 +fi + +docker stop --time 1 "$interrupt_container_id" >"$output_dir/interrupt-stop.txt" +docker inspect "$interrupt_container_id" | + jq '.[0] | { + id: .Id, + project: .Config.Labels["com.docker.compose.project"], + service: .Config.Labels["com.docker.compose.service"], + exit_code: .State.ExitCode, + state: .State.Status + }' >"$output_dir/interrupt-container.json" +docker logs "$interrupt_container_id" >"$output_dir/interrupt-backup.log" 2>&1 + +if ! jq -e '.exit_code != 0 and .state == "exited"' \ + "$output_dir/interrupt-container.json" >/dev/null; then + echo "The interrupted upload container did not exit unsuccessfully." >&2 + exit 1 +fi + +restic_run_repo "$interrupt_repository" snapshots \ + --json --tag "$LABEL-interrupted" \ + >"$output_dir/interrupt-snapshots.json" + +if ! jq -e 'length == 0' "$output_dir/interrupt-snapshots.json" >/dev/null; then + echo "An interrupted upload published a restorable snapshot." >&2 + exit 1 +fi + +mc_run ls --incomplete --recursive --json "local/$interrupt_bucket" \ + >"$output_dir/interrupt-incomplete-uploads.jsonl" 2>/dev/null || true +restic_run_repo "$interrupt_repository" check --read-data \ + >"$output_dir/interrupt-check-before-prune.txt" 2>&1 +restic_run_repo "$interrupt_repository" prune \ + >"$output_dir/interrupt-prune.txt" 2>&1 +restic_run_repo "$interrupt_repository" check --read-data \ + >"$output_dir/interrupt-check-after-prune.txt" 2>&1 + +docker rm "$interrupt_container_id" >"$output_dir/interrupt-remove.txt" +interrupt_container_id= +remove_bucket_and_verify "$interrupt_bucket" +interrupt_bucket_created=false + +database_snapshot "$output_dir/database-after.txt" + +if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \ + >"$output_dir/database-diff.txt"; then + echo "The encrypted backup drill changed source database counts." >&2 + exit 1 +fi + +minio_api=$("${compose[@]}" port minio 9000 | head -n 1) +minio_console=$("${compose[@]}" port minio 9001 | head -n 1) + +if [[ ! "$minio_api" =~ ^127[.]0[.]0[.]1:[1-9][0-9]*$ || + ! "$minio_console" =~ ^127[.]0[.]0[.]1:[1-9][0-9]*$ ]]; then + echo "MinIO ports are not bound to observed loopback addresses." >&2 + exit 1 +fi + +jq -n \ + --arg run_id "$LABEL" \ + --arg bucket "$bucket" \ + --arg snapshot_id "$snapshot_id" \ + --arg minio_api "http://$minio_api" \ + --arg minio_console "http://$minio_console" \ + --argjson corrupt_check_exit "$corrupt_check_status" \ + --argjson corrupt_restore_exit "$corrupt_restore_status" \ + --argjson interrupted_uploaded_objects "$uploaded_data_objects" \ + '{ + run_id: $run_id, + retained_bucket: $bucket, + snapshot_id: $snapshot_id, + encrypted_repository_check: "passed", + fresh_database_restore: "passed", + source_database_count_diff_bytes: 0, + corrupted_check_exit: $corrupt_check_exit, + corrupted_restore_exit: $corrupt_restore_exit, + interrupted_uploaded_objects: $interrupted_uploaded_objects, + interrupted_snapshot_count: 0, + interrupted_repository_check_after_prune: "passed", + minio_api: $minio_api, + minio_console: $minio_console + }' >"$output_dir/summary.json" + +trap - EXIT HUP INT TERM + +printf 'Encrypted S3 backup evidence: %s\n' "$output_dir" +printf 'Retained MinIO bucket: %s\nSnapshot: %s\n' "$bucket" "$snapshot_id" +printf 'MinIO API: http://%s\nMinIO console: http://%s\n' \ + "$minio_api" "$minio_console" diff --git a/scripts/ensure-local-load-env.sh b/scripts/ensure-local-load-env.sh index 4f72e94..1f893ab 100755 --- a/scripts/ensure-local-load-env.sh +++ b/scripts/ensure-local-load-env.sh @@ -28,6 +28,16 @@ if [ -f "$ENV_FILE" ]; then needs_observability_timeout=true needs_observability_grafana_user=true needs_observability_grafana_password=true + needs_backup_minio_api_port=true + needs_backup_minio_console_port=true + needs_backup_minio_root_user=true + needs_backup_minio_root_password=true + needs_backup_restic_password=true + needs_backup_bucket_prefix=true + needs_backup_timeout=true + needs_backup_interruption_chunks=true + needs_backup_interruption_chunk_bytes=true + needs_backup_interruption_interval=true grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" && @@ -53,6 +63,26 @@ if [ -f "$ENV_FILE" ]; then needs_observability_grafana_user=false grep -q '^OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=' "$ENV_FILE" && needs_observability_grafana_password=false + grep -q '^BACKUP_MINIO_API_PORT=' "$ENV_FILE" && + needs_backup_minio_api_port=false + grep -q '^BACKUP_MINIO_CONSOLE_PORT=' "$ENV_FILE" && + needs_backup_minio_console_port=false + grep -q '^BACKUP_MINIO_ROOT_USER=' "$ENV_FILE" && + needs_backup_minio_root_user=false + grep -q '^BACKUP_MINIO_ROOT_PASSWORD=' "$ENV_FILE" && + needs_backup_minio_root_password=false + grep -q '^BACKUP_RESTIC_PASSWORD=' "$ENV_FILE" && + needs_backup_restic_password=false + grep -q '^BACKUP_BUCKET_PREFIX=' "$ENV_FILE" && + needs_backup_bucket_prefix=false + grep -q '^BACKUP_TIMEOUT_SECONDS=' "$ENV_FILE" && + needs_backup_timeout=false + grep -q '^BACKUP_INTERRUPTION_CHUNKS=' "$ENV_FILE" && + needs_backup_interruption_chunks=false + grep -q '^BACKUP_INTERRUPTION_CHUNK_BYTES=' "$ENV_FILE" && + needs_backup_interruption_chunk_bytes=false + grep -q '^BACKUP_INTERRUPTION_INTERVAL_SECONDS=' "$ENV_FILE" && + needs_backup_interruption_interval=false if [ "$needs_fixture_password" = false ] && [ "$needs_resilience_timeout" = false ] && @@ -66,7 +96,17 @@ if [ -f "$ENV_FILE" ]; then [ "$needs_observability_evaluation_interval" = false ] && [ "$needs_observability_timeout" = false ] && [ "$needs_observability_grafana_user" = false ] && - [ "$needs_observability_grafana_password" = false ]; then + [ "$needs_observability_grafana_password" = false ] && + [ "$needs_backup_minio_api_port" = false ] && + [ "$needs_backup_minio_console_port" = false ] && + [ "$needs_backup_minio_root_user" = false ] && + [ "$needs_backup_minio_root_password" = false ] && + [ "$needs_backup_restic_password" = false ] && + [ "$needs_backup_bucket_prefix" = false ] && + [ "$needs_backup_timeout" = false ] && + [ "$needs_backup_interruption_chunks" = false ] && + [ "$needs_backup_interruption_chunk_bytes" = false ] && + [ "$needs_backup_interruption_interval" = false ]; then echo ".env.load already exists; no secret or experiment input was changed." exit 0 fi @@ -74,6 +114,9 @@ if [ -f "$ENV_FILE" ]; then umask 077 load_fixture_password= observability_grafana_admin_password= + backup_minio_root_user= + backup_minio_root_password= + backup_restic_password= if [ "$needs_fixture_password" = true ]; then load_fixture_password=$(openssl rand -hex 24) @@ -83,6 +126,18 @@ if [ -f "$ENV_FILE" ]; then observability_grafana_admin_password=$(openssl rand -hex 32) fi + if [ "$needs_backup_minio_root_user" = true ]; then + backup_minio_root_user="wnh$(openssl rand -hex 12)" + fi + + if [ "$needs_backup_minio_root_password" = true ]; then + backup_minio_root_password=$(openssl rand -hex 32) + fi + + if [ "$needs_backup_restic_password" = true ]; then + backup_restic_password=$(openssl rand -hex 32) + fi + { if [ "$needs_fixture_password" = true ]; then printf '\n# Added by the authenticated-load profile upgrade.\n' @@ -156,17 +211,76 @@ if [ -f "$ENV_FILE" ]; then printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' \ "$observability_grafana_admin_password" fi + + if [ "$needs_backup_minio_api_port" = true ] || + [ "$needs_backup_minio_console_port" = true ] || + [ "$needs_backup_minio_root_user" = true ] || + [ "$needs_backup_minio_root_password" = true ] || + [ "$needs_backup_restic_password" = true ] || + [ "$needs_backup_bucket_prefix" = true ] || + [ "$needs_backup_timeout" = true ] || + [ "$needs_backup_interruption_chunks" = true ] || + [ "$needs_backup_interruption_chunk_bytes" = true ] || + [ "$needs_backup_interruption_interval" = true ]; then + printf '\n# Added by the encrypted local backup-profile upgrade.\n' + fi + + if [ "$needs_backup_minio_api_port" = true ]; then + printf 'BACKUP_MINIO_API_PORT=0\n' + fi + + if [ "$needs_backup_minio_console_port" = true ]; then + printf 'BACKUP_MINIO_CONSOLE_PORT=0\n' + fi + + if [ "$needs_backup_minio_root_user" = true ]; then + printf 'BACKUP_MINIO_ROOT_USER=%s\n' "$backup_minio_root_user" + fi + + if [ "$needs_backup_minio_root_password" = true ]; then + printf 'BACKUP_MINIO_ROOT_PASSWORD=%s\n' "$backup_minio_root_password" + fi + + if [ "$needs_backup_restic_password" = true ]; then + printf 'BACKUP_RESTIC_PASSWORD=%s\n' "$backup_restic_password" + fi + + if [ "$needs_backup_bucket_prefix" = true ]; then + printf 'BACKUP_BUCKET_PREFIX=wnh-backup\n' + fi + + if [ "$needs_backup_timeout" = true ]; then + printf 'BACKUP_TIMEOUT_SECONDS=120\n' + fi + + if [ "$needs_backup_interruption_chunks" = true ]; then + printf 'BACKUP_INTERRUPTION_CHUNKS=300\n' + fi + + if [ "$needs_backup_interruption_chunk_bytes" = true ]; then + printf 'BACKUP_INTERRUPTION_CHUNK_BYTES=1048576\n' + fi + + if [ "$needs_backup_interruption_interval" = true ]; then + printf 'BACKUP_INTERRUPTION_INTERVAL_SECONDS=0.1\n' + fi } >>"$ENV_FILE" chmod 600 "$ENV_FILE" unset load_fixture_password observability_grafana_admin_password \ + backup_minio_root_user backup_minio_root_password backup_restic_password \ needs_fixture_password needs_resilience_timeout \ needs_resilience_interval needs_resilience_request_timeout \ needs_traefik_retry_attempts needs_observability_prometheus_port \ needs_observability_alertmanager_port needs_observability_grafana_port \ needs_observability_scrape_interval needs_observability_evaluation_interval \ needs_observability_timeout needs_observability_grafana_user \ - needs_observability_grafana_password - echo "Added missing load/resilience/observability inputs to ignored .env.load." + needs_observability_grafana_password needs_backup_minio_api_port \ + needs_backup_minio_console_port needs_backup_minio_root_user \ + needs_backup_minio_root_password needs_backup_restic_password \ + needs_backup_bucket_prefix needs_backup_timeout \ + needs_backup_interruption_chunks needs_backup_interruption_chunk_bytes \ + needs_backup_interruption_interval + echo "Added missing load/resilience/observability/backup inputs to ignored .env.load." exit 0 fi @@ -183,6 +297,9 @@ release_cookie=$(openssl rand -hex 64) metrics_token=$(openssl rand -hex 32) load_fixture_password=$(openssl rand -hex 24) observability_grafana_admin_password=$(openssl rand -hex 32) +backup_minio_root_user="wnh$(openssl rand -hex 12)" +backup_minio_root_password=$(openssl rand -hex 32) +backup_restic_password=$(openssl rand -hex 32) database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load" temporary=$(mktemp "${ENV_FILE}.XXXXXX") trap 'rm -f "$temporary"' EXIT HUP INT TERM @@ -195,6 +312,9 @@ RELEASE_COOKIE_VALUE=$release_cookie \ METRICS_TOKEN_VALUE=$metrics_token \ LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \ OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password \ +BACKUP_MINIO_ROOT_USER_VALUE=$backup_minio_root_user \ +BACKUP_MINIO_ROOT_PASSWORD_VALUE=$backup_minio_root_password \ +BACKUP_RESTIC_PASSWORD_VALUE=$backup_restic_password \ perl -0pe ' s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g; s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g; @@ -204,6 +324,9 @@ OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g; s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g; s/GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD/$ENV{OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE}/g; + s/GENERATE_BACKUP_MINIO_ROOT_USER/$ENV{BACKUP_MINIO_ROOT_USER_VALUE}/g; + s/GENERATE_BACKUP_MINIO_ROOT_PASSWORD/$ENV{BACKUP_MINIO_ROOT_PASSWORD_VALUE}/g; + s/GENERATE_BACKUP_RESTIC_PASSWORD/$ENV{BACKUP_RESTIC_PASSWORD_VALUE}/g; ' "$TEMPLATE" >"$temporary" if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then @@ -215,6 +338,8 @@ chmod 600 "$temporary" mv "$temporary" "$ENV_FILE" trap - EXIT HUP INT TERM unset postgres_password secret_key_base handover_secret release_cookie metrics_token \ - load_fixture_password observability_grafana_admin_password database_url + load_fixture_password observability_grafana_admin_password \ + backup_minio_root_user backup_minio_root_password backup_restic_password \ + database_url echo "Generated independent load-profile secrets in ignored .env.load." diff --git a/scripts/quality.sh b/scripts/quality.sh index 5c4ce46..ca7b500 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -18,6 +18,9 @@ quality_image="who-need-help:quality-$run_id" assets_image="who-need-help:assets-audit-$run_id" e2e_image="who-need-help:e2e-audit-$run_id" release_image="who-need-help:security-$run_id" +backup_image="who-need-help:backup-audit-$run_id" +minio_image="who-need-help:minio-audit-$run_id" +mc_image="who-need-help:mc-audit-$run_id" scan_dir=$(mktemp -d "${TMPDIR:-/tmp}/wnh-quality-scan.XXXXXX") scan_list="${scan_dir}.files" scan_tar="${scan_dir}.tar" @@ -32,6 +35,7 @@ compose="docker compose -p $project -f $ROOT/compose.quality.yaml" cleanup() { $compose down --volumes --remove-orphans >/dev/null 2>&1 || true docker image rm "$quality_image" "$assets_image" "$e2e_image" "$release_image" \ + "$backup_image" "$minio_image" "$mc_image" \ >/dev/null 2>&1 || true rm -rf "$scan_dir" "$scan_list" "$scan_tar" } @@ -49,7 +53,8 @@ docker run --rm \ $(find scripts -type f -name '*.sh' -print | sort) echo "Checking Dockerfiles with Hadolint 2.14.0" -for dockerfile in Dockerfile android/Dockerfile e2e/Dockerfile; do +for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \ + android/Dockerfile e2e/Dockerfile; do docker run --rm --interactive "$HADOLINT_IMAGE" \ hadolint --failure-threshold warning - <"$dockerfile" done @@ -73,6 +78,13 @@ OBSERVABILITY_RUNTIME_DIR="$scan_dir/observability-runtime" \ docker compose --env-file .env.load.example \ -f compose.yaml -f compose.load.yaml -f compose.observability.yaml \ --profile observability config --quiet +mkdir -p "$scan_dir/backup-runtime" +BACKUP_RUNTIME_DIR="$scan_dir/backup-runtime" \ +BACKUP_HOST_UID="$(id -u)" \ +BACKUP_HOST_GID="$(id -g)" \ + docker compose --env-file .env.load.example \ + -f compose.yaml -f compose.load.yaml -f compose.backup.yaml \ + --profile backup config --quiet docker compose -p "$project" -f compose.quality.yaml config --quiet echo "Validating local observability configuration" @@ -161,6 +173,76 @@ docker run --rm "$assets_image" npm audit --audit-level=high docker build --tag "$e2e_image" e2e docker run --rm "$e2e_image" npm audit --audit-level=high +echo "Building and scanning the pinned non-root backup tool image" +docker build --tag "$backup_image" --file Dockerfile.backup . +test "$(docker image inspect --format '{{.Config.User}}' "$backup_image")" = \ + "10001:10001" +backup_versions=$(docker run --rm \ + --user 10001:10001 \ + --read-only \ + --tmpfs /tmp \ + "$backup_image" \ + sh -euc 'restic version; pg_dump --version; test "$(id -u)" = 10001') +printf '%s\n' "$backup_versions" +printf '%s\n' "$backup_versions" | + grep -F 'restic 0.19.1 compiled with go1.26.5' >/dev/null +printf '%s\n' "$backup_versions" | + grep -F 'pg_dump (PostgreSQL) 18.4' >/dev/null +docker run --rm \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$ROOT/.tools/trivy-cache:/root/.cache/trivy" \ + "$TRIVY_IMAGE" image \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --exit-code 1 \ + "$backup_image" + +echo "Building and scanning the pinned non-root MinIO server and client images" +docker build --target server --tag "$minio_image" --file Dockerfile.minio . +docker build --target client --tag "$mc_image" --file Dockerfile.minio . +test "$(docker image inspect --format '{{.Config.User}}' "$minio_image")" = \ + "10001:10001" +test "$(docker image inspect --format '{{.Config.User}}' "$mc_image")" = \ + "10001:10001" +minio_version=$(docker run --rm \ + --user 10001:10001 \ + --read-only \ + --tmpfs /tmp \ + "$minio_image" \ + --version) +mc_version=$(docker run --rm \ + --user 10001:10001 \ + --read-only \ + --tmpfs /tmp \ + "$mc_image" \ + --version) +printf '%s\n' "$minio_version" +printf '%s\n' "$mc_version" +printf '%s\n' "$minio_version" | + grep -F 'RELEASE.2025-10-15T17-29-55Z' >/dev/null +printf '%s\n' "$minio_version" | + grep -F 'commit-id=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' >/dev/null +printf '%s\n' "$minio_version" | + grep -F 'Runtime: go1.26.5 linux/' >/dev/null +printf '%s\n' "$mc_version" | + grep -F 'RELEASE.2025-08-13T08-35-41Z' >/dev/null +printf '%s\n' "$mc_version" | + grep -F 'commit-id=7394ce0dd2a80935aded936b09fa12cbb3cb8096' >/dev/null +printf '%s\n' "$mc_version" | + grep -F 'Runtime: go1.26.5 linux/' >/dev/null +for image in "$minio_image" "$mc_image"; do + docker run --rm \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$ROOT/.tools/trivy-cache:/root/.cache/trivy" \ + "$TRIVY_IMAGE" image \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --exit-code 1 \ + "$image" +done + echo "Building and scanning the production release image" docker build --target release --tag "$release_image" . docker run --rm \