simpletest/who_need_help
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add verified encrypted S3 backups

Browse Source
This commit is contained in:
SimpleTest 2026-07-19 08:00:12 +03:00
parent 76a261b9ec
commit 1b60685454
12 changed files with 1197 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.env.load.example
View File

@ -67,3 +67,16 @@ OBSERVABILITY_EVALUATION_INTERVAL=1s
OBSERVABILITY_TIMEOUT_SECONDS=90 OBSERVABILITY_TIMEOUT_SECONDS=90
OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin
OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD
# The encrypted backup drill keeps its MinIO API and console on dynamically
# allocated loopback ports. Secrets are generated only in ignored .env.load.
BACKUP_MINIO_API_PORT=0
BACKUP_MINIO_CONSOLE_PORT=0
BACKUP_MINIO_ROOT_USER=GENERATE_BACKUP_MINIO_ROOT_USER
BACKUP_MINIO_ROOT_PASSWORD=GENERATE_BACKUP_MINIO_ROOT_PASSWORD
BACKUP_RESTIC_PASSWORD=GENERATE_BACKUP_RESTIC_PASSWORD
BACKUP_BUCKET_PREFIX=wnh-backup
BACKUP_TIMEOUT_SECONDS=120
BACKUP_INTERRUPTION_CHUNKS=300
BACKUP_INTERRUPTION_CHUNK_BYTES=1048576
BACKUP_INTERRUPTION_INTERVAL_SECONDS=0.1

20
Dockerfile.backup Normal file
View File

@ -0,0 +1,20 @@
FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS restic
RUN CGO_ENABLED=0 go install github.com/restic/restic/cmd/restic@v0.19.1
FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
RUN apk add --no-cache \
ca-certificates=20260611-r0 \
libcrypto3=3.5.7-r0 \
libssl3=3.5.7-r0 \
musl=1.2.5-r23 \
musl-utils=1.2.5-r23 \
postgresql18-client=18.4-r0 \
zlib=1.3.2-r0
COPY --from=restic /go/bin/restic /usr/local/bin/restic
ENTRYPOINT []
USER 10001:10001
CMD ["restic", "version"]

81
Dockerfile.minio Normal file
View File

@ -0,0 +1,81 @@
FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS minio_builder
ADD --checksum=sha256:45521908307306e925c98d629e1c17d78c8b72b6ee242b1bfb1409f7d8ee5841 \
https://github.com/minio/minio/archive/9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a.tar.gz \
/tmp/minio.tar.gz
WORKDIR /src
RUN tar --extract --gzip --file /tmp/minio.tar.gz \
--directory . --strip-components=1 \
&& go get \
github.com/apache/thrift@v0.23.0 \
github.com/buger/jsonparser@v1.1.2 \
github.com/prometheus/prometheus@v0.311.3 \
golang.org/x/crypto@v0.52.0 \
golang.org/x/net@v0.55.0 \
google.golang.org/grpc@v1.81.1 \
&& CGO_ENABLED=0 go build \
-mod=mod \
-trimpath \
-tags kqueue \
-ldflags="-s -w \
-X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z \
-X github.com/minio/minio/cmd.CopyrightYear=2025 \
-X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z \
-X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a \
-X github.com/minio/minio/cmd.ShortCommitID=9e49d5e7a648" \
-o /out/minio .
FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS mc_builder
ADD --checksum=sha256:95cd293c7119f16921a6dc515a1fb74a2227f19fd994b9c8b770a154e802ac44 \
https://github.com/minio/mc/archive/7394ce0dd2a80935aded936b09fa12cbb3cb8096.tar.gz \
/tmp/mc.tar.gz
WORKDIR /src
RUN tar --extract --gzip --file /tmp/mc.tar.gz \
--directory . --strip-components=1 \
&& go get \
github.com/prometheus/prometheus@v0.311.3 \
golang.org/x/crypto@v0.52.0 \
golang.org/x/net@v0.55.0 \
google.golang.org/grpc@v1.81.1 \
&& go mod tidy \
&& CGO_ENABLED=0 go build \
-trimpath \
-tags kqueue \
-ldflags="-s -w \
-X github.com/minio/mc/cmd.Version=2025-08-13T08:35:41Z \
-X github.com/minio/mc/cmd.CopyrightYear=2025 \
-X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-08-13T08-35-41Z \
-X github.com/minio/mc/cmd.CommitID=7394ce0dd2a80935aded936b09fa12cbb3cb8096 \
-X github.com/minio/mc/cmd.ShortCommitID=7394ce0dd2a8" \
-o /out/mc .
FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS runtime
RUN apk add --no-cache \
ca-certificates=20260611-r0 \
libcrypto3=3.5.7-r0 \
libssl3=3.5.7-r0 \
musl=1.2.5-r23 \
musl-utils=1.2.5-r23 \
zlib=1.3.2-r0
USER 10001:10001
FROM runtime AS server
COPY --from=minio_builder /out/minio /usr/local/bin/minio
ENTRYPOINT ["minio"]
CMD ["--version"]
FROM runtime AS client
COPY --from=mc_builder /out/mc /usr/local/bin/mc
ENTRYPOINT ["mc"]
CMD ["--version"]

23
README.md
View File

@ -92,6 +92,20 @@ backup_path=$(printf '%s\n' "$backup_output" | sed -n 's/^Backup: //p')
./scripts/restore-drill-compose.sh "$backup_path" ./scripts/restore-drill-compose.sh "$backup_path"
``` ```
For the reproducible encrypted S3-compatible drill, first start the isolated
load project and then run:
```bash
./scripts/load-stack-up.sh
./scripts/backup-s3-drill.sh local-encrypted-backup
```
The command generates MinIO and Restic secrets only in ignored mode-`0600`
`.env.load`, streams `pg_dump` directly into an encrypted Restic repository,
restores it into a new temporary database, checks corruption and interruption
failure paths, removes those temporary buckets, and retains the successful
encrypted bucket in local MinIO. It never writes a plaintext dump to the host.
The commands, boundaries, and unclaimed production properties are documented The commands, boundaries, and unclaimed production properties are documented
in [the operations runbook](docs/operations.md). in [the operations runbook](docs/operations.md).
@ -151,10 +165,11 @@ an isolated PostgreSQL volume:
It checks shell scripts, Dockerfiles, the GitHub Actions workflow, every Compose It checks shell scripts, Dockerfiles, the GitHub Actions workflow, every Compose
profile, the rendered Helm chart, tracked-source secrets and infrastructure profile, the rendered Helm chart, tracked-source secrets and infrastructure
misconfigurations, Elixir formatting/compilation/xref/Credo/Sobelow/Dialyzer, misconfigurations, Elixir formatting/compilation/xref/Credo/Sobelow/Dialyzer,
retired Hex packages, locked npm dependencies, all Phoenix tests, and the retired Hex packages, locked npm dependencies, all Phoenix tests, the pinned
production release image. Its generated database credentials are random and backup-tool, MinIO server/client, and production release images. Its generated
exist only for that run. The exact database volume, networks, temporary source database credentials are random and exist only for that run. The exact database
snapshot, and one-run images are removed automatically. volume, networks, temporary source snapshot, and one-run images are removed
automatically.
The cursor-pagination database benchmark also creates a one-run Compose The cursor-pagination database benchmark also creates a one-run Compose
project, random database credentials, and a separate PostgreSQL volume: project, random database credentials, and a separate PostgreSQL volume:

118
compose.backup.yaml Normal file
View File

@ -0,0 +1,118 @@
services:
minio-permissions:
image: who-need-help:minio-local
build:
context: .
dockerfile: Dockerfile.minio
target: server
entrypoint: ["/bin/sh", "-euc"]
command:
- chown -R 10001:10001 /data
volumes:
- minio_data:/data
user: "0:0"
read_only: true
cap_drop: [ALL]
cap_add: [CHOWN]
security_opt:
- no-new-privileges:true
networks: [internal]
restart: "no"
profiles: [backup]
minio:
image: who-need-help:minio-local
build:
context: .
dockerfile: Dockerfile.minio
target: server
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${BACKUP_MINIO_ROOT_USER:?Set BACKUP_MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${BACKUP_MINIO_ROOT_PASSWORD:?Set BACKUP_MINIO_ROOT_PASSWORD}
volumes:
- minio_data:/data
ports:
- target: 9000
published: "${BACKUP_MINIO_API_PORT:-0}"
host_ip: 127.0.0.1
protocol: tcp
- target: 9001
published: "${BACKUP_MINIO_CONSOLE_PORT:-0}"
host_ip: 127.0.0.1
protocol: tcp
healthcheck:
test: ["CMD", "wget", "--quiet", "--spider", "http://127.0.0.1:9000/minio/health/ready"]
interval: 2s
timeout: 3s
retries: 30
depends_on:
minio-permissions:
condition: service_completed_successfully
user: "10001:10001"
read_only: true
tmpfs:
- /tmp
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
networks: [internal, backup_host]
restart: unless-stopped
profiles: [backup]
minio-client:
image: who-need-help:mc-local
build:
context: .
dockerfile: Dockerfile.minio
target: client
entrypoint: ["mc"]
command: ["--version"]
environment:
MC_CONFIG_DIR: /tmp/mc
MC_HOST_local: http://${BACKUP_MINIO_ROOT_USER:?Set BACKUP_MINIO_ROOT_USER}:${BACKUP_MINIO_ROOT_PASSWORD:?Set BACKUP_MINIO_ROOT_PASSWORD}@minio:9000
volumes:
- ${BACKUP_RUNTIME_DIR:?Set BACKUP_RUNTIME_DIR for the backup profile}:/work
user: "${BACKUP_HOST_UID:?Set BACKUP_HOST_UID}:${BACKUP_HOST_GID:?Set BACKUP_HOST_GID}"
read_only: true
tmpfs:
- /tmp
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
networks: [internal]
profiles: [backup]
backup-tools:
image: who-need-help:backup-tools
build:
context: .
dockerfile: Dockerfile.backup
environment:
PGHOST: db
PGPORT: "5432"
PGUSER: ${POSTGRES_USER:?Set POSTGRES_USER}
PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
PGDATABASE: ${POSTGRES_DB:?Set POSTGRES_DB}
AWS_ACCESS_KEY_ID: ${BACKUP_MINIO_ROOT_USER:?Set BACKUP_MINIO_ROOT_USER}
AWS_SECRET_ACCESS_KEY: ${BACKUP_MINIO_ROOT_PASSWORD:?Set BACKUP_MINIO_ROOT_PASSWORD}
RESTIC_PASSWORD: ${BACKUP_RESTIC_PASSWORD:?Set BACKUP_RESTIC_PASSWORD}
RESTIC_CACHE_DIR: /tmp/restic-cache
BACKUP_INTERRUPTION_CHUNKS: ${BACKUP_INTERRUPTION_CHUNKS:?Set BACKUP_INTERRUPTION_CHUNKS}
BACKUP_INTERRUPTION_CHUNK_BYTES: ${BACKUP_INTERRUPTION_CHUNK_BYTES:?Set BACKUP_INTERRUPTION_CHUNK_BYTES}
BACKUP_INTERRUPTION_INTERVAL_SECONDS: ${BACKUP_INTERRUPTION_INTERVAL_SECONDS:?Set BACKUP_INTERRUPTION_INTERVAL_SECONDS}
user: "10001:10001"
read_only: true
tmpfs:
- /tmp
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
networks: [internal]
profiles: [backup]
networks:
backup_host:
volumes:
minio_data:

18
docs/dependency-baseline.md
View File

@ -42,6 +42,10 @@ package checksums are in `mix.lock` and `assets/package-lock.json`.
| Alertmanager | 0.33.1 | | Alertmanager | 0.33.1 |
| Grafana | 13.1.0 | | Grafana | 13.1.0 |
| Python alert-boundary runtime | 3.14.6 / Alpine 3.23 | | Python alert-boundary runtime | 3.14.6 / Alpine 3.23 |
| Restic | 0.19.1, rebuilt with Go 1.26.5 |
| MinIO server | RELEASE.2025-10-15T17-29-55Z, rebuilt with Go 1.26.5 |
| MinIO client | RELEASE.2025-08-13T08-35-41Z, rebuilt with Go 1.26.5 |
| Backup runtime | Alpine 3.23.3 / PostgreSQL client 18.4-r0 |
| Debian builder/runner snapshot | trixie-20260713-slim | | Debian builder/runner snapshot | trixie-20260713-slim |
Every external Compose/kind service image and every Dockerfile base image is Every external Compose/kind service image and every Dockerfile base image is
@ -49,6 +53,15 @@ locked to both an exact tag and an OCI digest. The observed local Docker tooling
was Docker Engine/CLI 29.6.2, Compose 5.3.1, and Buildx 0.35.0; their official was Docker Engine/CLI 29.6.2, Compose 5.3.1, and Buildx 0.35.0; their official
release feeds reported those same versions as current during verification. release feeds reported those same versions as current during verification.
`Dockerfile.minio` fetches checksum-pinned upstream source commits for MinIO
server and client and records every dependency override used to rebuild them.
The current overrides move Apache Thrift, jsonparser, Prometheus, Go crypto/net,
and gRPC dependencies to the versions checked by the image security gate.
Both projects are AGPLv3; anyone distributing or operating modified builds must
review and satisfy the applicable license obligations. This repository keeps
the exact upstream source identifiers and the complete modification/build
recipe; this statement is operational documentation, not legal advice.
## Android and local Kubernetes ## Android and local Kubernetes
| Component | Locked version | | Component | Locked version |
@ -106,6 +119,10 @@ Credo, Dialyxir, and Sobelow are locked in `mix.lock`.
- [Alertmanager releases](https://github.com/prometheus/alertmanager/releases) - [Alertmanager releases](https://github.com/prometheus/alertmanager/releases)
- [Grafana releases](https://github.com/grafana/grafana/releases) - [Grafana releases](https://github.com/grafana/grafana/releases)
- [Python releases](https://www.python.org/downloads/) - [Python releases](https://www.python.org/downloads/)
- [Go release history](https://go.dev/doc/devel/release)
- [Restic releases](https://github.com/restic/restic/releases)
- [MinIO releases](https://github.com/minio/minio/releases)
- [MinIO client releases](https://github.com/minio/mc/releases)
- [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes) - [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes)
- [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk) - [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk)
- [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests) - [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests)
@ -136,4 +153,5 @@ docker run --rm who-need-help:node-deps npm outdated --json
./scripts/android-build.sh ./scripts/android-build.sh
./scripts/android-instrumentation-test.sh ./scripts/android-instrumentation-test.sh
./scripts/observability-run.sh local-observability ./scripts/observability-run.sh local-observability
./scripts/backup-s3-drill.sh local-encrypted-backup
``` ```

9
docs/local-hardening-plan.md
View File

@ -14,7 +14,7 @@ item below unless the evidence column explicitly describes a local mock.
| Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset | | Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset |
| Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data | | Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data |
| Observability | Completed locally: protected per-process metrics feed a pinned Prometheus/Grafana/Alertmanager profile | All 3 direct web targets are up before/after the drill; a verified replica stop delivers firing and resolved webhooks; Grafana datasource/dashboard and an empty DB-count diff are retained | | Observability | Completed locally: protected per-process metrics feed a pinned Prometheus/Grafana/Alertmanager profile | All 3 direct web targets are up before/after the drill; a verified replica stop delivers firing and resolved webhooks; Grafana datasource/dashboard and an empty DB-count diff are retained |
| Backup | Validated local custom-format dump and restore drill exist | An encrypted artifact is uploaded to local S3-compatible MinIO and restored into a fresh database; corruption and interrupted-upload checks fail closed | | Backup | Completed locally: a plaintext-free Restic stream is retained in pinned local MinIO | The encrypted repository passes full-data checking and fresh-database restore; corrupted configuration and an interrupted upload fail closed, leave no snapshot, and their temporary buckets are removed |
| External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout | | External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout |
| Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits | | Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits |
@ -94,5 +94,12 @@ The goal remains open while any row lacks reproducible local evidence.
dashboard, and routes the `up == 0` rule through Alertmanager. The canonical dashboard, and routes the `up == 0` rule through Alertmanager. The canonical
drill observed both firing and resolved webhooks for the exact stopped drill observed both firing and resolved webhooks for the exact stopped
replica, restored all three targets, and left database counts unchanged. replica, restored all three targets, and left database counts unchanged.
- The encrypted backup profile streams PostgreSQL custom format through Restic
directly into local MinIO without a host plaintext archive. The canonical
drill restored 23 public tables with all 10 migrations and PostGIS 3.6.4,
rejected an intentionally corrupted repository, stopped a scoped upload
after an encrypted object reached MinIO, published no interrupted snapshot,
pruned the unreferenced packs, removed both temporary buckets and the
temporary restore database, and left source table counts unchanged.
- The remaining rows above are still pending; this document is not a - The remaining rows above are still pending; this document is not a
completion claim for the entire hardening goal. completion claim for the entire hardening goal.

56
docs/operations.md
View File

@ -26,6 +26,62 @@ globals such as roles and tablespaces are not part of `pg_dump`; deployment
credentials and database roles must be provisioned separately from secrets. credentials and database roles must be provisioned separately from secrets.
Local backup files on the same workstation are not an off-site backup. Local backup files on the same workstation are not an off-site backup.
## Encrypted local S3-compatible backup drill
The isolated load project can run a complete encrypted Restic/MinIO drill:
```bash
./scripts/load-stack-up.sh
./scripts/backup-s3-drill.sh local-encrypted-backup
```
The script refuses the staging Compose project and validates the project and
service labels of every pre-existing container in its scope. On first use,
`scripts/ensure-local-load-env.sh` generates independent random MinIO and
Restic credentials in ignored `.env.load` and restricts that file to mode
`0600`. MinIO publishes Docker-assigned ports only on `127.0.0.1`; the observed
API and console URLs are printed after a successful run.
The backup tool combines the matching PostgreSQL 18 client with pinned Restic
rebuilt on Go 1.26.5. MinIO server and client are also rebuilt as non-root
Alpine images from checksum-pinned upstream source commits with the exact
dependency updates recorded in `Dockerfile.minio`. The quality gate verifies
their reported release, commit, Go runtime, configured user, and current
HIGH/CRITICAL vulnerability scan. `restic backup --stdin-from-command` runs a
custom-format `pg_dump`, checks the producer exit status, encrypts the data,
and uploads it directly to MinIO. No plaintext database dump is written to the
host. The drill then:
1. runs `restic check --read-data`;
2. streams `restic dump` into `pg_restore --list`;
3. restores into a uniquely named database created from `template0`;
4. checks tables, current Ecto migrations, PostGIS, and release migration
readiness before removing that exact database;
5. clones and corrupts an isolated repository and requires both check and dump
to fail;
6. stops an exact scoped in-progress backup container only after encrypted
objects reach MinIO, requires zero published snapshots, prunes unreferenced
packs, and rechecks the repository;
7. removes and verifies removal of the corruption/interruption buckets and
requires source table counts to remain unchanged.
The successful encrypted bucket is deliberately retained in the named local
MinIO volume. Non-secret evidence is written under ignored
`output/backups-s3/<run-label>/`; the runtime scratch directory is under
ignored `tmp/backup-s3/`. Use a unique lowercase run label of at most 32
characters. The command refuses to replace an existing retained bucket.
MinIO server and client are AGPLv3. `Dockerfile.minio` identifies the exact
upstream source commits and contains the dependency changes and complete build
commands used here. Before distributing or publicly operating modified images,
review the license and make the corresponding source available as required;
this runbook does not provide legal advice.
This verifies encryption, local S3 protocol use, restore mechanics, and two
failure paths on the observed workstation. A MinIO volume on that same
workstation is not an off-site backup and does not establish production RPO,
RTO, retention, capacity, key custody, object locking, or database HA.
## Isolated restore drill ## Isolated restore drill
Run a real restore into a uniquely named temporary database: Run a real restore into a uniquely named temporary database:

25
docs/verification.md
View File

@ -21,6 +21,7 @@ results from product limits and unknown production properties.
| Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. | | Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. |
| Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. | | Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. |
| Local observability | Implemented and protocol-verified | Pinned Prometheus scraped all 3 direct load web targets with a file Bearer credential; Grafana provisioned a healthy datasource and four-panel dashboard; Alertmanager delivered firing and resolved webhooks for an induced scoped replica stop. | Local delivery does not establish production retention, notification-provider reliability, on-call policy, or measured alert thresholds. | | Local observability | Implemented and protocol-verified | Pinned Prometheus scraped all 3 direct load web targets with a file Bearer credential; Grafana provisioned a healthy datasource and four-panel dashboard; Alertmanager delivered firing and resolved webhooks for an induced scoped replica stop. | Local delivery does not establish production retention, notification-provider reliability, on-call policy, or measured alert thresholds. |
| Encrypted local backup | Implemented and failure-verified | Pinned Restic streamed PostgreSQL custom format into pinned local MinIO with no host plaintext dump, passed full-data checking and a fresh-database restore, rejected a corrupted repository, and published no snapshot for an interrupted upload. | The retained MinIO volume is on the same workstation; this is not off-site storage, database HA, or a production RPO/RTO/retention claim. |
## Reproducible checks ## Reproducible checks
@ -31,12 +32,15 @@ results from product limits and unknown production properties.
- `mix compile --force --warnings-as-errors` and - `mix compile --force --warnings-as-errors` and
`mix format --check-formatted`: passed against the same final source. `mix format --check-formatted`: passed against the same final source.
- `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning - `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning
threshold, actionlint 1.7.12, all five Compose renders, Helm lint, Trivy threshold, actionlint 1.7.12, all six Compose renders, Helm lint, Trivy
source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow
strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the
production-image vulnerability scan. The rendered Helm manifest and Debian backup/MinIO/mc/release-image vulnerability scans. The rendered Helm manifest
13.6 release image each reported zero HIGH/CRITICAL findings under the reported zero HIGH/CRITICAL misconfigurations; the Alpine backup, MinIO, and
configured gates. mc images and the Debian 13.6 release image each reported zero HIGH/CRITICAL
vulnerabilities under the configured gates. The backup binary reported
Restic 0.19.1 compiled with Go 1.26.5; MinIO and mc reported their pinned
commits and Go 1.26.5.
- Dialyzer passed with three path- and warning-specific documented filters and - Dialyzer passed with three path- and warning-specific documented filters and
zero unused filters. Two findings are opaque-type warnings at Ecto zero unused filters. Two findings are opaque-type warnings at Ecto
`Multi`/`MapSet` call sites; the third is the generated Gettext backend's `Multi`/`MapSet` call sites; the third is the generated Gettext backend's
@ -90,6 +94,19 @@ results from product limits and unknown production properties.
running, and the checked user/request/message counts were identical before running, and the checked user/request/message counts were identical before
and after. A subsequent full drill returned all monitoring services to and after. A subsequent full drill returned all monitoring services to
healthy. healthy.
- The encrypted S3 canonical drill restored 23 public application tables, all
10 current migrations, 14 categories, and PostGIS 3.6.4 into a fresh
temporary database. `restic check --read-data` passed for the retained
repository. Flipping the first byte of an isolated cloned repository made
both `check` and `dump` exit nonzero. Stopping the exact backup container
after uploaded encrypted objects produced exit 130 and zero snapshots;
Restic identified and pruned 32.809 MiB of unreferenced packs, then passed a
second full-data check. Only the retained canonical bucket remained, the
restore database and interruption container were absent, no configured
secret appeared in retained evidence, and the source database count diff was
empty. The obsolete successful-run bucket was then removed; only the final
canonical bucket remains. Evidence is retained at
`output/backups-s3/backup-canonical-20260719d`.
- The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium - The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium
specs against a fresh PostGIS volume with two web and two worker replicas on specs against a fresh PostGIS volume with two web and two worker replicas on
2026-07-19. The retained successful-run artifact directory is 2026-07-19. The retained successful-run artifact directory is

631
scripts/backup-s3-drill.sh Executable file
View File

@ -0,0 +1,631 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
ENV_FILE="$ROOT/.env.load"
LABEL=${1:-"backup-$(date -u +%Y%m%d%H%M%S)"}
if [[ ! -f "$ENV_FILE" ]]; then
echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
exit 1
fi
set -a
# shellcheck source=/dev/null
. "$ENV_FILE"
set +a
for name in LOAD_PROJECT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD DATABASE_URL \
BACKUP_MINIO_API_PORT BACKUP_MINIO_CONSOLE_PORT BACKUP_MINIO_ROOT_USER \
BACKUP_MINIO_ROOT_PASSWORD BACKUP_RESTIC_PASSWORD BACKUP_BUCKET_PREFIX \
BACKUP_TIMEOUT_SECONDS BACKUP_INTERRUPTION_CHUNKS \
BACKUP_INTERRUPTION_CHUNK_BYTES BACKUP_INTERRUPTION_INTERVAL_SECONDS; do
if [[ -z "${!name:-}" ]]; then
echo "$name is missing from .env.load" >&2
exit 1
fi
done
if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
echo "The encrypted backup drill must not use the staging Compose project." >&2
exit 1
fi
if [[ ! "$LABEL" =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]]; then
echo "Run label must be 1-32 lowercase letters, numbers, or dashes." >&2
exit 1
fi
if [[ ! "$BACKUP_BUCKET_PREFIX" =~ ^[a-z0-9][a-z0-9-]{1,19}$ ]]; then
echo "BACKUP_BUCKET_PREFIX must be 2-20 lowercase letters, numbers, or dashes." >&2
exit 1
fi
for name in BACKUP_MINIO_API_PORT BACKUP_MINIO_CONSOLE_PORT; do
if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then
echo "$name must be a TCP port number from 0 through 65535." >&2
exit 1
fi
done
for name in BACKUP_TIMEOUT_SECONDS BACKUP_INTERRUPTION_CHUNKS \
BACKUP_INTERRUPTION_CHUNK_BYTES; do
if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then
echo "$name must be a positive integer." >&2
exit 1
fi
done
if ! awk -v value="$BACKUP_INTERRUPTION_INTERVAL_SECONDS" \
'BEGIN {exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value > 0)}'; then
echo "BACKUP_INTERRUPTION_INTERVAL_SECONDS must be greater than zero." >&2
exit 1
fi
for command in awk dd diff docker grep head jq od sleep tr unlink; do
if ! command -v "$command" >/dev/null 2>&1; then
echo "Required command is unavailable: $command" >&2
exit 1
fi
done
bucket="$BACKUP_BUCKET_PREFIX-$LABEL"
corrupt_bucket="$bucket-corrupt"
interrupt_bucket="$bucket-interrupt"
for candidate in "$bucket" "$corrupt_bucket" "$interrupt_bucket"; do
if ((${#candidate} > 63)); then
echo "Derived S3 bucket name exceeds 63 characters: $candidate" >&2
exit 1
fi
done
runtime_dir="$ROOT/tmp/backup-s3/$LOAD_PROJECT/$LABEL"
output_dir="$ROOT/output/backups-s3/$LABEL"
mkdir -p "$runtime_dir" "$output_dir"
chmod 700 "$ROOT/tmp" "$ROOT/tmp/backup-s3" \
"$ROOT/tmp/backup-s3/$LOAD_PROJECT" "$runtime_dir" \
"$ROOT/output" "$ROOT/output/backups-s3" "$output_dir"
export BACKUP_RUNTIME_DIR="$runtime_dir"
export BACKUP_HOST_UID
export BACKUP_HOST_GID
BACKUP_HOST_UID=$(id -u)
BACKUP_HOST_GID=$(id -g)
compose=(
docker compose
--env-file "$ENV_FILE"
-p "$LOAD_PROJECT"
-f compose.yaml
-f compose.load.yaml
-f compose.backup.yaml
--profile backup
)
service_ids() {
"${compose[@]}" ps --all -q "$1"
}
assert_scope() {
local container_id=$1
local expected_service=$2
local observed_project observed_service
observed_project=$(
docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
"$container_id"
)
observed_service=$(
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
"$container_id"
)
if [[ "$observed_project" != "$LOAD_PROJECT" ||
"$observed_service" != "$expected_service" ]]; then
echo "Container scope mismatch for $container_id." >&2
exit 1
fi
}
mc_run() {
"${compose[@]}" run --rm --no-deps -T minio-client "$@"
}
restic_run_repo() {
local repository=$1
shift
"${compose[@]}" run --rm --no-deps -T \
--env "RESTIC_REPOSITORY=$repository" \
backup-tools restic "$@"
}
database_snapshot() {
# Variables are intentionally expanded inside the isolated PostGIS container.
# shellcheck disable=SC2016
"${compose[@]}" exec -T db sh -c \
'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
--username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL'
BEGIN READ ONLY;
SELECT 'users' AS table_name, count(*) AS row_count FROM users
UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
UNION ALL SELECT 'help_requests', count(*) FROM help_requests
UNION ALL SELECT 'messages', count(*) FROM messages
UNION ALL SELECT 'categories', count(*) FROM categories
UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
UNION ALL SELECT 'activities', count(*) FROM activities
UNION ALL SELECT 'reports', count(*) FROM reports
UNION ALL SELECT 'social_identities', count(*) FROM social_identities
UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations
ORDER BY table_name;
COMMIT;
SQL
}
wait_for_minio() {
local minio_id=$1
local deadline=$((SECONDS + BACKUP_TIMEOUT_SECONDS))
while ((SECONDS < deadline)); do
local state health
state=$(docker inspect --format '{{.State.Status}}' "$minio_id")
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \
"$minio_id")
if [[ "$state" == "running" && "$health" == "healthy" ]]; then
return 0
fi
sleep 1
done
echo "Timed out waiting for the scoped MinIO container." >&2
return 1
}
remove_bucket() {
local candidate=$1
mc_run rb --force "local/$candidate" >/dev/null 2>&1 || true
}
remove_bucket_and_verify() {
local candidate=$1
mc_run rb --force "local/$candidate" >/dev/null
if mc_run stat "local/$candidate" >/dev/null 2>&1; then
echo "Temporary MinIO bucket was not removed: $candidate" >&2
exit 1
fi
}
remove_interrupt_container() {
if [[ -n "${interrupt_container_id:-}" ]] &&
docker inspect "$interrupt_container_id" >/dev/null 2>&1; then
assert_scope "$interrupt_container_id" backup-tools
docker stop --time 1 "$interrupt_container_id" >/dev/null 2>&1 || true
docker rm "$interrupt_container_id" >/dev/null 2>&1 || true
fi
}
drop_drill_database() {
if [[ "${drill_database_created:-false}" == "true" ]]; then
"${compose[@]}" exec -T db \
dropdb --username "$POSTGRES_USER" --if-exists "$drill_db" >/dev/null 2>&1 || true
fi
}
cleanup() {
local status=$?
trap - EXIT HUP INT TERM
drop_drill_database
remove_interrupt_container
if [[ "${corrupt_bucket_created:-false}" == "true" ]]; then
remove_bucket "$corrupt_bucket"
fi
if [[ "${interrupt_bucket_created:-false}" == "true" ]]; then
remove_bucket "$interrupt_bucket"
fi
exit "$status"
}
trap cleanup EXIT HUP INT TERM
for service in minio-permissions minio minio-client backup-tools; do
while IFS= read -r existing_id; do
[[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service"
done < <(service_ids "$service")
done
database_snapshot "$output_dir/database-before.txt"
"${compose[@]}" build backup-tools minio minio-client \
>"$output_dir/backup-images-build.txt"
"${compose[@]}" up -d --wait minio >"$output_dir/minio-up.txt"
minio_id=$(service_ids minio | head -n 1)
assert_scope "$minio_id" minio
wait_for_minio "$minio_id"
if mc_run stat "local/$bucket" >/dev/null 2>&1; then
echo "Refusing to overwrite retained canonical bucket: $bucket" >&2
exit 1
fi
mc_run mb "local/$bucket" >"$output_dir/canonical-bucket-create.txt"
canonical_repository="s3:http://minio:9000/$bucket"
restic_run_repo "$canonical_repository" init >"$output_dir/restic-init.txt"
restic_run_repo "$canonical_repository" \
backup \
--json \
--host who-need-help-local \
--tag "$LABEL" \
--stdin-filename database.dump \
--stdin-from-command \
-- \
pg_dump \
--host db \
--username "$POSTGRES_USER" \
--dbname "$POSTGRES_DB" \
--format custom \
--no-owner \
>"$output_dir/backup.jsonl"
restic_run_repo "$canonical_repository" snapshots --json --tag "$LABEL" \
>"$output_dir/snapshots.json"
if ! jq -e --arg label "$LABEL" '
length == 1 and
.[0].tags == [$label] and
(.[0].id | length) == 64
' "$output_dir/snapshots.json" >/dev/null; then
echo "The canonical encrypted backup did not create exactly one tagged snapshot." >&2
exit 1
fi
snapshot_id=$(jq -r '.[0].id' "$output_dir/snapshots.json")
restic_run_repo "$canonical_repository" check --read-data \
>"$output_dir/canonical-check.txt" 2>&1
restic_run_repo "$canonical_repository" ls --json "$snapshot_id" \
>"$output_dir/snapshot-files.jsonl"
if ! jq -s -e '
any(
.[];
.struct_type == "node" and
.path == "/database.dump" and
.type == "file" and
.size > 0
)
' "$output_dir/snapshot-files.jsonl" >/dev/null; then
echo "The encrypted snapshot does not contain a non-empty database.dump." >&2
exit 1
fi
mc_run ls --recursive --json "local/$bucket" \
>"$output_dir/minio-objects.jsonl"
if ! jq -s -e '
length > 0 and
any(.[]; (.key // "") | startswith("data/"))
' "$output_dir/minio-objects.jsonl" >/dev/null; then
echo "MinIO did not retain encrypted Restic data objects." >&2
exit 1
fi
pack_key=$(
jq -sr '
[.[] | select((.key // "") | startswith("data/")) | .key]
| sort
| first
' "$output_dir/minio-objects.jsonl"
)
mc_run cat "local/$bucket/$pack_key" |
dd of="$runtime_dir/encrypted-prefix.bin" bs=1 count=16 status=none
od -An -tx1 -v "$runtime_dir/encrypted-prefix.bin" |
tr -d ' \n' >"$output_dir/encrypted-prefix.hex"
if grep -aFq 'PGDMP' "$runtime_dir/encrypted-prefix.bin"; then
echo "A Restic data object exposed the PostgreSQL custom-format header." >&2
exit 1
fi
restic_run_repo "$canonical_repository" dump "$snapshot_id" /database.dump |
"${compose[@]}" exec -T db pg_restore --list \
>"$output_dir/pg-restore-list.txt"
drill_db="wnh_s3_restore_${LABEL//-/_}_$$"
"${compose[@]}" exec -T db \
createdb \
--username "$POSTGRES_USER" \
--template template0 \
"$drill_db"
drill_database_created=true
restic_run_repo "$canonical_repository" dump "$snapshot_id" /database.dump |
"${compose[@]}" exec -T db \
pg_restore \
--username "$POSTGRES_USER" \
--dbname "$drill_db" \
--exit-on-error \
--no-owner \
--no-privileges
"${compose[@]}" exec -T db \
psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
--username "$POSTGRES_USER" --dbname "$drill_db" \
>"$output_dir/restored-database.json" <<'SQL'
SELECT json_build_object(
'public_table_count',
(
SELECT count(*)
FROM pg_tables
WHERE schemaname = 'public'
AND tablename <> 'spatial_ref_sys'
),
'migration_count', (SELECT count(*) FROM schema_migrations),
'user_count', (SELECT count(*) FROM users),
'request_count', (SELECT count(*) FROM help_requests),
'message_count', (SELECT count(*) FROM messages),
'category_count', (SELECT count(*) FROM categories),
'postgis_version', PostGIS_Lib_Version()
);
SQL
if ! jq -e '
.public_table_count > 0 and
.migration_count > 0 and
.category_count > 0 and
(.postgis_version | type == "string" and length > 0)
' "$output_dir/restored-database.json" >/dev/null; then
echo "The fresh restore database failed structural validation." >&2
exit 1
fi
case "$DATABASE_URL" in
*\?*)
database_base=${DATABASE_URL%%\?*}
database_query="?${DATABASE_URL#*\?}"
;;
*)
database_base=$DATABASE_URL
database_query=
;;
esac
database_prefix=${database_base%/*}
if [[ "$database_prefix" == "$database_base" ]]; then
echo "Could not derive the temporary drill DATABASE_URL." >&2
exit 1
fi
drill_database_url="$database_prefix/$drill_db$database_query"
"${compose[@]}" run --rm --no-deps \
--env APP_ROLE=migrate \
--env "DATABASE_URL=$drill_database_url" \
migrate /app/bin/migrate \
>"$output_dir/restore-migrate.txt"
"${compose[@]}" run --rm --no-deps \
--env APP_ROLE=migrate \
--env "DATABASE_URL=$drill_database_url" \
migrate /app/bin/who_need_help eval 'WhoNeedHelp.Release.await_migrations()' \
>"$output_dir/restore-migration-readiness.txt"
"${compose[@]}" exec -T db \
dropdb --username "$POSTGRES_USER" "$drill_db"
drill_database_created=false
remaining=$(
"${compose[@]}" exec -T db \
psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
--username "$POSTGRES_USER" --dbname postgres \
--command "SELECT count(*) FROM pg_database WHERE datname = '$drill_db';"
)
if [[ "$remaining" != "0" ]]; then
echo "Restore drill database was not removed: $drill_db" >&2
exit 1
fi
mc_run mb "local/$corrupt_bucket" >"$output_dir/corrupt-bucket-create.txt"
corrupt_bucket_created=true
mc_run mirror "local/$bucket" "local/$corrupt_bucket" \
>"$output_dir/corrupt-mirror.txt"
mc_run cp "local/$corrupt_bucket/config" /work/corrupt-config \
>"$output_dir/corrupt-download.txt"
first_byte=$(od -An -tu1 -N1 "$runtime_dir/corrupt-config" | tr -d ' ')
if [[ "$first_byte" == "0" ]]; then
printf '\001' | dd of="$runtime_dir/corrupt-config" bs=1 count=1 conv=notrunc status=none
else
printf '\000' | dd of="$runtime_dir/corrupt-config" bs=1 count=1 conv=notrunc status=none
fi
mc_run cp /work/corrupt-config "local/$corrupt_bucket/config" \
>"$output_dir/corrupt-upload.txt"
corrupt_repository="s3:http://minio:9000/$corrupt_bucket"
set +e
restic_run_repo "$corrupt_repository" check --read-data \
>"$output_dir/corrupt-check.txt" 2>&1
corrupt_check_status=$?
restic_run_repo "$corrupt_repository" dump "$snapshot_id" /database.dump \
>"/dev/null" 2>"$output_dir/corrupt-restore.txt"
corrupt_restore_status=$?
set -e
if [[ "$corrupt_check_status" -eq 0 || "$corrupt_restore_status" -eq 0 ]]; then
echo "The corrupted encrypted repository did not fail closed." >&2
exit 1
fi
remove_bucket_and_verify "$corrupt_bucket"
corrupt_bucket_created=false
unlink "$runtime_dir/corrupt-config"
mc_run mb "local/$interrupt_bucket" >"$output_dir/interrupt-bucket-create.txt"
interrupt_bucket_created=true
interrupt_repository="s3:http://minio:9000/$interrupt_bucket"
restic_run_repo "$interrupt_repository" init >"$output_dir/interrupt-init.txt"
interrupt_container="$LOAD_PROJECT-backup-interrupt-$LABEL"
if docker inspect "$interrupt_container" >/dev/null 2>&1; then
echo "Refusing to replace an existing interruption container: $interrupt_container" >&2
exit 1
fi
# The producer variables must expand inside the isolated backup-tools container.
# shellcheck disable=SC2016
interrupt_container_id=$(
"${compose[@]}" run -d \
--name "$interrupt_container" \
--no-deps \
--env "RESTIC_REPOSITORY=$interrupt_repository" \
backup-tools \
restic backup \
--json \
--host who-need-help-local \
--tag "$LABEL-interrupted" \
--stdin-filename interrupted.bin \
--stdin-from-command \
-- \
/bin/sh -euc \
'i=0
while [ "$i" -lt "$BACKUP_INTERRUPTION_CHUNKS" ]; do
head -c "$BACKUP_INTERRUPTION_CHUNK_BYTES" /dev/urandom
sleep "$BACKUP_INTERRUPTION_INTERVAL_SECONDS"
i=$((i + 1))
done'
)
assert_scope "$interrupt_container_id" backup-tools
upload_deadline=$((SECONDS + BACKUP_TIMEOUT_SECONDS))
uploaded_data_objects=0
while ((SECONDS < upload_deadline)); do
mc_run ls --recursive --json "local/$interrupt_bucket/data" \
>"$output_dir/interrupt-objects-current.jsonl" 2>/dev/null || true
uploaded_data_objects=$(
jq -s '[.[] | select(.type == "file")] | length' \
"$output_dir/interrupt-objects-current.jsonl"
)
if ((uploaded_data_objects > 0)); then
break
fi
if [[ "$(docker inspect --format '{{.State.Running}}' "$interrupt_container_id")" != "true" ]]; then
echo "Interruption producer exited before an uploaded data object was observed." >&2
exit 1
fi
sleep 1
done
if ((uploaded_data_objects == 0)); then
echo "Timed out waiting for the interruption upload to reach MinIO." >&2
exit 1
fi
docker stop --time 1 "$interrupt_container_id" >"$output_dir/interrupt-stop.txt"
docker inspect "$interrupt_container_id" |
jq '.[0] | {
id: .Id,
project: .Config.Labels["com.docker.compose.project"],
service: .Config.Labels["com.docker.compose.service"],
exit_code: .State.ExitCode,
state: .State.Status
}' >"$output_dir/interrupt-container.json"
docker logs "$interrupt_container_id" >"$output_dir/interrupt-backup.log" 2>&1
if ! jq -e '.exit_code != 0 and .state == "exited"' \
"$output_dir/interrupt-container.json" >/dev/null; then
echo "The interrupted upload container did not exit unsuccessfully." >&2
exit 1
fi
restic_run_repo "$interrupt_repository" snapshots \
--json --tag "$LABEL-interrupted" \
>"$output_dir/interrupt-snapshots.json"
if ! jq -e 'length == 0' "$output_dir/interrupt-snapshots.json" >/dev/null; then
echo "An interrupted upload published a restorable snapshot." >&2
exit 1
fi
mc_run ls --incomplete --recursive --json "local/$interrupt_bucket" \
>"$output_dir/interrupt-incomplete-uploads.jsonl" 2>/dev/null || true
restic_run_repo "$interrupt_repository" check --read-data \
>"$output_dir/interrupt-check-before-prune.txt" 2>&1
restic_run_repo "$interrupt_repository" prune \
>"$output_dir/interrupt-prune.txt" 2>&1
restic_run_repo "$interrupt_repository" check --read-data \
>"$output_dir/interrupt-check-after-prune.txt" 2>&1
docker rm "$interrupt_container_id" >"$output_dir/interrupt-remove.txt"
interrupt_container_id=
remove_bucket_and_verify "$interrupt_bucket"
interrupt_bucket_created=false
database_snapshot "$output_dir/database-after.txt"
if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \
>"$output_dir/database-diff.txt"; then
echo "The encrypted backup drill changed source database counts." >&2
exit 1
fi
minio_api=$("${compose[@]}" port minio 9000 | head -n 1)
minio_console=$("${compose[@]}" port minio 9001 | head -n 1)
if [[ ! "$minio_api" =~ ^127[.]0[.]0[.]1:[1-9][0-9]*$ ||
! "$minio_console" =~ ^127[.]0[.]0[.]1:[1-9][0-9]*$ ]]; then
echo "MinIO ports are not bound to observed loopback addresses." >&2
exit 1
fi
jq -n \
--arg run_id "$LABEL" \
--arg bucket "$bucket" \
--arg snapshot_id "$snapshot_id" \
--arg minio_api "http://$minio_api" \
--arg minio_console "http://$minio_console" \
--argjson corrupt_check_exit "$corrupt_check_status" \
--argjson corrupt_restore_exit "$corrupt_restore_status" \
--argjson interrupted_uploaded_objects "$uploaded_data_objects" \
'{
run_id: $run_id,
retained_bucket: $bucket,
snapshot_id: $snapshot_id,
encrypted_repository_check: "passed",
fresh_database_restore: "passed",
source_database_count_diff_bytes: 0,
corrupted_check_exit: $corrupt_check_exit,
corrupted_restore_exit: $corrupt_restore_exit,
interrupted_uploaded_objects: $interrupted_uploaded_objects,
interrupted_snapshot_count: 0,
interrupted_repository_check_after_prune: "passed",
minio_api: $minio_api,
minio_console: $minio_console
}' >"$output_dir/summary.json"
trap - EXIT HUP INT TERM
printf 'Encrypted S3 backup evidence: %s\n' "$output_dir"
printf 'Retained MinIO bucket: %s\nSnapshot: %s\n' "$bucket" "$snapshot_id"
printf 'MinIO API: http://%s\nMinIO console: http://%s\n' \
"$minio_api" "$minio_console"

133
scripts/ensure-local-load-env.sh
View File

@ -28,6 +28,16 @@ if [ -f "$ENV_FILE" ]; then
needs_observability_timeout=true needs_observability_timeout=true
needs_observability_grafana_user=true needs_observability_grafana_user=true
needs_observability_grafana_password=true needs_observability_grafana_password=true
needs_backup_minio_api_port=true
needs_backup_minio_console_port=true
needs_backup_minio_root_user=true
needs_backup_minio_root_password=true
needs_backup_restic_password=true
needs_backup_bucket_prefix=true
needs_backup_timeout=true
needs_backup_interruption_chunks=true
needs_backup_interruption_chunk_bytes=true
needs_backup_interruption_interval=true
grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false
grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" && grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" &&
@ -53,6 +63,26 @@ if [ -f "$ENV_FILE" ]; then
needs_observability_grafana_user=false needs_observability_grafana_user=false
grep -q '^OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=' "$ENV_FILE" && grep -q '^OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=' "$ENV_FILE" &&
needs_observability_grafana_password=false needs_observability_grafana_password=false
grep -q '^BACKUP_MINIO_API_PORT=' "$ENV_FILE" &&
needs_backup_minio_api_port=false
grep -q '^BACKUP_MINIO_CONSOLE_PORT=' "$ENV_FILE" &&
needs_backup_minio_console_port=false
grep -q '^BACKUP_MINIO_ROOT_USER=' "$ENV_FILE" &&
needs_backup_minio_root_user=false
grep -q '^BACKUP_MINIO_ROOT_PASSWORD=' "$ENV_FILE" &&
needs_backup_minio_root_password=false
grep -q '^BACKUP_RESTIC_PASSWORD=' "$ENV_FILE" &&
needs_backup_restic_password=false
grep -q '^BACKUP_BUCKET_PREFIX=' "$ENV_FILE" &&
needs_backup_bucket_prefix=false
grep -q '^BACKUP_TIMEOUT_SECONDS=' "$ENV_FILE" &&
needs_backup_timeout=false
grep -q '^BACKUP_INTERRUPTION_CHUNKS=' "$ENV_FILE" &&
needs_backup_interruption_chunks=false
grep -q '^BACKUP_INTERRUPTION_CHUNK_BYTES=' "$ENV_FILE" &&
needs_backup_interruption_chunk_bytes=false
grep -q '^BACKUP_INTERRUPTION_INTERVAL_SECONDS=' "$ENV_FILE" &&
needs_backup_interruption_interval=false
if [ "$needs_fixture_password" = false ] && if [ "$needs_fixture_password" = false ] &&
[ "$needs_resilience_timeout" = false ] && [ "$needs_resilience_timeout" = false ] &&
@ -66,7 +96,17 @@ if [ -f "$ENV_FILE" ]; then
[ "$needs_observability_evaluation_interval" = false ] && [ "$needs_observability_evaluation_interval" = false ] &&
[ "$needs_observability_timeout" = false ] && [ "$needs_observability_timeout" = false ] &&
[ "$needs_observability_grafana_user" = false ] && [ "$needs_observability_grafana_user" = false ] &&
[ "$needs_observability_grafana_password" = false ]; then [ "$needs_observability_grafana_password" = false ] &&
[ "$needs_backup_minio_api_port" = false ] &&
[ "$needs_backup_minio_console_port" = false ] &&
[ "$needs_backup_minio_root_user" = false ] &&
[ "$needs_backup_minio_root_password" = false ] &&
[ "$needs_backup_restic_password" = false ] &&
[ "$needs_backup_bucket_prefix" = false ] &&
[ "$needs_backup_timeout" = false ] &&
[ "$needs_backup_interruption_chunks" = false ] &&
[ "$needs_backup_interruption_chunk_bytes" = false ] &&
[ "$needs_backup_interruption_interval" = false ]; then
echo ".env.load already exists; no secret or experiment input was changed." echo ".env.load already exists; no secret or experiment input was changed."
exit 0 exit 0
fi fi
@ -74,6 +114,9 @@ if [ -f "$ENV_FILE" ]; then
umask 077 umask 077
load_fixture_password= load_fixture_password=
observability_grafana_admin_password= observability_grafana_admin_password=
backup_minio_root_user=
backup_minio_root_password=
backup_restic_password=
if [ "$needs_fixture_password" = true ]; then if [ "$needs_fixture_password" = true ]; then
load_fixture_password=$(openssl rand -hex 24) load_fixture_password=$(openssl rand -hex 24)
@ -83,6 +126,18 @@ if [ -f "$ENV_FILE" ]; then
observability_grafana_admin_password=$(openssl rand -hex 32) observability_grafana_admin_password=$(openssl rand -hex 32)
fi fi
if [ "$needs_backup_minio_root_user" = true ]; then
backup_minio_root_user="wnh$(openssl rand -hex 12)"
fi
if [ "$needs_backup_minio_root_password" = true ]; then
backup_minio_root_password=$(openssl rand -hex 32)
fi
if [ "$needs_backup_restic_password" = true ]; then
backup_restic_password=$(openssl rand -hex 32)
fi
{ {
if [ "$needs_fixture_password" = true ]; then if [ "$needs_fixture_password" = true ]; then
printf '\n# Added by the authenticated-load profile upgrade.\n' printf '\n# Added by the authenticated-load profile upgrade.\n'
@ -156,17 +211,76 @@ if [ -f "$ENV_FILE" ]; then
printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' \ printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' \
"$observability_grafana_admin_password" "$observability_grafana_admin_password"
fi fi
if [ "$needs_backup_minio_api_port" = true ] ||
[ "$needs_backup_minio_console_port" = true ] ||
[ "$needs_backup_minio_root_user" = true ] ||
[ "$needs_backup_minio_root_password" = true ] ||
[ "$needs_backup_restic_password" = true ] ||
[ "$needs_backup_bucket_prefix" = true ] ||
[ "$needs_backup_timeout" = true ] ||
[ "$needs_backup_interruption_chunks" = true ] ||
[ "$needs_backup_interruption_chunk_bytes" = true ] ||
[ "$needs_backup_interruption_interval" = true ]; then
printf '\n# Added by the encrypted local backup-profile upgrade.\n'
fi
if [ "$needs_backup_minio_api_port" = true ]; then
printf 'BACKUP_MINIO_API_PORT=0\n'
fi
if [ "$needs_backup_minio_console_port" = true ]; then
printf 'BACKUP_MINIO_CONSOLE_PORT=0\n'
fi
if [ "$needs_backup_minio_root_user" = true ]; then
printf 'BACKUP_MINIO_ROOT_USER=%s\n' "$backup_minio_root_user"
fi
if [ "$needs_backup_minio_root_password" = true ]; then
printf 'BACKUP_MINIO_ROOT_PASSWORD=%s\n' "$backup_minio_root_password"
fi
if [ "$needs_backup_restic_password" = true ]; then
printf 'BACKUP_RESTIC_PASSWORD=%s\n' "$backup_restic_password"
fi
if [ "$needs_backup_bucket_prefix" = true ]; then
printf 'BACKUP_BUCKET_PREFIX=wnh-backup\n'
fi
if [ "$needs_backup_timeout" = true ]; then
printf 'BACKUP_TIMEOUT_SECONDS=120\n'
fi
if [ "$needs_backup_interruption_chunks" = true ]; then
printf 'BACKUP_INTERRUPTION_CHUNKS=300\n'
fi
if [ "$needs_backup_interruption_chunk_bytes" = true ]; then
printf 'BACKUP_INTERRUPTION_CHUNK_BYTES=1048576\n'
fi
if [ "$needs_backup_interruption_interval" = true ]; then
printf 'BACKUP_INTERRUPTION_INTERVAL_SECONDS=0.1\n'
fi
} >>"$ENV_FILE" } >>"$ENV_FILE"
chmod 600 "$ENV_FILE" chmod 600 "$ENV_FILE"
unset load_fixture_password observability_grafana_admin_password \ unset load_fixture_password observability_grafana_admin_password \
backup_minio_root_user backup_minio_root_password backup_restic_password \
needs_fixture_password needs_resilience_timeout \ needs_fixture_password needs_resilience_timeout \
needs_resilience_interval needs_resilience_request_timeout \ needs_resilience_interval needs_resilience_request_timeout \
needs_traefik_retry_attempts needs_observability_prometheus_port \ needs_traefik_retry_attempts needs_observability_prometheus_port \
needs_observability_alertmanager_port needs_observability_grafana_port \ needs_observability_alertmanager_port needs_observability_grafana_port \
needs_observability_scrape_interval needs_observability_evaluation_interval \ needs_observability_scrape_interval needs_observability_evaluation_interval \
needs_observability_timeout needs_observability_grafana_user \ needs_observability_timeout needs_observability_grafana_user \
needs_observability_grafana_password needs_observability_grafana_password needs_backup_minio_api_port \
echo "Added missing load/resilience/observability inputs to ignored .env.load." needs_backup_minio_console_port needs_backup_minio_root_user \
needs_backup_minio_root_password needs_backup_restic_password \
needs_backup_bucket_prefix needs_backup_timeout \
needs_backup_interruption_chunks needs_backup_interruption_chunk_bytes \
needs_backup_interruption_interval
echo "Added missing load/resilience/observability/backup inputs to ignored .env.load."
exit 0 exit 0
fi fi
@ -183,6 +297,9 @@ release_cookie=$(openssl rand -hex 64)
metrics_token=$(openssl rand -hex 32) metrics_token=$(openssl rand -hex 32)
load_fixture_password=$(openssl rand -hex 24) load_fixture_password=$(openssl rand -hex 24)
observability_grafana_admin_password=$(openssl rand -hex 32) observability_grafana_admin_password=$(openssl rand -hex 32)
backup_minio_root_user="wnh$(openssl rand -hex 12)"
backup_minio_root_password=$(openssl rand -hex 32)
backup_restic_password=$(openssl rand -hex 32)
database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load" database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load"
temporary=$(mktemp "${ENV_FILE}.XXXXXX") temporary=$(mktemp "${ENV_FILE}.XXXXXX")
trap 'rm -f "$temporary"' EXIT HUP INT TERM trap 'rm -f "$temporary"' EXIT HUP INT TERM
@ -195,6 +312,9 @@ RELEASE_COOKIE_VALUE=$release_cookie \
METRICS_TOKEN_VALUE=$metrics_token \ METRICS_TOKEN_VALUE=$metrics_token \
LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \ LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \
OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password \ OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password \
BACKUP_MINIO_ROOT_USER_VALUE=$backup_minio_root_user \
BACKUP_MINIO_ROOT_PASSWORD_VALUE=$backup_minio_root_password \
BACKUP_RESTIC_PASSWORD_VALUE=$backup_restic_password \
perl -0pe ' perl -0pe '
s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g; s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g;
s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g; s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g;
@ -204,6 +324,9 @@ OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password
s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g; s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g;
s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g; s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g;
s/GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD/$ENV{OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE}/g; s/GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD/$ENV{OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE}/g;
s/GENERATE_BACKUP_MINIO_ROOT_USER/$ENV{BACKUP_MINIO_ROOT_USER_VALUE}/g;
s/GENERATE_BACKUP_MINIO_ROOT_PASSWORD/$ENV{BACKUP_MINIO_ROOT_PASSWORD_VALUE}/g;
s/GENERATE_BACKUP_RESTIC_PASSWORD/$ENV{BACKUP_RESTIC_PASSWORD_VALUE}/g;
' "$TEMPLATE" >"$temporary" ' "$TEMPLATE" >"$temporary"
if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then
@ -215,6 +338,8 @@ chmod 600 "$temporary"
mv "$temporary" "$ENV_FILE" mv "$temporary" "$ENV_FILE"
trap - EXIT HUP INT TERM trap - EXIT HUP INT TERM
unset postgres_password secret_key_base handover_secret release_cookie metrics_token \ unset postgres_password secret_key_base handover_secret release_cookie metrics_token \
load_fixture_password observability_grafana_admin_password database_url load_fixture_password observability_grafana_admin_password \
backup_minio_root_user backup_minio_root_password backup_restic_password \
database_url
echo "Generated independent load-profile secrets in ignored .env.load." echo "Generated independent load-profile secrets in ignored .env.load."

84
scripts/quality.sh
View File

@ -18,6 +18,9 @@ quality_image="who-need-help:quality-$run_id"
assets_image="who-need-help:assets-audit-$run_id" assets_image="who-need-help:assets-audit-$run_id"
e2e_image="who-need-help:e2e-audit-$run_id" e2e_image="who-need-help:e2e-audit-$run_id"
release_image="who-need-help:security-$run_id" release_image="who-need-help:security-$run_id"
backup_image="who-need-help:backup-audit-$run_id"
minio_image="who-need-help:minio-audit-$run_id"
mc_image="who-need-help:mc-audit-$run_id"
scan_dir=$(mktemp -d "${TMPDIR:-/tmp}/wnh-quality-scan.XXXXXX") scan_dir=$(mktemp -d "${TMPDIR:-/tmp}/wnh-quality-scan.XXXXXX")
scan_list="${scan_dir}.files" scan_list="${scan_dir}.files"
scan_tar="${scan_dir}.tar" scan_tar="${scan_dir}.tar"
@ -32,6 +35,7 @@ compose="docker compose -p $project -f $ROOT/compose.quality.yaml"
cleanup() { cleanup() {
$compose down --volumes --remove-orphans >/dev/null 2>&1 || true $compose down --volumes --remove-orphans >/dev/null 2>&1 || true
docker image rm "$quality_image" "$assets_image" "$e2e_image" "$release_image" \ docker image rm "$quality_image" "$assets_image" "$e2e_image" "$release_image" \
"$backup_image" "$minio_image" "$mc_image" \
>/dev/null 2>&1 || true >/dev/null 2>&1 || true
rm -rf "$scan_dir" "$scan_list" "$scan_tar" rm -rf "$scan_dir" "$scan_list" "$scan_tar"
} }
@ -49,7 +53,8 @@ docker run --rm \
$(find scripts -type f -name '*.sh' -print | sort) $(find scripts -type f -name '*.sh' -print | sort)
echo "Checking Dockerfiles with Hadolint 2.14.0" echo "Checking Dockerfiles with Hadolint 2.14.0"
for dockerfile in Dockerfile android/Dockerfile e2e/Dockerfile; do for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \
android/Dockerfile e2e/Dockerfile; do
docker run --rm --interactive "$HADOLINT_IMAGE" \ docker run --rm --interactive "$HADOLINT_IMAGE" \
hadolint --failure-threshold warning - <"$dockerfile" hadolint --failure-threshold warning - <"$dockerfile"
done done
@ -73,6 +78,13 @@ OBSERVABILITY_RUNTIME_DIR="$scan_dir/observability-runtime" \
docker compose --env-file .env.load.example \ docker compose --env-file .env.load.example \
-f compose.yaml -f compose.load.yaml -f compose.observability.yaml \ -f compose.yaml -f compose.load.yaml -f compose.observability.yaml \
--profile observability config --quiet --profile observability config --quiet
mkdir -p "$scan_dir/backup-runtime"
BACKUP_RUNTIME_DIR="$scan_dir/backup-runtime" \
BACKUP_HOST_UID="$(id -u)" \
BACKUP_HOST_GID="$(id -g)" \
docker compose --env-file .env.load.example \
-f compose.yaml -f compose.load.yaml -f compose.backup.yaml \
--profile backup config --quiet
docker compose -p "$project" -f compose.quality.yaml config --quiet docker compose -p "$project" -f compose.quality.yaml config --quiet
echo "Validating local observability configuration" echo "Validating local observability configuration"
@ -161,6 +173,76 @@ docker run --rm "$assets_image" npm audit --audit-level=high
docker build --tag "$e2e_image" e2e docker build --tag "$e2e_image" e2e
docker run --rm "$e2e_image" npm audit --audit-level=high docker run --rm "$e2e_image" npm audit --audit-level=high
echo "Building and scanning the pinned non-root backup tool image"
docker build --tag "$backup_image" --file Dockerfile.backup .
test "$(docker image inspect --format '{{.Config.User}}' "$backup_image")" = \
"10001:10001"
backup_versions=$(docker run --rm \
--user 10001:10001 \
--read-only \
--tmpfs /tmp \
"$backup_image" \
sh -euc 'restic version; pg_dump --version; test "$(id -u)" = 10001')
printf '%s\n' "$backup_versions"
printf '%s\n' "$backup_versions" |
grep -F 'restic 0.19.1 compiled with go1.26.5' >/dev/null
printf '%s\n' "$backup_versions" |
grep -F 'pg_dump (PostgreSQL) 18.4' >/dev/null
docker run --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$ROOT/.tools/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" image \
--scanners vuln \
--severity HIGH,CRITICAL \
--ignore-unfixed \
--exit-code 1 \
"$backup_image"
echo "Building and scanning the pinned non-root MinIO server and client images"
docker build --target server --tag "$minio_image" --file Dockerfile.minio .
docker build --target client --tag "$mc_image" --file Dockerfile.minio .
test "$(docker image inspect --format '{{.Config.User}}' "$minio_image")" = \
"10001:10001"
test "$(docker image inspect --format '{{.Config.User}}' "$mc_image")" = \
"10001:10001"
minio_version=$(docker run --rm \
--user 10001:10001 \
--read-only \
--tmpfs /tmp \
"$minio_image" \
--version)
mc_version=$(docker run --rm \
--user 10001:10001 \
--read-only \
--tmpfs /tmp \
"$mc_image" \
--version)
printf '%s\n' "$minio_version"
printf '%s\n' "$mc_version"
printf '%s\n' "$minio_version" |
grep -F 'RELEASE.2025-10-15T17-29-55Z' >/dev/null
printf '%s\n' "$minio_version" |
grep -F 'commit-id=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' >/dev/null
printf '%s\n' "$minio_version" |
grep -F 'Runtime: go1.26.5 linux/' >/dev/null
printf '%s\n' "$mc_version" |
grep -F 'RELEASE.2025-08-13T08-35-41Z' >/dev/null
printf '%s\n' "$mc_version" |
grep -F 'commit-id=7394ce0dd2a80935aded936b09fa12cbb3cb8096' >/dev/null
printf '%s\n' "$mc_version" |
grep -F 'Runtime: go1.26.5 linux/' >/dev/null
for image in "$minio_image" "$mc_image"; do
docker run --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$ROOT/.tools/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" image \
--scanners vuln \
--severity HIGH,CRITICAL \
--ignore-unfixed \
--exit-code 1 \
"$image"
done
echo "Building and scanning the production release image" echo "Building and scanning the production release image"
docker build --target release --tag "$release_image" . docker build --target release --tag "$release_image" .
docker run --rm \ docker run --rm \