who_need_help/scripts/backup-s3-drill.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
ENV_FILE="$ROOT/.env.load"
LABEL=${1:-"backup-$(date -u +%Y%m%d%H%M%S)"}

if [[ ! -f "$ENV_FILE" ]]; then
  echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
  exit 1
fi

set -a
# shellcheck source=/dev/null
. "$ENV_FILE"
set +a

for name in LOAD_PROJECT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD DATABASE_URL \
  BACKUP_MINIO_API_PORT BACKUP_MINIO_CONSOLE_PORT BACKUP_MINIO_ROOT_USER \
  BACKUP_MINIO_ROOT_PASSWORD BACKUP_RESTIC_PASSWORD BACKUP_BUCKET_PREFIX \
  BACKUP_TIMEOUT_SECONDS BACKUP_INTERRUPTION_CHUNKS \
  BACKUP_INTERRUPTION_CHUNK_BYTES BACKUP_INTERRUPTION_INTERVAL_SECONDS; do
  if [[ -z "${!name:-}" ]]; then
    echo "$name is missing from .env.load" >&2
    exit 1
  fi
done

if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
  echo "The encrypted backup drill must not use the staging Compose project." >&2
  exit 1
fi

if [[ ! "$LABEL" =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]]; then
  echo "Run label must be 1-32 lowercase letters, numbers, or dashes." >&2
  exit 1
fi

if [[ ! "$BACKUP_BUCKET_PREFIX" =~ ^[a-z0-9][a-z0-9-]{1,19}$ ]]; then
  echo "BACKUP_BUCKET_PREFIX must be 2-20 lowercase letters, numbers, or dashes." >&2
  exit 1
fi

for name in BACKUP_MINIO_API_PORT BACKUP_MINIO_CONSOLE_PORT; do
  if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then
    echo "$name must be a TCP port number from 0 through 65535." >&2
    exit 1
  fi
done

for name in BACKUP_TIMEOUT_SECONDS BACKUP_INTERRUPTION_CHUNKS \
  BACKUP_INTERRUPTION_CHUNK_BYTES; do
  if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then
    echo "$name must be a positive integer." >&2
    exit 1
  fi
done

if ! awk -v value="$BACKUP_INTERRUPTION_INTERVAL_SECONDS" \
  'BEGIN {exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value > 0)}'; then
  echo "BACKUP_INTERRUPTION_INTERVAL_SECONDS must be greater than zero." >&2
  exit 1
fi

for command in awk dd diff docker grep head jq od sleep tr unlink; do
  if ! command -v "$command" >/dev/null 2>&1; then
    echo "Required command is unavailable: $command" >&2
    exit 1
  fi
done

bucket="$BACKUP_BUCKET_PREFIX-$LABEL"
corrupt_bucket="$bucket-corrupt"
interrupt_bucket="$bucket-interrupt"

for candidate in "$bucket" "$corrupt_bucket" "$interrupt_bucket"; do
  if ((${#candidate} > 63)); then
    echo "Derived S3 bucket name exceeds 63 characters: $candidate" >&2
    exit 1
  fi
done

runtime_dir="$ROOT/tmp/backup-s3/$LOAD_PROJECT/$LABEL"
output_dir="$ROOT/output/backups-s3/$LABEL"
mkdir -p "$runtime_dir" "$output_dir"
chmod 700 "$ROOT/tmp" "$ROOT/tmp/backup-s3" \
  "$ROOT/tmp/backup-s3/$LOAD_PROJECT" "$runtime_dir" \
  "$ROOT/output" "$ROOT/output/backups-s3" "$output_dir"

export BACKUP_RUNTIME_DIR="$runtime_dir"
export BACKUP_HOST_UID
export BACKUP_HOST_GID
BACKUP_HOST_UID=$(id -u)
BACKUP_HOST_GID=$(id -g)

compose=(
  docker compose
  --env-file "$ENV_FILE"
  -p "$LOAD_PROJECT"
  -f compose.yaml
  -f compose.load.yaml
  -f compose.backup.yaml
  --profile backup
)

service_ids() {
  "${compose[@]}" ps --all -q "$1"
}

assert_scope() {
  local container_id=$1
  local expected_service=$2
  local observed_project observed_service

  observed_project=$(
    docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
      "$container_id"
  )
  observed_service=$(
    docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
      "$container_id"
  )

  if [[ "$observed_project" != "$LOAD_PROJECT" ||
    "$observed_service" != "$expected_service" ]]; then
    echo "Container scope mismatch for $container_id." >&2
    exit 1
  fi
}

mc_run() {
  "${compose[@]}" run --rm --no-deps -T minio-client "$@"
}

restic_run_repo() {
  local repository=$1
  shift
  "${compose[@]}" run --rm --no-deps -T \
    --env "RESTIC_REPOSITORY=$repository" \
    backup-tools restic "$@"
}

database_snapshot() {
  # Variables are intentionally expanded inside the isolated PostGIS container.
  # shellcheck disable=SC2016
  "${compose[@]}" exec -T db sh -c \
    'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
      --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL'
BEGIN READ ONLY;
SELECT 'users' AS table_name, count(*) AS row_count FROM users
UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
UNION ALL SELECT 'help_requests', count(*) FROM help_requests
UNION ALL SELECT 'messages', count(*) FROM messages
UNION ALL SELECT 'categories', count(*) FROM categories
UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
UNION ALL SELECT 'activities', count(*) FROM activities
UNION ALL SELECT 'reports', count(*) FROM reports
UNION ALL SELECT 'social_identities', count(*) FROM social_identities
UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations
ORDER BY table_name;
COMMIT;
SQL
}

wait_for_minio() {
  local minio_id=$1
  local deadline=$((SECONDS + BACKUP_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    local state health
    state=$(docker inspect --format '{{.State.Status}}' "$minio_id")
    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \
      "$minio_id")

    if [[ "$state" == "running" && "$health" == "healthy" ]]; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for the scoped MinIO container." >&2
  return 1
}

remove_bucket() {
  local candidate=$1
  mc_run rb --force "local/$candidate" >/dev/null 2>&1 || true
}

remove_bucket_and_verify() {
  local candidate=$1

  mc_run rb --force "local/$candidate" >/dev/null

  if mc_run stat "local/$candidate" >/dev/null 2>&1; then
    echo "Temporary MinIO bucket was not removed: $candidate" >&2
    exit 1
  fi
}

remove_interrupt_container() {
  if [[ -n "${interrupt_container_id:-}" ]] &&
    docker inspect "$interrupt_container_id" >/dev/null 2>&1; then
    assert_scope "$interrupt_container_id" backup-tools
    docker stop --time 1 "$interrupt_container_id" >/dev/null 2>&1 || true
    docker rm "$interrupt_container_id" >/dev/null 2>&1 || true
  fi
}

drop_drill_database() {
  if [[ "${drill_database_created:-false}" == "true" ]]; then
    "${compose[@]}" exec -T db \
      dropdb --username "$POSTGRES_USER" --if-exists "$drill_db" >/dev/null 2>&1 || true
  fi
}

cleanup() {
  local status=$?
  trap - EXIT HUP INT TERM
  drop_drill_database
  remove_interrupt_container

  if [[ "${corrupt_bucket_created:-false}" == "true" ]]; then
    remove_bucket "$corrupt_bucket"
  fi

  if [[ "${interrupt_bucket_created:-false}" == "true" ]]; then
    remove_bucket "$interrupt_bucket"
  fi

  exit "$status"
}

trap cleanup EXIT HUP INT TERM

for service in minio-permissions minio minio-client backup-tools; do
  while IFS= read -r existing_id; do
    [[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service"
  done < <(service_ids "$service")
done

database_snapshot "$output_dir/database-before.txt"

"${compose[@]}" build backup-tools minio minio-client \
  >"$output_dir/backup-images-build.txt"
"${compose[@]}" up -d --wait minio >"$output_dir/minio-up.txt"
minio_id=$(service_ids minio | head -n 1)
assert_scope "$minio_id" minio
wait_for_minio "$minio_id"

if mc_run stat "local/$bucket" >/dev/null 2>&1; then
  echo "Refusing to overwrite retained canonical bucket: $bucket" >&2
  exit 1
fi

mc_run mb "local/$bucket" >"$output_dir/canonical-bucket-create.txt"
canonical_repository="s3:http://minio:9000/$bucket"
restic_run_repo "$canonical_repository" init >"$output_dir/restic-init.txt"

restic_run_repo "$canonical_repository" \
  backup \
  --json \
  --host who-need-help-local \
  --tag "$LABEL" \
  --stdin-filename database.dump \
  --stdin-from-command \
  -- \
  pg_dump \
  --host db \
  --username "$POSTGRES_USER" \
  --dbname "$POSTGRES_DB" \
  --format custom \
  --no-owner \
  >"$output_dir/backup.jsonl"

restic_run_repo "$canonical_repository" snapshots --json --tag "$LABEL" \
  >"$output_dir/snapshots.json"

if ! jq -e --arg label "$LABEL" '
  length == 1 and
  .[0].tags == [$label] and
  (.[0].id | length) == 64
' "$output_dir/snapshots.json" >/dev/null; then
  echo "The canonical encrypted backup did not create exactly one tagged snapshot." >&2
  exit 1
fi

snapshot_id=$(jq -r '.[0].id' "$output_dir/snapshots.json")

restic_run_repo "$canonical_repository" check --read-data \
  >"$output_dir/canonical-check.txt" 2>&1
restic_run_repo "$canonical_repository" ls --json "$snapshot_id" \
  >"$output_dir/snapshot-files.jsonl"

if ! jq -s -e '
  any(
    .[];
    .struct_type == "node" and
    .path == "/database.dump" and
    .type == "file" and
    .size > 0
  )
' "$output_dir/snapshot-files.jsonl" >/dev/null; then
  echo "The encrypted snapshot does not contain a non-empty database.dump." >&2
  exit 1
fi

mc_run ls --recursive --json "local/$bucket" \
  >"$output_dir/minio-objects.jsonl"

if ! jq -s -e '
  length > 0 and
  any(.[]; (.key // "") | startswith("data/"))
' "$output_dir/minio-objects.jsonl" >/dev/null; then
  echo "MinIO did not retain encrypted Restic data objects." >&2
  exit 1
fi

pack_key=$(
  jq -sr '
    [.[] | select((.key // "") | startswith("data/")) | .key]
    | sort
    | first
  ' "$output_dir/minio-objects.jsonl"
)

mc_run cat "local/$bucket/$pack_key" |
  dd of="$runtime_dir/encrypted-prefix.bin" bs=1 count=16 status=none
od -An -tx1 -v "$runtime_dir/encrypted-prefix.bin" |
  tr -d ' \n' >"$output_dir/encrypted-prefix.hex"

if grep -aFq 'PGDMP' "$runtime_dir/encrypted-prefix.bin"; then
  echo "A Restic data object exposed the PostgreSQL custom-format header." >&2
  exit 1
fi

restic_run_repo "$canonical_repository" dump "$snapshot_id" /database.dump |
  "${compose[@]}" exec -T db pg_restore --list \
    >"$output_dir/pg-restore-list.txt"

drill_db="wnh_s3_restore_${LABEL//-/_}_$$"
"${compose[@]}" exec -T db \
  createdb \
  --username "$POSTGRES_USER" \
  --template template0 \
  "$drill_db"
drill_database_created=true

restic_run_repo "$canonical_repository" dump "$snapshot_id" /database.dump |
  "${compose[@]}" exec -T db \
    pg_restore \
    --username "$POSTGRES_USER" \
    --dbname "$drill_db" \
    --exit-on-error \
    --no-owner \
    --no-privileges

"${compose[@]}" exec -T db \
  psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
  --username "$POSTGRES_USER" --dbname "$drill_db" \
  >"$output_dir/restored-database.json" <<'SQL'
SELECT json_build_object(
  'public_table_count',
    (
      SELECT count(*)
      FROM pg_tables
      WHERE schemaname = 'public'
        AND tablename <> 'spatial_ref_sys'
    ),
  'migration_count', (SELECT count(*) FROM schema_migrations),
  'user_count', (SELECT count(*) FROM users),
  'request_count', (SELECT count(*) FROM help_requests),
  'message_count', (SELECT count(*) FROM messages),
  'category_count', (SELECT count(*) FROM categories),
  'postgis_version', PostGIS_Lib_Version()
);
SQL

if ! jq -e '
  .public_table_count > 0 and
  .migration_count > 0 and
  .category_count > 0 and
  (.postgis_version | type == "string" and length > 0)
' "$output_dir/restored-database.json" >/dev/null; then
  echo "The fresh restore database failed structural validation." >&2
  exit 1
fi

case "$DATABASE_URL" in
  *\?*)
    database_base=${DATABASE_URL%%\?*}
    database_query="?${DATABASE_URL#*\?}"
    ;;

  *)
    database_base=$DATABASE_URL
    database_query=
    ;;
esac

database_prefix=${database_base%/*}

if [[ "$database_prefix" == "$database_base" ]]; then
  echo "Could not derive the temporary drill DATABASE_URL." >&2
  exit 1
fi

drill_database_url="$database_prefix/$drill_db$database_query"

"${compose[@]}" run --rm --no-deps \
  --env APP_ROLE=migrate \
  --env "DATABASE_URL=$drill_database_url" \
  migrate /app/bin/migrate \
  >"$output_dir/restore-migrate.txt"

"${compose[@]}" run --rm --no-deps \
  --env APP_ROLE=migrate \
  --env "DATABASE_URL=$drill_database_url" \
  migrate /app/bin/who_need_help eval 'WhoNeedHelp.Release.await_migrations()' \
  >"$output_dir/restore-migration-readiness.txt"

"${compose[@]}" exec -T db \
  dropdb --username "$POSTGRES_USER" "$drill_db"
drill_database_created=false

remaining=$(
  "${compose[@]}" exec -T db \
    psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
    --username "$POSTGRES_USER" --dbname postgres \
    --command "SELECT count(*) FROM pg_database WHERE datname = '$drill_db';"
)

if [[ "$remaining" != "0" ]]; then
  echo "Restore drill database was not removed: $drill_db" >&2
  exit 1
fi

mc_run mb "local/$corrupt_bucket" >"$output_dir/corrupt-bucket-create.txt"
corrupt_bucket_created=true
mc_run mirror "local/$bucket" "local/$corrupt_bucket" \
  >"$output_dir/corrupt-mirror.txt"
mc_run cp "local/$corrupt_bucket/config" /work/corrupt-config \
  >"$output_dir/corrupt-download.txt"

first_byte=$(od -An -tu1 -N1 "$runtime_dir/corrupt-config" | tr -d ' ')

if [[ "$first_byte" == "0" ]]; then
  printf '\001' | dd of="$runtime_dir/corrupt-config" bs=1 count=1 conv=notrunc status=none
else
  printf '\000' | dd of="$runtime_dir/corrupt-config" bs=1 count=1 conv=notrunc status=none
fi

mc_run cp /work/corrupt-config "local/$corrupt_bucket/config" \
  >"$output_dir/corrupt-upload.txt"
corrupt_repository="s3:http://minio:9000/$corrupt_bucket"

set +e
restic_run_repo "$corrupt_repository" check --read-data \
  >"$output_dir/corrupt-check.txt" 2>&1
corrupt_check_status=$?
restic_run_repo "$corrupt_repository" dump "$snapshot_id" /database.dump \
  >"/dev/null" 2>"$output_dir/corrupt-restore.txt"
corrupt_restore_status=$?
set -e

if [[ "$corrupt_check_status" -eq 0 || "$corrupt_restore_status" -eq 0 ]]; then
  echo "The corrupted encrypted repository did not fail closed." >&2
  exit 1
fi

remove_bucket_and_verify "$corrupt_bucket"
corrupt_bucket_created=false
unlink "$runtime_dir/corrupt-config"

mc_run mb "local/$interrupt_bucket" >"$output_dir/interrupt-bucket-create.txt"
interrupt_bucket_created=true
interrupt_repository="s3:http://minio:9000/$interrupt_bucket"
restic_run_repo "$interrupt_repository" init >"$output_dir/interrupt-init.txt"

interrupt_container="$LOAD_PROJECT-backup-interrupt-$LABEL"

if docker inspect "$interrupt_container" >/dev/null 2>&1; then
  echo "Refusing to replace an existing interruption container: $interrupt_container" >&2
  exit 1
fi

# The producer variables must expand inside the isolated backup-tools container.
# shellcheck disable=SC2016
interrupt_container_id=$(
  "${compose[@]}" run -d \
    --name "$interrupt_container" \
    --no-deps \
    --env "RESTIC_REPOSITORY=$interrupt_repository" \
    backup-tools \
    restic backup \
    --json \
    --host who-need-help-local \
    --tag "$LABEL-interrupted" \
    --stdin-filename interrupted.bin \
    --stdin-from-command \
    -- \
    /bin/sh -euc \
    'i=0
     while [ "$i" -lt "$BACKUP_INTERRUPTION_CHUNKS" ]; do
       head -c "$BACKUP_INTERRUPTION_CHUNK_BYTES" /dev/urandom
       sleep "$BACKUP_INTERRUPTION_INTERVAL_SECONDS"
       i=$((i + 1))
     done'
)
assert_scope "$interrupt_container_id" backup-tools

upload_deadline=$((SECONDS + BACKUP_TIMEOUT_SECONDS))
uploaded_data_objects=0

while ((SECONDS < upload_deadline)); do
  mc_run ls --recursive --json "local/$interrupt_bucket/data" \
    >"$output_dir/interrupt-objects-current.jsonl" 2>/dev/null || true
  uploaded_data_objects=$(
    jq -s '[.[] | select(.type == "file")] | length' \
      "$output_dir/interrupt-objects-current.jsonl"
  )

  if ((uploaded_data_objects > 0)); then
    break
  fi

  if [[ "$(docker inspect --format '{{.State.Running}}' "$interrupt_container_id")" != "true" ]]; then
    echo "Interruption producer exited before an uploaded data object was observed." >&2
    exit 1
  fi

  sleep 1
done

if ((uploaded_data_objects == 0)); then
  echo "Timed out waiting for the interruption upload to reach MinIO." >&2
  exit 1
fi

docker stop --time 1 "$interrupt_container_id" >"$output_dir/interrupt-stop.txt"
docker inspect "$interrupt_container_id" |
  jq '.[0] | {
    id: .Id,
    project: .Config.Labels["com.docker.compose.project"],
    service: .Config.Labels["com.docker.compose.service"],
    exit_code: .State.ExitCode,
    state: .State.Status
  }' >"$output_dir/interrupt-container.json"
docker logs "$interrupt_container_id" >"$output_dir/interrupt-backup.log" 2>&1

if ! jq -e '.exit_code != 0 and .state == "exited"' \
  "$output_dir/interrupt-container.json" >/dev/null; then
  echo "The interrupted upload container did not exit unsuccessfully." >&2
  exit 1
fi

restic_run_repo "$interrupt_repository" snapshots \
  --json --tag "$LABEL-interrupted" \
  >"$output_dir/interrupt-snapshots.json"

if ! jq -e 'length == 0' "$output_dir/interrupt-snapshots.json" >/dev/null; then
  echo "An interrupted upload published a restorable snapshot." >&2
  exit 1
fi

mc_run ls --incomplete --recursive --json "local/$interrupt_bucket" \
  >"$output_dir/interrupt-incomplete-uploads.jsonl" 2>/dev/null || true
restic_run_repo "$interrupt_repository" check --read-data \
  >"$output_dir/interrupt-check-before-prune.txt" 2>&1
restic_run_repo "$interrupt_repository" prune \
  >"$output_dir/interrupt-prune.txt" 2>&1
restic_run_repo "$interrupt_repository" check --read-data \
  >"$output_dir/interrupt-check-after-prune.txt" 2>&1

docker rm "$interrupt_container_id" >"$output_dir/interrupt-remove.txt"
interrupt_container_id=
remove_bucket_and_verify "$interrupt_bucket"
interrupt_bucket_created=false

database_snapshot "$output_dir/database-after.txt"

if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \
  >"$output_dir/database-diff.txt"; then
  echo "The encrypted backup drill changed source database counts." >&2
  exit 1
fi

minio_api=$("${compose[@]}" port minio 9000 | head -n 1)
minio_console=$("${compose[@]}" port minio 9001 | head -n 1)

if [[ ! "$minio_api" =~ ^127[.]0[.]0[.]1:[1-9][0-9]*$ ||
  ! "$minio_console" =~ ^127[.]0[.]0[.]1:[1-9][0-9]*$ ]]; then
  echo "MinIO ports are not bound to observed loopback addresses." >&2
  exit 1
fi

jq -n \
  --arg run_id "$LABEL" \
  --arg bucket "$bucket" \
  --arg snapshot_id "$snapshot_id" \
  --arg minio_api "http://$minio_api" \
  --arg minio_console "http://$minio_console" \
  --argjson corrupt_check_exit "$corrupt_check_status" \
  --argjson corrupt_restore_exit "$corrupt_restore_status" \
  --argjson interrupted_uploaded_objects "$uploaded_data_objects" \
  '{
    run_id: $run_id,
    retained_bucket: $bucket,
    snapshot_id: $snapshot_id,
    encrypted_repository_check: "passed",
    fresh_database_restore: "passed",
    source_database_count_diff_bytes: 0,
    corrupted_check_exit: $corrupt_check_exit,
    corrupted_restore_exit: $corrupt_restore_exit,
    interrupted_uploaded_objects: $interrupted_uploaded_objects,
    interrupted_snapshot_count: 0,
    interrupted_repository_check_after_prune: "passed",
    minio_api: $minio_api,
    minio_console: $minio_console
  }' >"$output_dir/summary.json"

trap - EXIT HUP INT TERM

printf 'Encrypted S3 backup evidence: %s\n' "$output_dir"
printf 'Retained MinIO bucket: %s\nSnapshot: %s\n' "$bucket" "$snapshot_id"
printf 'MinIO API: http://%s\nMinIO console: http://%s\n' \
  "$minio_api" "$minio_console"