Reuse verified production release artifacts

docs/operations.md

@@ -1245,6 +1245,24 @@ WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \
   ./scripts/production-release.sh apply whoneedhelp
 ```
 
+The application must still be released from a clean checkout. If a verified
+bundle and image archive were prepared in another clean checkout, point the
+release process at their absolute parent directory instead of rebuilding them:
+
+```bash
+WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT=/absolute/path/to/output/releases \
+WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \
+  ./scripts/production-release.sh apply whoneedhelp
+```
+
+The selected directory must contain a child named with the exact full commit.
+The workflow rechecks the Git bundle HEAD, bundle checksum and manifest, image
+archive checksum, target platform, topology, immutable image names and image
+IDs before transfer. It does not accept a relative artifact path or silently
+fall back to a different commit. This allows a temporary clean worktree to
+reuse the already scanned workstation artifact while keeping production image
+compilation off the 4-GiB server.
+
 The apply path refuses tracked local or remote modifications. It then:
 
 1. creates and verifies a full Git bundle for exactly that clean commit;

scripts/prepare-production-images.sh

@@ -24,13 +24,23 @@ fi
 
 commit=$(git -C "$ROOT" rev-parse --verify HEAD)
 short_commit=${commit:0:12}
-release_dir="$ROOT/output/releases/$commit"
+artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"}
+case "$artifact_root" in
+  /*) ;;
+  *)
+    echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
+    exit 2
+    ;;
+esac
+mkdir -p "$artifact_root"
+artifact_root=$(realpath --canonicalize-existing "$artifact_root")
+release_dir="$artifact_root/$commit"
 archive="$release_dir/who_need_help-$commit-images-linux-amd64.tar.gz"
 checksum="$archive.sha256"
 manifest="$release_dir/who_need_help-$commit-images.manifest"
 
 mkdir -p "$release_dir"
-chmod 700 "$ROOT/output" "$ROOT/output/releases" "$release_dir"
+chmod 700 "$artifact_root" "$release_dir"
 
 build_env=$(mktemp "$release_dir/.production-image-build.XXXXXX")
 cleanup() {
@@ -115,8 +125,7 @@ if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then
     cd "$release_dir"
     sha256sum --check "$(basename -- "$checksum")" >/dev/null
   )
-  grep -Fx "commit=$commit" "$manifest" >/dev/null
+  echo "Production image package already exists; verifying all metadata."
-  echo "Production image package already exists and passed checksum verification."
 else
   "$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}"
 
@@ -155,8 +164,45 @@ fi
   sha256sum --check "$(basename -- "$checksum")" >/dev/null
 )
 gzip -t "$archive"
-grep -Fx 'platform=linux/amd64' "$manifest" >/dev/null
+
-grep -Fx "topology=$topology" "$manifest" >/dev/null
+archive_hash=$(sha256sum "$archive" | awk '{print $1}')
+expected_checksum="$archive_hash  $(basename -- "$archive")"
+if [[ "$(cat -- "$checksum")" != "$expected_checksum" ]]; then
+  echo "Production image checksum metadata does not name the exact archive." >&2
+  exit 2
+fi
+
+manifest_value() {
+  local key=$1
+  awk -F= -v key="$key" '
+    $1 == key { count += 1; value = substr($0, length(key) + 2) }
+    END {
+      if (count != 1) exit 1
+      print value
+    }
+  ' "$manifest"
+}
+
+[[ "$(manifest_value format)" == 1 ]] || {
+  echo "Production image manifest format is unsupported." >&2
+  exit 2
+}
+[[ "$(manifest_value commit)" == "$commit" ]] || {
+  echo "Production image manifest commit does not match the current commit." >&2
+  exit 2
+}
+[[ "$(manifest_value platform)" == linux/amd64 ]] || {
+  echo "Production image manifest platform is not linux/amd64." >&2
+  exit 2
+}
+[[ "$(manifest_value topology)" == "$topology" ]] || {
+  echo "Production image manifest topology does not match the environment." >&2
+  exit 2
+}
+[[ "$(manifest_value image_count)" == "${#images[@]}" ]] || {
+  echo "Production image manifest count does not match the required images." >&2
+  exit 2
+}
 test "$(grep -c '^image=' "$manifest")" = "${#images[@]}"
 for image in "${images[@]}"; do
   awk -F'|' -v image="$image" '

scripts/prepare-production-release.sh

@@ -11,13 +11,23 @@ fi
 
 commit=$(git -C "$ROOT" rev-parse --verify HEAD)
 short_commit=${commit:0:12}
-release_dir="$ROOT/output/releases/$commit"
+artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"}
+case "$artifact_root" in
+  /*) ;;
+  *)
+    echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
+    exit 2
+    ;;
+esac
+mkdir -p "$artifact_root"
+artifact_root=$(realpath --canonicalize-existing "$artifact_root")
+release_dir="$artifact_root/$commit"
 bundle="$release_dir/who_need_help-$commit.bundle"
 checksum="$bundle.sha256"
 manifest="$release_dir/manifest.txt"
 
 mkdir -p "$release_dir"
-chmod 700 "$ROOT/output" "$ROOT/output/releases" "$release_dir"
+chmod 700 "$artifact_root" "$release_dir"
 
 if [[ -e "$bundle" || -e "$checksum" || -e "$manifest" ]]; then
   echo "Release package already exists; verifying it instead of overwriting it."
@@ -41,6 +51,37 @@ fi
 )
 git -C "$ROOT" bundle verify "$bundle" >/dev/null
 
+bundle_hash=$(sha256sum "$bundle" | awk '{print $1}')
+expected_checksum="$bundle_hash  $(basename -- "$bundle")"
+if [[ "$(cat -- "$checksum")" != "$expected_checksum" ]]; then
+  echo "Release bundle checksum metadata does not name the exact bundle." >&2
+  exit 1
+fi
+
+manifest_value() {
+  local key=$1
+  awk -F= -v key="$key" '
+    $1 == key { count += 1; value = substr($0, length(key) + 2) }
+    END {
+      if (count != 1) exit 1
+      print value
+    }
+  ' "$manifest"
+}
+
+[[ "$(manifest_value commit)" == "$commit" ]] || {
+  echo "Release manifest commit does not match the current commit." >&2
+  exit 1
+}
+[[ "$(manifest_value short_commit)" == "$short_commit" ]] || {
+  echo "Release manifest short commit does not match the current commit." >&2
+  exit 1
+}
+[[ "$(manifest_value bundle_sha256)" == "$bundle_hash" ]] || {
+  echo "Release manifest bundle checksum does not match the bundle." >&2
+  exit 1
+}
+
 bundle_head=$(git -C "$ROOT" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
 if [[ "$bundle_head" != "$commit" ]]; then
   echo "Release bundle HEAD does not match the current commit." >&2

scripts/production-release.sh

@@ -15,7 +15,7 @@ case "$action" in
     ;;
 esac
 
-for command in docker git gzip mktemp pg_restore scp sha256sum ssh; do
+for command in docker git gzip mktemp pg_restore realpath scp sha256sum ssh; do
   command -v "$command" >/dev/null 2>&1 || {
     echo "Required command is unavailable: $command" >&2
     exit 2
@@ -125,7 +125,16 @@ if [[ "$migration_policy" == "forward_only" ]]; then
 fi
 
 "$ROOT/scripts/prepare-production-release.sh"
-release_dir="$ROOT/output/releases/$local_commit"
+artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"}
+case "$artifact_root" in
+  /*) ;;
+  *)
+    echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
+    exit 2
+    ;;
+esac
+artifact_root=$(realpath --canonicalize-existing "$artifact_root")
+release_dir="$artifact_root/$local_commit"
 bundle="$release_dir/who_need_help-$local_commit.bundle"
 image_archive="$release_dir/who_need_help-$local_commit-images-linux-amd64.tar.gz"
 image_checksum="$image_archive.sha256"

scripts/quality.sh

@@ -1556,6 +1556,7 @@ docker run --rm \
   --volume "$ROOT:/src:ro" \
   --workdir /src \
   "$PYTHON_IMAGE" python test/scripts/install_production_external_monitor_test.py
+python3 test/scripts/production_release_artifact_root_test.py
 docker run --rm \
   --volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \
   "$PYTHON_IMAGE" python -c \

test/scripts/production_release_artifact_root_test.py

@@ -0,0 +1,199 @@
+import gzip
+import hashlib
+import os
+import shutil
+import subprocess
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class ProductionReleaseArtifactRootTest(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.TemporaryDirectory()
+        self.base = Path(self.tempdir.name)
+        self.project = self.base / "project"
+        self.scripts = self.project / "scripts"
+        self.scripts.mkdir(parents=True)
+        for name in ("prepare-production-release.sh", "prepare-production-images.sh"):
+            destination = self.scripts / name
+            shutil.copy2(ROOT / "scripts" / name, destination)
+            destination.chmod(0o755)
+
+        self.run_command(["git", "init", "--quiet"], cwd=self.project)
+        self.run_command(
+            ["git", "config", "user.email", "release-test@example.invalid"],
+            cwd=self.project,
+        )
+        self.run_command(
+            ["git", "config", "user.name", "Release test"], cwd=self.project
+        )
+        self.run_command(["git", "add", "scripts"], cwd=self.project)
+        self.run_command(
+            ["git", "commit", "--quiet", "-m", "test fixture"], cwd=self.project
+        )
+        self.commit = self.run_command(
+            ["git", "rev-parse", "HEAD"], cwd=self.project
+        ).stdout.strip()
+        self.artifact_root = self.base / "verified-artifacts"
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+    def run_command(self, command, *, cwd=None, env=None, check=True):
+        return subprocess.run(
+            command,
+            cwd=cwd,
+            env=env,
+            capture_output=True,
+            text=True,
+            check=check,
+        )
+
+    def artifact_env(self):
+        env = os.environ.copy()
+        env["WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT"] = str(self.artifact_root)
+        return env
+
+    def test_bundle_is_reused_from_an_absolute_external_artifact_root(self):
+        first = self.run_command(
+            [str(self.scripts / "prepare-production-release.sh")],
+            cwd=self.project,
+            env=self.artifact_env(),
+        )
+        second = self.run_command(
+            [str(self.scripts / "prepare-production-release.sh")],
+            cwd=self.project,
+            env=self.artifact_env(),
+        )
+
+        release_dir = self.artifact_root / self.commit
+        bundle = release_dir / f"who_need_help-{self.commit}.bundle"
+        self.assertTrue(bundle.is_file())
+        self.assertIn(str(bundle), first.stdout)
+        self.assertIn("verifying it instead of overwriting it", second.stdout)
+        self.assertEqual(bundle.stat().st_mode & 0o777, 0o600)
+
+    def test_relative_artifact_root_is_rejected(self):
+        env = os.environ.copy()
+        env["WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT"] = "relative/releases"
+        result = self.run_command(
+            [str(self.scripts / "prepare-production-release.sh")],
+            cwd=self.project,
+            env=env,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("must be an absolute path", result.stderr)
+
+    def test_bundle_manifest_for_another_commit_is_rejected(self):
+        self.run_command(
+            [str(self.scripts / "prepare-production-release.sh")],
+            cwd=self.project,
+            env=self.artifact_env(),
+        )
+        manifest = self.artifact_root / self.commit / "manifest.txt"
+        manifest.write_text(
+            manifest.read_text(encoding="utf-8").replace(
+                f"commit={self.commit}", f"commit={'0' * 40}"
+            ),
+            encoding="utf-8",
+        )
+
+        result = self.run_command(
+            [str(self.scripts / "prepare-production-release.sh")],
+            cwd=self.project,
+            env=self.artifact_env(),
+            check=False,
+        )
+
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("manifest commit does not match", result.stderr)
+
+    def test_existing_compact_image_archive_is_reused_without_building(self):
+        fake_bin = self.base / "bin"
+        fake_bin.mkdir()
+        for name in ("docker", "jq"):
+            command = fake_bin / name
+            command.write_text("#!/bin/sh\nexit 97\n", encoding="utf-8")
+            command.chmod(0o755)
+
+        production_env = self.base / "production.env"
+        production_env.write_text(
+            textwrap.dedent(
+                """\
+                DEPLOYMENT_ENV=production
+                DATABASE_MODE=external
+                APP_TOPOLOGY=compact
+                APP_IMAGE=replace-me
+                SOCKET_PROXY_IMAGE=replace-me
+                POSTGIS_IMAGE=replace-me
+                """
+            ),
+            encoding="utf-8",
+        )
+        production_env.chmod(0o600)
+
+        release_dir = self.artifact_root / self.commit
+        release_dir.mkdir(parents=True)
+        archive = release_dir / f"who_need_help-{self.commit}-images-linux-amd64.tar.gz"
+        with archive.open("wb") as output:
+            with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed:
+                compressed.write(b"verified image archive fixture")
+        archive.chmod(0o600)
+        digest = hashlib.sha256(archive.read_bytes()).hexdigest()
+        checksum = Path(f"{archive}.sha256")
+        checksum.write_text(f"{digest}  {archive.name}\n", encoding="utf-8")
+        checksum.chmod(0o600)
+
+        short_commit = self.commit[:12]
+        manifest = release_dir / f"who_need_help-{self.commit}-images.manifest"
+        manifest.write_text(
+            textwrap.dedent(
+                f"""\
+                format=1
+                commit={self.commit}
+                platform=linux/amd64
+                topology=compact
+                image_count=1
+                image=who-need-help:production-{short_commit}|sha256:{'a' * 64}
+                """
+            ),
+            encoding="utf-8",
+        )
+        manifest.chmod(0o600)
+
+        env = self.artifact_env()
+        env["PATH"] = f"{fake_bin}:{env['PATH']}"
+        result = self.run_command(
+            [str(self.scripts / "prepare-production-images.sh"), str(production_env)],
+            cwd=self.project,
+            env=env,
+        )
+
+        self.assertIn("verifying all metadata", result.stdout)
+        self.assertIn(str(archive), result.stdout)
+
+        manifest.write_text(
+            manifest.read_text(encoding="utf-8").replace(
+                f"commit={self.commit}", f"commit={'f' * 40}"
+            ),
+            encoding="utf-8",
+        )
+        rejected = self.run_command(
+            [str(self.scripts / "prepare-production-images.sh"), str(production_env)],
+            cwd=self.project,
+            env=env,
+            check=False,
+        )
+        self.assertEqual(rejected.returncode, 2)
+        self.assertIn("manifest commit does not match", rejected.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()