diff --git a/docs/operations.md b/docs/operations.md index 4017675..956d787 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1245,6 +1245,24 @@ WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \ ./scripts/production-release.sh apply whoneedhelp ``` +The application must still be released from a clean checkout. If a verified +bundle and image archive were prepared in another clean checkout, point the +release process at their absolute parent directory instead of rebuilding them: + +```bash +WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT=/absolute/path/to/output/releases \ +WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \ + ./scripts/production-release.sh apply whoneedhelp +``` + +The selected directory must contain a child named with the exact full commit. +The workflow rechecks the Git bundle HEAD, bundle checksum and manifest, image +archive checksum, target platform, topology, immutable image names and image +IDs before transfer. It does not accept a relative artifact path or silently +fall back to a different commit. This allows a temporary clean worktree to +reuse the already scanned workstation artifact while keeping production image +compilation off the 4-GiB server. + The apply path refuses tracked local or remote modifications. It then: 1. creates and verifies a full Git bundle for exactly that clean commit; diff --git a/scripts/prepare-production-images.sh b/scripts/prepare-production-images.sh index c6c110e..08e1f36 100755 --- a/scripts/prepare-production-images.sh +++ b/scripts/prepare-production-images.sh @@ -24,13 +24,23 @@ fi commit=$(git -C "$ROOT" rev-parse --verify HEAD) short_commit=${commit:0:12} -release_dir="$ROOT/output/releases/$commit" +artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"} +case "$artifact_root" in + /*) ;; + *) + echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2 + exit 2 + ;; +esac +mkdir -p "$artifact_root" +artifact_root=$(realpath --canonicalize-existing "$artifact_root") +release_dir="$artifact_root/$commit" archive="$release_dir/who_need_help-$commit-images-linux-amd64.tar.gz" checksum="$archive.sha256" manifest="$release_dir/who_need_help-$commit-images.manifest" mkdir -p "$release_dir" -chmod 700 "$ROOT/output" "$ROOT/output/releases" "$release_dir" +chmod 700 "$artifact_root" "$release_dir" build_env=$(mktemp "$release_dir/.production-image-build.XXXXXX") cleanup() { @@ -115,8 +125,7 @@ if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then cd "$release_dir" sha256sum --check "$(basename -- "$checksum")" >/dev/null ) - grep -Fx "commit=$commit" "$manifest" >/dev/null - echo "Production image package already exists and passed checksum verification." + echo "Production image package already exists; verifying all metadata." else "$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}" @@ -155,8 +164,45 @@ fi sha256sum --check "$(basename -- "$checksum")" >/dev/null ) gzip -t "$archive" -grep -Fx 'platform=linux/amd64' "$manifest" >/dev/null -grep -Fx "topology=$topology" "$manifest" >/dev/null + +archive_hash=$(sha256sum "$archive" | awk '{print $1}') +expected_checksum="$archive_hash $(basename -- "$archive")" +if [[ "$(cat -- "$checksum")" != "$expected_checksum" ]]; then + echo "Production image checksum metadata does not name the exact archive." >&2 + exit 2 +fi + +manifest_value() { + local key=$1 + awk -F= -v key="$key" ' + $1 == key { count += 1; value = substr($0, length(key) + 2) } + END { + if (count != 1) exit 1 + print value + } + ' "$manifest" +} + +[[ "$(manifest_value format)" == 1 ]] || { + echo "Production image manifest format is unsupported." >&2 + exit 2 +} +[[ "$(manifest_value commit)" == "$commit" ]] || { + echo "Production image manifest commit does not match the current commit." >&2 + exit 2 +} +[[ "$(manifest_value platform)" == linux/amd64 ]] || { + echo "Production image manifest platform is not linux/amd64." >&2 + exit 2 +} +[[ "$(manifest_value topology)" == "$topology" ]] || { + echo "Production image manifest topology does not match the environment." >&2 + exit 2 +} +[[ "$(manifest_value image_count)" == "${#images[@]}" ]] || { + echo "Production image manifest count does not match the required images." >&2 + exit 2 +} test "$(grep -c '^image=' "$manifest")" = "${#images[@]}" for image in "${images[@]}"; do awk -F'|' -v image="$image" ' diff --git a/scripts/prepare-production-release.sh b/scripts/prepare-production-release.sh index 033333e..eadc152 100755 --- a/scripts/prepare-production-release.sh +++ b/scripts/prepare-production-release.sh @@ -11,13 +11,23 @@ fi commit=$(git -C "$ROOT" rev-parse --verify HEAD) short_commit=${commit:0:12} -release_dir="$ROOT/output/releases/$commit" +artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"} +case "$artifact_root" in + /*) ;; + *) + echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2 + exit 2 + ;; +esac +mkdir -p "$artifact_root" +artifact_root=$(realpath --canonicalize-existing "$artifact_root") +release_dir="$artifact_root/$commit" bundle="$release_dir/who_need_help-$commit.bundle" checksum="$bundle.sha256" manifest="$release_dir/manifest.txt" mkdir -p "$release_dir" -chmod 700 "$ROOT/output" "$ROOT/output/releases" "$release_dir" +chmod 700 "$artifact_root" "$release_dir" if [[ -e "$bundle" || -e "$checksum" || -e "$manifest" ]]; then echo "Release package already exists; verifying it instead of overwriting it." @@ -41,6 +51,37 @@ fi ) git -C "$ROOT" bundle verify "$bundle" >/dev/null +bundle_hash=$(sha256sum "$bundle" | awk '{print $1}') +expected_checksum="$bundle_hash $(basename -- "$bundle")" +if [[ "$(cat -- "$checksum")" != "$expected_checksum" ]]; then + echo "Release bundle checksum metadata does not name the exact bundle." >&2 + exit 1 +fi + +manifest_value() { + local key=$1 + awk -F= -v key="$key" ' + $1 == key { count += 1; value = substr($0, length(key) + 2) } + END { + if (count != 1) exit 1 + print value + } + ' "$manifest" +} + +[[ "$(manifest_value commit)" == "$commit" ]] || { + echo "Release manifest commit does not match the current commit." >&2 + exit 1 +} +[[ "$(manifest_value short_commit)" == "$short_commit" ]] || { + echo "Release manifest short commit does not match the current commit." >&2 + exit 1 +} +[[ "$(manifest_value bundle_sha256)" == "$bundle_hash" ]] || { + echo "Release manifest bundle checksum does not match the bundle." >&2 + exit 1 +} + bundle_head=$(git -C "$ROOT" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') if [[ "$bundle_head" != "$commit" ]]; then echo "Release bundle HEAD does not match the current commit." >&2 diff --git a/scripts/production-release.sh b/scripts/production-release.sh index fce211f..31b451f 100755 --- a/scripts/production-release.sh +++ b/scripts/production-release.sh @@ -15,7 +15,7 @@ case "$action" in ;; esac -for command in docker git gzip mktemp pg_restore scp sha256sum ssh; do +for command in docker git gzip mktemp pg_restore realpath scp sha256sum ssh; do command -v "$command" >/dev/null 2>&1 || { echo "Required command is unavailable: $command" >&2 exit 2 @@ -125,7 +125,16 @@ if [[ "$migration_policy" == "forward_only" ]]; then fi "$ROOT/scripts/prepare-production-release.sh" -release_dir="$ROOT/output/releases/$local_commit" +artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"} +case "$artifact_root" in + /*) ;; + *) + echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2 + exit 2 + ;; +esac +artifact_root=$(realpath --canonicalize-existing "$artifact_root") +release_dir="$artifact_root/$local_commit" bundle="$release_dir/who_need_help-$local_commit.bundle" image_archive="$release_dir/who_need_help-$local_commit-images-linux-amd64.tar.gz" image_checksum="$image_archive.sha256" diff --git a/scripts/quality.sh b/scripts/quality.sh index 76c0542..225e44a 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -1556,6 +1556,7 @@ docker run --rm \ --volume "$ROOT:/src:ro" \ --workdir /src \ "$PYTHON_IMAGE" python test/scripts/install_production_external_monitor_test.py +python3 test/scripts/production_release_artifact_root_test.py docker run --rm \ --volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \ "$PYTHON_IMAGE" python -c \ diff --git a/test/scripts/production_release_artifact_root_test.py b/test/scripts/production_release_artifact_root_test.py new file mode 100644 index 0000000..c12f2db --- /dev/null +++ b/test/scripts/production_release_artifact_root_test.py @@ -0,0 +1,199 @@ +import gzip +import hashlib +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +class ProductionReleaseArtifactRootTest(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.base = Path(self.tempdir.name) + self.project = self.base / "project" + self.scripts = self.project / "scripts" + self.scripts.mkdir(parents=True) + for name in ("prepare-production-release.sh", "prepare-production-images.sh"): + destination = self.scripts / name + shutil.copy2(ROOT / "scripts" / name, destination) + destination.chmod(0o755) + + self.run_command(["git", "init", "--quiet"], cwd=self.project) + self.run_command( + ["git", "config", "user.email", "release-test@example.invalid"], + cwd=self.project, + ) + self.run_command( + ["git", "config", "user.name", "Release test"], cwd=self.project + ) + self.run_command(["git", "add", "scripts"], cwd=self.project) + self.run_command( + ["git", "commit", "--quiet", "-m", "test fixture"], cwd=self.project + ) + self.commit = self.run_command( + ["git", "rev-parse", "HEAD"], cwd=self.project + ).stdout.strip() + self.artifact_root = self.base / "verified-artifacts" + + def tearDown(self): + self.tempdir.cleanup() + + def run_command(self, command, *, cwd=None, env=None, check=True): + return subprocess.run( + command, + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=check, + ) + + def artifact_env(self): + env = os.environ.copy() + env["WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT"] = str(self.artifact_root) + return env + + def test_bundle_is_reused_from_an_absolute_external_artifact_root(self): + first = self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=self.artifact_env(), + ) + second = self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=self.artifact_env(), + ) + + release_dir = self.artifact_root / self.commit + bundle = release_dir / f"who_need_help-{self.commit}.bundle" + self.assertTrue(bundle.is_file()) + self.assertIn(str(bundle), first.stdout) + self.assertIn("verifying it instead of overwriting it", second.stdout) + self.assertEqual(bundle.stat().st_mode & 0o777, 0o600) + + def test_relative_artifact_root_is_rejected(self): + env = os.environ.copy() + env["WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT"] = "relative/releases" + result = self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=env, + check=False, + ) + + self.assertEqual(result.returncode, 2) + self.assertIn("must be an absolute path", result.stderr) + + def test_bundle_manifest_for_another_commit_is_rejected(self): + self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=self.artifact_env(), + ) + manifest = self.artifact_root / self.commit / "manifest.txt" + manifest.write_text( + manifest.read_text(encoding="utf-8").replace( + f"commit={self.commit}", f"commit={'0' * 40}" + ), + encoding="utf-8", + ) + + result = self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=self.artifact_env(), + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("manifest commit does not match", result.stderr) + + def test_existing_compact_image_archive_is_reused_without_building(self): + fake_bin = self.base / "bin" + fake_bin.mkdir() + for name in ("docker", "jq"): + command = fake_bin / name + command.write_text("#!/bin/sh\nexit 97\n", encoding="utf-8") + command.chmod(0o755) + + production_env = self.base / "production.env" + production_env.write_text( + textwrap.dedent( + """\ + DEPLOYMENT_ENV=production + DATABASE_MODE=external + APP_TOPOLOGY=compact + APP_IMAGE=replace-me + SOCKET_PROXY_IMAGE=replace-me + POSTGIS_IMAGE=replace-me + """ + ), + encoding="utf-8", + ) + production_env.chmod(0o600) + + release_dir = self.artifact_root / self.commit + release_dir.mkdir(parents=True) + archive = release_dir / f"who_need_help-{self.commit}-images-linux-amd64.tar.gz" + with archive.open("wb") as output: + with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed: + compressed.write(b"verified image archive fixture") + archive.chmod(0o600) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + checksum = Path(f"{archive}.sha256") + checksum.write_text(f"{digest} {archive.name}\n", encoding="utf-8") + checksum.chmod(0o600) + + short_commit = self.commit[:12] + manifest = release_dir / f"who_need_help-{self.commit}-images.manifest" + manifest.write_text( + textwrap.dedent( + f"""\ + format=1 + commit={self.commit} + platform=linux/amd64 + topology=compact + image_count=1 + image=who-need-help:production-{short_commit}|sha256:{'a' * 64} + """ + ), + encoding="utf-8", + ) + manifest.chmod(0o600) + + env = self.artifact_env() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + result = self.run_command( + [str(self.scripts / "prepare-production-images.sh"), str(production_env)], + cwd=self.project, + env=env, + ) + + self.assertIn("verifying all metadata", result.stdout) + self.assertIn(str(archive), result.stdout) + + manifest.write_text( + manifest.read_text(encoding="utf-8").replace( + f"commit={self.commit}", f"commit={'f' * 40}" + ), + encoding="utf-8", + ) + rejected = self.run_command( + [str(self.scripts / "prepare-production-images.sh"), str(production_env)], + cwd=self.project, + env=env, + check=False, + ) + self.assertEqual(rejected.returncode, 2) + self.assertIn("manifest commit does not match", rejected.stderr) + + +if __name__ == "__main__": + unittest.main()