Maintain persistent development scale fixtures

docs/performance.md

@@ -893,6 +893,31 @@ users and records. Synthetic rows carry `structured_data.scale_fixture=true`,
 use deterministic UUIDs, and are inserted with conflict handling, so rerunning
 the command fills missing fixtures without duplicating them. The resulting
 summary is ignored at `output/dev-scale/summary.json`.
+
+The synthetic deadlines eventually expire even though the rows remain useful
+for viewport and clustering work. Refresh only the existing fixture rows with:
+
+```sh
+WNH_DEV_SCALE_REFRESH_CONFIRM=refresh-persistent-dev-scale \
+  ./scripts/dev-scale-refresh.sh
+```
+
+The refresh runner verifies the exact project-owned kind control plane,
+namespace, PostGIS pod, database name, and configured fixture counts before it
+changes anything. It updates only rows carrying
+`structured_data.scale_fixture=true`, keeps their IDs and coordinates intact,
+and writes a mode-`0600` summary to
+`output/dev-scale/refresh-summary.json`. It refuses a partial or differently
+sized fixture set instead of treating it as safe to repair.
+
+This is an explicit maintenance operation, not a startup hook. The observed
+two-million-row development refresh on 2026-08-14 created enough temporary
+memory pressure for all four development BEAM pods to be OOM-restarted once or
+twice before Kubernetes recovered them. Run it only when the development UI
+needs current synthetic deadlines, expect the development application to be
+temporarily unavailable, and verify pod readiness afterwards. It does not
+target the independent test or production databases.
+
 Overall HTTP duration was 2.566 ms median, 3.196 ms average, and 8.709 ms
 maximum. Every tracked table count was identical before and after. The
 discovery-query medians are the independently measured values in the table

lib/mix/tasks/wnh.scale_fixtures.ex

@@ -112,7 +112,7 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
 
     action =
       case rest do
-        [value] when value in ["prepare", "append", "verify"] -> value
+        [value] when value in ["prepare", "append", "refresh", "verify"] -> value
         _other -> Mix.raise(usage())
       end
 
@@ -125,12 +125,13 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
     case action do
       "prepare" -> prepare!(context, request_count, activity_count)
       "append" -> append!(context, request_count, activity_count)
+      "refresh" -> refresh!(context, request_count, activity_count)
       "verify" -> verify!(context, request_count, activity_count, samples)
     end
   end
 
   defp usage do
-    "usage: mix wnh.scale_fixtures prepare|append|verify " <>
+    "usage: mix wnh.scale_fixtures prepare|append|refresh|verify " <>
       "[--requests COUNT] [--activities COUNT] [--samples COUNT] --output /output/FILE.json"
   end
 
@@ -144,7 +145,9 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
     output = Path.expand(output)
 
     expected_confirmation =
-      if action == "append", do: @additive_confirmation, else: @isolated_confirmation
+      if action in ["append", "refresh"],
+        do: @additive_confirmation,
+        else: @isolated_confirmation
 
     unless System.get_env("WNH_SCALE_FIXTURE_CONFIRM") == expected_confirmation do
       Mix.raise("WNH_SCALE_FIXTURE_CONFIRM must equal #{expected_confirmation}")
@@ -212,6 +215,79 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
     )
   end
 
+  defp refresh!(context, request_count, activity_count) do
+    assert_synthetic_counts!(request_count, activity_count)
+    before_profile = fixture_profile()
+
+    {:ok, refreshed} =
+      Repo.transaction(
+        fn ->
+          %{num_rows: requests} =
+            Repo.query!(
+              """
+              UPDATE help_requests
+              SET status = 'open',
+                  expires_at = date_trunc('second', now()) +
+                    (mod(mod(hashtextextended(id::text, 0), 14) + 14, 14) + 1) *
+                      interval '1 day',
+                  updated_at = date_trunc('second', now())
+              WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+              """,
+              [],
+              timeout: :infinity,
+              log: false
+            )
+
+          %{num_rows: activities} =
+            Repo.query!(
+              """
+              WITH refreshed AS (
+                SELECT
+                  id,
+                  date_trunc('second', now()) +
+                    (mod(mod(hashtextextended(id::text, 0), 60) + 60, 60) + 2) *
+                      interval '1 day'
+                    AS refreshed_starts_at
+                FROM activities
+                WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+              )
+              UPDATE activities
+              SET status = 'open',
+                  starts_at = refreshed.refreshed_starts_at,
+                  join_deadline = refreshed.refreshed_starts_at - interval '1 day',
+                  updated_at = date_trunc('second', now())
+              FROM refreshed
+              WHERE activities.id = refreshed.id
+              """,
+              [],
+              timeout: :infinity,
+              log: false
+            )
+
+          %{help_requests: requests, activities: activities}
+        end,
+        timeout: :infinity
+      )
+
+    analyze_scale_tables!()
+    profile = assert_fixture_profile!(request_count, activity_count)
+
+    write_summary!(context, "refreshed", request_count, activity_count, %{
+      expected_kind: "synthetic",
+      refreshed_rows: refreshed,
+      public_open_rows_before: %{
+        help_requests: before_profile.help_requests.public_open_rows,
+        activities: before_profile.activities.public_open_rows
+      },
+      fixture_profile: profile
+    })
+
+    Mix.shell().info(
+      "refreshed synthetic discovery deadlines in #{context.database}: " <>
+        "#{refreshed.help_requests} requests and #{refreshed.activities} activities"
+    )
+  end
+
   defp prepare!(context, request_count, activity_count) do
     counts = discovery_counts()
 

scripts/dev-scale-refresh.sh

@@ -0,0 +1,159 @@
+#!/bin/sh
+set -eu
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+
+if [ ! -f "$ROOT/.env" ]; then
+  echo "Missing $ROOT/.env." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck source=/dev/null
+. "$ROOT/.env"
+set +a
+
+EXPECTED_DATABASE=${WNH_DEV_SCALE_EXPECTED_DATABASE:-who_need_help}
+REQUESTS=${WNH_DEV_SCALE_REQUESTS:-1000000}
+ACTIVITIES=${WNH_DEV_SCALE_ACTIVITIES:-1000000}
+OUTPUT_DIR="$ROOT/output/dev-scale"
+SUMMARY="$OUTPUT_DIR/refresh-summary.json"
+CLUSTER_CONTAINER=who-need-help-control-plane
+NAMESPACE=who-need-help
+DATABASE_POD=postgis-0
+
+if [ "${WNH_DEV_SCALE_REFRESH_CONFIRM:-}" != "refresh-persistent-dev-scale" ]; then
+  echo "WNH_DEV_SCALE_REFRESH_CONFIRM must equal refresh-persistent-dev-scale." >&2
+  exit 2
+fi
+
+case "$REQUESTS:$ACTIVITIES" in
+  *[!0-9:]*|:*|*:|0:*|*:0)
+    echo "The configured scale row counts must be positive integers." >&2
+    exit 2
+    ;;
+esac
+
+if ! docker inspect "$CLUSTER_CONTAINER" >/dev/null 2>&1; then
+  echo "The project-owned kind control-plane container does not exist." >&2
+  exit 1
+fi
+
+if [ "$(docker inspect "$CLUSTER_CONTAINER" --format '{{.State.Running}}')" != true ]; then
+  echo "The project-owned kind control plane is not running." >&2
+  exit 1
+fi
+
+if [ "$(docker inspect "$CLUSTER_CONTAINER" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}')" != "who-need-help" ]; then
+  echo "The running control plane is outside this project's kind cluster." >&2
+  exit 1
+fi
+
+kube() {
+  docker exec "$CLUSTER_CONTAINER" kubectl --namespace "$NAMESPACE" "$@"
+}
+
+if [ "$(kube get pod "$DATABASE_POD" -o jsonpath='{.status.phase}')" != "Running" ]; then
+  echo "The development PostGIS pod is not running." >&2
+  exit 1
+fi
+
+# Expansion is intentionally performed inside the database pod.
+# shellcheck disable=SC2016
+DB_USER=$(kube exec "$DATABASE_POD" -- sh -c 'printf %s "$POSTGRES_USER"')
+# shellcheck disable=SC2016
+DB_NAME=$(kube exec "$DATABASE_POD" -- sh -c 'printf %s "$POSTGRES_DB"')
+
+if [ "$DB_NAME" != "$EXPECTED_DATABASE" ]; then
+  echo "Refusing refresh: expected database $EXPECTED_DATABASE, observed $DB_NAME." >&2
+  exit 1
+fi
+
+observed=$(
+  kube exec "$DATABASE_POD" -- psql -U "$DB_USER" -d "$DB_NAME" -At \
+    -c "SELECT (SELECT count(*) FROM help_requests WHERE structured_data @> '{\"scale_fixture\": true}'::jsonb)::text || ':' || (SELECT count(*) FROM activities WHERE structured_data @> '{\"scale_fixture\": true}'::jsonb)::text"
+)
+
+if [ "$observed" != "$REQUESTS:$ACTIVITIES" ]; then
+  echo "Refusing refresh: expected $REQUESTS:$ACTIVITIES synthetic rows, observed $observed." >&2
+  exit 1
+fi
+
+mkdir -p "$OUTPUT_DIR"
+
+docker exec -i "$CLUSTER_CONTAINER" \
+  kubectl --namespace "$NAMESPACE" exec -i "$DATABASE_POD" -- \
+  psql -q -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -At <<'SQL' >"$SUMMARY"
+BEGIN;
+
+UPDATE help_requests
+SET status = 'open',
+    expires_at = date_trunc('second', now()) +
+      (mod(mod(hashtextextended(id::text, 0), 14) + 14, 14) + 1) * interval '1 day',
+    updated_at = date_trunc('second', now())
+WHERE structured_data @> '{"scale_fixture": true}'::jsonb;
+
+WITH refreshed AS (
+  SELECT
+    id,
+    date_trunc('second', now()) +
+      (mod(mod(hashtextextended(id::text, 0), 60) + 60, 60) + 2) * interval '1 day'
+      AS refreshed_starts_at
+  FROM activities
+  WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+)
+UPDATE activities
+SET status = 'open',
+    starts_at = refreshed.refreshed_starts_at,
+    join_deadline = refreshed.refreshed_starts_at - interval '1 day',
+    updated_at = date_trunc('second', now())
+FROM refreshed
+WHERE activities.id = refreshed.id;
+
+COMMIT;
+ANALYZE help_requests, activities;
+
+SELECT json_build_object(
+  'database', current_database(),
+  'synthetic_rows', json_build_object(
+    'help_requests', (
+      SELECT count(*) FROM help_requests
+      WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+    ),
+    'activities', (
+      SELECT count(*) FROM activities
+      WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+    )
+  ),
+  'public_open_rows', json_build_object(
+    'help_requests', (
+      SELECT count(*) FROM help_requests
+      WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+        AND status = 'open'
+        AND expires_at > now()
+        AND location_visibility <> 'hidden'
+    ),
+    'activities', (
+      SELECT count(*) FROM activities
+      WHERE structured_data @> '{"scale_fixture": true}'::jsonb
+        AND status = 'open'
+        AND starts_at > now()
+        AND location_visibility = 'approximate_public'
+    )
+  ),
+  'captured_at', date_trunc('second', now())
+);
+SQL
+
+chmod 600 "$SUMMARY"
+jq -e '
+  .database == $database and
+  .synthetic_rows.help_requests == ($requests | tonumber) and
+  .synthetic_rows.activities == ($activities | tonumber) and
+  .public_open_rows.help_requests > 0 and
+  .public_open_rows.activities > 0
+' --arg database "$EXPECTED_DATABASE" \
+  --arg requests "$REQUESTS" \
+  --arg activities "$ACTIVITIES" \
+  "$SUMMARY" >/dev/null
+printf 'Development scale refresh summary: %s\n' "$SUMMARY"