Harden production E2E fixture cleanup

lib/mix/tasks/wnh.staging_full_e2e.ex

@@ -4,14 +4,29 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
   import Ecto.Query
 
   alias Oban.Job
-  alias WhoNeedHelp.Accounts.{SocialIdentity, StaffRoleAssignment, User}
+
+  alias WhoNeedHelp.Accounts.{
+    AuthIdentity,
+    SocialIdentity,
+    StaffRoleAssignment,
+    User,
+    UserToken
+  }
+
   alias WhoNeedHelp.Activities.{Activity, Participant}
   alias WhoNeedHelp.Activities.Message, as: ActivityMessage
   alias WhoNeedHelp.Catalog.{CategoryProposal, CategoryVote}
   alias WhoNeedHelp.ContentRemoval.Notice
   alias WhoNeedHelp.Help.{Assignment, HelpRequest}
   alias WhoNeedHelp.Messaging.Message
-  alias WhoNeedHelp.Notifications.Notification
+
+  alias WhoNeedHelp.Notifications.{
+    NearbySubscription,
+    Notification,
+    Preference,
+    PushDevice
+  }
+
   alias WhoNeedHelp.Repo
   alias WhoNeedHelp.Support.SupportRequest
   alias WhoNeedHelp.Tracking.{Position, TrackingSession}
@@ -29,6 +44,36 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
   @precreated_roles ~w(requester helper replacement-helper activity-organizer activity-participant admin)
   @registered_roles ~w(auth-user auth-user-changed)
 
+  @target_schemas %{
+    users: User,
+    user_tokens: UserToken,
+    auth_identities: AuthIdentity,
+    social_identities: SocialIdentity,
+    staff_role_assignments: StaffRoleAssignment,
+    requests: HelpRequest,
+    assignments: Assignment,
+    messages: Message,
+    tracking_sessions: TrackingSession,
+    tracking_positions: Position,
+    reviews: Review,
+    activities: Activity,
+    activity_participants: Participant,
+    activity_messages: ActivityMessage,
+    reports: Report,
+    category_proposals: CategoryProposal,
+    category_votes: CategoryVote,
+    audit_events: AuditEvent,
+    abuse_signals: AbuseSignal,
+    blocks: Block,
+    notifications: Notification,
+    notification_preferences: Preference,
+    nearby_subscriptions: NearbySubscription,
+    push_devices: PushDevice,
+    support_requests: SupportRequest,
+    content_removal_notices: Notice,
+    push_jobs: Job
+  }
+
   @impl Mix.Task
   def run([action]) when action in ["prepare", "cleanup"] do
     Mix.Task.run("app.start")
@@ -121,8 +166,7 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
     }
 
     context.manifest_path |> Path.dirname() |> File.mkdir_p!()
-    File.write!(context.manifest_path, Jason.encode_to_iodata!(manifest, pretty: true))
+    write_manifest!(context.manifest_path, manifest)
-    File.chmod!(context.manifest_path, 0o600)
 
     Mix.shell().info("prepared exact full staging E2E users for #{context.run_id}")
   end
@@ -140,6 +184,21 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
     validate_precreated_users!(manifest, users)
 
     user_ids = Enum.map(users, & &1.id)
+    user_token_ids = ids(UserToken, :user_id, user_ids)
+    auth_identity_ids = ids(AuthIdentity, :user_id, user_ids)
+    social_identity_ids = ids(SocialIdentity, :user_id, user_ids)
+
+    staff_role_assignment_ids =
+      StaffRoleAssignment
+      |> where(
+        [assignment],
+        assignment.user_id in ^user_ids or assignment.assigned_by_id in ^user_ids
+      )
+      |> select([assignment], assignment.id)
+      |> Repo.all()
+
+    validate_staff_role_assignments!(staff_role_assignment_ids, user_ids)
+
     request_ids = ids(HelpRequest, :requester_id, user_ids)
 
     assignment_ids =
@@ -155,6 +214,7 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
 
     message_ids = ids(Message, :assignment_id, assignment_ids)
     tracking_session_ids = ids(TrackingSession, :assignment_id, assignment_ids)
+    tracking_position_ids = ids(Position, :tracking_session_id, tracking_session_ids)
 
     notification_ids =
       Notification
@@ -206,6 +266,36 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
       |> select([signal], signal.id)
       |> Repo.all()
 
+    review_ids =
+      Review
+      |> where(
+        [review],
+        review.assignment_id in ^assignment_ids or review.reviewer_id in ^user_ids or
+          review.reviewee_id in ^user_ids
+      )
+      |> select([review], review.id)
+      |> Repo.all()
+
+    category_vote_ids =
+      CategoryVote
+      |> where(
+        [vote],
+        vote.proposal_id in ^proposal_ids or vote.user_id in ^user_ids
+      )
+      |> select([vote], vote.id)
+      |> Repo.all()
+
+    block_ids =
+      Block
+      |> where([block], block.blocker_id in ^user_ids or block.blocked_id in ^user_ids)
+      |> select([block], block.id)
+      |> Repo.all()
+
+    audit_event_ids = ids(AuditEvent, :actor_id, user_ids)
+    notification_preference_ids = ids(Preference, :user_id, user_ids)
+    nearby_subscription_ids = ids(NearbySubscription, :user_id, user_ids)
+    push_device_ids = ids(PushDevice, :user_id, user_ids)
+
     validate_audits(%{
       users: user_ids,
       requests: request_ids,
@@ -221,54 +311,73 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
 
     job_ids = fixture_job_ids(user_ids, request_ids, notification_ids)
 
+    targets = %{
+      users: user_ids,
+      user_tokens: user_token_ids,
+      auth_identities: auth_identity_ids,
+      social_identities: social_identity_ids,
+      staff_role_assignments: staff_role_assignment_ids,
+      requests: request_ids,
+      assignments: assignment_ids,
+      messages: message_ids,
+      tracking_sessions: tracking_session_ids,
+      tracking_positions: tracking_position_ids,
+      reviews: review_ids,
+      activities: activity_ids,
+      activity_participants: activity_participant_ids,
+      activity_messages: activity_message_ids,
+      reports: report_ids,
+      category_proposals: proposal_ids,
+      category_votes: category_vote_ids,
+      audit_events: audit_event_ids,
+      abuse_signals: abuse_signal_ids,
+      blocks: block_ids,
+      notifications: notification_ids,
+      notification_preferences: notification_preference_ids,
+      nearby_subscriptions: nearby_subscription_ids,
+      push_devices: push_device_ids,
+      support_requests: support_request_ids,
+      content_removal_notices: content_removal_notice_ids,
+      push_jobs: job_ids
+    }
+
+    manifest =
+      manifest
+      |> Map.put("observed_users", Map.new(users, &{&1.email, &1.id}))
+      |> Map.put("cleanup_targets", stringify_keys(targets))
+      |> Map.put("cleanup_started_at", DateTime.utc_now() |> DateTime.to_iso8601())
+
+    write_manifest!(context.manifest_path, manifest)
+
     {:ok, deleted} =
       Repo.transaction(fn ->
         %{
           push_jobs: delete_ids(Job, job_ids),
           notifications: delete_ids(Notification, notification_ids),
-          audit_events:
+          audit_events: delete_ids(AuditEvent, audit_event_ids),
-            AuditEvent
-            |> where([event], event.actor_id in ^user_ids)
-            |> delete_count(),
           support_requests: delete_ids(SupportRequest, support_request_ids),
           content_removal_notices: delete_ids(Notice, content_removal_notice_ids),
           reports: delete_ids(Report, report_ids),
           activity_messages: delete_ids(ActivityMessage, activity_message_ids),
           activity_participants: delete_ids(Participant, activity_participant_ids),
           activities: delete_ids(Activity, activity_ids),
-          tracking_positions:
+          tracking_positions: delete_ids(Position, tracking_position_ids),
-            Position
-            |> where([position], position.tracking_session_id in ^tracking_session_ids)
-            |> delete_count(),
           tracking_sessions: delete_ids(TrackingSession, tracking_session_ids),
           messages: delete_ids(Message, message_ids),
           abuse_signals: delete_ids(AbuseSignal, abuse_signal_ids),
-          reviews:
+          reviews: delete_ids(Review, review_ids),
-            Review
-            |> where(
-              [review],
-              review.assignment_id in ^assignment_ids or review.reviewer_id in ^user_ids or
-                review.reviewee_id in ^user_ids
-            )
-            |> delete_count(),
           assignments: delete_ids(Assignment, assignment_ids),
           requests: delete_ids(HelpRequest, request_ids),
-          category_votes:
+          category_votes: delete_ids(CategoryVote, category_vote_ids),
-            CategoryVote
-            |> where(
-              [vote],
-              vote.proposal_id in ^proposal_ids or vote.user_id in ^user_ids
-            )
-            |> delete_count(),
           category_proposals: delete_ids(CategoryProposal, proposal_ids),
-          blocks:
+          blocks: delete_ids(Block, block_ids),
-            Block
+          nearby_subscriptions: delete_ids(NearbySubscription, nearby_subscription_ids),
-            |> where([block], block.blocker_id in ^user_ids or block.blocked_id in ^user_ids)
+          notification_preferences: delete_ids(Preference, notification_preference_ids),
-            |> delete_count(),
+          push_devices: delete_ids(PushDevice, push_device_ids),
-          social_identities:
+          staff_role_assignments: delete_ids(StaffRoleAssignment, staff_role_assignment_ids),
-            SocialIdentity
+          social_identities: delete_ids(SocialIdentity, social_identity_ids),
-            |> where([identity], identity.user_id in ^user_ids)
+          auth_identities: delete_ids(AuthIdentity, auth_identity_ids),
-            |> delete_count(),
+          user_tokens: delete_ids(UserToken, user_token_ids),
           users:
             User
             |> where([user], user.id in ^user_ids)
@@ -282,7 +391,17 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
       )
     end
 
+    assert_targets_absent!(targets)
+    assert_no_late_fixture_jobs!(user_ids, request_ids, notification_ids)
     assert_prefix_unused!(context)
+
+    manifest =
+      manifest
+      |> Map.put("cleanup_deleted_counts", stringify_keys(deleted))
+      |> Map.put("cleanup_verified", true)
+      |> Map.put("cleanup_verified_at", DateTime.utc_now() |> DateTime.to_iso8601())
+
+    write_manifest!(context.manifest_path, manifest)
     Mix.shell().info("removed exact full staging E2E fixture: #{inspect(deleted)}")
   end
 
@@ -334,6 +453,19 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
       do: Mix.raise("a full staging fixture assignment crosses into non-fixture data")
   end
 
+  defp validate_staff_role_assignments!(assignment_ids, user_ids) do
+    unexpected? =
+      Repo.exists?(
+        from(assignment in StaffRoleAssignment,
+          where: assignment.id in ^assignment_ids and assignment.user_id not in ^user_ids
+        )
+      )
+
+    if unexpected? do
+      Mix.raise("the run-scoped administrator assigned a role to a non-fixture user")
+    end
+  end
+
   defp validate_activity_membership!(activity_ids, user_ids) do
     unexpected_participant? =
       Repo.exists?(
@@ -465,6 +597,30 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
     |> Repo.all()
   end
 
+  defp assert_targets_absent!(targets) do
+    remaining =
+      Enum.reduce(@target_schemas, %{}, fn {name, schema}, acc ->
+        ids = Map.fetch!(targets, name)
+
+        if ids != [] and Repo.exists?(from(row in schema, where: row.id in ^ids)) do
+          Map.put(acc, name, ids)
+        else
+          acc
+        end
+      end)
+
+    if remaining != %{} do
+      Mix.raise("run-scoped records remain after cleanup: #{inspect(Map.keys(remaining))}")
+    end
+  end
+
+  defp assert_no_late_fixture_jobs!(user_ids, request_ids, notification_ids) do
+    case fixture_job_ids(user_ids, request_ids, notification_ids) do
+      [] -> :ok
+      ids -> Mix.raise("run-scoped Oban jobs appeared during cleanup: #{inspect(ids)}")
+    end
+  end
+
   defp assert_prefix_unused!(context) do
     if Repo.exists?(from(user in User, where: like(user.email, ^"#{prefix(context.run_id)}%"))) do
       Mix.raise("staging E2E users still exist for #{inspect(context.run_id)}")
@@ -503,6 +659,17 @@ defmodule Mix.Tasks.Wnh.StagingFullE2e do
 
   defp delete_count(query), do: query |> Repo.delete_all() |> elem(0)
 
+  defp stringify_keys(map) do
+    Map.new(map, fn {key, value} -> {Atom.to_string(key), value} end)
+  end
+
+  defp write_manifest!(path, manifest) do
+    temporary_path = path <> ".tmp"
+    File.write!(temporary_path, Jason.encode_to_iodata!(manifest, pretty: true))
+    File.chmod!(temporary_path, 0o600)
+    File.rename!(temporary_path, path)
+  end
+
   defp insert_user!(email, display_name, password_hash, now) do
     %User{}
     |> User.registration_changeset(%{

scripts/production-full-e2e.sh

@@ -34,7 +34,7 @@ if [[ ! "$RUN_ID" =~ ^[a-z0-9-]+$ ]]; then
   exit 1
 fi
 
-for command in curl docker git jq mktemp openssl scp sha256sum ssh tar; do
+for command in curl docker git jq mktemp openssl sha256sum ssh tar; do
   command -v "$command" >/dev/null 2>&1 || {
     echo "Required command is unavailable: $command" >&2
     exit 2
@@ -292,12 +292,34 @@ REMOTE
 cleanup() {
   local status=$?
   trap - EXIT HUP INT TERM
+  local copied_manifest=0
 
   if [[ "$prepared" -eq 1 ]]; then
     if ! run_fixture cleanup >"$output_dir/fixture-cleanup.log" 2>&1; then
       echo "Exact production E2E fixture cleanup failed; inspect $output_dir." >&2
       status=1
     fi
+
+    if ssh -o BatchMode=yes "$SSH_TARGET" docker run --rm \
+      --volume "$remote_output:/output:ro" \
+      "$tools_image" \
+      cat /output/fixture.json >"$output_dir/fixture.json"; then
+      copied_manifest=1
+    else
+      echo "Production E2E cleanup manifest could not be copied." >&2
+      status=1
+    fi
+
+    if [[ "$copied_manifest" -eq 1 ]] &&
+      ! jq -e '
+        .cleanup_verified == true and
+        (.cleanup_targets.users | length) >= 6 and
+        (.cleanup_deleted_counts.users | type) == "number" and
+        .cleanup_verified_at != null
+      ' "$output_dir/fixture.json" >/dev/null; then
+      echo "Production E2E cleanup manifest lacks exact run-scoped verification." >&2
+      status=1
+    fi
   fi
 
   snapshot "$output_dir/database-after.json" 2>"$output_dir/database-after-error.log" || status=1
@@ -308,7 +330,6 @@ cleanup() {
     status=1
   fi
 
-  scp -q "$SSH_TARGET:$remote_output/fixture.json" "$output_dir/fixture.json" 2>/dev/null || true
   ssh -o BatchMode=yes "$SSH_TARGET" \
     "rm -rf -- '$remote_output'; docker image rm '$tools_image' >/dev/null 2>&1 || true" || status=1
   docker image rm "$tools_image" "$browser_image" >/dev/null 2>&1 || true