Monitor email failures by purpose

docs/operations.md

@@ -738,11 +738,14 @@ HTTP exception, failed Oban job, and failed/exceptional email-delivery counters.
 It sends one aggregate notification when one of those counters increases. A
 new node establishes a baseline without alerting, and a lower value is treated
 as a counter reset rather than a failure. The metric labels are deliberately
-bounded to queue and delivery status; recipient addresses, message content,
+bounded to queue, delivery status, and the fixed application-owned email
-request payloads, and user identifiers are never included. With multiple
+purposes listed by `WhoNeedHelp.EmailDelivery`. This lets an operator
-application replicas, this external check samples only the node that answers
+distinguish an authentication-delivery failure from a support-update failure
-each request; use the full Prometheus deployment when every replica must be
+without exposing recipient addresses, message content, request payloads, or
-scraped continuously.
+user identifiers. Unknown email-purpose labels are ignored rather than
+creating unbounded monitor state. With multiple application replicas, this
+external check samples only the node that answers each request; use the full
+Prometheus deployment when every replica must be scraped continuously.
 
 Install it from the production SMTP configuration without printing the SMTP
 credential:

scripts/production-external-monitor.py

@@ -24,6 +24,21 @@ MONITORED_COUNTERS = {
     "who_need_help_oban_jobs_failed_total",
     "who_need_help_email_deliveries_total",
     "who_need_help_email_delivery_exceptions_total",
+    "who_need_help_email_by_kind_deliveries_total",
+}
+
+MONITORED_EMAIL_KINDS = {
+    "auth_email_change",
+    "auth_google_link",
+    "auth_login",
+    "auth_registration",
+    "content_removal_confirmation",
+    "content_removal_operator",
+    "content_removal_received",
+    "content_removal_update",
+    "support_confirmation",
+    "support_operator",
+    "support_update",
 }
 
 PROMETHEUS_SAMPLE = re.compile(
@@ -104,6 +119,13 @@ def parse_monitored_counters(payload: str) -> dict[str, float]:
             if labels.get("status") != "error":
                 continue
             key = f"{name}|status=error"
+        elif name == "who_need_help_email_by_kind_deliveries_total":
+            kind = labels.get("kind")
+            if labels.get("status") not in {"error", "exception"}:
+                continue
+            if kind not in MONITORED_EMAIL_KINDS:
+                continue
+            key = f"{name}|kind={kind}|status={labels['status']}"
         elif name == "who_need_help_oban_jobs_failed_total":
             key = f"{name}|queue={labels.get('queue', 'unknown')}"
         else:

test/scripts/production_external_monitor_test.py

@@ -115,6 +115,10 @@ who_need_help_oban_jobs_failed_total{queue="mailers"} 4
 who_need_help_email_deliveries_total{status="ok"} 50
 who_need_help_email_deliveries_total{status="error"} 5
 who_need_help_email_delivery_exceptions_total 1
+who_need_help_email_by_kind_deliveries_total{kind="auth_login",status="ok"} 40
+who_need_help_email_by_kind_deliveries_total{kind="auth_login",status="error"} 2
+who_need_help_email_by_kind_deliveries_total{kind="support_update",status="exception"} 1
+who_need_help_email_by_kind_deliveries_total{kind="user@example.test",status="error"} 99
 who_need_help_http_requests_total 999
 """
 
@@ -126,6 +130,8 @@ who_need_help_http_requests_total 999
                 "who_need_help_oban_jobs_failed_total|queue=mailers": 4.0,
                 "who_need_help_email_deliveries_total|status=error": 5.0,
                 "who_need_help_email_delivery_exceptions_total": 1.0,
+                "who_need_help_email_by_kind_deliveries_total|kind=auth_login|status=error": 2.0,
+                "who_need_help_email_by_kind_deliveries_total|kind=support_update|status=exception": 1.0,
             },
         )