feat: add scheduler and mail outbox

SMTP adapter with TLS and a fake, outbox with retries, backoff, rate limit and restart safety, scheduler tick with job-run records started by gunicorn, and a send_test_mail command.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N3t4WL7bd9bdHHNjEsxzqC

portal/mailer/services.py

@@ -0,0 +1,151 @@
+"""The outbox: other modules call `enqueue`, the scheduler calls `flush` (AD-11).
+
+A mail is claimed (queued -> sending) before it is handed to the server, so two runs never
+send the same row. If the process dies in between, the row stays `sending`; later it is marked
+failed with an unknown outcome and is not sent again, because a lost mail is better than a
+duplicate one for a password or sign-in notice.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.db.models import F
+from django.template.loader import render_to_string
+from django.utils import timezone, translation
+
+from portal.adapters.mail import MailMessage, MailRejected, MailUnavailable
+from portal.adapters.registry import get_mailer
+
+from .models import OutboxMail
+
+logger = logging.getLogger(__name__)
+
+MAX_ATTEMPTS = 5
+BACKOFF_SECONDS = (60, 5 * 60, 15 * 60, 60 * 60)  # wait after the 1st, 2nd, ... failed attempt
+RATE_PER_MINUTE = 20
+LEASE_SECONDS = 10 * 60  # a row that has been `sending` this long is given up on
+STOP_AFTER_CONSECUTIVE_OUTAGES = 3
+
+_ADDRESS = re.compile(r"^[^@\s<>,;\"()\[\]\\]+@[^@\s<>,;\"()\[\]\\]+\.[^@\s<>,;\"()\[\]\\]+$")
+
+
+@dataclass(frozen=True)
+class FlushResult:
+    sent: int = 0
+    retried: int = 0
+    failed: int = 0
+
+    def __str__(self) -> str:
+        return f"sent={self.sent} retried={self.retried} failed={self.failed}"
+
+
+def is_valid_address(value: str) -> bool:
+    return len(value) <= 254 and bool(_ADDRESS.match(value))
+
+
+def enqueue(template: str, recipient: str, context: dict | None = None) -> OutboxMail:
+    """Render the e-mail in the portal-wide language and queue it.
+
+    Call it inside the transaction of the change that causes the mail, so a rolled-back change
+    queues nothing.
+    """
+    if not is_valid_address(recipient):
+        raise ValueError("Not a usable e-mail address")
+    with translation.override(settings.MAIL_LANGUAGE):
+        subject = render_to_string(f"mail/{template}/subject.txt", context or {})
+        body = render_to_string(f"mail/{template}/body.txt", context or {})
+    subject = " ".join(subject.split())  # one line, whatever the template did
+    now = timezone.now()
+    return OutboxMail.objects.create(
+        template=template,
+        recipient=recipient,
+        subject=subject[:300],
+        body_text=body.strip() + "\n",
+        created_at=now,
+        next_attempt_at=now,
+    )
+
+
+def _give_up_on_stale(now: datetime) -> None:
+    stale = OutboxMail.objects.filter(
+        status=OutboxMail.Status.SENDING, claimed_at__lt=now - timedelta(seconds=LEASE_SECONDS)
+    )
+    count = stale.update(
+        status=OutboxMail.Status.FAILED,
+        body_text="",
+        last_error="Outcome unknown after a restart; not sent again",
+    )
+    if count:
+        logger.error("Gave up on %s mails that were being sent when the process stopped", count)
+
+
+def _backoff(attempts: int) -> timedelta:
+    return timedelta(seconds=BACKOFF_SECONDS[min(attempts, len(BACKOFF_SECONDS)) - 1])
+
+
+def flush(now: datetime | None = None) -> FlushResult:
+    """Send what is due, within the rate limit. Safe to call repeatedly."""
+    now = now or timezone.now()
+    _give_up_on_stale(now)
+    recent = OutboxMail.objects.filter(
+        status=OutboxMail.Status.SENT, sent_at__gt=now - timedelta(minutes=1)
+    ).count()
+    budget = max(RATE_PER_MINUTE - recent, 0)
+    due = list(
+        OutboxMail.objects.filter(status=OutboxMail.Status.QUEUED, next_attempt_at__lte=now)
+        .order_by("id")
+        .values_list("pk", flat=True)[:budget]
+    )
+    sent = retried = failed = outages = 0
+    mailer = get_mailer()
+    for pk in due:
+        claimed = OutboxMail.objects.filter(pk=pk, status=OutboxMail.Status.QUEUED).update(
+            status=OutboxMail.Status.SENDING, claimed_at=now, attempts=F("attempts") + 1
+        )
+        if not claimed:  # another run took it
+            continue
+        mail = OutboxMail.objects.get(pk=pk)
+        message = MailMessage(
+            sender=settings.SMTP.sender,
+            recipient=mail.recipient,
+            subject=mail.subject,
+            body_text=mail.body_text,
+        )
+        try:
+            mailer.send(message)
+        except MailUnavailable as exc:
+            outages += 1
+            if mail.attempts >= MAX_ATTEMPTS:
+                _finish(mail, OutboxMail.Status.FAILED, str(exc))
+                failed += 1
+            else:
+                mail.status = OutboxMail.Status.QUEUED
+                mail.next_attempt_at = now + _backoff(mail.attempts)
+                mail.last_error = str(exc)[:200]
+                mail.save(update_fields=["status", "next_attempt_at", "last_error"])
+                retried += 1
+            logger.warning("Mail %s not sent (attempt %s)", pk, mail.attempts)
+            if outages >= STOP_AFTER_CONSECUTIVE_OUTAGES:
+                break
+        except MailRejected as exc:
+            _finish(mail, OutboxMail.Status.FAILED, str(exc))
+            failed += 1
+            logger.warning("Mail %s was refused", pk)
+        else:
+            outages = 0
+            _finish(mail, OutboxMail.Status.SENT, "", sent_at=now)
+            sent += 1
+    return FlushResult(sent, retried, failed)
+
+
+def _finish(mail: OutboxMail, status: str, error: str, sent_at: datetime | None = None) -> None:
+    mail.status = status
+    mail.last_error = error[:200]
+    mail.body_text = ""
+    mail.sent_at = sent_at
+    mail.save(update_fields=["status", "last_error", "body_text", "sent_at"])