ping-mee-self-service-platform/portal/mailer/services.py

"""The outbox: other modules call `enqueue`, the scheduler calls `flush` (AD-11).

A mail is claimed (queued -> sending) before it is handed to the server, so two runs never
send the same row. If the process dies in between, the row stays `sending`; later it is marked
failed with an unknown outcome and is not sent again, because a lost mail is better than a
duplicate one for a password or sign-in notice.
"""

from __future__ import annotations

import logging
import re
from dataclasses import dataclass
from datetime import datetime, timedelta

from django.conf import settings
from django.db.models import F
from django.template.loader import render_to_string
from django.utils import timezone, translation

from portal.adapters.mail import MailMessage, MailRejected, MailUnavailable
from portal.adapters.registry import get_mailer

from .models import OutboxMail

logger = logging.getLogger(__name__)

MAX_ATTEMPTS = 5
BACKOFF_SECONDS = (60, 5 * 60, 15 * 60, 60 * 60)  # wait after the 1st, 2nd, ... failed attempt
RATE_PER_MINUTE = 20
LEASE_SECONDS = 10 * 60  # a row that has been `sending` this long is given up on
STOP_AFTER_CONSECUTIVE_OUTAGES = 3

_ADDRESS = re.compile(r"^[^@\s<>,;\"()\[\]\\]+@[^@\s<>,;\"()\[\]\\]+\.[^@\s<>,;\"()\[\]\\]+$")


@dataclass(frozen=True)
class FlushResult:
    sent: int = 0
    retried: int = 0
    failed: int = 0

    def __str__(self) -> str:
        return f"sent={self.sent} retried={self.retried} failed={self.failed}"


def is_valid_address(value: str) -> bool:
    return len(value) <= 254 and bool(_ADDRESS.match(value))


def enqueue(template: str, recipient: str, context: dict | None = None) -> OutboxMail:
    """Render the e-mail in the portal-wide language and queue it.

    Call it inside the transaction of the change that causes the mail, so a rolled-back change
    queues nothing.
    """
    if not is_valid_address(recipient):
        raise ValueError("Not a usable e-mail address")
    with translation.override(settings.MAIL_LANGUAGE):
        subject = render_to_string(f"mail/{template}/subject.txt", context or {})
        body = render_to_string(f"mail/{template}/body.txt", context or {})
    subject = " ".join(subject.split())  # one line, whatever the template did
    now = timezone.now()
    return OutboxMail.objects.create(
        template=template,
        recipient=recipient,
        subject=subject[:300],
        body_text=body.strip() + "\n",
        created_at=now,
        next_attempt_at=now,
    )


def _give_up_on_stale(now: datetime) -> None:
    stale = OutboxMail.objects.filter(
        status=OutboxMail.Status.SENDING, claimed_at__lt=now - timedelta(seconds=LEASE_SECONDS)
    )
    count = stale.update(
        status=OutboxMail.Status.FAILED,
        body_text="",
        last_error="Outcome unknown after a restart; not sent again",
    )
    if count:
        logger.error("Gave up on %s mails that were being sent when the process stopped", count)


def _backoff(attempts: int) -> timedelta:
    return timedelta(seconds=BACKOFF_SECONDS[min(attempts, len(BACKOFF_SECONDS)) - 1])


def flush(now: datetime | None = None) -> FlushResult:
    """Send what is due, within the rate limit. Safe to call repeatedly."""
    now = now or timezone.now()
    _give_up_on_stale(now)
    recent = OutboxMail.objects.filter(
        status=OutboxMail.Status.SENT, sent_at__gt=now - timedelta(minutes=1)
    ).count()
    budget = max(RATE_PER_MINUTE - recent, 0)
    due = list(
        OutboxMail.objects.filter(status=OutboxMail.Status.QUEUED, next_attempt_at__lte=now)
        .order_by("id")
        .values_list("pk", flat=True)[:budget]
    )
    sent = retried = failed = outages = 0
    mailer = get_mailer()
    for pk in due:
        claimed = OutboxMail.objects.filter(pk=pk, status=OutboxMail.Status.QUEUED).update(
            status=OutboxMail.Status.SENDING, claimed_at=now, attempts=F("attempts") + 1
        )
        if not claimed:  # another run took it
            continue
        mail = OutboxMail.objects.get(pk=pk)
        message = MailMessage(
            sender=settings.SMTP.sender,
            recipient=mail.recipient,
            subject=mail.subject,
            body_text=mail.body_text,
        )
        try:
            mailer.send(message)
        except MailUnavailable as exc:
            outages += 1
            if mail.attempts >= MAX_ATTEMPTS:
                _finish(mail, OutboxMail.Status.FAILED, str(exc))
                failed += 1
            else:
                mail.status = OutboxMail.Status.QUEUED
                mail.next_attempt_at = now + _backoff(mail.attempts)
                mail.last_error = str(exc)[:200]
                mail.save(update_fields=["status", "next_attempt_at", "last_error"])
                retried += 1
            logger.warning("Mail %s not sent (attempt %s)", pk, mail.attempts)
            if outages >= STOP_AFTER_CONSECUTIVE_OUTAGES:
                break
        except MailRejected as exc:
            _finish(mail, OutboxMail.Status.FAILED, str(exc))
            failed += 1
            logger.warning("Mail %s was refused", pk)
        else:
            outages = 0
            _finish(mail, OutboxMail.Status.SENT, "", sent_at=now)
            sent += 1
    return FlushResult(sent, retried, failed)


def _finish(mail: OutboxMail, status: str, error: str, sent_at: datetime | None = None) -> None:
    mail.status = status
    mail.last_error = error[:200]
    mail.body_text = ""
    mail.sent_at = sent_at
    mail.save(update_fields=["status", "last_error", "body_text", "sent_at"])