Add archive scraping, webcal feed, and scrape schedule reporting
- scraper.py: --archive flag to scrape https://fundbureau.de/archiv.html (past events), fixing a container-specific wait selector and a crash on events with no door time that this surfaced. - api: EventStore now merges an optional archive JSON file into the main event list (deduped by date+name); adds GET /events/archive. - api: GET /calendar.ics serves an RFC 5545 feed of all events for webcal subscriptions, with an all-day fallback when no door time is parseable. - api: GET /health reports last-scrape time (from file mtime) and, via new SCRAPE_CRON/ARCHIVE_SCRAPE_CRON env vars, the next scheduled run and seconds until it, for both the regular and archive scrape. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
"""Reports when a scrape last ran and when cron will next run it.
|
||||
|
||||
"Last ran" is read from the mtime of the scraper's output file - true
|
||||
regardless of how/when the scrape was actually invoked. "Next run" requires
|
||||
telling this module the cron schedule via an env var (there's no reliable,
|
||||
portable way to introspect another process's crontab from here), given as
|
||||
a standard 5-field cron expression, e.g. "0 * * * *".
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
|
||||
def _now_local() -> datetime:
|
||||
return datetime.now().astimezone()
|
||||
|
||||
|
||||
def _file_mtime_local(path: Path) -> Optional[datetime]:
|
||||
try:
|
||||
return datetime.fromtimestamp(path.stat().st_mtime).astimezone()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def _next_cron_run(cron_expr: str, after: datetime) -> Optional[datetime]:
|
||||
try:
|
||||
naive_after = after.replace(tzinfo=None)
|
||||
return croniter(cron_expr, naive_after).get_next(datetime).astimezone()
|
||||
except (ValueError, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def get_schedule_info(path: Path, cron_expr: Optional[str]) -> dict:
|
||||
"""Build the last/next-scrape block for one scraper output file.
|
||||
|
||||
``cron_expr`` is that scrape's cron expression (e.g. "0 * * * *");
|
||||
None/invalid -> next-run fields are null.
|
||||
"""
|
||||
now = _now_local()
|
||||
cron_expr = (cron_expr or "").strip() or None
|
||||
|
||||
last_scrape_at = _file_mtime_local(path)
|
||||
next_scrape_at = _next_cron_run(cron_expr, now) if cron_expr else None
|
||||
|
||||
return {
|
||||
"cron_schedule": cron_expr,
|
||||
"last_scrape_at": last_scrape_at.isoformat() if last_scrape_at else None,
|
||||
"seconds_since_last_scrape": (
|
||||
round((now - last_scrape_at).total_seconds()) if last_scrape_at else None
|
||||
),
|
||||
"next_scrape_at": next_scrape_at.isoformat() if next_scrape_at else None,
|
||||
"seconds_until_next_scrape": (
|
||||
round((next_scrape_at - now).total_seconds()) if next_scrape_at else None
|
||||
),
|
||||
}
|
||||
Reference in New Issue
Block a user