Add archive scraping, webcal feed, and scrape schedule reporting

- scraper.py: --archive flag to scrape https://fundbureau.de/archiv.html
  (past events), fixing a container-specific wait selector and a crash
  on events with no door time that this surfaced.
- api: EventStore now merges an optional archive JSON file into the
  main event list (deduped by date+name); adds GET /events/archive.
- api: GET /calendar.ics serves an RFC 5545 feed of all events for
  webcal subscriptions, with an all-day fallback when no door time is
  parseable.
- api: GET /health reports last-scrape time (from file mtime) and, via
  new SCRAPE_CRON/ARCHIVE_SCRAPE_CRON env vars, the next scheduled run
  and seconds until it, for both the regular and archive scrape.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-05 13:42:51 +02:00
co-authored by Claude Sonnet 5
parent e4ffa61988
commit eaed7d4ce6
8 changed files with 436 additions and 46 deletions
+54 -11
View File
@@ -1,7 +1,14 @@
"""Loads the scraped events JSON file and keeps it in memory.
"""Loads the scraped events JSON file(s) and keeps them in memory.
The file is treated as read-only. Call ``reload()`` to pick up a fresh
Files are treated as read-only. Call ``reload()`` to pick up a fresh
scrape without restarting the server.
Two files are read:
- the main data file (upcoming events, produced by a plain
``scraper.py`` run) - required, resolved from DATA_FILE.
- the archive file (past events, produced by ``scraper.py --archive``)
- optional, resolved from ARCHIVE_DATA_FILE. Missing archive file
just means no archived events are available yet.
"""
import json
@@ -13,28 +20,54 @@ from typing import List, Optional
from models import Event
# Default: the fundi-scraped-output.json sitting in the repo root, one
# directory above this file. Override with the DATA_FILE env var.
# Defaults: the fundi-*-output.json files sitting in the repo root, one
# directory above this file. Override with the DATA_FILE / ARCHIVE_DATA_FILE
# env vars.
_DEFAULT_DATA_FILE = Path(__file__).resolve().parent.parent / "fundi-scraped-output.json"
_DEFAULT_ARCHIVE_DATA_FILE = Path(__file__).resolve().parent.parent / "fundi-archive-output.json"
class EventStore:
def __init__(self, path: Optional[os.PathLike] = None):
def __init__(self, path: Optional[os.PathLike] = None, archive_path: Optional[os.PathLike] = None):
self.path = Path(path or os.environ.get("DATA_FILE") or _DEFAULT_DATA_FILE)
self.archive_path = Path(archive_path or os.environ.get("ARCHIVE_DATA_FILE") or _DEFAULT_ARCHIVE_DATA_FILE)
self._lock = Lock()
self._events: List[Event] = []
self._current_count = 0
self._archive_count = 0
self.reload()
@staticmethod
def _load_file(path: Path) -> List[Event]:
if not path.exists():
return []
with path.open(encoding="utf-8") as fh:
raw = json.load(fh)
return [Event.model_validate(item) for item in raw]
def reload(self) -> int:
"""Re-read the JSON file from disk. Returns the number of events loaded."""
"""Re-read the data file(s) from disk. Returns the number of events loaded."""
if not self.path.exists():
raise FileNotFoundError(f"Data file not found: {self.path}")
with self.path.open(encoding="utf-8") as fh:
raw = json.load(fh)
events = [Event.model_validate(item) for item in raw]
current = self._load_file(self.path)
archive = self._load_file(self.archive_path)
# Merge, de-duplicating on (date, name) in case an event that was
# scraped while upcoming was later also picked up by an archive
# scrape - the current-file copy wins.
seen = {(event.event_date, event.event_name) for event in current}
merged = list(current)
for event in archive:
key = (event.event_date, event.event_name)
if key not in seen:
merged.append(event)
seen.add(key)
with self._lock:
self._events = events
return len(events)
self._events = merged
self._current_count = len(current)
self._archive_count = len(merged) - len(current)
return len(merged)
def all(self) -> List[Event]:
with self._lock:
@@ -46,6 +79,16 @@ class EventStore:
return self._events[index]
return None
@property
def current_count(self) -> int:
with self._lock:
return self._current_count
@property
def archive_count(self) -> int:
with self._lock:
return self._archive_count
def parse_event_date(value: str) -> Optional[date]:
"""Parse the scraper's ``dd.mm.yy`` date strings. Returns None if unparseable."""