Add archive scraping, webcal feed, and scrape schedule reporting
- scraper.py: --archive flag to scrape https://fundbureau.de/archiv.html (past events), fixing a container-specific wait selector and a crash on events with no door time that this surfaced. - api: EventStore now merges an optional archive JSON file into the main event list (deduped by date+name); adds GET /events/archive. - api: GET /calendar.ics serves an RFC 5545 feed of all events for webcal subscriptions, with an all-day fallback when no door time is parseable. - api: GET /health reports last-scrape time (from file mtime) and, via new SCRAPE_CRON/ARCHIVE_SCRAPE_CRON env vars, the next scheduled run and seconds until it, for both the regular and archive scrape. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+54
-11
@@ -1,7 +1,14 @@
|
||||
"""Loads the scraped events JSON file and keeps it in memory.
|
||||
"""Loads the scraped events JSON file(s) and keeps them in memory.
|
||||
|
||||
The file is treated as read-only. Call ``reload()`` to pick up a fresh
|
||||
Files are treated as read-only. Call ``reload()`` to pick up a fresh
|
||||
scrape without restarting the server.
|
||||
|
||||
Two files are read:
|
||||
- the main data file (upcoming events, produced by a plain
|
||||
``scraper.py`` run) - required, resolved from DATA_FILE.
|
||||
- the archive file (past events, produced by ``scraper.py --archive``)
|
||||
- optional, resolved from ARCHIVE_DATA_FILE. Missing archive file
|
||||
just means no archived events are available yet.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -13,28 +20,54 @@ from typing import List, Optional
|
||||
|
||||
from models import Event
|
||||
|
||||
# Default: the fundi-scraped-output.json sitting in the repo root, one
|
||||
# directory above this file. Override with the DATA_FILE env var.
|
||||
# Defaults: the fundi-*-output.json files sitting in the repo root, one
|
||||
# directory above this file. Override with the DATA_FILE / ARCHIVE_DATA_FILE
|
||||
# env vars.
|
||||
_DEFAULT_DATA_FILE = Path(__file__).resolve().parent.parent / "fundi-scraped-output.json"
|
||||
_DEFAULT_ARCHIVE_DATA_FILE = Path(__file__).resolve().parent.parent / "fundi-archive-output.json"
|
||||
|
||||
|
||||
class EventStore:
|
||||
def __init__(self, path: Optional[os.PathLike] = None):
|
||||
def __init__(self, path: Optional[os.PathLike] = None, archive_path: Optional[os.PathLike] = None):
|
||||
self.path = Path(path or os.environ.get("DATA_FILE") or _DEFAULT_DATA_FILE)
|
||||
self.archive_path = Path(archive_path or os.environ.get("ARCHIVE_DATA_FILE") or _DEFAULT_ARCHIVE_DATA_FILE)
|
||||
self._lock = Lock()
|
||||
self._events: List[Event] = []
|
||||
self._current_count = 0
|
||||
self._archive_count = 0
|
||||
self.reload()
|
||||
|
||||
@staticmethod
|
||||
def _load_file(path: Path) -> List[Event]:
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open(encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
return [Event.model_validate(item) for item in raw]
|
||||
|
||||
def reload(self) -> int:
|
||||
"""Re-read the JSON file from disk. Returns the number of events loaded."""
|
||||
"""Re-read the data file(s) from disk. Returns the number of events loaded."""
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"Data file not found: {self.path}")
|
||||
with self.path.open(encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
events = [Event.model_validate(item) for item in raw]
|
||||
current = self._load_file(self.path)
|
||||
archive = self._load_file(self.archive_path)
|
||||
|
||||
# Merge, de-duplicating on (date, name) in case an event that was
|
||||
# scraped while upcoming was later also picked up by an archive
|
||||
# scrape - the current-file copy wins.
|
||||
seen = {(event.event_date, event.event_name) for event in current}
|
||||
merged = list(current)
|
||||
for event in archive:
|
||||
key = (event.event_date, event.event_name)
|
||||
if key not in seen:
|
||||
merged.append(event)
|
||||
seen.add(key)
|
||||
|
||||
with self._lock:
|
||||
self._events = events
|
||||
return len(events)
|
||||
self._events = merged
|
||||
self._current_count = len(current)
|
||||
self._archive_count = len(merged) - len(current)
|
||||
return len(merged)
|
||||
|
||||
def all(self) -> List[Event]:
|
||||
with self._lock:
|
||||
@@ -46,6 +79,16 @@ class EventStore:
|
||||
return self._events[index]
|
||||
return None
|
||||
|
||||
@property
|
||||
def current_count(self) -> int:
|
||||
with self._lock:
|
||||
return self._current_count
|
||||
|
||||
@property
|
||||
def archive_count(self) -> int:
|
||||
with self._lock:
|
||||
return self._archive_count
|
||||
|
||||
|
||||
def parse_event_date(value: str) -> Optional[date]:
|
||||
"""Parse the scraper's ``dd.mm.yy`` date strings. Returns None if unparseable."""
|
||||
|
||||
Reference in New Issue
Block a user