ebbe.bassandClaude Sonnet 5
eaed7d4ce6
- scraper.py: --archive flag to scrape https://fundbureau.de/archiv.html (past events), fixing a container-specific wait selector and a crash on events with no door time that this surfaced. - api: EventStore now merges an optional archive JSON file into the main event list (deduped by date+name); adds GET /events/archive. - api: GET /calendar.ics serves an RFC 5545 feed of all events for webcal subscriptions, with an all-day fallback when no door time is parseable. - api: GET /health reports last-scrape time (from file mtime) and, via new SCRAPE_CRON/ARCHIVE_SCRAPE_CRON env vars, the next scheduled run and seconds until it, for both the regular and archive scrape. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
112 lines
3.8 KiB
Python
112 lines
3.8 KiB
Python
"""Loads the scraped events JSON file(s) and keeps them in memory.
|
|
|
|
Files are treated as read-only. Call ``reload()`` to pick up a fresh
|
|
scrape without restarting the server.
|
|
|
|
Two files are read:
|
|
- the main data file (upcoming events, produced by a plain
|
|
``scraper.py`` run) - required, resolved from DATA_FILE.
|
|
- the archive file (past events, produced by ``scraper.py --archive``)
|
|
- optional, resolved from ARCHIVE_DATA_FILE. Missing archive file
|
|
just means no archived events are available yet.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from datetime import date
|
|
from pathlib import Path
|
|
from threading import Lock
|
|
from typing import List, Optional
|
|
|
|
from models import Event
|
|
|
|
# Defaults: the fundi-*-output.json files sitting in the repo root, one
|
|
# directory above this file. Override with the DATA_FILE / ARCHIVE_DATA_FILE
|
|
# env vars.
|
|
_DEFAULT_DATA_FILE = Path(__file__).resolve().parent.parent / "fundi-scraped-output.json"
|
|
_DEFAULT_ARCHIVE_DATA_FILE = Path(__file__).resolve().parent.parent / "fundi-archive-output.json"
|
|
|
|
|
|
class EventStore:
|
|
def __init__(self, path: Optional[os.PathLike] = None, archive_path: Optional[os.PathLike] = None):
|
|
self.path = Path(path or os.environ.get("DATA_FILE") or _DEFAULT_DATA_FILE)
|
|
self.archive_path = Path(archive_path or os.environ.get("ARCHIVE_DATA_FILE") or _DEFAULT_ARCHIVE_DATA_FILE)
|
|
self._lock = Lock()
|
|
self._events: List[Event] = []
|
|
self._current_count = 0
|
|
self._archive_count = 0
|
|
self.reload()
|
|
|
|
@staticmethod
|
|
def _load_file(path: Path) -> List[Event]:
|
|
if not path.exists():
|
|
return []
|
|
with path.open(encoding="utf-8") as fh:
|
|
raw = json.load(fh)
|
|
return [Event.model_validate(item) for item in raw]
|
|
|
|
def reload(self) -> int:
|
|
"""Re-read the data file(s) from disk. Returns the number of events loaded."""
|
|
if not self.path.exists():
|
|
raise FileNotFoundError(f"Data file not found: {self.path}")
|
|
current = self._load_file(self.path)
|
|
archive = self._load_file(self.archive_path)
|
|
|
|
# Merge, de-duplicating on (date, name) in case an event that was
|
|
# scraped while upcoming was later also picked up by an archive
|
|
# scrape - the current-file copy wins.
|
|
seen = {(event.event_date, event.event_name) for event in current}
|
|
merged = list(current)
|
|
for event in archive:
|
|
key = (event.event_date, event.event_name)
|
|
if key not in seen:
|
|
merged.append(event)
|
|
seen.add(key)
|
|
|
|
with self._lock:
|
|
self._events = merged
|
|
self._current_count = len(current)
|
|
self._archive_count = len(merged) - len(current)
|
|
return len(merged)
|
|
|
|
def all(self) -> List[Event]:
|
|
with self._lock:
|
|
return list(self._events)
|
|
|
|
def get(self, index: int) -> Optional[Event]:
|
|
with self._lock:
|
|
if 0 <= index < len(self._events):
|
|
return self._events[index]
|
|
return None
|
|
|
|
@property
|
|
def current_count(self) -> int:
|
|
with self._lock:
|
|
return self._current_count
|
|
|
|
@property
|
|
def archive_count(self) -> int:
|
|
with self._lock:
|
|
return self._archive_count
|
|
|
|
|
|
def parse_event_date(value: str) -> Optional[date]:
|
|
"""Parse the scraper's ``dd.mm.yy`` date strings. Returns None if unparseable."""
|
|
try:
|
|
day, month, year = (int(part) for part in value.strip().split("."))
|
|
except (ValueError, AttributeError):
|
|
return None
|
|
if year < 100:
|
|
year += 2000
|
|
try:
|
|
return date(year, month, day)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def iter_artist_names(event: Event):
|
|
for group in event.event_artists:
|
|
for artist in group:
|
|
if artist.artist_name:
|
|
yield artist.artist_name
|