Add archive scraping, webcal feed, and scrape schedule reporting
- scraper.py: --archive flag to scrape https://fundbureau.de/archiv.html (past events), fixing a container-specific wait selector and a crash on events with no door time that this surfaced. - api: EventStore now merges an optional archive JSON file into the main event list (deduped by date+name); adds GET /events/archive. - api: GET /calendar.ics serves an RFC 5545 feed of all events for webcal subscriptions, with an all-day fallback when no door time is parseable. - api: GET /health reports last-scrape time (from file mtime) and, via new SCRAPE_CRON/ARCHIVE_SCRAPE_CRON env vars, the next scheduled run and seconds until it, for both the regular and archive scrape. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+56
-3
@@ -8,12 +8,16 @@ Run with:
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.responses import PlainTextResponse
|
||||
|
||||
from calendar_feed import build_ics
|
||||
from datasource import EventStore, iter_artist_names, parse_event_date
|
||||
from models import Event, EventList
|
||||
from scrape_schedule import get_schedule_info
|
||||
|
||||
app = FastAPI(
|
||||
title="Fundi Scraper API",
|
||||
@@ -55,17 +59,34 @@ def _matches(
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok", "events_loaded": len(store.all()), "data_file": str(store.path)}
|
||||
scrape_cron = os.environ.get("SCRAPE_CRON")
|
||||
archive_scrape_cron = os.environ.get("ARCHIVE_SCRAPE_CRON") or scrape_cron
|
||||
return {
|
||||
"status": "ok",
|
||||
"events_loaded": len(store.all()),
|
||||
"data_file": str(store.path),
|
||||
"current_events": store.current_count,
|
||||
"archive_file": str(store.archive_path),
|
||||
"archive_file_found": store.archive_path.exists(),
|
||||
"archive_events": store.archive_count,
|
||||
"scrape": get_schedule_info(store.path, scrape_cron),
|
||||
"archive_scrape": get_schedule_info(store.archive_path, archive_scrape_cron),
|
||||
}
|
||||
|
||||
|
||||
@app.post("/reload")
|
||||
def reload():
|
||||
"""Re-read the JSON file from disk (e.g. after a fresh scrape)."""
|
||||
"""Re-read the data file(s) from disk (e.g. after a fresh scrape)."""
|
||||
try:
|
||||
count = store.reload()
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
return {"reloaded": True, "events_loaded": count}
|
||||
return {
|
||||
"reloaded": True,
|
||||
"events_loaded": count,
|
||||
"current_events": store.current_count,
|
||||
"archive_events": store.archive_count,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/events", response_model=EventList)
|
||||
@@ -94,6 +115,38 @@ def list_artists():
|
||||
return sorted(names, key=str.lower)
|
||||
|
||||
|
||||
@app.get("/events/archive", response_model=EventList)
|
||||
def list_archive_events(
|
||||
free: Optional[bool] = Query(None, description="Filter by free admission"),
|
||||
name: Optional[str] = Query(None, description="Case-insensitive substring match on event name"),
|
||||
artist: Optional[str] = Query(None, description="Case-insensitive substring match on any artist name"),
|
||||
date: Optional[str] = Query(None, description="Exact match on the raw event date (dd.mm.yy)"),
|
||||
):
|
||||
"""Past events only - shorthand for ``/events?upcoming=false``."""
|
||||
today = datetime.date.today()
|
||||
events = [
|
||||
e
|
||||
for e in store.all()
|
||||
if _matches(e, free, name, artist, date, upcoming=False, today=today)
|
||||
]
|
||||
return EventList(count=len(events), events=events)
|
||||
|
||||
|
||||
@app.get("/calendar.ics")
|
||||
def calendar_ics():
|
||||
"""iCalendar feed of all known events (upcoming + archived).
|
||||
|
||||
Subscribe from a calendar app with ``webcal://<host>/calendar.ics``
|
||||
(or the plain https URL) to get an auto-refreshing calendar.
|
||||
"""
|
||||
ics = build_ics(store.all())
|
||||
return PlainTextResponse(
|
||||
content=ics,
|
||||
media_type="text/calendar",
|
||||
headers={"Content-Disposition": "inline; filename=fundbureau-events.ics"},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/events/{index}", response_model=Event)
|
||||
def get_event(index: int):
|
||||
event = store.get(index)
|
||||
|
||||
Reference in New Issue
Block a user