Add archive scraping, webcal feed, and scrape schedule reporting
- scraper.py: --archive flag to scrape https://fundbureau.de/archiv.html (past events), fixing a container-specific wait selector and a crash on events with no door time that this surfaced. - api: EventStore now merges an optional archive JSON file into the main event list (deduped by date+name); adds GET /events/archive. - api: GET /calendar.ics serves an RFC 5545 feed of all events for webcal subscriptions, with an all-day fallback when no door time is parseable. - api: GET /health reports last-scrape time (from file mtime) and, via new SCRAPE_CRON/ARCHIVE_SCRAPE_CRON env vars, the next scheduled run and seconds until it, for both the regular and archive scrape. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+15
-3
@@ -8,6 +8,8 @@ load_local_file = False
|
||||
output_file = "fundi-scraped-output.json"
|
||||
output_path = "./"
|
||||
ignore_ticket_link = False
|
||||
ARCHIVE_URL = "https://fundbureau.de/archiv.html"
|
||||
ARCHIVE_OUTPUT_FILE = "fundi-archive-output.json"
|
||||
|
||||
argp = argparse.ArgumentParser()
|
||||
argp.add_argument("--url", type=str, help="URL to scrape (default: https://fundbureau.de/)")
|
||||
@@ -15,8 +17,12 @@ argp.add_argument("--output-file", type=str, help="Output JSON file name (defaul
|
||||
argp.add_argument("--output-path", type=str, help="Output path for the JSON file (default: current directory)")
|
||||
argp.add_argument("--load-local-file", action="store_true", help="Load local HTML file instead of scraping the website (default: False)")
|
||||
argp.add_argument("--ignore-ticket-link", action="store_true", help="Ignore ticket link extraction (will leave it empty in the output JSON)")
|
||||
argp.add_argument("--archive", action="store_true", help=f"Scrape the past-events archive ({ARCHIVE_URL}) instead of upcoming events. Changes the default url/output-file, both can still be overridden with --url/--output-file.")
|
||||
args = argp.parse_args()
|
||||
|
||||
if vars(args)["archive"]:
|
||||
scrape_url = ARCHIVE_URL
|
||||
output_file = ARCHIVE_OUTPUT_FILE
|
||||
if vars(args)["url"]:
|
||||
scrape_url = vars(args)["url"]
|
||||
if vars(args)["output_file"]:
|
||||
@@ -42,8 +48,13 @@ def scrape_website(url, load_local_file):
|
||||
page = browser.new_page()
|
||||
page.goto(url, wait_until="networkidle")
|
||||
try:
|
||||
# Wait until at least one .event div actually renders in the DOM
|
||||
page.wait_for_selector("#upcoming-events-container .event", timeout=15000)
|
||||
# Wait until at least one .event div actually renders in the DOM.
|
||||
# Upcoming events live in #upcoming-events-container, past/archive
|
||||
# events in #past-events-container - match either.
|
||||
page.wait_for_selector(
|
||||
"#upcoming-events-container .event, #past-events-container .event",
|
||||
timeout=15000,
|
||||
)
|
||||
except Exception:
|
||||
print("Warning: no .event elements appeared before timeout.")
|
||||
html = page.content()
|
||||
@@ -86,7 +97,8 @@ def extract_events(soup, ignore_ticket_link):
|
||||
})
|
||||
|
||||
|
||||
event_starttime = event.find("div", class_="event-starttime").text.strip()
|
||||
event_starttime_tag = event.find("div", class_="event-starttime")
|
||||
event_starttime = event_starttime_tag.text.strip() if event_starttime_tag else ""
|
||||
if ignore_ticket_link:
|
||||
event_ticket_link = ""
|
||||
event_free = None
|
||||
|
||||
Reference in New Issue
Block a user