diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..d8cd896 Binary files /dev/null and b/.DS_Store differ diff --git a/scraper/scraper.py b/scraper/scraper.py new file mode 100644 index 0000000..8967d6b --- /dev/null +++ b/scraper/scraper.py @@ -0,0 +1,112 @@ +import argparse +from bs4 import BeautifulSoup +import json +from playwright.sync_api import sync_playwright + +scrape_url = "https://fundbureau.de/" +load_local_file = False +output_file = "fundi-scraped-output.json" +output_path = "./" +ignore_ticket_link = False + +argp = argparse.ArgumentParser() +argp.add_argument("--url", type=str, help="URL to scrape (default: https://fundbureau.de/)") +argp.add_argument("--output-file", type=str, help="Output JSON file name (default: fundi-scraped-output.json)") +argp.add_argument("--output-path", type=str, help="Output path for the JSON file (default: current directory)") +argp.add_argument("--load-local-file", action="store_true", help="Load local HTML file instead of scraping the website (default: False)") +argp.add_argument("--ignore-ticket-link", action="store_true", help="Ignore ticket link extraction (will leave it empty in the output JSON)") +args = argp.parse_args() + +if vars(args)["url"]: + scrape_url = vars(args)["url"] +if vars(args)["output_file"]: + output_file = vars(args)["output_file"] +if vars(args)["output_path"]: + output_path = vars(args)["output_path"] +if vars(args)["load_local_file"]: + load_local_file = True +if vars(args)["ignore_ticket_link"]: + ignore_ticket_link = True + +# ------------- +# START OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium) + +def scrape_website(url, load_local_file): + if load_local_file == True: + with open(url) as file: + soup = BeautifulSoup(file, "html.parser") + return soup + else: + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + page.goto(url, wait_until="networkidle") + try: + # Wait until at least one .event div actually renders in the DOM + page.wait_for_selector("#upcoming-events-container .event", timeout=15000) + except Exception: + print("Warning: no .event elements appeared before timeout.") + html = page.content() + browser.close() + + soup = BeautifulSoup(html, "html.parser") + return soup + +# END OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium) +# ------------- + +def extract_events(soup, ignore_ticket_link): + events = soup.find_all("div", class_="event") + events_processed = [] + for event in events: + event_date = event.find("div", class_="event-date").text.strip() + event_name = event.find("div", class_="event-name").text.strip() + + event_artists_datalist = event.find("div", class_="event-artists") + event_artists_datalist = str(event_artists_datalist).removeprefix('