import argparse from bs4 import BeautifulSoup import json from playwright.sync_api import sync_playwright scrape_url = "https://fundbureau.de/" load_local_file = False output_file = "fundi-scraped-output.json" output_path = "./" ignore_ticket_link = False argp = argparse.ArgumentParser() argp.add_argument("--url", type=str, help="URL to scrape (default: https://fundbureau.de/)") argp.add_argument("--output-file", type=str, help="Output JSON file name (default: fundi-scraped-output.json)") argp.add_argument("--output-path", type=str, help="Output path for the JSON file (default: current directory)") argp.add_argument("--load-local-file", action="store_true", help="Load local HTML file instead of scraping the website (default: False)") argp.add_argument("--ignore-ticket-link", action="store_true", help="Ignore ticket link extraction (will leave it empty in the output JSON)") args = argp.parse_args() if vars(args)["url"]: scrape_url = vars(args)["url"] if vars(args)["output_file"]: output_file = vars(args)["output_file"] if vars(args)["output_path"]: output_path = vars(args)["output_path"] if vars(args)["load_local_file"]: load_local_file = True if vars(args)["ignore_ticket_link"]: ignore_ticket_link = True # ------------- # START OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium) def scrape_website(url, load_local_file): if load_local_file == True: with open(url) as file: soup = BeautifulSoup(file, "html.parser") return soup else: with sync_playwright() as p: browser = p.chromium.launch(headless=True) page = browser.new_page() page.goto(url, wait_until="networkidle") try: # Wait until at least one .event div actually renders in the DOM page.wait_for_selector("#upcoming-events-container .event", timeout=15000) except Exception: print("Warning: no .event elements appeared before timeout.") html = page.content() browser.close() soup = BeautifulSoup(html, "html.parser") return soup # END OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium) # ------------- def extract_events(soup, ignore_ticket_link): events = soup.find_all("div", class_="event") events_processed = [] for event in events: event_date = event.find("div", class_="event-date").text.strip() event_name = event.find("div", class_="event-name").text.strip() event_artists_datalist = event.find("div", class_="event-artists") event_artists_datalist = str(event_artists_datalist).removeprefix('
').removesuffix("
").split(", ") event_artists = [] for event_artist_data in event_artists_datalist: if "" in event_artist_data: event_artist_data = event_artist_data.split("") artist_name = event_artist_data[0] artist_time = event_artist_data[1].removesuffix("").replace("?", "Open end").split("-") event_artists.append({ "artist_name": event_artist_data[0], "artist_play_time_start": artist_time[0], "artist_play_time_end": artist_time[1] }) else: artist_name = event_artist_data event_artists.append({ "artist_name": event_artist_data, "artist_play_time_start": "n/a", "artist_play_time_end": "n/a" }) event_starttime = event.find("div", class_="event-starttime").text.strip() if ignore_ticket_link: event_ticket_link = "" else: event_ticket_link = event.find("a", class_="event-ticket-link")["href"] json_data = { "event_date": event_date, "event_name": event_name, "event_artists": [event_artists], "event_starttime": event_starttime, "event_ticket_link": event_ticket_link } events_processed.append(json_data) with open(output_path + output_file, "w") as json_file: json.dump(events_processed, json_file, indent=4) def main(): soup = scrape_website(scrape_url, load_local_file) extract_events(soup, ignore_ticket_link) if __name__ == "__main__": main()