Wrote scraper for fundbureau website

2026-08-25 23:55:40 +02:00
parent 64a21da282
commit 2f5ed60ffb
2 changed files with 112 additions and 0 deletions
@@ -0,0 +1,112 @@
+import argparse
+from bs4 import BeautifulSoup
+import json
+from playwright.sync_api import sync_playwright
+
+scrape_url = "https://fundbureau.de/"
+load_local_file = False
+output_file = "fundi-scraped-output.json"
+output_path = "./"
+ignore_ticket_link = False
+
+argp = argparse.ArgumentParser()
+argp.add_argument("--url", type=str, help="URL to scrape (default: https://fundbureau.de/)")
+argp.add_argument("--output-file", type=str, help="Output JSON file name (default: fundi-scraped-output.json)")
+argp.add_argument("--output-path", type=str, help="Output path for the JSON file (default: current directory)")
+argp.add_argument("--load-local-file", action="store_true", help="Load local HTML file instead of scraping the website (default: False)")
+argp.add_argument("--ignore-ticket-link", action="store_true", help="Ignore ticket link extraction (will leave it empty in the output JSON)")
+args = argp.parse_args()
+
+if vars(args)["url"]:
+    scrape_url = vars(args)["url"]
+if vars(args)["output_file"]:
+    output_file = vars(args)["output_file"]
+if vars(args)["output_path"]:
+    output_path = vars(args)["output_path"]
+if vars(args)["load_local_file"]:
+    load_local_file = True
+if vars(args)["ignore_ticket_link"]:
+    ignore_ticket_link = True
+
+# -------------
+# START OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium)
+
+def scrape_website(url, load_local_file):
+    if load_local_file == True:
+        with open(url) as file:
+            soup = BeautifulSoup(file, "html.parser")
+        return soup
+    else:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+            page.goto(url, wait_until="networkidle")
+            try:
+                # Wait until at least one .event div actually renders in the DOM
+                page.wait_for_selector("#upcoming-events-container .event", timeout=15000)
+            except Exception:
+                print("Warning: no .event elements appeared before timeout.")
+            html = page.content()
+            browser.close()
+
+        soup = BeautifulSoup(html, "html.parser")
+        return soup
+
+# END OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium)
+# -------------
+        
+def extract_events(soup, ignore_ticket_link):
+    events = soup.find_all("div", class_="event")
+    events_processed = []
+    for event in events:
+        event_date = event.find("div", class_="event-date").text.strip()
+        event_name = event.find("div", class_="event-name").text.strip()
+        
+        event_artists_datalist = event.find("div", class_="event-artists")
+        event_artists_datalist = str(event_artists_datalist).removeprefix('<div class="event-artists">').removesuffix("</div>").split(", ")
+        
+        event_artists = []
+
+        for event_artist_data in event_artists_datalist:
+            if "<sup>" in event_artist_data:
+                event_artist_data = event_artist_data.split("<sup>")
+                artist_name = event_artist_data[0]
+                artist_time = event_artist_data[1].removesuffix("</sup>").replace("?", "Open end").split("-")
+                event_artists.append({
+                    "artist_name": event_artist_data[0],
+                    "artist_play_time_start": artist_time[0],
+                    "artist_play_time_end": artist_time[1]
+                })
+            else:
+                artist_name = event_artist_data
+                event_artists.append({
+                    "artist_name": event_artist_data,
+                    "artist_play_time_start": "n/a",
+                    "artist_play_time_end": "n/a"
+                })
+
+
+        event_starttime = event.find("div", class_="event-starttime").text.strip()
+        if ignore_ticket_link:
+            event_ticket_link = ""
+        else:
+            event_ticket_link = event.find("a", class_="event-ticket-link")["href"]
+
+        json_data = {
+            "event_date": event_date,
+            "event_name": event_name,
+            "event_artists": [event_artists],
+            "event_starttime": event_starttime,
+            "event_ticket_link": event_ticket_link
+        }
+
+        events_processed.append(json_data)
+    with open(output_path + output_file, "w") as json_file:
+        json.dump(events_processed, json_file, indent=4)
+
+def main():
+    soup = scrape_website(scrape_url, load_local_file)
+    extract_events(soup, ignore_ticket_link)
+
+if __name__ == "__main__":
+    main()