diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..d8cd896 Binary files /dev/null and b/.DS_Store differ diff --git a/scraper/scraper.py b/scraper/scraper.py new file mode 100644 index 0000000..8967d6b --- /dev/null +++ b/scraper/scraper.py @@ -0,0 +1,112 @@ +import argparse +from bs4 import BeautifulSoup +import json +from playwright.sync_api import sync_playwright + +scrape_url = "https://fundbureau.de/" +load_local_file = False +output_file = "fundi-scraped-output.json" +output_path = "./" +ignore_ticket_link = False + +argp = argparse.ArgumentParser() +argp.add_argument("--url", type=str, help="URL to scrape (default: https://fundbureau.de/)") +argp.add_argument("--output-file", type=str, help="Output JSON file name (default: fundi-scraped-output.json)") +argp.add_argument("--output-path", type=str, help="Output path for the JSON file (default: current directory)") +argp.add_argument("--load-local-file", action="store_true", help="Load local HTML file instead of scraping the website (default: False)") +argp.add_argument("--ignore-ticket-link", action="store_true", help="Ignore ticket link extraction (will leave it empty in the output JSON)") +args = argp.parse_args() + +if vars(args)["url"]: + scrape_url = vars(args)["url"] +if vars(args)["output_file"]: + output_file = vars(args)["output_file"] +if vars(args)["output_path"]: + output_path = vars(args)["output_path"] +if vars(args)["load_local_file"]: + load_local_file = True +if vars(args)["ignore_ticket_link"]: + ignore_ticket_link = True + +# ------------- +# START OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium) + +def scrape_website(url, load_local_file): + if load_local_file == True: + with open(url) as file: + soup = BeautifulSoup(file, "html.parser") + return soup + else: + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + page.goto(url, wait_until="networkidle") + try: + # Wait until at least one .event div actually renders in the DOM + page.wait_for_selector("#upcoming-events-container .event", timeout=15000) + except Exception: + print("Warning: no .event elements appeared before timeout.") + html = page.content() + browser.close() + + soup = BeautifulSoup(html, "html.parser") + return soup + +# END OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium) +# ------------- + +def extract_events(soup, ignore_ticket_link): + events = soup.find_all("div", class_="event") + events_processed = [] + for event in events: + event_date = event.find("div", class_="event-date").text.strip() + event_name = event.find("div", class_="event-name").text.strip() + + event_artists_datalist = event.find("div", class_="event-artists") + event_artists_datalist = str(event_artists_datalist).removeprefix('
').removesuffix("
").split(", ") + + event_artists = [] + + for event_artist_data in event_artists_datalist: + if "" in event_artist_data: + event_artist_data = event_artist_data.split("") + artist_name = event_artist_data[0] + artist_time = event_artist_data[1].removesuffix("").replace("?", "Open end").split("-") + event_artists.append({ + "artist_name": event_artist_data[0], + "artist_play_time_start": artist_time[0], + "artist_play_time_end": artist_time[1] + }) + else: + artist_name = event_artist_data + event_artists.append({ + "artist_name": event_artist_data, + "artist_play_time_start": "n/a", + "artist_play_time_end": "n/a" + }) + + + event_starttime = event.find("div", class_="event-starttime").text.strip() + if ignore_ticket_link: + event_ticket_link = "" + else: + event_ticket_link = event.find("a", class_="event-ticket-link")["href"] + + json_data = { + "event_date": event_date, + "event_name": event_name, + "event_artists": [event_artists], + "event_starttime": event_starttime, + "event_ticket_link": event_ticket_link + } + + events_processed.append(json_data) + with open(output_path + output_file, "w") as json_file: + json.dump(events_processed, json_file, indent=4) + +def main(): + soup = scrape_website(scrape_url, load_local_file) + extract_events(soup, ignore_ticket_link) + +if __name__ == "__main__": + main() \ No newline at end of file