123 lines
4.8 KiB
Python
123 lines
4.8 KiB
Python
import argparse
|
|
from bs4 import BeautifulSoup
|
|
import json
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
scrape_url = "https://fundbureau.de/"
|
|
load_local_file = False
|
|
output_file = "fundi-scraped-output.json"
|
|
output_path = "./"
|
|
ignore_ticket_link = False
|
|
|
|
argp = argparse.ArgumentParser()
|
|
argp.add_argument("--url", type=str, help="URL to scrape (default: https://fundbureau.de/)")
|
|
argp.add_argument("--output-file", type=str, help="Output JSON file name (default: fundi-scraped-output.json)")
|
|
argp.add_argument("--output-path", type=str, help="Output path for the JSON file (default: current directory)")
|
|
argp.add_argument("--load-local-file", action="store_true", help="Load local HTML file instead of scraping the website (default: False)")
|
|
argp.add_argument("--ignore-ticket-link", action="store_true", help="Ignore ticket link extraction (will leave it empty in the output JSON)")
|
|
args = argp.parse_args()
|
|
|
|
if vars(args)["url"]:
|
|
scrape_url = vars(args)["url"]
|
|
if vars(args)["output_file"]:
|
|
output_file = vars(args)["output_file"]
|
|
if vars(args)["output_path"]:
|
|
output_path = vars(args)["output_path"]
|
|
if vars(args)["load_local_file"]:
|
|
load_local_file = True
|
|
if vars(args)["ignore_ticket_link"]:
|
|
ignore_ticket_link = True
|
|
|
|
# -------------
|
|
# START OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium)
|
|
|
|
def scrape_website(url, load_local_file):
|
|
if load_local_file == True:
|
|
with open(url) as file:
|
|
soup = BeautifulSoup(file, "html.parser")
|
|
return soup
|
|
else:
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
page.goto(url, wait_until="networkidle")
|
|
try:
|
|
# Wait until at least one .event div actually renders in the DOM
|
|
page.wait_for_selector("#upcoming-events-container .event", timeout=15000)
|
|
except Exception:
|
|
print("Warning: no .event elements appeared before timeout.")
|
|
html = page.content()
|
|
browser.close()
|
|
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
return soup
|
|
|
|
# END OF AI WRITTEN SECTION (Claude Pro, Sonnet 5 medium)
|
|
# -------------
|
|
|
|
def extract_events(soup, ignore_ticket_link):
|
|
events = soup.find_all("div", class_="event")
|
|
events_processed = []
|
|
for event in events:
|
|
event_date = event.find("div", class_="event-date").text.strip()
|
|
event_name = event.find("div", class_="event-name").text.strip()
|
|
|
|
event_artists_datalist = event.find("div", class_="event-artists")
|
|
event_artists_datalist = str(event_artists_datalist).removeprefix('<div class="event-artists">').removesuffix("</div>").split(", ")
|
|
|
|
event_artists = []
|
|
|
|
for event_artist_data in event_artists_datalist:
|
|
if "<sup>" in event_artist_data:
|
|
event_artist_data = event_artist_data.split("<sup>")
|
|
artist_name = event_artist_data[0]
|
|
artist_time = event_artist_data[1].removesuffix("</sup>").replace("?", "Open end").split("-")
|
|
event_artists.append({
|
|
"artist_name": event_artist_data[0],
|
|
"artist_play_time_start": artist_time[0],
|
|
"artist_play_time_end": artist_time[1]
|
|
})
|
|
else:
|
|
artist_name = event_artist_data
|
|
event_artists.append({
|
|
"artist_name": event_artist_data,
|
|
"artist_play_time_start": "n/a",
|
|
"artist_play_time_end": "n/a"
|
|
})
|
|
|
|
|
|
event_starttime = event.find("div", class_="event-starttime").text.strip()
|
|
if ignore_ticket_link:
|
|
event_ticket_link = ""
|
|
event_free = None
|
|
else:
|
|
try:
|
|
event_ticket_link = event.find("a", class_="event-ticket-link")["href"]
|
|
event_free = False
|
|
except KeyError:
|
|
if event.find("a", class_="event-ticket-link").text.strip() == "Eintritt frei":
|
|
event_ticket_link = "n/a"
|
|
event_free = True
|
|
else:
|
|
event_ticket_link = ""
|
|
event_free = False
|
|
|
|
json_data = {
|
|
"event_date": event_date,
|
|
"event_name": event_name,
|
|
"event_artists": [event_artists],
|
|
"event_starttime": event_starttime,
|
|
"event_ticket_link": event_ticket_link,
|
|
"event_free": event_free
|
|
}
|
|
|
|
events_processed.append(json_data)
|
|
with open(output_path + output_file, "w") as json_file:
|
|
json.dump(events_processed, json_file, indent=4)
|
|
|
|
def main():
|
|
soup = scrape_website(scrape_url, load_local_file)
|
|
extract_events(soup, ignore_ticket_link)
|
|
|
|
if __name__ == "__main__":
|
|
main() |