diff --git a/api/README.md b/api/README.md index 8e4da39..0bf8991 100644 --- a/api/README.md +++ b/api/README.md @@ -31,8 +31,17 @@ DATA_FILE=/path/to/events.json ARCHIVE_DATA_FILE=/path/to/archive.json uvicorn m ### Scrape schedule reporting `/health` reports when each scraper output file was last written (its mtime) -and, if you tell it the cron schedule, when it's next due. This doesn't read -your crontab — set the same expression(s) you put there as env vars: +and, if it can determine the cron schedule, when it's next due. The schedule +is sourced in order: + +1. `SCRAPE_CRON` / `ARCHIVE_SCRAPE_CRON` env vars, if set — always wins, and + the only option that works when the API doesn't run as the same user/host + as the cron job. +2. Otherwise, the API's own OS user's crontab (`crontab -l`), looked up for a + line invoking `scraper.py` (with vs. without `--archive` picks archive vs. + plain). Only works when the API process runs as the same user whose + personal crontab holds the scrape job — not a system crontab/cron.d entry, + not a job scheduled under a different user or host. ```bash SCRAPE_CRON="0 * * * *" ARCHIVE_SCRAPE_CRON="0 4 * * *" uvicorn main:app @@ -40,8 +49,9 @@ SCRAPE_CRON="0 * * * *" ARCHIVE_SCRAPE_CRON="0 4 * * *" uvicorn main:app `SCRAPE_CRON` covers the plain scrape (`fundi-scraped-output.json`); `ARCHIVE_SCRAPE_CRON` covers `--archive` (`fundi-archive-output.json`) and -falls back to `SCRAPE_CRON` if unset — set it separately only if the archive -scrape runs on its own cron line. Leave both unset to just get the +falls back to `SCRAPE_CRON` (env or crontab-discovered) if unset — set it +separately only if the archive scrape runs on its own cron line. If neither +an env var nor a matching crontab line is found, you just get the last-scrape info with `next_scrape_at` / `seconds_until_next_scrape` as `null`. diff --git a/api/__pycache__/scrape_schedule.cpython-313.pyc b/api/__pycache__/scrape_schedule.cpython-313.pyc new file mode 100644 index 0000000..fac45a5 Binary files /dev/null and b/api/__pycache__/scrape_schedule.cpython-313.pyc differ diff --git a/api/main.py b/api/main.py index 4cb50fa..6e959bd 100644 --- a/api/main.py +++ b/api/main.py @@ -17,7 +17,7 @@ from fastapi.responses import PlainTextResponse from calendar_feed import build_ics from datasource import EventStore, iter_artist_names, parse_event_date from models import Event, EventList -from scrape_schedule import get_schedule_info +from scrape_schedule import discover_cron_expr, get_schedule_info app = FastAPI( title="Fundi Scraper API", @@ -59,8 +59,14 @@ def _matches( @app.get("/health") def health(): - scrape_cron = os.environ.get("SCRAPE_CRON") - archive_scrape_cron = os.environ.get("ARCHIVE_SCRAPE_CRON") or scrape_cron + scrape_cron = os.environ.get("SCRAPE_CRON") or discover_cron_expr( + "scraper.py", archive=False + ) + archive_scrape_cron = ( + os.environ.get("ARCHIVE_SCRAPE_CRON") + or discover_cron_expr("scraper.py", archive=True) + or scrape_cron + ) return { "status": "ok", "events_loaded": len(store.all()), diff --git a/api/scrape_schedule.py b/api/scrape_schedule.py index d2e876d..e584219 100644 --- a/api/scrape_schedule.py +++ b/api/scrape_schedule.py @@ -1,15 +1,22 @@ """Reports when a scrape last ran and when cron will next run it. "Last ran" is read from the mtime of the scraper's output file - true -regardless of how/when the scrape was actually invoked. "Next run" requires -telling this module the cron schedule via an env var (there's no reliable, -portable way to introspect another process's crontab from here), given as -a standard 5-field cron expression, e.g. "0 * * * *". +regardless of how/when the scrape was actually invoked. "Next run" needs the +cron schedule, sourced in order of preference: + +1. An explicit env var (SCRAPE_CRON / ARCHIVE_SCRAPE_CRON) - always wins, + works anywhere. +2. The current OS user's crontab (``discover_cron_expr``) - only works when + this process runs as the same user whose personal crontab (``crontab -e``) + contains the scrape job, and the ``crontab`` binary is available. Not + portable beyond that (different user, no cron, system crontab/cron.d + instead of a personal one, containers split across hosts, ...). """ +import subprocess from datetime import datetime from pathlib import Path -from typing import Optional +from typing import List, Optional, Tuple from croniter import croniter @@ -18,6 +25,58 @@ def _now_local() -> datetime: return datetime.now().astimezone() +def _read_crontab_lines() -> List[str]: + """Return the invoking OS user's crontab lines, or [] if unavailable.""" + try: + result = subprocess.run( + ["crontab", "-l"], + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return [] + if result.returncode != 0: + return [] + return result.stdout.splitlines() + + +def _parse_crontab_line(line: str) -> Optional[Tuple[str, str]]: + """Split one crontab line into (schedule, command); None if not a job.""" + line = line.strip() + if not line or line.startswith("#"): + return None + if line.startswith("@"): + parts = line.split(maxsplit=1) + if len(parts) != 2: + return None + return parts[0], parts[1] + parts = line.split(maxsplit=5) + if len(parts) != 6: + return None + return " ".join(parts[:5]), parts[5] + + +def discover_cron_expr(command_hint: str, *, archive: bool) -> Optional[str]: + """Find a matching scrape job in the current user's crontab. + + ``command_hint`` is a substring identifying the scrape command (e.g. + "scraper.py"); ``archive`` picks the "--archive" line vs. the plain one. + Returns None if the crontab can't be read or no line matches. + """ + for line in _read_crontab_lines(): + parsed = _parse_crontab_line(line) + if parsed is None: + continue + schedule, command = parsed + if command_hint not in command: + continue + if ("--archive" in command) != archive: + continue + return schedule + return None + + def _file_mtime_local(path: Path) -> Optional[datetime]: try: return datetime.fromtimestamp(path.stat().st_mtime).astimezone()