basic BMAD setup

2026-10-15 08:57:23 +02:00
parent 44029127dc
commit 5e747654b9
438 changed files with 41818 additions and 0 deletions
@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""recon_kit — deterministic helpers for bmad-deep-recon.
+
+The mechanical half of the research workflow: everything here is exact,
+repeatable work the LLM should never re-derive by hand. All subcommands
+print one JSON object to stdout; diagnostics go to stderr. Exit codes:
+0 = pass, 1 = findings that need attention, 2 = usage/parse error.
+
+Subcommands:
+  citations RESEARCH_MD
+      Cross-check inline [n] markers against the source-appendix table:
+      dangling markers (no appendix row) and orphaned rows (never cited).
+  tally MEMLOG_MD
+      Count memlog entries by type, and claim entries by status.
+      Claim lines carry `status=<word>` and optionally `ref=[n]`; for a
+      given ref the LAST status wins, so status changes are appends.
+  staleness CLAIMS_JSON --windows JSON [--today YYYY-MM-DD]
+      Given claims [{claim, class, pub_date}] and a months-per-class map
+      (e.g. '{"size/growth": 18, "pricing": 3}'), compute each claim's
+      re-check date, flag stale ones, and report the earliest re-check.
+  slug TOPIC --type TYPE [--pattern P] [--date YYYY-MM-DD]
+      Expand the run-folder pattern deterministically so the same topic
+      always lands in the same folder across draft -> process -> refresh.
+  escape-sources RESEARCH_MD
+      Emit the source-appendix table as HTML with every cell escaped and
+      only validated http(s) URLs turned into links, for the briefing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import calendar
+import html
+import json
+import re
+import sys
+import unicodedata
+from datetime import date, datetime
+from pathlib import Path
+from urllib.parse import urlparse
+
+MARKER_RE = re.compile(r"\[(\d+)\](?!\()")  # [3] but not a [3](url) link
+# URLs may hold one level of balanced parentheses, as Wikipedia's often do.
+MD_LINK_RE = re.compile(r"\[([^\]]*)\]\(((?:[^\s()]|\([^\s()]*\))+)\)")
+BARE_URL_RE = re.compile(r"https?://(?:[^\s()|\]]|\([^\s()]*\))+")
+ROW_ID_RE = re.compile(r"\[(\d+)\]")  # appendix row id: [n], never a bare number
+FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")  # any indent: fences nest in list items
+
+
+def out(payload: dict, exit_code: int) -> int:
+    print(json.dumps(payload, indent=2, ensure_ascii=False, default=str))
+    return exit_code
+
+
+def read_text(path_arg: str) -> str:
+    if path_arg == "-":
+        # Decode the bytes ourselves: the locale's encoding may not be UTF-8.
+        buffer = getattr(sys.stdin, "buffer", None)
+        return buffer.read().decode("utf-8") if buffer is not None else sys.stdin.read()
+    return Path(path_arg).read_text(encoding="utf-8")
+
+
+def strip_fences(text: str) -> str:
+    """Blank out fenced code blocks so their contents never count as markers or rows.
+
+    Fences pair CommonMark-style: a block closes only on a line of the opener's
+    character, at least as long, with nothing after it.
+    """
+    lines, open_fence = [], None  # (char, length) while inside a fenced block
+    for ln in text.splitlines():
+        fence = FENCE_RE.match(ln)
+        if open_fence is None:
+            if fence:
+                open_fence = (fence.group(1)[0], len(fence.group(1)))
+            lines.append("" if fence else ln)
+            continue
+        marker = fence.group(1) if fence else ""
+        if marker[:1] == open_fence[0] and len(marker) >= open_fence[1] and ln.strip() == marker:
+            open_fence = None
+        lines.append("")
+    return "\n".join(lines)
+
+
+def table_cells(line: str) -> list[str]:
+    return [c.strip() for c in line.strip().strip("|").split("|")]
+
+
+def appendix_rows(text: str) -> dict[int, list[str]]:
+    """Source-appendix rows: markdown table rows whose first cell is [n]."""
+    rows: dict[int, list[str]] = {}
+    for ln in text.splitlines():
+        stripped = ln.strip()
+        if not stripped.startswith("|"):
+            continue
+        cells = table_cells(stripped)
+        if not cells or len(cells) < 2:
+            continue
+        m = ROW_ID_RE.fullmatch(cells[0])
+        if m:
+            rows[int(m.group(1))] = cells
+    return rows
+
+
+# --- citations ---------------------------------------------------------------
+
+
+def cmd_citations(args) -> int:
+    text = strip_fences(read_text(args.file))
+    rows = appendix_rows(text)
+    markers: set[int] = set()
+    for ln in text.splitlines():
+        stripped = ln.strip()
+        if stripped.startswith("|"):
+            cells = table_cells(stripped)
+            if cells and ROW_ID_RE.fullmatch(cells[0]):
+                continue  # an appendix row is not a citation of itself
+        markers.update(int(n) for n in MARKER_RE.findall(ln))
+    dangling = sorted(markers - set(rows))
+    orphaned = sorted(set(rows) - markers)
+    ok = not dangling and not orphaned
+    return out(
+        {
+            "markers": sorted(markers),
+            "appendix_rows": sorted(rows),
+            "dangling_markers": dangling,
+            "orphaned_rows": orphaned,
+            "ok": ok,
+        },
+        0 if ok else 1,
+    )
+
+
+# --- tally -------------------------------------------------------------------
+
+ENTRY_RE = re.compile(r"^- (?:\(([\w-]+)(?: by [^)]*)?\)\s*)?(.*)$")
+
+
+def cmd_tally(args) -> int:
+    text = read_text(args.file)
+    body = text.split("---", 2)[-1] if text.startswith("---") else text
+    by_type: dict[str, int] = {}
+    by_ref: dict[int, str] = {}
+    unref_status: dict[str, int] = {}
+    entries = 0
+    for ln in body.splitlines():
+        m = ENTRY_RE.match(ln)
+        if not m or not ln.startswith("- "):
+            continue
+        entries += 1
+        etype = m.group(1) or "note"
+        by_type[etype] = by_type.get(etype, 0) + 1
+        if etype == "claim":
+            status_m = re.search(r"status=([\w-]+)", m.group(2))
+            status = status_m.group(1) if status_m else "unknown"
+            ref_m = re.search(r"ref=\[?(\d+)\]?", m.group(2))
+            if ref_m:
+                by_ref[int(ref_m.group(1))] = status  # last status wins per ref
+            else:
+                unref_status[status] = unref_status.get(status, 0) + 1
+    claims: dict[str, int] = dict(unref_status)
+    for status in by_ref.values():
+        claims[status] = claims.get(status, 0) + 1
+    return out(
+        {
+            "entries": entries,
+            "by_type": dict(sorted(by_type.items())),
+            "claims": dict(sorted(claims.items())),
+            "claims_total": sum(claims.values()),
+        },
+        0,
+    )
+
+
+# --- staleness ---------------------------------------------------------------
+
+
+def parse_date(raw: str) -> date:
+    raw = raw.strip()
+    for fmt in ("%Y-%m-%d", "%Y-%m", "%Y"):
+        try:
+            return datetime.strptime(raw, fmt).date()
+        except ValueError:
+            continue
+    raise ValueError(f"unparseable date: {raw!r} (want YYYY[-MM[-DD]])")
+
+
+def add_months(d: date, months: int) -> date:
+    total = d.month - 1 + months
+    year, month = d.year + total // 12, total % 12 + 1
+    return date(year, month, min(d.day, calendar.monthrange(year, month)[1]))
+
+
+def cmd_staleness(args) -> int:
+    try:
+        payload = json.loads(read_text(args.file))
+        raw_windows = json.loads(args.windows)
+        if not isinstance(raw_windows, dict):
+            raise ValueError("--windows must be a JSON object of class -> months")
+        windows = {k.lower(): int(v) for k, v in raw_windows.items()}
+        today = parse_date(args.today) if args.today else date.today()
+    except (TypeError, ValueError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+    claims = payload.get("claims") if isinstance(payload, dict) else payload
+    if not isinstance(claims, list) or not all(isinstance(c, dict) for c in claims):
+        print('error: claims must be a JSON array of objects, or {"claims": [...]}', file=sys.stderr)
+        return 2
+    results, no_window, stale_count = [], set(), 0
+    earliest: date | None = None
+    for c in claims:
+        cls = str(c.get("class", "")).lower()
+        try:
+            pub = parse_date(str(c["pub_date"]))
+        except (KeyError, ValueError) as e:
+            print(f"error in claim {c!r}: {e}", file=sys.stderr)
+            return 2
+        months = windows.get(cls)
+        if months is None:
+            no_window.add(cls)
+            results.append({**c, "recheck": None, "stale": None})
+            continue
+        recheck = add_months(pub, months)
+        stale = recheck <= today
+        stale_count += stale
+        earliest = recheck if earliest is None or recheck < earliest else earliest
+        results.append({**c, "recheck": recheck.isoformat(), "stale": stale})
+    return out(
+        {
+            "today": today.isoformat(),
+            "claims": results,
+            "stale_count": stale_count,
+            "earliest_recheck": earliest.isoformat() if earliest else None,
+            "no_window_classes": sorted(no_window),
+        },
+        1 if stale_count else 0,
+    )
+
+
+# --- slug --------------------------------------------------------------------
+
+
+def slugify(text: str, max_len: int = 40) -> str:
+    text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
+    text = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
+    return re.sub(r"-{2,}", "-", text)[:max_len].rstrip("-")
+
+
+def cmd_slug(args) -> int:
+    slug = slugify(args.topic)
+    if not slug:
+        print("error: topic slugified to an empty string", file=sys.stderr)
+        return 2
+    folder = (
+        args.pattern.replace("{research_type}", args.type)
+        .replace("{topic_slug}", slug)
+        .replace("{date}", args.date or date.today().isoformat())
+    )
+    return out({"topic_slug": slug, "folder": folder}, 0)
+
+
+# --- escape-sources ----------------------------------------------------------
+
+
+def safe_url(raw: str) -> str | None:
+    parsed = urlparse(raw)
+    return raw if parsed.scheme in ("http", "https") and parsed.netloc else None
+
+
+def cell_html(cell: str, invalid: list[str]) -> str:
+    """Escape a cell; a markdown link or bare URL becomes an <a> only when http(s)."""
+    link = MD_LINK_RE.search(cell)
+    if link:
+        url = safe_url(link.group(2))
+        label = html.escape(link.group(1) or link.group(2))
+        if url:
+            return (
+                html.escape(cell[: link.start()])
+                + f'<a href="{html.escape(url, quote=True)}" target="_blank" rel="noopener">{label}</a>'
+                + html.escape(cell[link.end() :])
+            )
+        invalid.append(link.group(2))
+        return html.escape(cell.replace(link.group(0), link.group(1) or link.group(2)))
+    bare = BARE_URL_RE.search(cell)
+    if bare:
+        url = safe_url(bare.group(0))
+        if url:
+            escaped = html.escape(url, quote=True)
+            return (
+                html.escape(cell[: bare.start()])
+                + f'<a href="{escaped}" target="_blank" rel="noopener">{escaped}</a>'
+                + html.escape(cell[bare.end() :])
+            )
+        invalid.append(bare.group(0))
+    return html.escape(cell)
+
+
+def cmd_escape_sources(args) -> int:
+    text = strip_fences(read_text(args.file))
+    rows = appendix_rows(text)
+    if not rows:
+        print("error: no source-appendix table rows found", file=sys.stderr)
+        return 2
+    invalid: list[str] = []
+    body_rows = []
+    for n in sorted(rows):
+        cells = rows[n]
+        tds = "".join(f"<td>{cell_html(c, invalid)}</td>" for c in cells[1:])
+        body_rows.append(f'<tr id="src-{n}"><td>[{n}]</td>{tds}</tr>')
+    table = '<table class="sources"><tbody>' + "".join(body_rows) + "</tbody></table>"
+    return out({"rows": len(rows), "invalid_urls": invalid, "html": table}, 1 if invalid else 0)
+
+
+# --- entry point -------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    pc = sub.add_parser("citations", help="cross-check [n] markers vs the source appendix")
+    pc.add_argument("file", help="path to the research report (or - for stdin)")
+    pc.set_defaults(func=cmd_citations)
+
+    pt = sub.add_parser("tally", help="count memlog entries by type and claims by status")
+    pt.add_argument("file", help="path to .memlog.md (or - for stdin)")
+    pt.set_defaults(func=cmd_tally)
+
+    ps = sub.add_parser("staleness", help="compute re-check dates from freshness windows")
+    ps.add_argument("file", help="claims JSON: [{claim, class, pub_date}] (or - for stdin)")
+    ps.add_argument("--windows", required=True, help="JSON months-per-class map, e.g. '{\"pricing\": 3}'")
+    ps.add_argument("--today", help="override today's date (YYYY-MM-DD)")
+    ps.set_defaults(func=cmd_staleness)
+
+    pg = sub.add_parser("slug", help="expand the run-folder pattern deterministically")
+    pg.add_argument("topic", help="research topic text")
+    pg.add_argument("--type", required=True, help="research type code (e.g. market)")
+    pg.add_argument(
+        "--pattern",
+        default="research-{topic_slug}",
+        help="folder pattern (default: research-{topic_slug})",
+    )
+    pg.add_argument("--date", help="override date (YYYY-MM-DD; default today)")
+    pg.set_defaults(func=cmd_slug)
+
+    pe = sub.add_parser("escape-sources", help="source appendix as escaped HTML with validated links")
+    pe.add_argument("file", help="path to the research report (or - for stdin)")
+    pe.set_defaults(func=cmd_escape_sources)
+
+    args = p.parse_args(argv)
+    try:
+        return args.func(args)
+    except FileNotFoundError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    if sys.platform == "win32":
+        # Piped output on Windows defaults to a legacy code page, not UTF-8.
+        sys.stdout.reconfigure(encoding="utf-8")
+        sys.stderr.reconfigure(encoding="utf-8")
+    sys.exit(main())
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.11"
+# ///
+"""Tests for recon_kit.py."""
+
+import io
+import json
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from datetime import date
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from recon_kit import add_months, main, parse_date, slugify
+
+REPORT = """---
+title: 'market research: widgets'
+---
+
+# Report
+
+The market is growing fast [1] and pricing clusters at $20 [2].
+An uncited assertion sits here [4].
+
+```
+[9] inside a fence never counts
+```
+
+## Source appendix
+
+| [n] | Supports | Publisher | Pub date | Accessed | Confidence |
+| --- | --- | --- | --- | --- | --- |
+| [1] | market growth | [Gartner](https://example.com/g) | 2026-01 | 2026-07-01 | high |
+| [2] | pricing | [Acme](javascript:alert(1)) | 2026-05 | 2026-07-01 | medium |
+| [3] | never cited | https://example.com/x | 2025-11 | 2026-07-01 | low |
+"""
+
+MEMLOG = """---
+topic: widgets
+updated: 2026-07-22T10:00
+---
+
+- (decision) plan approved
+- (source) round 1 batch: 4 sources
+- (claim) ref=[1] status=verified class=size/growth pub=2026-01 — market growing 12% CAGR
+- (claim) ref=[2] status=unverified class=pricing pub=2026-05 — pricing clusters at $20
+- (claim) ref=[2] status=verified class=pricing pub=2026-05 — confirmed by second source
+- (claim) status=unverified class=behavior pub=2025-03 — users churn at day 8
+- (event) dimension 1 complete
+"""
+
+
+def run(argv):
+    buf = io.StringIO()
+    with redirect_stdout(buf):
+        code = main(argv)
+    return code, json.loads(buf.getvalue())
+
+
+class CitationsTest(unittest.TestCase):
+    def test_cross_check(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            report = Path(tmp) / "report.md"
+            report.write_text(REPORT, encoding="utf-8")
+            code, result = run(["citations", str(report)])
+        self.assertEqual(result["dangling_markers"], [4])
+        self.assertEqual(result["orphaned_rows"], [3])
+        self.assertNotIn(9, result["markers"])  # fenced content ignored
+        self.assertEqual(code, 1)
+
+
+def run_text(argv, text, name="report.md"):
+    with tempfile.TemporaryDirectory() as tmp:
+        f = Path(tmp) / name
+        f.write_text(text, encoding="utf-8")
+        return run([argv[0], str(f), *argv[1:]])
+
+
+class CitationsEdgeTest(unittest.TestCase):
+    def test_a_bare_number_table_is_not_the_source_appendix(self):
+        text = (
+            "Growth is fast [1].\n\n| Rank | Vendor |\n| --- | --- |\n| 1 | Acme |\n| 2 | Beta |\n| 3 | Gamma |\n\n"
+            "| [n] | Supports |\n| --- | --- |\n| [1] | growth |\n"
+        )
+        code, result = run_text(["citations"], text)
+        self.assertEqual(result["appendix_rows"], [1])
+        self.assertTrue(result["ok"])
+        self.assertEqual(code, 0)
+
+    def test_tilde_and_long_backtick_fences(self):
+        text = (
+            "~~~\n[7] tilde fenced\n~~~\n\n"
+            "````\n```\n[8] still fenced after an inner three-backtick line\n```\n````\n\n"
+            "Read after the close [1].\n\n| [n] | Supports |\n| --- | --- |\n| [1] | x |\n"
+        )
+        code, result = run_text(["citations"], text)
+        self.assertEqual(result["markers"], [1])
+        self.assertTrue(result["ok"])
+
+    def test_a_fence_nested_in_a_list_item_hides_its_markers(self):
+        text = (
+            "- Example:\n\n    ```\n    [5] inside a list-nested fence\n    ```\n\n"
+            "Cited [1].\n\n| [n] | Supports |\n| --- | --- |\n| [1] | x |\n"
+        )
+        code, result = run_text(["citations"], text)
+        self.assertEqual(result["markers"], [1])
+        self.assertTrue(result["ok"])
+
+
+class TallyTest(unittest.TestCase):
+    def test_last_status_wins_per_ref(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log = Path(tmp) / "memlog.md"
+            log.write_text(MEMLOG, encoding="utf-8")
+            code, result = run(["tally", str(log)])
+        self.assertEqual(result["by_type"]["claim"], 4)
+        self.assertEqual(result["claims"], {"unverified": 1, "verified": 2})
+        self.assertEqual(result["claims_total"], 3)  # ref=[2] counted once
+        self.assertEqual(code, 0)
+
+
+class StalenessTest(unittest.TestCase):
+    def test_dates(self):
+        self.assertEqual(parse_date("2026-01"), date(2026, 1, 1))
+        self.assertEqual(add_months(date(2026, 1, 31), 1), date(2026, 2, 28))
+
+    def test_windows(self):
+        claims = json.dumps(
+            [
+                {"claim": "sizing", "class": "size/growth", "pub_date": "2024-06"},
+                {"claim": "pricing", "class": "pricing", "pub_date": "2026-06"},
+                {"claim": "odd", "class": "unmapped", "pub_date": "2026-06"},
+            ]
+        )
+        with tempfile.TemporaryDirectory() as tmp:
+            f = Path(tmp) / "claims.json"
+            f.write_text(claims, encoding="utf-8")
+            code, result = run(
+                [
+                    "staleness",
+                    str(f),
+                    "--windows",
+                    '{"size/growth": 18, "pricing": 3}',
+                    "--today",
+                    "2026-07-22",
+                ]
+            )
+        self.assertEqual(result["stale_count"], 1)  # sizing recheck 2025-12 < today
+        self.assertEqual(result["earliest_recheck"], "2025-12-01")
+        self.assertEqual(result["no_window_classes"], ["unmapped"])
+        self.assertEqual(code, 1)
+
+    def test_malformed_claims_exit_2(self):
+        for payload in ('{"x": 1}', '{"claims": [5]}'):
+            err = io.StringIO()
+            with tempfile.TemporaryDirectory() as tmp:
+                f = Path(tmp) / "claims.json"
+                f.write_text(payload, encoding="utf-8")
+                with redirect_stderr(err), redirect_stdout(io.StringIO()):
+                    code = main(["staleness", str(f), "--windows", '{"pricing": 3}'])
+            self.assertEqual(code, 2, payload)
+            self.assertTrue(err.getvalue().startswith("error: "), payload)
+
+    def test_stdin_is_read_as_utf8_whatever_the_locale(self):
+        claims = json.dumps([{"claim": "café prices", "class": "pricing", "pub_date": "2026-06"}], ensure_ascii=False)
+        stdin = io.TextIOWrapper(io.BytesIO(claims.encode("utf-8")), encoding="latin-1")
+        original = sys.stdin
+        sys.stdin = stdin
+        try:
+            code, result = run(["staleness", "-", "--windows", '{"pricing": 3}', "--today", "2026-07-01"])
+        finally:
+            sys.stdin = original
+        self.assertEqual(result["claims"][0]["claim"], "café prices")
+        self.assertEqual(code, 0)
+
+
+class SlugTest(unittest.TestCase):
+    def test_deterministic_folder(self):
+        self.assertEqual(slugify("Créme Brûlée: AI Tools!"), "creme-brulee-ai-tools")
+        code, result = run(["slug", "SMB Accounting SaaS", "--type", "market"])
+        self.assertEqual(result["folder"], "research-smb-accounting-saas")
+        self.assertEqual(code, 0)
+        pattern = "{research_type}-{topic_slug}-{date}"
+        code, result = run(
+            ["slug", "SMB Accounting SaaS", "--type", "market", "--date", "2026-07-22", "--pattern", pattern]
+        )
+        self.assertEqual(result["folder"], "market-smb-accounting-saas-2026-07-22")
+        self.assertEqual(code, 0)
+
+
+class EscapeSourcesTest(unittest.TestCase):
+    def test_escaping_and_url_validation(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            report = Path(tmp) / "report.md"
+            report.write_text(REPORT, encoding="utf-8")
+            code, result = run(["escape-sources", str(report)])
+        self.assertEqual(result["rows"], 3)
+        self.assertTrue(any(u.startswith("javascript:") for u in result["invalid_urls"]))
+        self.assertNotIn("javascript:", result["html"])  # never linked
+        self.assertIn('href="https://example.com/g"', result["html"])
+        self.assertIn('id="src-1"', result["html"])
+        self.assertEqual(code, 1)
+
+    def test_urls_keep_balanced_parentheses(self):
+        text = (
+            "| [n] | Supports | Publisher |\n| --- | --- | --- |\n"
+            "| [1] | a | [Rust](https://en.wikipedia.org/wiki/Rust_(programming_language)) |\n"
+            "| [2] | b | https://en.wikipedia.org/wiki/Go_(programming_language) |\n"
+        )
+        code, result = run_text(["escape-sources"], text)
+        self.assertIn('href="https://en.wikipedia.org/wiki/Rust_(programming_language)"', result["html"])
+        self.assertIn('href="https://en.wikipedia.org/wiki/Go_(programming_language)"', result["html"])
+        self.assertEqual(result["invalid_urls"], [])
+        self.assertEqual(code, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()