g4borg
cfd3cfb357
Tome is opt-in per project — each project declares its own knowledge management approach. Spire retains a small knowledge-awareness skill that explains what tome does and provides a fallback strategy for projects without it. - New plugins/tome/ with own manifests, README, all 5 sub-skills - Spire keeps skills/knowledge/SKILL.md as awareness stub - Updated all READMEs and marketplace JSONs - tome: namespace refs now correct (tome is its own plugin)
2187 lines
80 KiB
Python
2187 lines
80 KiB
Python
#!/usr/bin/env python3
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = []
|
|
# ///
|
|
"""
|
|
tome knowledge.py — Deterministic knowledge-layer tool for the tome skill family.
|
|
|
|
Subcommands:
|
|
bootstrap Scaffold wiki/ and navigate skill supporting files.
|
|
refresh Re-generate status:generated files; report authored/stub candidates.
|
|
context <path> Resolve navigation context for a target path.
|
|
check Validate knowledge-layer health; exit 1 on errors.
|
|
impact [paths...] Analyze which wiki docs are affected by changed source paths.
|
|
|
|
Usage (from repo root, via Justfile module):
|
|
just tome <subcommand> [args]
|
|
|
|
Or directly:
|
|
uv run -q --script <path-to-tome>/bin/knowledge.py <subcommand> [args]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, NamedTuple
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
EXCLUDED_DIRS = {
|
|
".git", "target", "node_modules", "wiki", "dist", "archive", "__pycache__",
|
|
}
|
|
|
|
CONTROL_NAMES = ("CLAUDE.md", "AGENTS.md")
|
|
WIKI_SUBDIR = "wiki"
|
|
CODEBASE_SUBDIR = "codebase"
|
|
BOOT_MD_REL = "wiki/boot.md"
|
|
WIKI_README_REL = "wiki/README.md"
|
|
WIKI_NAV_REL = "wiki/NAV.md"
|
|
WIKI_CODEBASE_NAV_REL = "wiki/codebase/NAV.md"
|
|
WIKI_INVENTORY_REL = "wiki/.inventory.md"
|
|
WIKI_CODEBASE_INVENTORY_REL = "wiki/codebase/.inventory.md"
|
|
|
|
# Canonical H2 order
|
|
CANONICAL_SECTIONS = ["Load", "Warnings", "Handoff"]
|
|
|
|
# Top-level dirs that are clearly source/code (not meta)
|
|
TOP_LEVEL_SOURCE_DIRS_HINTS = {"src", "crates", "scripts", "assets", "docs", "experiments"}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Exit codes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
EXIT_OK = 0
|
|
EXIT_CHECK_ERRORS = 1 # check: >=1 error-severity finding
|
|
EXIT_USAGE = 2 # context: path outside repo / nonexistent / invalid args
|
|
EXIT_REPO_DISCOVERY = 3 # not a git repo
|
|
EXIT_PARSE = 4 # parse failure (context)
|
|
EXIT_INTERNAL = 5 # internal error (bootstrap/refresh)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Repository root discovery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def find_repo_root() -> Path:
|
|
"""Discover repo root via git rev-parse --show-toplevel."""
|
|
try:
|
|
result = subprocess.run(
|
|
["git", "rev-parse", "--show-toplevel"],
|
|
capture_output=True, text=True, check=True
|
|
)
|
|
return Path(result.stdout.strip())
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
print("Repository discovery failed: not a git repository", file=sys.stderr)
|
|
sys.exit(EXIT_REPO_DISCOVERY)
|
|
|
|
|
|
def to_posix_rel(path: Path, root: Path) -> str:
|
|
"""Return a POSIX-separator, repo-relative string for a path."""
|
|
try:
|
|
rel = path.relative_to(root)
|
|
except ValueError:
|
|
rel = path
|
|
return rel.as_posix()
|
|
|
|
|
|
def posix_rel(path: Path, root: Path) -> str:
|
|
return to_posix_rel(path, root)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Filesystem traversal helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def should_exclude(name: str) -> bool:
|
|
"""Return True if a directory name should be excluded from traversal."""
|
|
return name in EXCLUDED_DIRS or name.startswith(".")
|
|
|
|
|
|
def walk_dirs(root: Path, include_root: bool = False):
|
|
"""Yield all non-excluded directories under root (breadth-first), optionally root itself."""
|
|
if include_root:
|
|
yield root
|
|
stack = [root]
|
|
while stack:
|
|
current = stack.pop()
|
|
try:
|
|
entries = sorted(current.iterdir(), key=lambda p: p.name)
|
|
except PermissionError:
|
|
continue
|
|
for entry in entries:
|
|
if entry.is_dir() and not should_exclude(entry.name):
|
|
yield entry
|
|
stack.append(entry)
|
|
|
|
|
|
def top_level_dirs(root: Path) -> list[Path]:
|
|
"""Return sorted top-level directories that are not excluded."""
|
|
result = []
|
|
try:
|
|
for entry in sorted(root.iterdir(), key=lambda p: p.name):
|
|
if entry.is_dir() and not should_exclude(entry.name):
|
|
result.append(entry)
|
|
except PermissionError:
|
|
pass
|
|
return result
|
|
|
|
|
|
def find_control_file(directory: Path) -> Path | None:
|
|
"""Return the effective control file for a directory (CLAUDE.md wins, else AGENTS.md)."""
|
|
claude = directory / "CLAUDE.md"
|
|
agents = directory / "AGENTS.md"
|
|
if claude.exists():
|
|
return claude
|
|
if agents.exists():
|
|
return agents
|
|
return None
|
|
|
|
|
|
def has_both_control_files(directory: Path) -> bool:
|
|
return (directory / "CLAUDE.md").exists() and (directory / "AGENTS.md").exists()
|
|
|
|
|
|
def find_all_control_files(root: Path) -> list[Path]:
|
|
"""Return sorted list of all control files in the repo (excluding excluded dirs)."""
|
|
result = []
|
|
for d in walk_dirs(root, include_root=True):
|
|
for name in CONTROL_NAMES:
|
|
f = d / name
|
|
if f.exists():
|
|
result.append(f)
|
|
return sorted(result)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Front-matter parsing (hand-rolled, stdlib only)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class FrontMatter(NamedTuple):
|
|
status: str | None # generated / authored / stub / None
|
|
source_paths: list[str] # list of repo-relative POSIX paths
|
|
raw: str # the raw front-matter block (without --- delimiters)
|
|
present: bool # was a front-matter block found at all?
|
|
|
|
|
|
def parse_frontmatter(text: str) -> tuple[FrontMatter, str]:
|
|
"""
|
|
Parse YAML front-matter from a markdown file.
|
|
Returns (FrontMatter, body_without_frontmatter).
|
|
Only handles 'status:' and 'source-paths:' fields.
|
|
"""
|
|
if not text.startswith("---"):
|
|
return FrontMatter(None, [], "", False), text
|
|
|
|
# Find closing ---
|
|
rest = text[3:]
|
|
# skip optional newline
|
|
if rest.startswith("\n"):
|
|
rest = rest[1:]
|
|
end = rest.find("\n---")
|
|
if end == -1:
|
|
return FrontMatter(None, [], "", False), text
|
|
|
|
raw = rest[:end]
|
|
body = rest[end + 4:] # skip \n---
|
|
if body.startswith("\n"):
|
|
body = body[1:]
|
|
|
|
status = None
|
|
source_paths: list[str] = []
|
|
|
|
in_source_paths = False
|
|
for line in raw.splitlines():
|
|
stripped = line.strip()
|
|
# Continuation of a multi-line YAML list (indented "- item" lines)
|
|
if in_source_paths:
|
|
if stripped.startswith("- "):
|
|
val = stripped[2:].strip().strip('"').strip("'")
|
|
if val:
|
|
source_paths.append(val)
|
|
continue
|
|
else:
|
|
in_source_paths = False
|
|
if stripped.startswith("status:"):
|
|
val = stripped[len("status:"):].strip().strip('"').strip("'")
|
|
status = val
|
|
elif stripped.startswith("source-paths:"):
|
|
val = stripped[len("source-paths:"):].strip()
|
|
# parse simple inline list: [a, b] or []
|
|
if val.startswith("[") and val.endswith("]"):
|
|
inner = val[1:-1].strip()
|
|
if inner:
|
|
source_paths = [p.strip().strip('"').strip("'") for p in inner.split(",") if p.strip()]
|
|
elif not val:
|
|
# Multi-line YAML list follows on subsequent lines
|
|
in_source_paths = True
|
|
|
|
return FrontMatter(status, source_paths, raw, True), body
|
|
|
|
|
|
def render_frontmatter(status: str, source_paths: list[str]) -> str:
|
|
"""Render a front-matter block string (without outer --- lines)."""
|
|
if source_paths:
|
|
paths_str = ", ".join(f"{p}" for p in source_paths)
|
|
sp_line = f"source-paths: [{paths_str}]"
|
|
else:
|
|
sp_line = "source-paths: []"
|
|
return f"status: {status}\n{sp_line}"
|
|
|
|
|
|
def render_frontmatter_block(status: str, source_paths: list[str]) -> str:
|
|
"""Render complete ---...--- front-matter block."""
|
|
return f"---\n{render_frontmatter(status, source_paths)}\n---\n"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Control file parsing (CLAUDE.md / AGENTS.md)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
INCLUDE_LINE_RE = re.compile(r"^@([A-Za-z0-9_./\\-]+)\s*$")
|
|
|
|
|
|
def expand_includes(text: str, base_dir: Path, depth: int = 0) -> str:
|
|
"""Expand `@path` include lines (the Claude Code @-include convention).
|
|
|
|
A line consisting solely of `@relative/path` is replaced by that file's
|
|
content, resolved relative to base_dir. Recursion is capped to avoid cycles.
|
|
Unresolvable includes are kept verbatim.
|
|
"""
|
|
if depth >= 3 or "@" not in text:
|
|
return text
|
|
out_lines = []
|
|
for line in text.splitlines():
|
|
m = INCLUDE_LINE_RE.match(line.strip())
|
|
if m:
|
|
inc_path = (base_dir / m.group(1)).resolve()
|
|
if inc_path.is_file():
|
|
try:
|
|
inc_text = inc_path.read_text(encoding="utf-8", errors="replace")
|
|
out_lines.append(expand_includes(inc_text, inc_path.parent, depth + 1))
|
|
continue
|
|
except OSError:
|
|
pass
|
|
out_lines.append(line)
|
|
return "\n".join(out_lines)
|
|
|
|
|
|
def is_pure_include(path: Path) -> bool:
|
|
"""True if a control file's non-blank lines are all `@path` includes."""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return False
|
|
lines = [l.strip() for l in text.splitlines() if l.strip()]
|
|
return bool(lines) and all(INCLUDE_LINE_RE.match(l) for l in lines)
|
|
|
|
class LoadItem(NamedTuple):
|
|
kind: str # wiki_nav / domain / pattern / layer / link / prose
|
|
label: str # original label text
|
|
target: str | None # resolved target path or None for prose
|
|
raw: str # original bullet text
|
|
|
|
|
|
class ControlFile(NamedTuple):
|
|
path: Path
|
|
rel: str # repo-relative POSIX
|
|
title: str | None
|
|
area: str | None # Area: value
|
|
knowledge: str | None # Knowledge: link target
|
|
load_items: list[LoadItem]
|
|
warnings: list[str]
|
|
handoff: list[str]
|
|
first_para: str | None # first paragraph after H1 (for purpose seeding)
|
|
section_order: list[str] # recognized sections in parse order
|
|
|
|
|
|
LOAD_LABEL_MAP = {
|
|
"Local navigation:": "wiki_nav",
|
|
"Domain semantics:": "domain",
|
|
"Required pattern:": "pattern",
|
|
"Layer context:": "layer",
|
|
}
|
|
|
|
|
|
def classify_load_label(label: str) -> str:
|
|
for prefix, kind in LOAD_LABEL_MAP.items():
|
|
if label.strip().startswith(prefix.rstrip(":")):
|
|
return kind
|
|
return "link"
|
|
|
|
|
|
MD_LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)]*)\)")
|
|
|
|
|
|
def parse_md_link(text: str) -> tuple[str, str] | None:
|
|
"""Return (label, target) or None."""
|
|
m = MD_LINK_RE.search(text)
|
|
if m:
|
|
return m.group(1), m.group(2)
|
|
return None
|
|
|
|
|
|
def parse_control_file(path: Path, root: Path) -> ControlFile:
|
|
"""Parse a control file, gracefully degrading on free-form files."""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return ControlFile(path, posix_rel(path, root), None, None, None, [], [], [], None, [])
|
|
|
|
rel = posix_rel(path, root)
|
|
control_dir = path.parent
|
|
text = expand_includes(text, control_dir)
|
|
|
|
lines = text.splitlines()
|
|
|
|
title: str | None = None
|
|
area: str | None = None
|
|
knowledge: str | None = None
|
|
load_items: list[LoadItem] = []
|
|
warnings_list: list[str] = []
|
|
handoff_list: list[str] = []
|
|
first_para: str | None = None
|
|
section_order: list[str] = []
|
|
|
|
current_section: str | None = None
|
|
in_preamble = False
|
|
preamble_lines: list[str] = []
|
|
after_h1 = False
|
|
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
|
|
# H1
|
|
if stripped.startswith("# ") and not stripped.startswith("## "):
|
|
if title is None:
|
|
raw_title = stripped[2:].strip()
|
|
# Strip suffix after " — "
|
|
if " — " in raw_title:
|
|
raw_title = raw_title.split(" — ")[0].strip()
|
|
title = raw_title
|
|
in_preamble = True
|
|
after_h1 = True
|
|
current_section = None
|
|
continue
|
|
|
|
# H2 section headers
|
|
if stripped.startswith("## "):
|
|
section_name = stripped[3:].strip()
|
|
if in_preamble:
|
|
in_preamble = False
|
|
if section_name in CANONICAL_SECTIONS:
|
|
section_order.append(section_name)
|
|
current_section = section_name
|
|
else:
|
|
current_section = None
|
|
continue
|
|
|
|
# Deeper headers — end current section processing but track for H3+
|
|
if stripped.startswith("#"):
|
|
if in_preamble:
|
|
in_preamble = False
|
|
current_section = None
|
|
continue
|
|
|
|
# Preamble: Area: and Knowledge: lines
|
|
if in_preamble and after_h1:
|
|
if stripped.startswith("Area:"):
|
|
val = stripped[len("Area:"):].strip()
|
|
if val.startswith("`") and val.endswith("`"):
|
|
val = val[1:-1]
|
|
area = val if val else None
|
|
elif stripped.startswith("Knowledge:"):
|
|
val = stripped[len("Knowledge:"):].strip()
|
|
link = parse_md_link(val)
|
|
if link:
|
|
target = link[1]
|
|
# Resolve: /path = repo-root-relative, else relative to control_dir
|
|
if target.startswith("/"):
|
|
resolved = (root / target.lstrip("/")).resolve()
|
|
else:
|
|
resolved = (control_dir / target).resolve()
|
|
knowledge = posix_rel(resolved, root)
|
|
else:
|
|
knowledge = val if val else None
|
|
elif stripped and not stripped.startswith("#"):
|
|
preamble_lines.append(stripped)
|
|
|
|
# Section content
|
|
if current_section == "Load" and stripped.startswith("- "):
|
|
bullet = stripped[2:].strip()
|
|
link = parse_md_link(bullet)
|
|
if link:
|
|
label, target = link
|
|
kind = classify_load_label(label)
|
|
# Resolve target: /path = repo-root-relative, else relative to control_dir
|
|
if not target.startswith(("http://", "https://", "mailto:")):
|
|
if target.startswith("/"):
|
|
resolved = (root / target.lstrip("/")).resolve()
|
|
else:
|
|
resolved = (control_dir / target).resolve()
|
|
target_rel = posix_rel(resolved, root)
|
|
else:
|
|
target_rel = target
|
|
load_items.append(LoadItem(kind, label, target_rel, bullet))
|
|
else:
|
|
load_items.append(LoadItem("prose", "", None, bullet))
|
|
|
|
elif current_section == "Warnings" and stripped.startswith("- "):
|
|
warnings_list.append(stripped[2:].strip())
|
|
|
|
elif current_section == "Handoff" and stripped.startswith("- "):
|
|
handoff_list.append(stripped[2:].strip())
|
|
|
|
# Extract first paragraph from preamble
|
|
if preamble_lines:
|
|
# Join consecutive non-empty preamble lines into paragraph
|
|
para_parts = []
|
|
for pl in preamble_lines:
|
|
if pl:
|
|
para_parts.append(pl)
|
|
else:
|
|
break
|
|
if para_parts:
|
|
first_para = " ".join(para_parts)
|
|
|
|
return ControlFile(path, rel, title, area, knowledge, load_items, warnings_list, handoff_list, first_para, section_order)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wiki file helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def read_wiki_file(path: Path) -> tuple[FrontMatter, str]:
|
|
"""Read a wiki file, returning parsed front-matter and body."""
|
|
if not path.exists():
|
|
return FrontMatter(None, [], "", False), ""
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
return parse_frontmatter(text)
|
|
|
|
|
|
def write_generated_file(path: Path, content: str, status: str = "generated",
|
|
source_paths: list[str] | None = None):
|
|
"""Write a generated file with front-matter, only if status allows.
|
|
|
|
Returns:
|
|
True = file was created or updated
|
|
False = file was skipped (authored/stub or identical content)
|
|
"""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
full_content = render_frontmatter_block(status, source_paths or []) + "\n" + content
|
|
|
|
# If file exists, check status and content
|
|
if path.exists():
|
|
existing = path.read_text(encoding="utf-8", errors="replace")
|
|
fm, _ = parse_frontmatter(existing)
|
|
if fm.present and fm.status in ("authored", "stub"):
|
|
return False # never clobber authored/stub
|
|
if existing == full_content:
|
|
return False # already up to date; report as skipped
|
|
|
|
path.write_text(full_content, encoding="utf-8")
|
|
return True
|
|
|
|
|
|
def file_is_generated(path: Path) -> bool:
|
|
"""Return True if file has status: generated front-matter."""
|
|
if not path.exists():
|
|
return False
|
|
fm, _ = parse_frontmatter(path.read_text(encoding="utf-8", errors="replace"))
|
|
return fm.present and fm.status == "generated"
|
|
|
|
|
|
def find_wiki_docs_for_path(target_dir: Path, root: Path, wiki_root: Path) -> list[dict]:
|
|
"""Find wiki documents relevant to a source path.
|
|
|
|
Searches:
|
|
1. Codebase mirror: wiki/codebase/<rel>/ — README.md, NAV.md, DOMAIN.md, any .md
|
|
2. All wiki branches: scan wiki/<branch>/ trees for docs whose source-paths
|
|
include the target (or an ancestor of it).
|
|
3. Knowledge: links from the target's control file.
|
|
|
|
Returns list of dicts: {path, label, kind, status}
|
|
"""
|
|
target_rel = posix_rel(target_dir, root)
|
|
results = []
|
|
seen_paths: set[str] = set()
|
|
|
|
def add_doc(doc_path: Path, label: str, kind: str):
|
|
p = posix_rel(doc_path, root)
|
|
if p not in seen_paths and doc_path.exists():
|
|
fm, _ = read_wiki_file(doc_path)
|
|
results.append({
|
|
"path": p,
|
|
"label": label,
|
|
"kind": kind,
|
|
"status": fm.status or "unknown",
|
|
})
|
|
seen_paths.add(p)
|
|
|
|
# 1. Codebase mirror — walk up from target to root looking for mirror docs
|
|
check_dir = target_dir
|
|
while True:
|
|
rel = posix_rel(check_dir, root)
|
|
mirror_dir = wiki_root / CODEBASE_SUBDIR / rel if rel and rel != "." else wiki_root / CODEBASE_SUBDIR
|
|
if mirror_dir.exists():
|
|
for md_file in sorted(mirror_dir.glob("*.md")):
|
|
name = md_file.stem.upper()
|
|
if name == "README":
|
|
add_doc(md_file, "Code orientation", "wiki_readme")
|
|
elif name == "NAV":
|
|
add_doc(md_file, "Local navigation", "wiki_nav")
|
|
elif name == "DOMAIN":
|
|
add_doc(md_file, "Domain overview", "wiki_domain")
|
|
else:
|
|
add_doc(md_file, md_file.stem, "wiki_doc")
|
|
if any(mirror_dir.glob("*.md")):
|
|
break # found the nearest mirror with content
|
|
if check_dir == root:
|
|
break
|
|
parent = check_dir.parent
|
|
if parent == check_dir:
|
|
break
|
|
check_dir = parent
|
|
|
|
# 2. Scan non-codebase wiki branches for docs with matching source-paths
|
|
if wiki_root.exists():
|
|
for branch_dir in sorted(wiki_root.iterdir()):
|
|
if not branch_dir.is_dir():
|
|
continue
|
|
if branch_dir.name == CODEBASE_SUBDIR:
|
|
continue # already handled above
|
|
if branch_dir.name.startswith("."):
|
|
continue
|
|
# Walk this branch for .md files with source-paths that match target
|
|
for md_file in branch_dir.rglob("*.md"):
|
|
fm, _ = read_wiki_file(md_file)
|
|
if not fm.source_paths:
|
|
continue
|
|
for sp in fm.source_paths:
|
|
sp_normalized = sp.rstrip("/")
|
|
target_normalized = target_rel.rstrip("/")
|
|
# Match if source-path is the target, an ancestor, or a child
|
|
if (sp_normalized == target_normalized
|
|
or target_normalized.startswith(sp_normalized + "/")
|
|
or sp_normalized.startswith(target_normalized + "/")):
|
|
name = md_file.stem.upper()
|
|
branch = branch_dir.name
|
|
if name == "README":
|
|
add_doc(md_file, f"Domain knowledge ({branch})", "domain_readme")
|
|
elif name == "NAV":
|
|
add_doc(md_file, f"Navigation ({branch})", "domain_nav")
|
|
else:
|
|
add_doc(md_file, f"{md_file.stem} ({branch})", "domain_doc")
|
|
break # don't add same file twice for multiple source-paths
|
|
|
|
# 3. Knowledge: link from control file
|
|
cf = find_control_file(target_dir)
|
|
if cf:
|
|
parsed = parse_control_file(cf, root)
|
|
if parsed.knowledge:
|
|
knowledge_path = root / parsed.knowledge
|
|
if knowledge_path.is_dir():
|
|
# Look for README.md or index in the knowledge dir
|
|
for candidate in ["README.md", "readme.md", "NAV.md"]:
|
|
kf = knowledge_path / candidate
|
|
if kf.exists():
|
|
add_doc(kf, "Knowledge link", "knowledge_ref")
|
|
elif knowledge_path.exists():
|
|
add_doc(knowledge_path, "Knowledge link", "knowledge_ref")
|
|
|
|
return results
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Justfile parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def parse_justfile(root: Path) -> list[tuple[str, str]]:
|
|
"""
|
|
Parse a Justfile and return list of (recipe_name, comment) pairs.
|
|
Comments are the # line immediately preceding a recipe.
|
|
"""
|
|
justfile = root / "Justfile"
|
|
if not justfile.exists():
|
|
return []
|
|
|
|
try:
|
|
text = justfile.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return []
|
|
|
|
recipes = []
|
|
lines = text.splitlines()
|
|
last_comment = ""
|
|
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
if stripped.startswith("#"):
|
|
last_comment = stripped[1:].strip()
|
|
elif stripped and not stripped.startswith(" ") and not stripped.startswith("\t"):
|
|
# Possible recipe line: name [params]:
|
|
m = re.match(r"^([a-zA-Z0-9_-]+)(?:\s+[^:]+)?:", stripped)
|
|
if m and not stripped.startswith("set ") and not stripped.startswith("export"):
|
|
recipe_name = m.group(1)
|
|
if recipe_name not in ("default",):
|
|
recipes.append((recipe_name, last_comment))
|
|
last_comment = ""
|
|
else:
|
|
last_comment = ""
|
|
else:
|
|
if not stripped:
|
|
last_comment = ""
|
|
|
|
return recipes
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Toolchain detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def detect_toolchains(root: Path) -> dict[str, Any]:
|
|
"""Detect toolchains from manifests and files."""
|
|
toolchains = {}
|
|
|
|
if (root / "Cargo.toml").exists():
|
|
toolchains["cargo"] = True
|
|
# Read rust edition / toolchain
|
|
rust_toolchain = root / "rust-toolchain.toml"
|
|
if rust_toolchain.exists():
|
|
try:
|
|
t = rust_toolchain.read_text(encoding="utf-8")
|
|
m = re.search(r'channel\s*=\s*"([^"]+)"', t)
|
|
if m:
|
|
toolchains["rust_channel"] = m.group(1)
|
|
except OSError:
|
|
pass
|
|
|
|
if (root / "scripts" / "pyproject.toml").exists():
|
|
toolchains["uv"] = True
|
|
toolchains["uv_project"] = "scripts"
|
|
|
|
if (root / "pyproject.toml").exists():
|
|
toolchains["uv"] = True
|
|
|
|
if (root / "package.json").exists():
|
|
toolchains["node"] = True
|
|
|
|
if (root / "Justfile").exists():
|
|
toolchains["just"] = True
|
|
|
|
return toolchains
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Navigation table generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_MD_LINK_RE = re.compile(r"\[([^\]]+)\]\([^)]*\)")
|
|
|
|
|
|
def sanitize_purpose(text: str) -> str:
|
|
"""Make text safe for a Markdown table Purpose cell.
|
|
|
|
Control-file paragraphs can contain Markdown links, pipes, and newlines.
|
|
Embedding those verbatim corrupts the table and creates false broken-link
|
|
findings (the links resolve relative to the wiki file, not the source file).
|
|
Flatten links to their label, escape pipes, collapse whitespace, truncate.
|
|
"""
|
|
if not text:
|
|
return ""
|
|
text = _MD_LINK_RE.sub(r"\1", text) # [label](url) -> label
|
|
text = " ".join(text.split()) # collapse whitespace/newlines
|
|
text = text.replace("|", "\\|") # don't break table columns
|
|
if len(text) > 100:
|
|
text = text[:100].rstrip() + "..."
|
|
return text
|
|
|
|
|
|
def get_dir_purpose(directory: Path, root: Path) -> str:
|
|
"""
|
|
Return a purpose string for a directory.
|
|
Source: Area: or Knowledge: label in control file, else H1/first_para, else "".
|
|
NEVER invent from dir name. Output is sanitized for table cells.
|
|
"""
|
|
cf = find_control_file(directory)
|
|
if cf:
|
|
parsed = parse_control_file(cf, root)
|
|
if parsed.area:
|
|
return sanitize_purpose(parsed.area)
|
|
if parsed.knowledge:
|
|
return sanitize_purpose(parsed.knowledge)
|
|
if parsed.first_para:
|
|
return sanitize_purpose(parsed.first_para)
|
|
if parsed.title:
|
|
return sanitize_purpose(parsed.title)
|
|
return ""
|
|
|
|
|
|
def generate_nav_table(dirs: list[Path], root: Path, wiki_root: Path,
|
|
nav_from: Path, *, include_empty: bool = False) -> str:
|
|
"""
|
|
Generate a markdown nav table using repo-root-relative /path links.
|
|
nav_from: the file that will contain this table (kept for signature compat).
|
|
include_empty: if False, skip rows with no purpose AND no knowledge link.
|
|
"""
|
|
rows = []
|
|
for d in dirs:
|
|
rel = posix_rel(d, root)
|
|
purpose = get_dir_purpose(d, root)
|
|
# Local control link — repo-root-relative /path
|
|
cf = find_control_file(d)
|
|
if cf:
|
|
cf_repo_rel = posix_rel(cf, root)
|
|
cf_cell = f"[{cf.name}](/{cf_repo_rel})"
|
|
else:
|
|
cf_cell = "—"
|
|
|
|
# Knowledge link: wiki/codebase/<rel>/ if exists — repo-root-relative /path
|
|
wiki_dir = wiki_root / CODEBASE_SUBDIR / rel
|
|
if wiki_dir.exists() and any(wiki_dir.glob("*.md")):
|
|
wiki_repo_rel = posix_rel(wiki_dir, root)
|
|
knowledge_cell = f"[wiki](/{wiki_repo_rel}/)"
|
|
else:
|
|
knowledge_cell = "—"
|
|
|
|
# Skip rows with no useful content unless include_empty is True
|
|
if not include_empty and not purpose and knowledge_cell == "—":
|
|
continue
|
|
|
|
rows.append((f"`{rel}/`", purpose, cf_cell, knowledge_cell))
|
|
|
|
lines = [
|
|
"| Path | Purpose | Local control | Knowledge |",
|
|
"|------|---------|---------------|-----------|",
|
|
]
|
|
for path_cell, purpose, cf_cell, knowledge_cell in rows:
|
|
lines.append(f"| {path_cell} | {purpose} | {cf_cell} | {knowledge_cell} |")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Content generators
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def generate_boot_md(root: Path) -> str:
|
|
"""Generate content for boot.md (without front-matter)."""
|
|
sections = []
|
|
|
|
# --- What This Project Is ---
|
|
readme = root / "README.md"
|
|
project_name = root.name
|
|
project_desc = ""
|
|
if readme.exists():
|
|
try:
|
|
rtext = readme.read_text(encoding="utf-8", errors="replace")
|
|
rlines = rtext.splitlines()
|
|
for line in rlines:
|
|
s = line.strip()
|
|
if s.startswith("# "):
|
|
project_name = s[2:].strip()
|
|
break
|
|
# First non-empty paragraph after H1
|
|
found_h1 = False
|
|
para_lines = []
|
|
for line in rlines:
|
|
s = line.strip()
|
|
if not found_h1:
|
|
if s.startswith("# "):
|
|
found_h1 = True
|
|
continue
|
|
if s.startswith("#"):
|
|
break
|
|
if s:
|
|
para_lines.append(s)
|
|
elif para_lines:
|
|
break
|
|
if para_lines:
|
|
project_desc = " ".join(para_lines)
|
|
except OSError:
|
|
pass
|
|
|
|
what_section = f"## What This Project Is\n\n**{project_name}**"
|
|
if project_desc:
|
|
what_section += f"\n\n{project_desc}"
|
|
sections.append(what_section)
|
|
|
|
# --- Toolchains ---
|
|
toolchains = detect_toolchains(root)
|
|
tc_lines = ["## Toolchains", ""]
|
|
if toolchains.get("cargo"):
|
|
channel = toolchains.get("rust_channel", "stable")
|
|
tc_lines.append(f"- **Rust / Cargo** — `{channel}` channel")
|
|
if toolchains.get("uv"):
|
|
uv_proj = toolchains.get("uv_project", "scripts")
|
|
tc_lines.append(f"- **Python / uv** — project: `{uv_proj}/`")
|
|
if toolchains.get("node"):
|
|
tc_lines.append("- **Node.js / npm** — `package.json` present")
|
|
if toolchains.get("just"):
|
|
tc_lines.append("- **Just** — task runner (`just --list` for all recipes)")
|
|
sections.append("\n".join(tc_lines))
|
|
|
|
# --- Key Folders ---
|
|
dirs = top_level_dirs(root)
|
|
folder_rows = []
|
|
for d in dirs:
|
|
purpose = get_dir_purpose(d, root)
|
|
if purpose:
|
|
folder_rows.append(f"| `{d.name}/` | {purpose} |")
|
|
folder_lines = ["## Key Folders", ""]
|
|
if folder_rows:
|
|
folder_lines.extend(["| Folder | Purpose |", "|--------|---------|"])
|
|
folder_lines.extend(folder_rows)
|
|
folder_lines.append("")
|
|
folder_lines.append("Run `ls` (or check `wiki/codebase/NAV.md`) for the full directory listing.")
|
|
else:
|
|
folder_lines.append("No folder descriptions available yet. Run `ls` to see the directory structure.")
|
|
sections.append("\n".join(folder_lines))
|
|
|
|
# --- Essential Commands ---
|
|
recipes = parse_justfile(root)
|
|
cmd_lines = ["## Essential Commands", ""]
|
|
|
|
if toolchains.get("just") and recipes:
|
|
essential = {"run", "dev", "server", "check", "build", "test", "test-net", "test-one"}
|
|
cmd_lines.append("**Core recipes** (run via `just <recipe>`):")
|
|
cmd_lines.append("")
|
|
for name, comment in recipes:
|
|
if name in essential:
|
|
if comment:
|
|
cmd_lines.append(f"- `just {name}` — {comment}")
|
|
else:
|
|
cmd_lines.append(f"- `just {name}`")
|
|
cmd_lines.append("")
|
|
cmd_lines.append("Run `just --list` for the full set.")
|
|
cmd_lines.append("")
|
|
|
|
if toolchains.get("cargo"):
|
|
cmd_lines.append("**Cargo** (Rust build system):")
|
|
cmd_lines.append("")
|
|
cmd_lines.append("- `cargo check --workspace` — type-check all crates")
|
|
cmd_lines.append("- `cargo test --workspace` — run all tests")
|
|
cmd_lines.append("- `cargo build --workspace` — build all crates")
|
|
cmd_lines.append("")
|
|
|
|
if toolchains.get("uv"):
|
|
uv_proj = toolchains.get("uv_project", "scripts")
|
|
cmd_lines.append("**Python scripts** (via uv):")
|
|
cmd_lines.append("")
|
|
cmd_lines.append(f"- `uv run --project {uv_proj} scripts/<name>.py` — run a Python script")
|
|
cmd_lines.append("")
|
|
|
|
sections.append("\n".join(cmd_lines))
|
|
|
|
# --- Where the Knowledge Layer Lives ---
|
|
kl_section = (
|
|
"## Where the Knowledge Layer Lives\n\n"
|
|
"- `wiki/` — Authored and generated wiki documents (navigation, concepts, patterns)\n"
|
|
"- `wiki/boot.md` — This file (project identity and orientation)\n\n"
|
|
"Navigate the wiki via `wiki/NAV.md` or `wiki/codebase/NAV.md`.\n"
|
|
"See the root `CLAUDE.md` for full project guidelines, conventions, and platform notes."
|
|
)
|
|
sections.append(kl_section)
|
|
|
|
# --- Where Executable Truth Lives ---
|
|
et_section = (
|
|
"## Where Executable Truth Lives\n\n"
|
|
"Source code is authoritative for current behavior. "
|
|
"Wiki documents describe concepts and provide navigation — they may lag implementation.\n\n"
|
|
"Always inspect source before making claims about current executable behavior."
|
|
)
|
|
sections.append(et_section)
|
|
|
|
# --- Critical Universal Warnings ---
|
|
root_cf = find_control_file(root)
|
|
universal_warnings = []
|
|
if root_cf:
|
|
parsed = parse_control_file(root_cf, root)
|
|
universal_warnings = parsed.warnings
|
|
|
|
warn_section = "## Critical Universal Warnings\n\n"
|
|
if universal_warnings:
|
|
warn_section += "\n".join(f"- {w}" for w in universal_warnings)
|
|
else:
|
|
warn_section += "(none — see root CLAUDE.md for full project guidelines)"
|
|
sections.append(warn_section)
|
|
|
|
body = "# Repository Boot\n\n" + "\n\n".join(sections)
|
|
return body
|
|
|
|
|
|
def generate_wiki_readme(root: Path, wiki_root: Path) -> str:
|
|
"""Generate wiki/README.md content (without front-matter)."""
|
|
return (
|
|
"# Wiki\n\n"
|
|
"This directory contains the knowledge layer for the codebase: "
|
|
"navigation maps, authored wiki documents, and generated resources.\n\n"
|
|
"| Section | Description |\n"
|
|
"|---------|-------------|\n"
|
|
"| [`codebase/`](/wiki/codebase/) | Codebase navigation — mirrors the source tree |\n\n"
|
|
"**Note**: Wiki documents describe concepts and provide navigation. "
|
|
"Source code is authoritative for current behavior.\n\n"
|
|
"See [`NAV.md`](/wiki/NAV.md) for a top-level navigation table."
|
|
)
|
|
|
|
|
|
def generate_wiki_nav(root: Path, wiki_root: Path) -> str:
|
|
"""Generate wiki/NAV.md content (without front-matter)."""
|
|
dirs = top_level_dirs(root)
|
|
nav_file = wiki_root / "NAV.md"
|
|
table = generate_nav_table(dirs, root, wiki_root, nav_file)
|
|
return (
|
|
"# Repository Navigation\n\n"
|
|
f"{table}\n\n"
|
|
"Only directories with known purpose or wiki coverage are listed. "
|
|
"Run `ls` for the complete directory listing."
|
|
)
|
|
|
|
|
|
def generate_wiki_codebase_nav(root: Path, wiki_root: Path) -> str:
|
|
"""Generate wiki/codebase/NAV.md content (without front-matter).
|
|
|
|
Maps every directory that holds a control file (the existing CLAUDE.md /
|
|
AGENTS.md structure), so the wiki is a real index of authored guidance from
|
|
day one. The repository root control file is excluded — it is covered by
|
|
boot.md. Directories are listed in repository order.
|
|
"""
|
|
cf_dirs = sorted(
|
|
{cf.parent for cf in find_all_control_files(root) if cf.parent != root},
|
|
key=lambda d: posix_rel(d, root),
|
|
)
|
|
nav_file = wiki_root / CODEBASE_SUBDIR / "NAV.md"
|
|
table = generate_nav_table(cf_dirs, root, wiki_root, nav_file)
|
|
intro = (
|
|
"Directories below carry a local control file (`CLAUDE.md` / `AGENTS.md`). "
|
|
"This is the existing authored guidance the wiki ingests and grows from. "
|
|
"Use `just tome context <path>` to resolve the full ancestor chain for any path."
|
|
)
|
|
return f"# Codebase Navigation\n\n{intro}\n\n{table}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Inventory generation (gitignored, curation input)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def generate_codebase_inventory(root: Path, wiki_root: Path) -> str:
|
|
"""Generate exhaustive codebase inventory for curation agents.
|
|
|
|
Lists every directory with a control file, plus metadata to help curation
|
|
agents decide what belongs in NAV.md without extra tool calls.
|
|
"""
|
|
cf_dirs = sorted(
|
|
{cf.parent for cf in find_all_control_files(root) if cf.parent != root},
|
|
key=lambda d: posix_rel(d, root),
|
|
)
|
|
|
|
lines = [
|
|
"# Codebase Inventory",
|
|
"",
|
|
"Generated by `knowledge.py refresh`. **Do not edit** — this file is gitignored",
|
|
"and regenerated on every refresh. Curation agents read this to discover what",
|
|
"exists; they write to `NAV.md` only what earns its place.",
|
|
"",
|
|
"| Path | Purpose | Control file | Wiki mirror | Wiki branches |",
|
|
"|------|---------|-------------|-------------|---------------|",
|
|
]
|
|
|
|
for d in cf_dirs:
|
|
rel = posix_rel(d, root)
|
|
purpose = get_dir_purpose(d, root)
|
|
|
|
# Control file info
|
|
cf = find_control_file(d)
|
|
cf_name = cf.name if cf else "—"
|
|
|
|
# Wiki codebase mirror
|
|
mirror_dir = wiki_root / CODEBASE_SUBDIR / rel
|
|
has_mirror = mirror_dir.exists() and any(mirror_dir.glob("*.md"))
|
|
mirror_cell = "yes" if has_mirror else "—"
|
|
|
|
# Non-codebase wiki branches that reference this path
|
|
branch_hits = []
|
|
if wiki_root.exists():
|
|
for branch_dir in sorted(wiki_root.iterdir()):
|
|
if not branch_dir.is_dir() or branch_dir.name == CODEBASE_SUBDIR:
|
|
continue
|
|
if branch_dir.name.startswith("."):
|
|
continue
|
|
for md_file in branch_dir.rglob("*.md"):
|
|
fm, _ = read_wiki_file(md_file)
|
|
for sp in fm.source_paths:
|
|
sp_n = sp.rstrip("/")
|
|
rel_n = rel.rstrip("/")
|
|
if sp_n == rel_n or rel_n.startswith(sp_n + "/") or sp_n.startswith(rel_n + "/"):
|
|
branch_hits.append(branch_dir.name)
|
|
break
|
|
else:
|
|
continue
|
|
break
|
|
branches_cell = ", ".join(sorted(set(branch_hits))) if branch_hits else "—"
|
|
|
|
lines.append(f"| `{rel}/` | {sanitize_purpose(purpose)} | {cf_name} | {mirror_cell} | {branches_cell} |")
|
|
|
|
lines.append("")
|
|
lines.append(f"Total: {len(cf_dirs)} directories with control files.")
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def generate_wiki_inventory(root: Path, wiki_root: Path) -> str:
|
|
"""Generate top-level wiki inventory for curation agents.
|
|
|
|
Lists all wiki branches and their contents to help curators understand
|
|
wiki coverage without globbing.
|
|
"""
|
|
lines = [
|
|
"# Wiki Inventory",
|
|
"",
|
|
"Generated by `knowledge.py refresh`. **Do not edit** — this file is gitignored",
|
|
"and regenerated on every refresh.",
|
|
"",
|
|
]
|
|
|
|
if not wiki_root.exists():
|
|
lines.append("Wiki directory does not exist yet. Run `tome init` to bootstrap.")
|
|
return "\n".join(lines)
|
|
|
|
# List wiki branches
|
|
branches = []
|
|
for entry in sorted(wiki_root.iterdir()):
|
|
if not entry.is_dir() or entry.name.startswith("."):
|
|
continue
|
|
md_files = list(entry.rglob("*.md"))
|
|
authored = sum(1 for f in md_files if _file_status(f) == "authored")
|
|
generated = sum(1 for f in md_files if _file_status(f) == "generated")
|
|
stub = sum(1 for f in md_files if _file_status(f) == "stub")
|
|
branches.append((entry.name, len(md_files), authored, generated, stub))
|
|
|
|
if branches:
|
|
lines.append("## Branches")
|
|
lines.append("")
|
|
lines.append("| Branch | Files | Authored | Generated | Stub |")
|
|
lines.append("|--------|-------|----------|-----------|------|")
|
|
for name, total, auth, gen, stb in branches:
|
|
lines.append(f"| `{name}/` | {total} | {auth} | {gen} | {stb} |")
|
|
lines.append("")
|
|
|
|
# List top-level .md files
|
|
top_files = sorted(f for f in wiki_root.glob("*.md") if not f.name.startswith("."))
|
|
if top_files:
|
|
lines.append("## Top-level files")
|
|
lines.append("")
|
|
for f in top_files:
|
|
status = _file_status(f)
|
|
lines.append(f"- `{f.name}` ({status})")
|
|
lines.append("")
|
|
|
|
# Top-level source dirs without any wiki coverage
|
|
lines.append("## Uncovered source directories")
|
|
lines.append("")
|
|
lines.append("Top-level source directories with control files but no wiki branch coverage:")
|
|
lines.append("")
|
|
uncovered = []
|
|
for d in top_level_dirs(root):
|
|
if not find_control_file(d):
|
|
continue
|
|
rel = posix_rel(d, root)
|
|
mirror = wiki_root / CODEBASE_SUBDIR / rel
|
|
has_mirror = mirror.exists() and any(mirror.glob("*.md"))
|
|
if not has_mirror:
|
|
uncovered.append(rel)
|
|
if uncovered:
|
|
for rel in uncovered:
|
|
lines.append(f"- `{rel}/`")
|
|
else:
|
|
lines.append("(all covered)")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _file_status(path: Path) -> str:
|
|
"""Return the front-matter status of a wiki file, or 'unknown'."""
|
|
fm, _ = read_wiki_file(path)
|
|
return fm.status or "unknown"
|
|
|
|
|
|
def write_inventory_file(path: Path, content: str):
|
|
"""Write an inventory file (always overwritten, no front-matter)."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content, encoding="utf-8")
|
|
|
|
|
|
def seed_nav_stub(path: Path, title: str, intro: str) -> bool:
|
|
"""Create a NAV.md as status:stub if it doesn't exist. Never overwrites.
|
|
|
|
Returns True if created, False if already exists.
|
|
"""
|
|
if path.exists():
|
|
return False
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
content = render_frontmatter_block("stub", []) + "\n"
|
|
content += f"# {title}\n\n"
|
|
content += f"{intro}\n\n"
|
|
content += "<!-- Curate this file: read .inventory.md for the full listing, -->\n"
|
|
content += "<!-- then add only entries that earn their place here. -->\n\n"
|
|
content += "| Path | Why it matters |\n"
|
|
content += "|------|----------------|\n"
|
|
path.write_text(content, encoding="utf-8")
|
|
return True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# bootstrap subcommand
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def cmd_bootstrap(root: Path) -> int:
|
|
"""Create missing scaffold files. Never clobber status:authored files."""
|
|
created = []
|
|
skipped = []
|
|
inventories = []
|
|
|
|
wiki_root = root / WIKI_SUBDIR
|
|
|
|
# 1. boot.md
|
|
boot_path = root / BOOT_MD_REL
|
|
boot_content = generate_boot_md(root)
|
|
if write_generated_file(boot_path, boot_content):
|
|
created.append(BOOT_MD_REL)
|
|
else:
|
|
skipped.append(BOOT_MD_REL)
|
|
|
|
# 2. wiki/README.md
|
|
wiki_readme_path = root / WIKI_README_REL
|
|
wiki_readme_content = generate_wiki_readme(root, wiki_root)
|
|
if write_generated_file(wiki_readme_path, wiki_readme_content):
|
|
created.append(WIKI_README_REL)
|
|
else:
|
|
skipped.append(WIKI_README_REL)
|
|
|
|
# 3. wiki/NAV.md — seed as stub if missing, never overwrite
|
|
wiki_nav_path = root / WIKI_NAV_REL
|
|
if seed_nav_stub(wiki_nav_path,
|
|
"Repository Navigation",
|
|
"Selective pointers to the most important areas. Not a complete listing — use `ls` for that."):
|
|
created.append(WIKI_NAV_REL)
|
|
else:
|
|
skipped.append(WIKI_NAV_REL)
|
|
|
|
# 4. wiki/codebase/NAV.md — seed as stub if missing, never overwrite
|
|
wiki_codebase_nav_path = root / WIKI_CODEBASE_NAV_REL
|
|
if seed_nav_stub(wiki_codebase_nav_path,
|
|
"Codebase Navigation",
|
|
"Key entry points into the source tree. Curated from `.inventory.md` — only directories that matter for orientation."):
|
|
created.append(WIKI_CODEBASE_NAV_REL)
|
|
else:
|
|
skipped.append(WIKI_CODEBASE_NAV_REL)
|
|
|
|
# 5. Inventory files (always regenerated, gitignored)
|
|
inv_path = root / WIKI_INVENTORY_REL
|
|
write_inventory_file(inv_path, generate_wiki_inventory(root, wiki_root))
|
|
inventories.append(WIKI_INVENTORY_REL)
|
|
|
|
cb_inv_path = root / WIKI_CODEBASE_INVENTORY_REL
|
|
write_inventory_file(cb_inv_path, generate_codebase_inventory(root, wiki_root))
|
|
inventories.append(WIKI_CODEBASE_INVENTORY_REL)
|
|
|
|
# Report conceptual knowledge: control files without wiki mirrors
|
|
control_files = find_all_control_files(root)
|
|
no_mirror = []
|
|
for cf in control_files:
|
|
cf_dir = cf.parent
|
|
rel = posix_rel(cf_dir, root)
|
|
wiki_mirror = wiki_root / CODEBASE_SUBDIR / rel
|
|
if not wiki_mirror.exists():
|
|
no_mirror.append(rel if rel else "(root)")
|
|
|
|
# Print report
|
|
print("=" * 60)
|
|
print("tome bootstrap report")
|
|
print("=" * 60)
|
|
print()
|
|
if created:
|
|
print("Created:")
|
|
for f in created:
|
|
print(f" + {f}")
|
|
print()
|
|
if skipped:
|
|
print("Skipped (already exist or status:authored/stub):")
|
|
for f in skipped:
|
|
print(f" ~ {f}")
|
|
print()
|
|
if inventories:
|
|
print("Inventories (gitignored, for curation agents):")
|
|
for f in inventories:
|
|
print(f" * {f}")
|
|
print()
|
|
if no_mirror:
|
|
print(f"Conceptual knowledge available for authoring later ({len(no_mirror)} dirs):")
|
|
for rel in sorted(no_mirror)[:20]: # cap display at 20
|
|
print(f" ? {rel}/CLAUDE.md -> wiki/codebase/{rel}/")
|
|
if len(no_mirror) > 20:
|
|
print(f" ... and {len(no_mirror) - 20} more")
|
|
print()
|
|
print(f"Control files found: {len(control_files)}")
|
|
print()
|
|
|
|
return EXIT_OK
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# refresh subcommand
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def cmd_refresh(root: Path) -> int:
|
|
"""Refresh status:generated files and inventories; report authored/stub for review."""
|
|
wiki_root = root / WIKI_SUBDIR
|
|
updated = []
|
|
inventories = []
|
|
proposed = []
|
|
|
|
# Files to auto-refresh if status:generated (NAV files excluded — they are
|
|
# now authored/stub and curated by agents, not regenerated by the tool)
|
|
generated_files = [
|
|
(root / BOOT_MD_REL, lambda: generate_boot_md(root), []),
|
|
]
|
|
|
|
for path, content_fn, src_paths in generated_files:
|
|
rel = posix_rel(path, root)
|
|
if not path.exists():
|
|
proposed.append(f"{rel} — missing (run bootstrap first)")
|
|
continue
|
|
if not file_is_generated(path):
|
|
proposed.append(f"{rel} — status is not 'generated'; skipped")
|
|
continue
|
|
new_content = content_fn()
|
|
full_content = render_frontmatter_block("generated", src_paths) + "\n" + new_content
|
|
existing = path.read_text(encoding="utf-8", errors="replace")
|
|
if existing != full_content:
|
|
path.write_text(full_content, encoding="utf-8")
|
|
updated.append(rel)
|
|
|
|
# Inventory files (always regenerated, gitignored)
|
|
inv_path = root / WIKI_INVENTORY_REL
|
|
write_inventory_file(inv_path, generate_wiki_inventory(root, wiki_root))
|
|
inventories.append(WIKI_INVENTORY_REL)
|
|
|
|
cb_inv_path = root / WIKI_CODEBASE_INVENTORY_REL
|
|
write_inventory_file(cb_inv_path, generate_codebase_inventory(root, wiki_root))
|
|
inventories.append(WIKI_CODEBASE_INVENTORY_REL)
|
|
|
|
# Scan control files
|
|
control_files = find_all_control_files(root)
|
|
dirs = top_level_dirs(root)
|
|
|
|
# Stale checks: wiki mirrors for removed source dirs
|
|
wiki_codebase = wiki_root / CODEBASE_SUBDIR
|
|
if wiki_codebase.exists():
|
|
for wiki_sub in sorted(wiki_codebase.iterdir()):
|
|
if wiki_sub.is_dir():
|
|
source_candidate = root / wiki_sub.name
|
|
if not source_candidate.exists():
|
|
proposed.append(
|
|
f"wiki/codebase/{wiki_sub.name}/ — "
|
|
f"source dir '{wiki_sub.name}/' no longer exists"
|
|
)
|
|
|
|
# New top-level dirs without wiki coverage
|
|
for d in dirs:
|
|
rel = posix_rel(d, root)
|
|
mirror = wiki_root / CODEBASE_SUBDIR / rel
|
|
if not mirror.exists() and find_control_file(d):
|
|
proposed.append(
|
|
f"{rel}/ — has CLAUDE.md but no wiki mirror (candidate for wiki/codebase/{rel}/)"
|
|
)
|
|
|
|
# NAV files that are still status:generated (legacy — should be curated)
|
|
for nav_rel in [WIKI_NAV_REL, WIKI_CODEBASE_NAV_REL]:
|
|
nav_path = root / nav_rel
|
|
if nav_path.exists() and file_is_generated(nav_path):
|
|
proposed.append(
|
|
f"{nav_rel} — still status:generated (should be curated to status:authored)"
|
|
)
|
|
|
|
# Print report
|
|
print("=" * 60)
|
|
print("tome refresh report")
|
|
print("=" * 60)
|
|
print()
|
|
if updated:
|
|
print("Automatically updated:")
|
|
for f in updated:
|
|
print(f" * {f}")
|
|
print()
|
|
else:
|
|
print("Automatically updated: (none — all generated files are up to date)")
|
|
print()
|
|
|
|
if inventories:
|
|
print("Inventories regenerated (gitignored, for curation agents):")
|
|
for f in inventories:
|
|
print(f" * {f}")
|
|
print()
|
|
|
|
if proposed:
|
|
print("Proposed for review (not automatically changed):")
|
|
for msg in proposed:
|
|
print(f" ? {msg}")
|
|
print()
|
|
|
|
print(f"Control-file inventory: {len(control_files)} files found")
|
|
print()
|
|
|
|
return EXIT_OK
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# context subcommand
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class Finding(NamedTuple):
|
|
id: str
|
|
severity: str # error / warning
|
|
path: str
|
|
message: str
|
|
target: str = ""
|
|
|
|
|
|
def cmd_context(root: Path, target_str: str, as_json: bool = False) -> int:
|
|
"""Resolve navigation context for a target path."""
|
|
# 1. Normalize path. Relative paths are repository-relative (the documented
|
|
# contract), not cwd-relative — the tool is usually invoked from outside
|
|
# the repo root (e.g. the `just tome` module runs from the plugin dir).
|
|
target = Path(target_str)
|
|
if not target.is_absolute():
|
|
target = root / target
|
|
target = target.resolve()
|
|
|
|
# Check outside repo
|
|
try:
|
|
target.relative_to(root)
|
|
except ValueError:
|
|
msg = f"Path '{target_str}' is outside the repository root '{root}'"
|
|
if as_json:
|
|
print(json.dumps({"error": msg, "exit_code": EXIT_USAGE}))
|
|
else:
|
|
print(f"Error: {msg}", file=sys.stderr)
|
|
return EXIT_USAGE
|
|
|
|
# Check exists
|
|
if not target.exists():
|
|
msg = f"Path '{target_str}' does not exist"
|
|
if as_json:
|
|
print(json.dumps({"error": msg, "exit_code": EXIT_USAGE}))
|
|
else:
|
|
print(f"Error: {msg}", file=sys.stderr)
|
|
return EXIT_USAGE
|
|
|
|
target_rel = posix_rel(target, root)
|
|
target_kind = "directory" if target.is_dir() else "file"
|
|
|
|
# 2. Target dir
|
|
target_dir = target if target.is_dir() else target.parent
|
|
|
|
# 3. Enumerate ancestor dirs root -> target (inclusive)
|
|
ancestors: list[Path] = []
|
|
current = target_dir
|
|
while True:
|
|
ancestors.append(current)
|
|
if current == root:
|
|
break
|
|
parent = current.parent
|
|
if parent == current:
|
|
break
|
|
current = parent
|
|
ancestors.reverse() # root first
|
|
|
|
# 4. Per ancestor: pick effective control file
|
|
effective_controls: list[dict] = []
|
|
findings: list[Finding] = []
|
|
|
|
all_warnings: list[dict] = []
|
|
all_handoff: list[dict] = []
|
|
all_load: list[dict] = []
|
|
|
|
for anc in ancestors:
|
|
anc_rel = posix_rel(anc, root)
|
|
# Root is represented as "" not "."
|
|
if anc_rel == ".":
|
|
anc_rel = ""
|
|
cf = find_control_file(anc)
|
|
if cf is None:
|
|
continue
|
|
|
|
cf_rel = posix_rel(cf, root)
|
|
effective_controls.append({"path": cf_rel, "ancestor": anc_rel})
|
|
|
|
# 5. Parse
|
|
parsed = parse_control_file(cf, root)
|
|
|
|
# Aggregate warnings
|
|
for w in parsed.warnings:
|
|
all_warnings.append({"text": w, "source": cf_rel})
|
|
|
|
# Aggregate handoff
|
|
for h in parsed.handoff:
|
|
all_handoff.append({"text": h, "source": cf_rel})
|
|
|
|
# Aggregate load items
|
|
for item in parsed.load_items:
|
|
if item.target:
|
|
target_path = root / item.target
|
|
if not target_path.exists() and not item.target.startswith(("http://", "https://")):
|
|
findings.append(Finding(
|
|
"control.load-link.broken", "warning", cf_rel,
|
|
f"Load link target not found: '{item.target}'",
|
|
item.target
|
|
))
|
|
all_load.append({
|
|
"label": item.label,
|
|
"kind": item.kind,
|
|
"target": item.target,
|
|
"source": cf_rel,
|
|
})
|
|
|
|
# 7. Wiki document discovery
|
|
wiki_root = root / WIKI_SUBDIR
|
|
wiki_docs = find_wiki_docs_for_path(target_dir, root, wiki_root)
|
|
|
|
# Mirror info for backward compat (JSON output)
|
|
mirror_info: dict | None = None
|
|
if wiki_docs:
|
|
# First codebase mirror doc determines the mirror path
|
|
for wd in wiki_docs:
|
|
if wd["kind"].startswith("wiki_"):
|
|
mirror_path = str(PurePosixPath(wd["path"]).parent)
|
|
mirror_info = {
|
|
"path": mirror_path,
|
|
"documents": wiki_docs,
|
|
}
|
|
break
|
|
if not mirror_info:
|
|
mirror_info = {"path": "", "documents": wiki_docs}
|
|
|
|
# 8. Required reading (resolvable Load link targets + wiki docs)
|
|
required_reading = []
|
|
for item in all_load:
|
|
if item["target"] and not item["target"].startswith(("http://", "https://")):
|
|
t_path = root / item["target"]
|
|
if t_path.exists():
|
|
required_reading.append({
|
|
"path": item["target"],
|
|
"label": item["label"],
|
|
"source": item["source"],
|
|
"kind": item["kind"],
|
|
})
|
|
|
|
# Add wiki documents to required reading — authored docs first, then generated
|
|
for wd in wiki_docs:
|
|
if not any(r["path"] == wd["path"] for r in required_reading):
|
|
required_reading.append({
|
|
"path": wd["path"],
|
|
"label": wd["label"],
|
|
"source": "wiki",
|
|
"kind": wd["kind"],
|
|
})
|
|
|
|
# 9. Quick top commands from Justfile
|
|
recipes = parse_justfile(root)
|
|
top_commands = []
|
|
important_recipes = ["check", "test", "build", "run", "dev", "tome-check", "tome-context"]
|
|
for name, comment in recipes:
|
|
if name in important_recipes:
|
|
top_commands.append(f"just {name}")
|
|
if not top_commands and recipes:
|
|
top_commands = [f"just {recipes[0][0]}"]
|
|
|
|
# Sort findings
|
|
findings.sort(key=lambda f: (0 if f.severity == "error" else 1, f.path))
|
|
|
|
if as_json:
|
|
# Enrich required_reading and effective_controls with root_relative_path (/ prefix)
|
|
rr_enriched = [
|
|
{**r, "root_relative_path": f"/{r['path']}"}
|
|
for r in required_reading
|
|
]
|
|
ec_enriched = [
|
|
{**ec, "root_relative_path": f"/{ec['path']}"}
|
|
for ec in effective_controls
|
|
]
|
|
output = {
|
|
"schema_version": 2,
|
|
"repository_root": root.as_posix(),
|
|
"target": target_rel,
|
|
"target_kind": target_kind,
|
|
"effective_controls": ec_enriched,
|
|
"warnings": all_warnings,
|
|
"handoff": all_handoff,
|
|
"required_reading": rr_enriched,
|
|
"mirror": mirror_info,
|
|
"top_commands": top_commands,
|
|
"findings": [
|
|
{
|
|
"id": f.id,
|
|
"severity": f.severity,
|
|
"path": f.path,
|
|
"message": f.message,
|
|
"target": f.target,
|
|
}
|
|
for f in findings
|
|
],
|
|
}
|
|
print(json.dumps(output, indent=2))
|
|
else:
|
|
# Human output
|
|
print(f"Context: {target_rel} ({target_kind})")
|
|
print()
|
|
|
|
print("Warnings:")
|
|
if all_warnings:
|
|
for w in all_warnings:
|
|
print(f" - {w['text']} [from {w['source']}]")
|
|
else:
|
|
print(" (none)")
|
|
print()
|
|
|
|
print("Read first:")
|
|
if required_reading:
|
|
for r in required_reading:
|
|
status_hint = ""
|
|
# Show authored wiki docs prominently
|
|
if r["source"] == "wiki":
|
|
wd_match = next((w for w in wiki_docs if w["path"] == r["path"]), None)
|
|
if wd_match and wd_match["status"] == "authored":
|
|
status_hint = " (authored)"
|
|
elif wd_match and wd_match["status"] == "generated":
|
|
status_hint = " (generated)"
|
|
print(f" /{r['path']}{status_hint} — {r['label']}")
|
|
else:
|
|
print(" (none)")
|
|
print()
|
|
|
|
if wiki_docs:
|
|
print("Wiki knowledge:")
|
|
for wd in wiki_docs:
|
|
print(f" /{wd['path']} [{wd['status']}] — {wd['label']}")
|
|
print(" Use `ls` to verify current directory contents — wiki may lag.")
|
|
else:
|
|
print("Wiki knowledge: (none — no wiki documents cover this path yet)")
|
|
print()
|
|
|
|
print("Useful commands:")
|
|
if top_commands:
|
|
for cmd in top_commands:
|
|
print(f" {cmd}")
|
|
else:
|
|
print(" (run `just --list` for available recipes)")
|
|
print()
|
|
|
|
print("Relevant control files:")
|
|
if effective_controls:
|
|
for ec in effective_controls:
|
|
print(f" /{ec['path']}")
|
|
else:
|
|
print(" (none)")
|
|
print()
|
|
|
|
if findings:
|
|
print("Findings:")
|
|
for f in findings:
|
|
print(f" [{f.severity.upper()}] {f.id}: {f.message} ({f.path})")
|
|
print()
|
|
|
|
return EXIT_OK
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check subcommand
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def strip_fenced_code(text: str) -> str:
|
|
"""Return a version of text with fenced code block contents replaced by blank lines."""
|
|
lines = text.splitlines(keepends=True)
|
|
result = []
|
|
in_fence = False
|
|
fence_marker = ""
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
if not in_fence:
|
|
if stripped.startswith("```") or stripped.startswith("~~~"):
|
|
in_fence = True
|
|
fence_marker = stripped[:3]
|
|
result.append(line) # keep the fence line itself
|
|
else:
|
|
result.append(line)
|
|
else:
|
|
if stripped.startswith(fence_marker) and stripped == fence_marker:
|
|
in_fence = False
|
|
result.append(line)
|
|
else:
|
|
# Replace with blank line to preserve line numbers
|
|
result.append("\n")
|
|
return "".join(result)
|
|
|
|
|
|
def count_h1(text: str) -> int:
|
|
"""Count H1 headings in markdown text, skipping fenced code blocks."""
|
|
clean = strip_fenced_code(text)
|
|
count = 0
|
|
for line in clean.splitlines():
|
|
s = line.strip()
|
|
if s.startswith("# ") and not s.startswith("## "):
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def resolve_link_target(target: str, file_dir: Path, root: Path) -> Path:
|
|
"""Resolve a markdown link target to an absolute Path.
|
|
|
|
Resolution rules:
|
|
- ``/path`` — repo-root-relative (strip leading ``/``, join with ``root``)
|
|
- ``http://`` / ``https://`` — external (caller must skip before calling this)
|
|
- anything else — relative to ``file_dir`` (existing behaviour)
|
|
"""
|
|
if target.startswith("/"):
|
|
return (root / target.lstrip("/")).resolve()
|
|
return (file_dir / target).resolve()
|
|
|
|
|
|
def check_markdown_links(path: Path, root: Path) -> list[Finding]:
|
|
"""Check local markdown links in a file, skipping fenced code blocks."""
|
|
findings = []
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return findings
|
|
|
|
# Strip code blocks so we don't check links inside them
|
|
text = strip_fenced_code(text)
|
|
|
|
file_dir = path.parent
|
|
# Find all markdown links
|
|
for m in MD_LINK_RE.finditer(text):
|
|
target = m.group(2).strip()
|
|
# Skip external links, mailto, anchors-only
|
|
if target.startswith(("http://", "https://", "mailto:", "#")):
|
|
continue
|
|
# Strip fragment
|
|
if "#" in target:
|
|
target = target[:target.index("#")]
|
|
if not target:
|
|
continue
|
|
# Resolve: /path = repo-root-relative, else relative to file dir
|
|
resolved = resolve_link_target(target, file_dir, root)
|
|
if not resolved.exists():
|
|
rel_path = posix_rel(path, root)
|
|
findings.append(Finding(
|
|
"link.broken", "error", rel_path,
|
|
f"Broken link target: '{target}'",
|
|
target
|
|
))
|
|
return findings
|
|
|
|
|
|
def cmd_check(root: Path, as_json: bool = False) -> int:
|
|
"""Validate knowledge-layer health."""
|
|
findings: list[Finding] = []
|
|
wiki_root = root / WIKI_SUBDIR
|
|
|
|
# --- Check nav resources ---
|
|
boot_path = root / BOOT_MD_REL
|
|
|
|
if not boot_path.exists():
|
|
findings.append(Finding(
|
|
"nav.resource.missing", "error",
|
|
BOOT_MD_REL,
|
|
"boot.md is missing — run 'knowledge.py bootstrap' to create it",
|
|
))
|
|
|
|
wiki_readme = root / WIKI_README_REL
|
|
wiki_nav = root / WIKI_NAV_REL
|
|
if not wiki_readme.exists():
|
|
findings.append(Finding(
|
|
"nav.wiki.missing", "warning",
|
|
WIKI_README_REL,
|
|
"wiki/README.md is missing — run 'knowledge.py bootstrap'",
|
|
))
|
|
if not wiki_nav.exists():
|
|
findings.append(Finding(
|
|
"nav.wiki.missing", "warning",
|
|
WIKI_NAV_REL,
|
|
"wiki/NAV.md is missing — run 'knowledge.py bootstrap'",
|
|
))
|
|
|
|
# --- Check control files ---
|
|
control_files = find_all_control_files(root)
|
|
checked_dirs: set[Path] = set()
|
|
|
|
for cf in control_files:
|
|
cf_dir = cf.parent
|
|
rel = posix_rel(cf, root)
|
|
|
|
# coexist check — the @AGENTS.md router pattern (CLAUDE.md that is a
|
|
# pure include of AGENTS.md) is a recommended convention, not a clash
|
|
if cf_dir not in checked_dirs:
|
|
checked_dirs.add(cf_dir)
|
|
if has_both_control_files(cf_dir):
|
|
if not (is_pure_include(cf_dir / "CLAUDE.md") or is_pure_include(cf_dir / "AGENTS.md")):
|
|
findings.append(Finding(
|
|
"control.coexist", "warning",
|
|
posix_rel(cf_dir, root),
|
|
"Both CLAUDE.md and AGENTS.md exist in this directory",
|
|
))
|
|
|
|
# Pure-include routers delegate all content; the included file is
|
|
# checked on its own, so skip structural checks here
|
|
if is_pure_include(cf):
|
|
continue
|
|
|
|
# Parse
|
|
parsed = parse_control_file(cf, root)
|
|
|
|
# H1 checks (on include-expanded text, matching how agents see the file)
|
|
try:
|
|
text = expand_includes(cf.read_text(encoding="utf-8", errors="replace"), cf_dir)
|
|
except OSError:
|
|
continue
|
|
|
|
h1_count = count_h1(text)
|
|
if h1_count == 0:
|
|
findings.append(Finding(
|
|
"control.h1.missing", "error", rel,
|
|
"Control file has no H1 heading",
|
|
))
|
|
elif h1_count > 1:
|
|
findings.append(Finding(
|
|
"control.h1.multiple", "error", rel,
|
|
f"Control file has {h1_count} H1 headings (expected 1)",
|
|
))
|
|
|
|
# Section order
|
|
if len(parsed.section_order) > 1:
|
|
expected = [s for s in CANONICAL_SECTIONS if s in parsed.section_order]
|
|
if parsed.section_order != expected:
|
|
findings.append(Finding(
|
|
"control.section.order", "warning", rel,
|
|
f"Recognized sections out of canonical order: {parsed.section_order} (expected {expected})",
|
|
))
|
|
|
|
# Load link broken (only for files that actually have Load sections)
|
|
for item in parsed.load_items:
|
|
if item.kind != "prose" and item.target:
|
|
if not item.target.startswith(("http://", "https://")):
|
|
t_path = root / item.target
|
|
if not t_path.exists():
|
|
findings.append(Finding(
|
|
"control.load-link.broken", "error", rel,
|
|
f"Load link target not found: '{item.target}'",
|
|
item.target,
|
|
))
|
|
|
|
# Check links in control files
|
|
link_findings = check_markdown_links(cf, root)
|
|
# Filter out same findings already caught by load-link.broken
|
|
for lf in link_findings:
|
|
findings.append(lf)
|
|
|
|
# --- Check wiki files ---
|
|
if wiki_root.exists():
|
|
for wiki_file in sorted(wiki_root.rglob("*.md")):
|
|
if wiki_file.name.startswith("."):
|
|
continue # skip inventory files (.inventory.md)
|
|
wrel = posix_rel(wiki_file, root)
|
|
try:
|
|
text = wiki_file.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
continue
|
|
|
|
fm, body = parse_frontmatter(text)
|
|
|
|
if not fm.present:
|
|
findings.append(Finding(
|
|
"wiki.frontmatter.missing", "warning",
|
|
wrel,
|
|
"Wiki file has no front-matter block (add ---\\nstatus: generated\\nsource-paths: []\\n---)",
|
|
))
|
|
else:
|
|
if fm.status not in ("generated", "authored", "stub"):
|
|
findings.append(Finding(
|
|
"wiki.frontmatter.status.invalid", "error",
|
|
wrel,
|
|
f"Invalid status value '{fm.status}' (must be generated/authored/stub)",
|
|
))
|
|
for sp in fm.source_paths:
|
|
sp_path = root / sp
|
|
if not sp_path.exists():
|
|
findings.append(Finding(
|
|
"wiki.source-path.missing", "warning",
|
|
wrel,
|
|
f"source-paths entry '{sp}' does not exist",
|
|
))
|
|
|
|
# Check links in wiki files
|
|
link_findings = check_markdown_links(wiki_file, root)
|
|
for lf in link_findings:
|
|
findings.append(lf)
|
|
|
|
# --- nav.stale: check nav tables ---
|
|
dirs = top_level_dirs(root)
|
|
dir_names = {d.name for d in dirs}
|
|
dir_rels = {posix_rel(d, root) for d in dirs}
|
|
|
|
# Check wiki/NAV.md for stale entries
|
|
for nav_path_rel in [WIKI_NAV_REL, WIKI_CODEBASE_NAV_REL]:
|
|
nav_path = root / nav_path_rel
|
|
if not nav_path.exists():
|
|
continue
|
|
fm_nav, nav_body = parse_frontmatter(
|
|
nav_path.read_text(encoding="utf-8", errors="replace")
|
|
)
|
|
# Extract dir entries from nav table
|
|
for line in nav_body.splitlines():
|
|
m = re.match(r"\|\s*`([^`/]+)/`", line)
|
|
if m:
|
|
listed_name = m.group(1)
|
|
listed_path = root / listed_name
|
|
if not listed_path.exists():
|
|
findings.append(Finding(
|
|
"nav.stale", "warning",
|
|
nav_path_rel,
|
|
f"Nav table lists '{listed_name}/' but directory does not exist",
|
|
))
|
|
|
|
# Check for dirs present but not in nav (only top-level that have control
|
|
# files). Applies only to status:generated NAVs — authored NAVs are
|
|
# selective by design (curated pointers, not complete inventories).
|
|
for nav_path_rel in [WIKI_NAV_REL, WIKI_CODEBASE_NAV_REL]:
|
|
nav_path = root / nav_path_rel
|
|
if not nav_path.exists() or not file_is_generated(nav_path):
|
|
continue
|
|
fm_nav, nav_body = parse_frontmatter(
|
|
nav_path.read_text(encoding="utf-8", errors="replace")
|
|
)
|
|
listed_names: set[str] = set()
|
|
for line in nav_body.splitlines():
|
|
m = re.match(r"\|\s*`([^`/]+)/`", line)
|
|
if m:
|
|
listed_names.add(m.group(1))
|
|
for d in dirs:
|
|
if d.name not in listed_names and find_control_file(d):
|
|
findings.append(Finding(
|
|
"nav.stale", "warning",
|
|
nav_path_rel,
|
|
f"Nav table omits '{d.name}/' which has a control file",
|
|
))
|
|
|
|
# Sort: errors first, then by path
|
|
findings.sort(key=lambda f: (0 if f.severity == "error" else 1, f.path))
|
|
|
|
error_count = sum(1 for f in findings if f.severity == "error")
|
|
warning_count = sum(1 for f in findings if f.severity == "warning")
|
|
|
|
if as_json:
|
|
output = {
|
|
"schema_version": 2,
|
|
"findings": [
|
|
{
|
|
"id": f.id,
|
|
"severity": f.severity,
|
|
"path": f.path,
|
|
"message": f.message,
|
|
"remediation": _remediation(f.id),
|
|
}
|
|
for f in findings
|
|
],
|
|
"summary": {
|
|
"errors": error_count,
|
|
"warnings": warning_count,
|
|
},
|
|
}
|
|
print(json.dumps(output, indent=2))
|
|
else:
|
|
if error_count > 0:
|
|
print("Errors:")
|
|
for f in findings:
|
|
if f.severity == "error":
|
|
print(f" {f.path} [{f.id}]")
|
|
print(f" {f.message}")
|
|
rem = _remediation(f.id)
|
|
if rem:
|
|
print(f" -> {rem}")
|
|
print()
|
|
|
|
if warning_count > 0:
|
|
print("Warnings:")
|
|
for f in findings:
|
|
if f.severity == "warning":
|
|
print(f" {f.path} [{f.id}]")
|
|
print(f" {f.message}")
|
|
print()
|
|
|
|
if error_count == 0 and warning_count == 0:
|
|
print("No findings. Knowledge layer is healthy.")
|
|
print()
|
|
|
|
print(f"Summary: {error_count} errors, {warning_count} warnings")
|
|
|
|
return EXIT_CHECK_ERRORS if error_count > 0 else EXIT_OK
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# impact subcommand
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ImpactHit(NamedTuple):
|
|
wiki_doc: str # repo-relative POSIX path to the wiki doc
|
|
changed_file: str # the changed path that triggered the match
|
|
hit_type: str # direct, directory, control, potential
|
|
|
|
|
|
def cmd_impact(root: Path, changed_paths: list[str]) -> int:
|
|
"""Analyze which wiki documents are affected by a set of changed source paths."""
|
|
wiki_root = root / WIKI_SUBDIR
|
|
|
|
# Normalize input paths: strip whitespace, make repo-relative POSIX, skip empty
|
|
normalized: list[str] = []
|
|
for p in changed_paths:
|
|
p = p.strip()
|
|
if not p:
|
|
continue
|
|
# Normalize to forward slashes, strip leading ./ or /
|
|
p = p.replace("\\", "/")
|
|
if p.startswith("./"):
|
|
p = p[2:]
|
|
if p.startswith("/"):
|
|
p = p[1:]
|
|
normalized.append(p)
|
|
|
|
if not normalized:
|
|
print("No changed paths provided.")
|
|
print()
|
|
print("Usage:")
|
|
print(" knowledge.py impact <path1> [path2 ...]")
|
|
print(" git diff --name-only | knowledge.py impact")
|
|
return EXIT_OK
|
|
|
|
# Collect all wiki docs with their source-paths
|
|
wiki_docs: list[tuple[str, list[str]]] = [] # (wiki_rel_path, source_paths)
|
|
if wiki_root.exists():
|
|
for md_file in sorted(wiki_root.rglob("*.md")):
|
|
if md_file.name.startswith("."):
|
|
continue
|
|
fm, _ = read_wiki_file(md_file)
|
|
if fm.source_paths:
|
|
wiki_docs.append((posix_rel(md_file, root), fm.source_paths))
|
|
|
|
hits: list[ImpactHit] = []
|
|
seen: set[tuple[str, str, str]] = set() # (wiki_doc, changed_file, hit_type)
|
|
|
|
def add_hit(wiki_doc: str, changed_file: str, hit_type: str):
|
|
key = (wiki_doc, changed_file, hit_type)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
hits.append(ImpactHit(wiki_doc, changed_file, hit_type))
|
|
|
|
for changed in normalized:
|
|
changed_normalized = changed.rstrip("/")
|
|
|
|
# 1. Direct hits: wiki doc source-paths lists the exact changed file
|
|
for wiki_path, source_paths in wiki_docs:
|
|
for sp in source_paths:
|
|
sp_normalized = sp.rstrip("/")
|
|
if sp_normalized == changed_normalized:
|
|
add_hit(wiki_path, changed, "direct")
|
|
|
|
# 2. Directory hits: wiki doc source-paths lists a parent directory
|
|
for wiki_path, source_paths in wiki_docs:
|
|
for sp in source_paths:
|
|
sp_normalized = sp.rstrip("/")
|
|
# sp is a parent directory of the changed file
|
|
if changed_normalized.startswith(sp_normalized + "/"):
|
|
add_hit(wiki_path, changed, "directory")
|
|
|
|
# 3. Control file hits: CLAUDE.md in directories containing changed files
|
|
# Walk ancestor directories of the changed file
|
|
changed_path = PurePosixPath(changed_normalized)
|
|
parents = list(changed_path.parents)
|
|
# Include the file's directory (parent of the file)
|
|
for parent in parents:
|
|
parent_str = str(parent)
|
|
if parent_str == ".":
|
|
parent_dir = root
|
|
else:
|
|
parent_dir = root / parent_str
|
|
cf = find_control_file(parent_dir)
|
|
if cf:
|
|
cf_rel = posix_rel(cf, root)
|
|
add_hit(cf_rel, changed, "control")
|
|
|
|
# 4. Potential stale: wiki docs whose source-paths reference files in
|
|
# the same module (same top-level directory subtree) as changed files
|
|
# Determine the module prefix of the changed file (first two path segments, or first one)
|
|
parts = changed_normalized.split("/")
|
|
if len(parts) >= 2:
|
|
module_prefix = parts[0] + "/" + parts[1]
|
|
elif len(parts) == 1:
|
|
module_prefix = parts[0]
|
|
else:
|
|
continue
|
|
|
|
for wiki_path, source_paths in wiki_docs:
|
|
for sp in source_paths:
|
|
sp_normalized = sp.rstrip("/")
|
|
# Same module but not already a direct or directory hit
|
|
if (sp_normalized.startswith(module_prefix + "/") or sp_normalized == module_prefix):
|
|
# Only add if not already matched as direct or directory
|
|
if (wiki_path, changed, "direct") not in seen and \
|
|
(wiki_path, changed, "directory") not in seen:
|
|
add_hit(wiki_path, changed, "potential")
|
|
break # one match per wiki doc per changed file is enough
|
|
|
|
# Output report
|
|
if not hits:
|
|
print("No impact detected.")
|
|
print()
|
|
print(f"Checked {len(normalized)} changed path(s) against {len(wiki_docs)} wiki document(s).")
|
|
return EXIT_OK
|
|
|
|
# Group by hit type
|
|
direct_hits = [h for h in hits if h.hit_type == "direct"]
|
|
directory_hits = [h for h in hits if h.hit_type == "directory"]
|
|
control_hits = [h for h in hits if h.hit_type == "control"]
|
|
potential_hits = [h for h in hits if h.hit_type == "potential"]
|
|
|
|
print("=" * 60)
|
|
print("tome impact report")
|
|
print("=" * 60)
|
|
print()
|
|
print(f"Changed paths: {len(normalized)}")
|
|
print(f"Wiki documents checked: {len(wiki_docs)}")
|
|
print()
|
|
|
|
if direct_hits:
|
|
print("Direct hits (wiki doc source-paths lists the exact changed file):")
|
|
for h in direct_hits:
|
|
print(f" {h.wiki_doc}")
|
|
print(f" <- {h.changed_file}")
|
|
print()
|
|
|
|
if directory_hits:
|
|
print("Directory hits (wiki doc source-paths lists a parent directory):")
|
|
for h in directory_hits:
|
|
print(f" {h.wiki_doc}")
|
|
print(f" <- {h.changed_file}")
|
|
print()
|
|
|
|
if control_hits:
|
|
print("Control file hits (CLAUDE.md in ancestor directories of changed files):")
|
|
for h in control_hits:
|
|
print(f" {h.wiki_doc}")
|
|
print(f" <- {h.changed_file}")
|
|
print()
|
|
|
|
if potential_hits:
|
|
print("Potential stale (wiki docs referencing same module as changed files):")
|
|
for h in potential_hits:
|
|
print(f" {h.wiki_doc}")
|
|
print(f" <- {h.changed_file}")
|
|
print()
|
|
|
|
# Summary
|
|
total = len(direct_hits) + len(directory_hits) + len(control_hits) + len(potential_hits)
|
|
print(f"Total: {total} impact(s) — "
|
|
f"{len(direct_hits)} direct, {len(directory_hits)} directory, "
|
|
f"{len(control_hits)} control, {len(potential_hits)} potential")
|
|
|
|
return EXIT_OK
|
|
|
|
|
|
def _remediation(finding_id: str) -> str:
|
|
return {
|
|
"control.h1.missing": "Add a '# Title' heading as the first H1 in the control file",
|
|
"control.h1.multiple": "Keep exactly one H1 heading in the control file",
|
|
"control.section.order": "Reorder recognized sections: Load -> Warnings -> Handoff",
|
|
"control.load-link.broken": "Fix the link target path in the ## Load section",
|
|
"control.coexist": "Remove one of CLAUDE.md or AGENTS.md from this directory",
|
|
"wiki.frontmatter.missing": "Add YAML front-matter: ---\\nstatus: generated\\nsource-paths: []\\n---",
|
|
"wiki.frontmatter.status.invalid": "Set status to one of: generated, authored, stub",
|
|
"wiki.source-path.missing": "Update source-paths to list existing repo paths",
|
|
"link.broken": "Fix or remove the broken markdown link",
|
|
"nav.resource.missing": "Run 'knowledge.py bootstrap' to create missing resources",
|
|
"nav.wiki.missing": "Run 'knowledge.py bootstrap' to create missing wiki files",
|
|
"nav.stale": "Run 'knowledge.py refresh' to regenerate nav tables",
|
|
}.get(finding_id, "")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main():
|
|
# Emit UTF-8 regardless of the host console code page (Windows consoles
|
|
# default to cp1252/cp437 and would mangle em-dashes in reports).
|
|
for stream in (sys.stdout, sys.stderr):
|
|
try:
|
|
stream.reconfigure(encoding="utf-8")
|
|
except (AttributeError, ValueError):
|
|
pass
|
|
|
|
parser = argparse.ArgumentParser(
|
|
prog="knowledge.py",
|
|
description="tome — deterministic knowledge-layer tool",
|
|
)
|
|
subparsers = parser.add_subparsers(dest="subcommand", required=True)
|
|
|
|
# bootstrap
|
|
subparsers.add_parser("bootstrap", help="Scaffold wiki/ and navigate skill supporting files")
|
|
|
|
# refresh
|
|
subparsers.add_parser("refresh", help="Refresh status:generated files; report stale knowledge")
|
|
|
|
# context
|
|
ctx_parser = subparsers.add_parser("context", help="Resolve navigation context for a path")
|
|
ctx_parser.add_argument("path", help="Target path (repo-relative or absolute)")
|
|
ctx_parser.add_argument("--json", action="store_true", help="Output as JSON")
|
|
|
|
# check
|
|
chk_parser = subparsers.add_parser("check", help="Validate knowledge-layer health")
|
|
chk_parser.add_argument("--json", action="store_true", help="Output as JSON")
|
|
|
|
# impact
|
|
imp_parser = subparsers.add_parser("impact", help="Analyze wiki impact of changed source paths")
|
|
imp_parser.add_argument("paths", nargs="*", help="Changed file paths (or pipe via stdin)")
|
|
|
|
args = parser.parse_args()
|
|
|
|
root = find_repo_root()
|
|
|
|
if args.subcommand == "bootstrap":
|
|
sys.exit(cmd_bootstrap(root))
|
|
elif args.subcommand == "refresh":
|
|
sys.exit(cmd_refresh(root))
|
|
elif args.subcommand == "context":
|
|
sys.exit(cmd_context(root, args.path, getattr(args, "json", False)))
|
|
elif args.subcommand == "check":
|
|
sys.exit(cmd_check(root, getattr(args, "json", False)))
|
|
elif args.subcommand == "impact":
|
|
# Accept paths from positional args or stdin
|
|
paths = args.paths if args.paths else []
|
|
if not paths and not sys.stdin.isatty():
|
|
paths = [line for line in sys.stdin.read().splitlines() if line.strip()]
|
|
sys.exit(cmd_impact(root, paths))
|
|
else:
|
|
parser.print_help()
|
|
sys.exit(EXIT_USAGE)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|