✨ Gremlins MVP: store, launch, stream, call modules and templates
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
"""The only subprocess code: run one `claude -p` call in the foreground and tee its stream."""
|
||||
import json
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
POLL_S = 0.2
|
||||
READER_JOIN_S = 10.0 # an orphaned grandchild can hold stdout open after claude exits
|
||||
STOP_SIGNALS = tuple(s for s in (signal.SIGTERM, signal.SIGINT, getattr(signal, "SIGBREAK", None))
|
||||
if s is not None)
|
||||
|
||||
|
||||
def _feed(stdin, data: bytes) -> None:
|
||||
"""Write the prompt and close stdin; a child that exits early is not an error here."""
|
||||
try:
|
||||
stdin.write(data)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
try:
|
||||
stdin.close()
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _read(stdout, raw, ss) -> None:
|
||||
"""Tee every stream line to the raw file and feed each JSON event to `ss`."""
|
||||
try:
|
||||
for line in iter(stdout.readline, b""):
|
||||
raw.write(line)
|
||||
raw.flush()
|
||||
try:
|
||||
ev = json.loads(line.decode("utf-8", "replace"))
|
||||
except ValueError:
|
||||
continue
|
||||
ss.feed(ev, time.time())
|
||||
except (OSError, ValueError):
|
||||
pass # the raw file or pipe was closed after READER_JOIN_S
|
||||
|
||||
|
||||
def run_claude(argv, prompt: str, env, cwd, raw_path, on_start, ss=None):
|
||||
"""Run one call to its end. Returns (ss, exit_code, stopped)."""
|
||||
if ss is None:
|
||||
from grm.stream import StreamState
|
||||
ss = StreamState()
|
||||
raw_path = Path(raw_path)
|
||||
with open(raw_path, "wb") as raw, open(raw_path.with_suffix(".err"), "wb") as err:
|
||||
child = subprocess.Popen(argv, cwd=cwd, env=env, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=err)
|
||||
stopped = False
|
||||
|
||||
def _stop(signum, frame):
|
||||
nonlocal stopped
|
||||
try:
|
||||
if stopped: # a second signal: stop waiting for a graceful exit
|
||||
child.kill()
|
||||
else:
|
||||
child.terminate()
|
||||
except OSError:
|
||||
pass
|
||||
stopped = True
|
||||
|
||||
old = {}
|
||||
if threading.current_thread() is threading.main_thread():
|
||||
old = {s: signal.signal(s, _stop) for s in STOP_SIGNALS}
|
||||
try:
|
||||
reader = threading.Thread(target=_read, args=(child.stdout, raw, ss), daemon=True)
|
||||
reader.start()
|
||||
threading.Thread(target=_feed, args=(child.stdin, prompt.encode("utf-8")),
|
||||
daemon=True).start()
|
||||
on_start(child.pid)
|
||||
while child.poll() is None: # a bare wait() would block signal handlers on Windows
|
||||
time.sleep(POLL_S)
|
||||
except BaseException:
|
||||
child.kill()
|
||||
child.wait()
|
||||
raise
|
||||
finally:
|
||||
for s, h in old.items():
|
||||
if h is not None:
|
||||
signal.signal(s, h)
|
||||
reader.join(READER_JOIN_S)
|
||||
if not reader.is_alive():
|
||||
child.stdout.close()
|
||||
return ss, child.returncode, stopped
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Launch fingerprint, env scrub, binary lookup and the `claude -p` argv (B1, B2, B7, B8)."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
MODEL_RE = re.compile(r"^claude-[a-z0-9-]+(\[1m\])?$")
|
||||
EFFORTS = ("low", "medium", "high", "xhigh", "max")
|
||||
PERMISSION_MODE = "dontAsk"
|
||||
DEFAULT_TOOLS = ["Bash", "Read", "Edit", "Write", "Glob", "Grep", "Skill"]
|
||||
DEFAULT_DISALLOWED = ["Bash(git push:*)", "Bash(git rebase:*)", "Bash(git reset --hard:*)",
|
||||
"Bash(claude:*)"]
|
||||
|
||||
# Env pinned into every launch; part of the fingerprint.
|
||||
ENV_PINNED_LAUNCH = {"CLAUDE_CODE_PROMPT_CACHE_TTL": "1h", "DISABLE_AUTOUPDATER": "1"}
|
||||
|
||||
# Host env that must not reach a child `claude -p`.
|
||||
SCRUB_DROP = frozenset({
|
||||
# Desktop host vars
|
||||
"CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_CODE_HOST_SESSION_ID",
|
||||
"CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_DESKTOP_APP_VERSION",
|
||||
"CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH", "CLAUDE_CODE_TERMINAL_MCP_TOOLS",
|
||||
"CLAUDE_CODE_REPORT_FINDINGS", "CLAUDE_CODE_SESSION_ID", "CLAUDE_AGENT_SDK_VERSION",
|
||||
"CLAUDE_CODE_ENABLE_ASK_USER_QUESTION_TOOL", "CLAUDE_CODE_ENABLE_SDK_FILE_CHECKPOINTING",
|
||||
# fingerprint and auth path
|
||||
"CLAUDE_EFFORT", "ANTHROPIC_MODEL", "ANTHROPIC_SMALL_FAST_MODEL", "CLAUDE_CODE_SUBAGENT_MODEL",
|
||||
"ENABLE_PROMPT_CACHING_1H", "FORCE_PROMPT_CACHING_5M", "CLAUDE_CODE_SUBAGENT_PROMPT_CACHE_TTL",
|
||||
"ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX",
|
||||
# uv
|
||||
"VIRTUAL_ENV",
|
||||
})
|
||||
SCRUB_DROP_PATTERNS = (re.compile(r"^ANTHROPIC_DEFAULT_.+_MODEL$"), re.compile(r"^UV_"))
|
||||
SCRUB_KEEP = frozenset({
|
||||
"CLAUDECODE", "CLAUDE_CODE_CHILD_SESSION", "AI_AGENT", "CLAUDE_PID", "CLAUDE_CODE_GIT_BASH_PATH",
|
||||
"PATH", "HOME", "USERPROFILE",
|
||||
"ANTHROPIC_BASE_URL", # kept: gateways need it
|
||||
})
|
||||
SCRUB_PINNED = {**ENV_PINNED_LAUNCH, "SPIRE_GREMLIN_CHILD": "1"}
|
||||
SCRUB_PINNED_BINARY = "CLAUDE_CODE_EXECPATH" # set to the stored absolute binary path
|
||||
|
||||
|
||||
def canonical_json(obj) -> str:
|
||||
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
|
||||
|
||||
def _sha256_bytes(b: bytes) -> str:
|
||||
return hashlib.sha256(b).hexdigest()
|
||||
|
||||
|
||||
def _sha256_file(path: str) -> str:
|
||||
with open(path, "rb") as f:
|
||||
return _sha256_bytes(f.read())
|
||||
|
||||
|
||||
def scrub_class(name: str) -> str | None:
|
||||
"""'drop', 'pinned', 'keep' or None (on no list; kept)."""
|
||||
n = name.upper()
|
||||
if n in SCRUB_DROP or any(p.match(n) for p in SCRUB_DROP_PATTERNS):
|
||||
return "drop"
|
||||
if n in SCRUB_PINNED or n == SCRUB_PINNED_BINARY:
|
||||
return "pinned"
|
||||
if n in SCRUB_KEEP:
|
||||
return "keep"
|
||||
return None
|
||||
|
||||
|
||||
def _venv_prefixes(environ) -> list[str]:
|
||||
out = []
|
||||
venv = environ.get("VIRTUAL_ENV")
|
||||
if venv:
|
||||
out.append(venv)
|
||||
if os.path.normcase(os.path.abspath(sys.prefix)) != os.path.normcase(os.path.abspath(sys.base_prefix)):
|
||||
out.append(sys.prefix)
|
||||
return [os.path.normcase(os.path.normpath(os.path.abspath(x))) for x in out]
|
||||
|
||||
|
||||
def _under(path: str, prefixes) -> bool:
|
||||
try:
|
||||
p = os.path.normcase(os.path.normpath(os.path.abspath(path)))
|
||||
except ValueError:
|
||||
return False
|
||||
return any(p == px or p.startswith(px.rstrip("\\/") + os.sep) for px in prefixes)
|
||||
|
||||
|
||||
def scrubbed_env(binary: str, environ=None) -> dict:
|
||||
"""The child env: host env minus the drop list, venv PATH entries removed, pinned vars on
|
||||
top, `CLAUDE_CODE_EXECPATH` = the stored binary path."""
|
||||
if environ is None:
|
||||
environ = os.environ
|
||||
prefixes = _venv_prefixes(environ)
|
||||
env = {}
|
||||
for k, v in environ.items():
|
||||
if scrub_class(k) in ("drop", "pinned"):
|
||||
continue
|
||||
if k.upper() == "PATH":
|
||||
v = os.pathsep.join(e for e in v.split(os.pathsep) if e and not _under(e, prefixes))
|
||||
env[k] = v
|
||||
env.update(SCRUB_PINNED)
|
||||
env[SCRUB_PINNED_BINARY] = binary
|
||||
return env
|
||||
|
||||
|
||||
def resolve_binary(explicit: str | None = None) -> str | None:
|
||||
"""Absolute path of `claude`: `explicit`, else $CLAUDE_CODE_EXECPATH, else PATH."""
|
||||
path = explicit or os.environ.get("CLAUDE_CODE_EXECPATH") or shutil.which("claude")
|
||||
return os.path.abspath(path) if path else None
|
||||
|
||||
|
||||
def binary_version(path: str) -> str | None:
|
||||
"""First token of `<path> --version`, or None on any failure."""
|
||||
try:
|
||||
r = subprocess.run([path, "--version"], stdin=subprocess.DEVNULL, capture_output=True,
|
||||
timeout=60)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
toks = r.stdout.decode("utf-8", "replace").split()
|
||||
return toks[0] if r.returncode == 0 and toks else None
|
||||
|
||||
|
||||
def _argv(fp: dict) -> list[str]:
|
||||
"""The fingerprinted command line; its order must never change (cache hit)."""
|
||||
a = [fp["binary"], "-p", "--output-format", "stream-json", "--verbose",
|
||||
"--model", fp["model"], "--effort", fp["effort"],
|
||||
"--permission-mode", fp["permission_mode"], "--tools", ",".join(fp["tools"])]
|
||||
if fp.get("allowed_tools"):
|
||||
a += ["--allowedTools", ",".join(fp["allowed_tools"])]
|
||||
if fp.get("disallowed_tools"):
|
||||
a += ["--disallowedTools", ",".join(fp["disallowed_tools"])]
|
||||
a += ["--strict-mcp-config", "--mcp-config", fp["mcp_config"],
|
||||
"--append-system-prompt-file", fp["append_prompt"]["path"]]
|
||||
if fp.get("agent"):
|
||||
a += ["--agent", fp["agent"]]
|
||||
return a
|
||||
|
||||
|
||||
def build_fingerprint(*, binary, model, effort, cwd, agent, preamble_path, mcp_config) -> dict:
|
||||
"""Validate the launch fields; return {"fingerprint", "argv", "fp_hash"}. Writes nothing."""
|
||||
if not MODEL_RE.match(model or ""):
|
||||
raise ValueError("model must be a full id (aliases are refused): %r" % model)
|
||||
if effort not in EFFORTS:
|
||||
raise ValueError("unknown effort %r (one of %s)" % (effort, ", ".join(EFFORTS)))
|
||||
if not os.path.isdir(cwd):
|
||||
raise ValueError("cwd is not a directory: %s" % cwd)
|
||||
if not os.path.isfile(preamble_path):
|
||||
raise ValueError("preamble not found: %s" % preamble_path)
|
||||
if not binary or not os.path.isfile(binary):
|
||||
raise ValueError("claude binary not found: %s" % binary)
|
||||
preamble_path = os.path.abspath(preamble_path)
|
||||
fp = {
|
||||
"binary": os.path.abspath(binary),
|
||||
"binary_version": binary_version(binary),
|
||||
"model": model,
|
||||
"effort": effort,
|
||||
"permission_mode": PERMISSION_MODE,
|
||||
"tools": list(DEFAULT_TOOLS),
|
||||
# under dontAsk every tool not allowed is denied
|
||||
"allowed_tools": list(DEFAULT_TOOLS),
|
||||
"disallowed_tools": list(DEFAULT_DISALLOWED),
|
||||
"agent": agent or None,
|
||||
"cwd": os.path.abspath(cwd),
|
||||
"append_prompt": {"path": preamble_path, "sha256": _sha256_file(preamble_path)},
|
||||
"mcp_config": os.path.abspath(mcp_config),
|
||||
"env_pinned": dict(ENV_PINNED_LAUNCH),
|
||||
}
|
||||
return {"fingerprint": fp, "argv": _argv(fp),
|
||||
"fp_hash": _sha256_bytes(canonical_json(fp).encode("utf-8"))}
|
||||
|
||||
|
||||
def train_argv(run: dict, sid: str) -> list[str]:
|
||||
return list(run["argv"]) + ["--session-id", sid]
|
||||
|
||||
|
||||
def fork_argv(run: dict, base_sid: str, sid: str) -> list[str]:
|
||||
return list(run["argv"]) + ["--resume", base_sid, "--fork-session", "--session-id", sid]
|
||||
|
||||
|
||||
def resume_argv(run: dict, base_sid: str) -> list[str]:
|
||||
"""No `--session-id`: claude refuses it on a resume without a fork."""
|
||||
return list(run["argv"]) + ["--resume", base_sid]
|
||||
@@ -0,0 +1,116 @@
|
||||
"""State dir layout, atomic JSON files and the append-only ledger."""
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
SUBDIRS = ("bases", "jobs", "raw", "notes", "logs", "briefs")
|
||||
RETRY_TRIES = 40
|
||||
RETRY_SLEEP_S = 0.05
|
||||
|
||||
_ledger_lock = threading.Lock()
|
||||
|
||||
|
||||
def state_dir(run) -> Path:
|
||||
"""`<x>/run.md` -> `<x>/run-gremlins` (a sibling of the run log)."""
|
||||
p = os.path.abspath(str(run))
|
||||
if not p.lower().endswith(".md"):
|
||||
raise ValueError("--run must name the run log .md file: %s" % p)
|
||||
return Path(p[:-3] + "-gremlins")
|
||||
|
||||
|
||||
def init_state(run) -> Path:
|
||||
"""Create the state dir and its subdirs (idempotent)."""
|
||||
state = state_dir(run)
|
||||
for d in SUBDIRS:
|
||||
(state / d).mkdir(parents=True, exist_ok=True)
|
||||
return state
|
||||
|
||||
|
||||
def _retry(fn, tries: int = RETRY_TRIES, sleep_s: float = RETRY_SLEEP_S):
|
||||
"""Call fn(); retry PermissionError (Windows sharing violations)."""
|
||||
for attempt in range(1, tries + 1):
|
||||
try:
|
||||
return fn()
|
||||
except PermissionError:
|
||||
if attempt == tries:
|
||||
raise
|
||||
time.sleep(sleep_s)
|
||||
|
||||
|
||||
def atomic_write_bytes(path, data: bytes) -> None:
|
||||
"""Write to a temp file in the same dir, then `os.replace` with retries."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.parent / (".%s.%d.%s.tmp" % (path.name, os.getpid(), uuid.uuid4().hex[:8]))
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(data)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
try:
|
||||
_retry(lambda: os.replace(tmp, path))
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(tmp)
|
||||
raise
|
||||
|
||||
|
||||
def read_bytes_retry(path) -> bytes:
|
||||
def _read():
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
return _retry(_read)
|
||||
|
||||
|
||||
def read_json(p) -> dict | None:
|
||||
"""The parsed file, or None if it does not exist. Corrupt content raises ValueError."""
|
||||
try:
|
||||
data = read_bytes_retry(p)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
|
||||
def write_json(p, obj) -> None:
|
||||
atomic_write_bytes(p, (json.dumps(obj, indent=1, sort_keys=True) + "\n").encode("utf-8"))
|
||||
|
||||
|
||||
def _ledger_path(run) -> Path:
|
||||
return state_dir(run) / "ledger.jsonl"
|
||||
|
||||
|
||||
def ledger_append(run, line: dict) -> None:
|
||||
"""Append one JSON line with a single write; repairs a torn last line first."""
|
||||
path = _ledger_path(run)
|
||||
body = (json.dumps(line, sort_keys=True) + "\n").encode("utf-8")
|
||||
|
||||
def _append():
|
||||
torn = False
|
||||
with contextlib.suppress(FileNotFoundError), open(path, "rb") as r:
|
||||
if r.seek(0, os.SEEK_END) > 0:
|
||||
r.seek(-1, os.SEEK_END)
|
||||
torn = r.read(1) != b"\n"
|
||||
with open(path, "ab") as f:
|
||||
f.write((b"\n" if torn else b"") + body)
|
||||
with _ledger_lock:
|
||||
_retry(_append)
|
||||
|
||||
|
||||
def ledger_lines(run) -> list[dict]:
|
||||
"""Every parsable line of the ledger; [] when there is none yet."""
|
||||
try:
|
||||
data = read_bytes_retry(_ledger_path(run))
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
out = []
|
||||
for ln in data.splitlines():
|
||||
try:
|
||||
obj = json.loads(ln.decode("utf-8", "replace"))
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
out.append(obj)
|
||||
return out
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Stream-json reader and accounting for one `claude -p` call: pure functions, no IO."""
|
||||
import datetime as _dt
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
||||
TTL_1H, TTL_5M = 3600, 300
|
||||
CACHE_HIT_RATIO = 0.9
|
||||
MU_FIELDS = ("inputTokens", "outputTokens", "cacheReadInputTokens", "cacheCreationInputTokens")
|
||||
TOKEN_KEYS = ("input", "output", "cache_creation", "cache_read", "cache_creation_1h", "cache_creation_5m")
|
||||
|
||||
|
||||
def iso(t: float | None = None) -> str:
|
||||
"""ISO 8601 with the local offset, from epoch seconds (now if not given)."""
|
||||
if t is None:
|
||||
t = time.time()
|
||||
return _dt.datetime.fromtimestamp(t).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def parse_iso(s) -> float | None:
|
||||
"""Epoch seconds of an ISO string written by `iso`, or None."""
|
||||
if not s or not isinstance(s, str):
|
||||
return None
|
||||
try:
|
||||
return _dt.datetime.fromisoformat(s).timestamp()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class StreamState:
|
||||
"""What one `claude -p --output-format stream-json --verbose` stream told us so far."""
|
||||
|
||||
def __init__(self):
|
||||
self.init = None # the system/init event
|
||||
self.session_id = None
|
||||
self.msg_ids: list = []
|
||||
self._ids: set = set()
|
||||
self.first_usage = None # usage of the first assistant event (cache verdict)
|
||||
self.first_at = None # its arrival time (the base prefix read)
|
||||
self.last_usage = None # usage of the last assistant event (base context)
|
||||
self.last_assistant_at = None
|
||||
self.result = None # the result event
|
||||
self.refused = None # text of a model-restricted warning
|
||||
self.events = 0
|
||||
|
||||
def feed(self, ev: dict, t: float | None) -> None:
|
||||
if not isinstance(ev, dict):
|
||||
return
|
||||
self.events += 1
|
||||
ty = ev.get("type")
|
||||
if ty == "system":
|
||||
st = ev.get("subtype")
|
||||
if st == "init" and self.init is None:
|
||||
self.init = ev
|
||||
self.session_id = ev.get("session_id") or self.session_id
|
||||
elif (st == "informational" and ev.get("level") == "warning"
|
||||
and "is restricted" in str(ev.get("content", ""))):
|
||||
self.refused = str(ev.get("content"))
|
||||
elif ty == "assistant":
|
||||
m = ev.get("message") or {}
|
||||
mid = m.get("id")
|
||||
if mid not in self._ids:
|
||||
self._ids.add(mid)
|
||||
self.msg_ids.append(mid)
|
||||
u = m.get("usage") or {}
|
||||
if self.first_usage is None:
|
||||
self.first_usage, self.first_at = u, t
|
||||
self.last_usage, self.last_assistant_at = u, t
|
||||
elif ty == "result":
|
||||
self.result = ev
|
||||
self.session_id = ev.get("session_id") or self.session_id
|
||||
|
||||
@property
|
||||
def turns(self) -> int:
|
||||
return len(self.msg_ids)
|
||||
|
||||
|
||||
def context_tokens(usage: dict | None) -> int:
|
||||
"""Prompt-side tokens of one request: input + cache creation + cache read."""
|
||||
if not usage:
|
||||
return 0
|
||||
return (int(usage.get("input_tokens") or 0) + int(usage.get("cache_creation_input_tokens") or 0)
|
||||
+ int(usage.get("cache_read_input_tokens") or 0))
|
||||
|
||||
|
||||
_API_ERROR_TEXT = ( # checked in this order, after the status code
|
||||
("prompt_too_long", re.compile(r"prompt is too long|prompt too long|prompt_too_long", re.I)),
|
||||
("usage_limit", re.compile(r"usage limit|rate[ _-]?limit|hit your limit|api error:?\s*429", re.I)),
|
||||
("overloaded", re.compile(r"overloaded|api error:?\s*529", re.I)),
|
||||
("api_error", re.compile(r"api error", re.I)),
|
||||
)
|
||||
|
||||
|
||||
def api_error_reason(res: dict | None) -> tuple[str, str] | None:
|
||||
"""(reason, detail) for a result that carries an API error, else None. `reason` is
|
||||
`usage_limit` (429), `overloaded` (529), `prompt_too_long` or `api_error`."""
|
||||
if not res:
|
||||
return None
|
||||
status = res.get("api_error_status")
|
||||
try:
|
||||
status = int(status) if status not in (None, "") else None
|
||||
except (TypeError, ValueError):
|
||||
status = None
|
||||
if not res.get("is_error") and status is None:
|
||||
return None
|
||||
parts = [res.get("result"), res.get("error"), res.get("errors")]
|
||||
text = " | ".join(p if isinstance(p, str) else json.dumps(p) for p in parts if p)
|
||||
detail = "api_error_status %s: %s" % (status, text[:200] if text else "(no text)")
|
||||
if status == 429:
|
||||
return "usage_limit", detail
|
||||
if status == 529:
|
||||
return "overloaded", detail
|
||||
for reason, rx in _API_ERROR_TEXT:
|
||||
if rx.search(text):
|
||||
return reason, detail
|
||||
if status is not None:
|
||||
return "api_error", detail
|
||||
return None
|
||||
|
||||
|
||||
def classify(ss: StreamState, exit_code, stopped: bool) -> tuple[str, str | None]:
|
||||
"""(status, reason) of a finished call; the first matching case wins."""
|
||||
res = ss.result
|
||||
match (bool(stopped), ss.refused, res, api_error_reason(res), exit_code):
|
||||
case (True, _, _, _, _):
|
||||
return "stopped", "stopped"
|
||||
case (_, str(), _, _, _):
|
||||
return "error", "model_refused"
|
||||
case (_, _, None, _, code):
|
||||
return "error", "exit_code" if code else "no_result"
|
||||
case (_, _, _, (reason, _), _):
|
||||
return "error", reason
|
||||
case (_, _, dict() as r, _, _) if r.get("is_error"):
|
||||
return "error", "is_error"
|
||||
case (_, _, _, _, code) if code:
|
||||
return "error", "exit_code"
|
||||
case _:
|
||||
return "ok", None
|
||||
|
||||
|
||||
def usage_from_result(result: dict, before_mu: dict, cost_before) -> dict:
|
||||
"""Tokens (with the 1h/5m split) from `result.usage`; `models_used` = modelUsage entries that
|
||||
grew against the session's totals before the call, largest first; cost delta against the
|
||||
session's cost before the call. Costs are rounded here, once, at 6 decimals."""
|
||||
result = result or {}
|
||||
usage = result.get("usage") or {}
|
||||
cc = usage.get("cache_creation") or {}
|
||||
tokens = {
|
||||
"input": int(usage.get("input_tokens") or 0),
|
||||
"output": int(usage.get("output_tokens") or 0),
|
||||
"cache_creation": int(usage.get("cache_creation_input_tokens") or 0),
|
||||
"cache_read": int(usage.get("cache_read_input_tokens") or 0),
|
||||
"cache_creation_1h": int(cc.get("ephemeral_1h_input_tokens") or 0),
|
||||
"cache_creation_5m": int(cc.get("ephemeral_5m_input_tokens") or 0),
|
||||
}
|
||||
after = result.get("modelUsage") or {}
|
||||
before_mu = before_mu or {}
|
||||
delta = {}
|
||||
for m, v in after.items():
|
||||
b = before_mu.get(m) or {}
|
||||
delta[m] = {f: int((v or {}).get(f) or 0) - int(b.get(f) or 0) for f in MU_FIELDS}
|
||||
grew = [m for m, d in delta.items() if any(x > 0 for x in d.values())]
|
||||
models_used = sorted(grew, key=lambda m: (-sum(delta[m].values()), m))
|
||||
sums = {f: sum(d[f] for d in delta.values()) for f in MU_FIELDS}
|
||||
mismatch = (sums["inputTokens"], sums["outputTokens"], sums["cacheCreationInputTokens"],
|
||||
sums["cacheReadInputTokens"]) != (tokens["input"], tokens["output"],
|
||||
tokens["cache_creation"], tokens["cache_read"])
|
||||
total = result.get("total_cost_usd")
|
||||
if total is None:
|
||||
cost_delta = None
|
||||
else:
|
||||
cost_delta = round(float(total) - float(cost_before or 0.0), 6)
|
||||
total = round(float(total), 6)
|
||||
return {"tokens": tokens, "models_used": models_used, "mismatch": mismatch,
|
||||
"cost_delta": cost_delta, "total": total, "model_usage": after}
|
||||
|
||||
|
||||
def cache_verdict(first_usage: dict | None, base_ctx) -> str | None:
|
||||
"""The first request's cache read against the base's context tokens."""
|
||||
if first_usage is None:
|
||||
return None
|
||||
cr = int(first_usage.get("cache_read_input_tokens") or 0)
|
||||
if base_ctx and cr >= CACHE_HIT_RATIO * float(base_ctx):
|
||||
return "hit"
|
||||
return "partial" if cr > 0 else "miss"
|
||||
|
||||
|
||||
def infer_ttl(tokens: dict | None, prev):
|
||||
"""The TTL of a base's cache entries from one call's tokens: 1h written -> 3600, only 5m
|
||||
written -> 300, otherwise (read only, or nothing) `prev`."""
|
||||
if not tokens:
|
||||
return prev
|
||||
if int(tokens.get("cache_creation_1h") or 0) > 0:
|
||||
return TTL_1H
|
||||
if int(tokens.get("cache_creation_5m") or 0) > 0:
|
||||
return TTL_5M
|
||||
return prev
|
||||
|
||||
|
||||
def wrote_only_5m(tokens: dict | None) -> bool:
|
||||
"""The call wrote 5-minute cache entries and no 1-hour ones."""
|
||||
return (bool(tokens) and int(tokens.get("cache_creation_5m") or 0) > 0
|
||||
and int(tokens.get("cache_creation_1h") or 0) == 0)
|
||||
|
||||
|
||||
def warmth(last_hit_ts: float | None, ttl_s, now: float) -> dict:
|
||||
"""warm until `margin` before last_hit + ttl, expiring until last_hit + ttl, then cold;
|
||||
margin = max(120, ttl/10). No hit at all is cold."""
|
||||
if last_hit_ts is None:
|
||||
return {"base_state": "cold", "warm_until": None, "ttl_left_s": None}
|
||||
ttl = float(ttl_s or TTL_1H)
|
||||
margin = max(120.0, ttl / 10.0)
|
||||
wu = float(last_hit_ts) + ttl
|
||||
state = "warm" if now < wu - margin else ("expiring" if now < wu else "cold")
|
||||
return {"base_state": state, "warm_until": iso(wu), "ttl_left_s": int(max(0.0, wu - now))}
|
||||
@@ -0,0 +1,22 @@
|
||||
# Headless implementer (spire:gremlins)
|
||||
|
||||
You run headless under `claude -p` in a Spire build: first as the trained base, then as gremlins spawned from it once per block (each a fork of the base's session). Any session of yours may later be resumed or forked again. Nobody can answer a question mid-run: decide, act, and record what you decided.
|
||||
|
||||
- Do the work yourself. Never dispatch agents, and never start `claude` or the gremlins helper (`gremlins.py`).
|
||||
- Never use `run_in_background` on Bash. Run every command in the foreground.
|
||||
- Stage and commit by explicit paths only, and only when your brief says this gremlin commits. Never push, rebase or reset.
|
||||
- Write gate and test output to `.txt` files under the state dir's `logs/` folder (the brief names the state dir).
|
||||
- When you finish a block, write the handoff note to the path the brief names, in this format, and stamp your exact model id:
|
||||
|
||||
```markdown
|
||||
### Block <N.M> — <title>
|
||||
|
||||
- Model: <exact model id>
|
||||
- Start / end: <`date` output> / <`date` output>
|
||||
- Done: <what changed, in a few lines>
|
||||
- Deviations from plan: <what, and why> — or "none"
|
||||
- Flagged for review: <anything the reviewer should look at>
|
||||
- Left for later: <work deferred to a later block or phase>
|
||||
- Expected broken: <areas that do not build or pass yet, and which block fixes them>
|
||||
- Commits: <hashes> — or "none" when your brief says you do not commit
|
||||
```
|
||||
@@ -0,0 +1,17 @@
|
||||
# Rebase (spire:gremlins)
|
||||
|
||||
This session is a fork of base `${base}` (run log `${run}`). When you finish, this session becomes the new base `${new_base}` (phase ${phase}): every later gremlin of this build starts from what you know at the end of it. Your job is to absorb what changed since `${base}` was trained or last rebased.
|
||||
|
||||
Your system prompt was snapshotted when the base was trained: the git status, the CLAUDE.md files and the environment it shows are **stale**. The delta header below lists what changed in the repository since then. Trust it and the files on disk over the snapshot and over what you remember.
|
||||
|
||||
- Read the handoff notes and the diffs the brief lists (`git --no-optional-locks diff`, `git --no-optional-locks log -p`), and re-read the files they touch. Handoff notes live under `${notes_dir}`.
|
||||
- Do not change code and do not commit. If the brief names a trunk-notes path, update that file; it is the only file you write.
|
||||
- Finish with one line `READY`, then a short summary of what changed and anything the brief asked for that you could not do.
|
||||
|
||||
## Delta since the base commit
|
||||
|
||||
${delta}
|
||||
|
||||
## Brief
|
||||
|
||||
${brief}
|
||||
@@ -0,0 +1,19 @@
|
||||
# Block ${block} (spire:gremlins)
|
||||
|
||||
You are a gremlin spawned from base `${base}` (a fork of its session; run log `${run}`), now working on block `${block}`.
|
||||
|
||||
Your system prompt was snapshotted when the base was trained: the git status, the CLAUDE.md files and the environment it shows are **stale**. The delta header below lists what changed in the repository since then. Trust it and the files on disk over the snapshot and over what you remember from training; re-read every file before you change it.
|
||||
|
||||
## Delta since training
|
||||
|
||||
${delta}
|
||||
|
||||
## Rules for this gremlin
|
||||
|
||||
- Commit only if the brief below says so.
|
||||
- Write your handoff note to `${note_path}`.
|
||||
- Write gate and test output to `.txt` files under `${logs_dir}`.
|
||||
|
||||
## Brief
|
||||
|
||||
${brief}
|
||||
@@ -0,0 +1,11 @@
|
||||
# Training (spire:gremlins)
|
||||
|
||||
You are base `${base}` of the build whose run log is `${run}`. This session is trained once; then one gremlin per block is spawned from it by forking it, and every gremlin starts from what you know at the end of this session. Build an accurate picture of the code the brief below points you at: read it, trace the seams, note the invariants and the traps.
|
||||
|
||||
- Do not change code in this session. If the brief asks for trunk notes, write them to the path it names; that is the only file you write.
|
||||
- The helper's state dir is `${state_dir}`. Gremlins write handoff notes under `${notes_dir}` and gate and test output under `${logs_dir}`.
|
||||
- Finish with one line `READY`, then a short summary of what you read and anything the brief asked for that you could not do.
|
||||
|
||||
## Brief
|
||||
|
||||
${brief}
|
||||
Reference in New Issue
Block a user