dirigence/reliquary
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
72723707315ba3864bf96919b3e287e1c4f12f08
reliquary/plugins/spire/skills/gremlins/tests/test_gremlins.py
T

4236 lines
227 KiB
Python
Raw Blame History

# /// script
# requires-python = ">=3.10"
# dependencies = []
# ///
"""Unit tests for the spire:gremlins helper.
Run from the repo root: `uv run plugins/spire/skills/gremlins/tests/test_gremlins.py`.
Imports gremlins.py by path. Everything generated (stub binary, state dirs, settings files)
lives under `tempfile`; nothing touches ~/.claude or the repo's own state.
"""
from __future__ import annotations
import contextlib
import importlib.util
import io
import json
import os
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from unittest import mock
HERE = os.path.dirname(os.path.abspath(__file__))
GREMLINS_PY = os.path.join(os.path.dirname(HERE), "gremlins.py")
def _load_gremlins():
spec = importlib.util.spec_from_file_location("gremlins", GREMLINS_PY)
mod = importlib.util.module_from_spec(spec)
sys.modules["gremlins"] = mod
spec.loader.exec_module(mod)
return mod
gremlins = _load_gremlins()
IS_WINDOWS = os.name == "nt" # test-only: generated wrappers (.cmd vs sh) and the module-end sweep
WIN_PLATFORM = gremlins.PLATFORM.name == "windows" # platform behaviour under test (digest §3.5)
STUB_VERSION = "2.1.999"
def make_stub_binary(d: str, version: str = STUB_VERSION) -> str:
"""A fake `claude` that answers --version. Generated at runtime (A18), never committed."""
if IS_WINDOWS:
p = os.path.join(d, "claude-stub.cmd")
with open(p, "w", newline="\r\n") as f:
f.write("@echo off\necho %s (Claude Code stub)\n" % version)
else:
p = os.path.join(d, "claude-stub")
with open(p, "w", newline="\n") as f:
f.write("#!/bin/sh\necho '%s (Claude Code stub)'\n" % version)
os.chmod(p, 0o755)
return p
def call(argv):
"""Run gremlins.main in-process. Returns (exit code, parsed JSON, raw stdout)."""
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = gremlins.main(list(argv))
raw = out.getvalue()
return code, json.loads(raw), raw
def child_env():
env = dict(os.environ)
env.pop(gremlins.CHILD_MARKER, None)
return env
def gate_warning(out: dict) -> list[str]:
"""The warning `where` adds when the platform gate is overridden (C13 on a closed platform;
never on Windows), so that the exact `warnings` assertions hold on every host."""
return [gremlins.GATE_OVERRIDE_WARNING] if out["platform"]["override"] else []
class Base(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp(prefix="gremlins-test-")
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
env_patch = mock.patch.dict(os.environ)
env_patch.start()
self.addCleanup(env_patch.stop)
os.environ.pop(gremlins.CHILD_MARKER, None)
# C13: the suite runs with the platform gate overridden, so it exercises the POSIX code on
# the porting hosts; on Windows (supported) the variable changes nothing. GateTests drop it.
os.environ[gremlins.UNSUPPORTED_OK_ENV] = "1"
def assert_one_json_line(self, raw: str):
self.assertTrue(raw.endswith("\n"), repr(raw[-40:]))
self.assertEqual(raw.count("\n"), 1, raw)
obj = json.loads(raw)
self.assertIsInstance(obj, dict)
return obj
# --------------------------------------------------------------------------------------
class PathTests(Base):
def test_windows_forms(self):
cwd = "G:\\dev\\proj"
cases = {
"docs\\a.md": "G:\\dev\\proj\\docs\\a.md",
"docs/a.md": "G:\\dev\\proj\\docs\\a.md",
"G:\\x\\y.md": "G:\\x\\y.md",
"G:/x/y.md": "G:\\x\\y.md",
"g:/x/y.md": "G:\\x\\y.md",
"/g/x/y.md": "G:\\x\\y.md",
"/c/Users/me": "C:\\Users\\me",
"/c": "C:\\",
"/c/": "C:\\",
"../other/./z.md": "G:\\dev\\other\\z.md",
"\\rooted\\a": "G:\\rooted\\a", # native rooted path: the cwd's drive (Windows meaning)
" G:/x/y.md ": "G:\\x\\y.md",
}
for src, want in cases.items():
with self.subTest(src=src):
self.assertEqual(gremlins.norm_path(src, cwd=cwd, windows=True), want)
def fake_cygpath(self, code=0):
"""A fake `cygpath` (python behind a wrapper) that maps `/a/b` to `C:\\Fake\\a\\b` and logs
its arguments."""
d = os.path.join(self.tmp, "cyg")
os.makedirs(d, exist_ok=True)
py = os.path.join(d, "cygpath.py")
log = os.path.join(d, "calls.txt")
with open(py, "w", newline="\n") as f:
f.write("import sys\nopen(%r, 'a').write(' '.join(sys.argv[1:]) + '\\n')\n"
"print('C:\\\\Fake' + sys.argv[-1].replace('/', '\\\\'))\nsys.exit(%d)\n" % (log, code))
if IS_WINDOWS:
exe = os.path.join(d, "cygpath.cmd")
with open(exe, "w", newline="\r\n") as f:
f.write('@echo off\n"%s" -I "%s" %%*\n' % (stub_interpreter(), py))
else:
exe = os.path.join(d, "cygpath")
with open(exe, "w", newline="\n") as f:
f.write('#!/bin/sh\nexec "%s" -I "%s" "$@"\n' % (stub_interpreter(), py))
os.chmod(exe, 0o755)
return exe, log
def test_gitbash_rooted_without_cygpath(self):
"""M13: a Git Bash rooted path that is not a drive form is refused (exit 2) when cygpath is
not on PATH; it is never mapped onto the cwd's drive. Drive forms need no cygpath."""
cwd = "G:\\dev\\proj"
with mock.patch.object(gremlins.WindowsPlatform, "find_cygpath", lambda: None):
for src in ("/tmp/x", "/usr/share/doc", "/tmp/x-run.md"):
with self.subTest(src=src):
with self.assertRaises(gremlins.GremlinsError) as cm:
gremlins.norm_path(src, cwd=cwd, windows=True)
self.assertEqual(cm.exception.code, 2)
self.assertIn("cygpath", cm.exception.payload["error"])
self.assertEqual(cm.exception.payload["path"], src)
self.assertEqual(gremlins.norm_path("/g/x/y.md", cwd=cwd, windows=True), "G:\\x\\y.md")
self.assertEqual(gremlins.norm_path("/c", cwd=cwd, windows=True), "C:\\")
self.assertEqual(gremlins.norm_path("\\rooted\\a", cwd=cwd, windows=True), "G:\\rooted\\a")
if WIN_PLATFORM: # end to end: --run /tmp/... is refused, nothing written on the cwd's drive
drive = os.path.splitdrive(os.getcwd())[0]
code, out, raw = call(["init", "--run", "/tmp/gremlins-m13-x-run.md", "--model",
"claude-opus-5-5", "--cwd", self.tmp])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertIn("cygpath", out["error"])
self.assertFalse(os.path.exists(gremlins.state_dir_for(drive + "\\tmp\\gremlins-m13-x-run.md")))
def test_gitbash_rooted_with_cygpath(self):
"""M13: with cygpath on PATH the path is converted by `cygpath -w`; a failing cygpath
refuses."""
exe, log = self.fake_cygpath()
with mock.patch.object(gremlins.WindowsPlatform, "find_cygpath", lambda: exe):
self.assertEqual(gremlins.norm_path("/tmp/x", cwd="G:\\dev\\proj", windows=True),
"C:\\Fake\\tmp\\x")
self.assertEqual(gremlins.norm_path("/g/x", cwd="G:\\dev\\proj", windows=True), "G:\\x")
with open(log) as f:
self.assertEqual(f.read().splitlines(), ["-w /tmp/x"]) # drive forms skip cygpath
bad, _ = self.fake_cygpath(code=1)
with mock.patch.object(gremlins.WindowsPlatform, "find_cygpath", lambda: bad):
with self.assertRaises(gremlins.GremlinsError) as cm:
gremlins.norm_path("/tmp/x", cwd="G:\\dev\\proj", windows=True)
self.assertEqual(cm.exception.payload["error"], "cygpath failed")
real = shutil.which("cygpath")
if WIN_PLATFORM and real: # the real one (Git Bash): never the cwd's drive + \tmp
got = gremlins.norm_path("/tmp/x", cwd="G:\\dev\\proj", windows=True)
self.assertNotEqual(got, "G:\\tmp\\x")
self.assertRegex(got, r"^[A-Z]:\\.+\\x$")
def test_windows_cwd_in_gitbash_form(self):
self.assertEqual(gremlins.norm_path("a.md", cwd="/g/dev/proj", windows=True), "G:\\dev\\proj\\a.md")
def test_posix_forms(self):
cwd = "/home/u/proj"
self.assertEqual(gremlins.norm_path("a/b.md", cwd=cwd, windows=False), "/home/u/proj/a/b.md")
self.assertEqual(gremlins.norm_path("/abs/x.md", cwd=cwd, windows=False), "/abs/x.md")
self.assertEqual(gremlins.norm_path("../z", cwd=cwd, windows=False), "/home/u/z")
def test_native_relative(self):
self.assertEqual(os.path.normcase(gremlins.norm_path("x/y.md")),
os.path.normcase(os.path.abspath(os.path.join("x", "y.md"))))
def test_empty_refused(self):
with self.assertRaises(gremlins.GremlinsError):
gremlins.norm_path(" ")
@unittest.skipUnless(WIN_PLATFORM, "Windows path forms")
def test_state_dir_windows_forms(self):
want = "G:\\w\\reports\\2026-10-16-08-x-run-gremlins"
for src in ("G:\\w\\reports\\2026-10-16-08-x-run.md", "G:/w/reports/2026-10-16-08-x-run.md",
"/g/w/reports/2026-10-16-08-x-run.md"):
with self.subTest(src=src):
self.assertEqual(gremlins.state_dir_for(src), want)
def test_state_dir_relative(self):
got = gremlins.state_dir_for(os.path.join("reports", "2026-10-16-08-x-run.md"))
self.assertEqual(os.path.normcase(got),
os.path.normcase(os.path.abspath(os.path.join("reports", "2026-10-16-08-x-run-gremlins"))))
def test_state_dir_needs_md(self):
with self.assertRaises(gremlins.GremlinsError) as cm:
gremlins.state_dir_for(os.path.join(self.tmp, "run.txt"))
self.assertEqual(cm.exception.code, 2)
# --------------------------------------------------------------------------------------
class PathMatrixTests(Base):
"""Digest §3.4 (amendment A5): every path form x both platform classes, on any host. Pure: the
cygpath lookup and its one call are faked, nothing touches the filesystem."""
WIN_CWD = "G:\\dev\\proj"
POSIX_CWD = "/home/u/proj"
# (row, input, Windows result, POSIX result). A result is the normalised path, or
# ("refused", <word the error names>).
ROWS = (
(1, "C:\\x\\y.md", "C:\\x\\y.md", ("refused", "drive")),
(2, "c:/x/y.md", "C:\\x\\y.md", ("refused", "drive")),
(3, "C:", ("refused", "cannot normalise"), ("refused", "drive")),
(3, "C:x", ("refused", "cannot normalise"), ("refused", "drive")),
(4, "/c/x/y.md", "C:\\x\\y.md", "/c/x/y.md"),
(5, "/c", "C:\\", "/c"),
(6, "/tmp/x", "C:\\Fake\\tmp\\x", "/tmp/x"),
(7, "\\\\srv\\share\\x\\y.md", "\\\\srv\\share\\x\\y.md", ("refused", "backslash")),
(8, "//srv/share/x/y.md", "\\\\srv\\share\\x\\y.md", "//srv/share/x/y.md"),
(9, "/mnt/c/x/y.md", ("refused", "WSL"), "/mnt/c/x/y.md"),
(10, "docs/a.md", "G:\\dev\\proj\\docs\\a.md", "/home/u/proj/docs/a.md"),
(11, "docs\\a.md", "G:\\dev\\proj\\docs\\a.md", ("refused", "backslash")),
(12, "\\rooted\\a", "G:\\rooted\\a", ("refused", "backslash")),
(13, "../z", "G:\\dev\\z", "/home/u/z"),
(14, " G:/x/y.md ", "G:\\x\\y.md", ("refused", "drive")),
(15, "", ("refused", "empty"), ("refused", "empty")),
(15, " ", ("refused", "empty"), ("refused", "empty")),
)
FAKE_CYGPATH = "fake-cygpath"
def setUp(self):
super().setUp()
self.cyg_calls = []
def fake_run(args, cwd=None, timeout=60, env=None):
self.assertEqual(list(args[:2]), [self.FAKE_CYGPATH, "-w"]) # the only process call
self.cyg_calls.append(args[2])
out = "C:\\Fake" + args[2].replace("/", "\\") + "\r\n"
return subprocess.CompletedProcess(args, 0, stdout=out.encode(), stderr=b"")
for target, name, fn in ((gremlins.WindowsPlatform, "find_cygpath", lambda: self.FAKE_CYGPATH),
(gremlins, "_run", fake_run)):
p = mock.patch.object(target, name, fn)
p.start()
self.addCleanup(p.stop)
def check(self, fn, cwd, src, want):
if isinstance(want, tuple):
with self.assertRaises(gremlins.GremlinsError) as cm:
fn(src, cwd)
self.assertEqual(cm.exception.code, 2)
self.assertIn(want[1], cm.exception.payload["error"])
self.assertEqual(cm.exception.payload["path"], src)
else:
self.assertEqual(fn(src, cwd), want)
def test_windows_rows(self):
for row, src, want, _ in self.ROWS:
with self.subTest(row=row, src=src):
self.check(gremlins.WindowsPlatform.norm_path, self.WIN_CWD, src, want)
self.assertEqual(self.cyg_calls, ["/tmp/x"]) # row 6 only; drive forms skip cygpath
def test_posix_rows(self):
for row, src, _, want in self.ROWS:
with self.subTest(row=row, src=src):
self.check(gremlins.PosixPlatform.norm_path, self.POSIX_CWD, src, want)
self.assertEqual(self.cyg_calls, [])
def test_row6_without_cygpath(self):
"""M13: without cygpath a Git Bash rooted non-drive path is refused on Windows."""
with mock.patch.object(gremlins.WindowsPlatform, "find_cygpath", lambda: None):
self.check(gremlins.WindowsPlatform.norm_path, self.WIN_CWD, "/tmp/x", ("refused", "cygpath"))
self.assertEqual(self.cyg_calls, [])
def test_delegate_selects_the_class(self):
"""The module-level `norm_path(windows=…)` is a selector over the two classes."""
self.assertEqual(gremlins.norm_path("../z", cwd=self.WIN_CWD, windows=True), "G:\\dev\\z")
self.assertEqual(gremlins.norm_path("../z", cwd=self.POSIX_CWD, windows=False), "/home/u/z")
with self.assertRaises(gremlins.GremlinsError):
gremlins.norm_path("C:\\x", cwd=self.POSIX_CWD, windows=False)
def test_both_classes_build_anywhere(self):
"""No ctypes at construction: both classes and their pure parts work on any host."""
w, p = gremlins.WindowsPlatform(), gremlins.PosixPlatform()
self.assertIsNone(w._api)
self.assertEqual((w.name, w.detach_method), ("windows", "cim-hidden"))
self.assertEqual((p.name, p.detach_method), ("posix", "setsid"))
self.assertEqual(w.popen_kwargs(), {"creationflags": 0x08000000})
self.assertEqual(w.popen_kwargs(new_group=True), {"creationflags": 0x08000000})
self.assertEqual(p.popen_kwargs(), {"start_new_session": False})
self.assertEqual(p.popen_kwargs(new_group=True), {"start_new_session": True})
self.assertEqual(w.epoch_to_ctime(0), 116444736000000000)
self.assertIsNone(w.epoch_to_ctime(None))
self.assertIsNone(p.visible_windows())
for d, name in ((w.describe(), "windows"), (p.describe(), "posix")):
self.assertEqual(set(d), {"name", "sys_platform", "wsl", "supported", "detach"})
self.assertEqual((d["name"], d["sys_platform"]), (name, sys.platform))
self.assertIsNone(w._api)
# select_platform: the test's own os.name check is the reference here (IS_WINDOWS)
self.assertIs(type(gremlins.PLATFORM), gremlins.WindowsPlatform if IS_WINDOWS
else gremlins.PosixPlatform)
# --------------------------------------------------------------------------------------
class ScrubTests(Base):
def test_scrubbed_env(self):
venv = os.path.join(self.tmp, "venv")
keep_dir = os.path.join(self.tmp, "bin")
host = {
"CLAUDE_CODE_MESSAGING_SOCKET": "s", "CLAUDE_CODE_SESSION_ID": "x", "CLAUDE_EFFORT": "high",
"ANTHROPIC_API_KEY": "k", "ANTHROPIC_DEFAULT_OPUS_MODEL": "m", "UV_CACHE_DIR": "c",
"VIRTUAL_ENV": venv, "CLAUDECODE": "1", "CLAUDE_CODE_CHILD_SESSION": "1", "AI_AGENT": "a",
"CLAUDE_PID": "1", "ANTHROPIC_BASE_URL": "u", "HOME": "h",
"CLAUDE_CODE_PROMPT_CACHE_TTL": "5m", "CLAUDE_CODE_EXECPATH": "old",
"PATH": os.pathsep.join([os.path.join(venv, "Scripts"), keep_dir]),
}
env = gremlins.scrubbed_env("BIN", environ=host)
for k in ("CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_SESSION_ID", "CLAUDE_EFFORT",
"ANTHROPIC_API_KEY", "ANTHROPIC_DEFAULT_OPUS_MODEL", "UV_CACHE_DIR", "VIRTUAL_ENV"):
self.assertNotIn(k, env)
for k in ("CLAUDECODE", "CLAUDE_CODE_CHILD_SESSION", "AI_AGENT", "CLAUDE_PID",
"ANTHROPIC_BASE_URL", "HOME"):
self.assertEqual(env[k], host[k])
self.assertEqual(env["CLAUDE_CODE_PROMPT_CACHE_TTL"], "1h")
self.assertEqual(env["DISABLE_AUTOUPDATER"], "1")
self.assertEqual(env["SPIRE_GREMLIN_CHILD"], "1")
self.assertEqual(env["CLAUDE_CODE_EXECPATH"], "BIN")
self.assertEqual(env["PATH"], keep_dir)
def test_unknown(self):
host = {"CLAUDECODE": "1", "CLAUDE_EFFORT": "x", "CLAUDE_CODE_NEW_THING": "1",
"ANTHROPIC_BASE_URL": "u", "OTHER": "1", "CLAUDE_CODE_EXECPATH": "p"}
# M9: ANTHROPIC_BASE_URL is on the keep list now, so it is no longer "unknown"
self.assertEqual(gremlins.scrub_unknown(host), ["CLAUDE_CODE_NEW_THING"])
self.assertEqual(gremlins.scrub_class("ANTHROPIC_BASE_URL"), "keep")
# --------------------------------------------------------------------------------------
class LockTests(Base):
def setUp(self):
super().setUp()
self.state = os.path.join(self.tmp, "st")
os.makedirs(self.state)
def test_contention_times_out(self):
with gremlins.RegistryLock(self.state):
t0 = time.monotonic()
with self.assertRaises(gremlins.GremlinsError) as cm:
gremlins.RegistryLock(self.state, timeout=0.3).acquire()
self.assertGreaterEqual(time.monotonic() - t0, 0.25)
self.assertEqual(cm.exception.payload["error"], "registry locked")
with gremlins.RegistryLock(self.state, timeout=0.3):
pass
self.assertFalse(os.path.exists(gremlins.lock_path(self.state)))
def test_waiter_gets_lock_after_release(self):
l1 = gremlins.RegistryLock(self.state).acquire()
threading.Timer(0.2, l1.release).start()
t0 = time.monotonic()
with gremlins.RegistryLock(self.state, timeout=5) as l2:
self.assertGreaterEqual(time.monotonic() - t0, 0.15)
self.assertFalse(l2.broke_stale)
def test_contention_across_processes(self):
code = ("import importlib.util,sys;"
"s=importlib.util.spec_from_file_location('gremlins',sys.argv[1]);m=importlib.util.module_from_spec(s);"
"s.loader.exec_module(m)\n"
"try:\n m.RegistryLock(sys.argv[2],timeout=0.3).acquire(); sys.exit(0)\n"
"except m.GremlinsError: sys.exit(7)\n")
with gremlins.RegistryLock(self.state):
r = subprocess.run([sys.executable, "-c", code, GREMLINS_PY, self.state], capture_output=True)
self.assertEqual(r.returncode, 7, r.stderr)
def dead_holder(self) -> dict:
p = subprocess.Popen([sys.executable, "-c", "pass"])
rec = gremlins.pid_record(p.pid)
p.wait()
return rec
def write_lock(self, holder: dict, at: float, token: str):
with open(gremlins.lock_path(self.state), "w") as f:
json.dump({"pid": holder["pid"], "ctime": holder["ctime"], "at": at, "token": token}, f)
def test_dead_holder_lock_broken_at_once(self):
# M2: a lock whose holder (pid + creation time) is dead is stale even when it is fresh
self.write_lock(self.dead_holder(), time.time(), "old")
t0 = time.monotonic()
with gremlins.RegistryLock(self.state, timeout=2) as lk:
self.assertTrue(lk.broke_stale)
self.assertLess(time.monotonic() - t0, 1.5)
with open(gremlins.lock_path(self.state)) as f:
self.assertEqual(json.load(f)["token"], lk.token)
self.assertEqual([n for n in os.listdir(self.state) if n.endswith(".lock")], [])
def test_abandoned_lock_broken_by_age(self):
me = gremlins.pid_record(os.getpid()) # a live holder: only the age rule applies
self.write_lock(me, time.time() - (gremlins.LOCK_ABANDONED_S + 1), "old")
with gremlins.RegistryLock(self.state, timeout=2) as lk:
self.assertTrue(lk.broke_stale)
def test_garbage_stale_lock_broken_by_mtime(self):
p = gremlins.lock_path(self.state)
with open(p, "w") as f:
f.write("")
old = time.time() - (gremlins.LOCK_ABANDONED_S + 5)
os.utime(p, (old, old))
with gremlins.RegistryLock(self.state, timeout=2) as lk:
self.assertTrue(lk.broke_stale)
def test_fresh_lock_not_broken(self):
# a live holder's lock is never broken before LOCK_ABANDONED_S (garbage content neither)
self.write_lock(gremlins.pid_record(os.getpid()), time.time() - 60, "other")
with self.assertRaises(gremlins.GremlinsError):
gremlins.RegistryLock(self.state, timeout=0.3).acquire()
with open(gremlins.lock_path(self.state), "w") as f:
f.write("")
with self.assertRaises(gremlins.GremlinsError):
gremlins.RegistryLock(self.state, timeout=0.3).acquire()
def test_stale_under_injected_clock_and_safe_release(self):
l1 = gremlins.RegistryLock(self.state).acquire()
real = time.time()
with mock.patch.object(gremlins, "_now", lambda: real + gremlins.LOCK_ABANDONED_S + 1):
l2 = gremlins.RegistryLock(self.state, timeout=2).acquire()
self.assertTrue(l2.broke_stale)
l1.release() # must not remove l2's lock
self.assertTrue(os.path.exists(gremlins.lock_path(self.state)))
l2.release()
self.assertFalse(os.path.exists(gremlins.lock_path(self.state)))
def test_unbreakable_stale_lock_respects_deadline(self):
"""M2: a stale lock that cannot be removed or renamed (AV, sync client) must not spin:
the call ends with its "registry locked" error once the deadline passes."""
self.write_lock(self.dead_holder(), time.time() - 1000, "old") # stale by holder and by age
lp = os.path.normcase(gremlins.lock_path(self.state))
real_remove, real_rename = os.remove, os.rename
def no_remove(p, *a, **k):
if os.path.normcase(str(p)) == lp:
raise PermissionError(13, "held by another program", p)
return real_remove(p, *a, **k)
def no_rename(src, dst, *a, **k):
if os.path.normcase(str(src)) == lp:
raise PermissionError(13, "held by another program", src)
return real_rename(src, dst, *a, **k)
box = {}
def attempt():
t0 = time.monotonic()
try:
gremlins.RegistryLock(self.state, timeout=0.5).acquire()
box["result"] = "acquired"
except gremlins.GremlinsError as e:
box["result"] = e.payload["error"]
box["took"] = time.monotonic() - t0
with mock.patch.object(os, "remove", no_remove), mock.patch.object(os, "rename", no_rename):
th = threading.Thread(target=attempt, daemon=True)
th.start()
th.join(5)
alive = th.is_alive()
self.assertFalse(alive, "acquire spins on a lock it cannot break (hot loop, no deadline)")
self.assertEqual(box["result"], "registry locked")
self.assertLess(box["took"], 2.0)
def test_fresh_lock_taken_during_break_survives(self):
"""M2: between a waiter's judgement (stale) and its break, another waiter breaks the stale
lock and takes a fresh one. The first waiter must not delete the fresh lock."""
self.write_lock(self.dead_holder(), time.time(), "stale")
lp = os.path.normcase(gremlins.lock_path(self.state))
fresh = {"pid": os.getpid(), "ctime": gremlins.proc_ctime(os.getpid()), "at": time.time(),
"token": "fresh"}
real_remove, real_rename = os.remove, os.rename
swapped = []
def swap_in_fresh():
if not swapped:
swapped.append(1)
with open(gremlins.lock_path(self.state), "w") as f:
json.dump(dict(fresh, at=time.time()), f)
def hook_remove(p, *a, **k):
if os.path.normcase(str(p)) == lp:
swap_in_fresh()
return real_remove(p, *a, **k)
def hook_rename(src, dst, *a, **k):
if os.path.normcase(str(src)) == lp:
swap_in_fresh()
return real_rename(src, dst, *a, **k)
with mock.patch.object(os, "remove", hook_remove), mock.patch.object(os, "rename", hook_rename):
with self.assertRaises(gremlins.GremlinsError):
gremlins.RegistryLock(self.state, timeout=0.5).acquire()
self.assertEqual(swapped, [1], "the break was never attempted")
with open(gremlins.lock_path(self.state)) as f:
self.assertEqual(json.load(f)["token"], "fresh")
self.assertEqual(sorted(n for n in os.listdir(self.state) if n.endswith(".lock")),
["registry.lock"])
# --------------------------------------------------------------------------------------
class AtomicTests(Base):
def test_replace_while_reader_holds_target(self):
target = os.path.join(self.tmp, "registry.json")
gremlins.write_json_atomic(target, {"v": 1})
opened, done = threading.Event(), threading.Event()
def reader():
with open(target, "rb") as f:
opened.set()
time.sleep(0.4)
f.read()
done.set()
th = threading.Thread(target=reader)
th.start()
opened.wait(2)
attempts = gremlins.write_json_atomic(target, {"v": 2})
th.join()
self.assertEqual(gremlins.read_json(target), {"v": 2})
if IS_WINDOWS:
self.assertGreater(attempts, 1, "replace should have hit a sharing violation and retried")
self.assertEqual([n for n in os.listdir(self.tmp) if n.endswith(".tmp")], [])
def test_read_json_missing_is_none(self):
self.assertIsNone(gremlins.read_json(os.path.join(self.tmp, "nope.json")))
def test_registry_roundtrip_and_schema(self):
st = os.path.join(self.tmp, "st")
gremlins.ensure_state(st)
with gremlins.RegistryLock(st):
gremlins.write_registry(st, gremlins.new_registry())
reg = gremlins.read_registry(st)
self.assertEqual(reg["schema"], 1)
with open(gremlins.registry_path(st), "rb") as f:
self.assertNotIn(b"\r\n", f.read())
gremlins.write_json_atomic(gremlins.registry_path(st), {"schema": 99})
with self.assertRaises(gremlins.GremlinsError):
gremlins.read_registry(st)
def test_ledger_append(self):
st = os.path.join(self.tmp, "st")
gremlins.ensure_state(st)
with gremlins.RegistryLock(st):
gremlins.ledger_append(st, {"job": "j0001"})
gremlins.ledger_append(st, {"job": "j0002"})
with open(gremlins.ledger_path(st), "rb") as f:
lines = f.read().decode().splitlines()
self.assertEqual([json.loads(x)["job"] for x in lines], ["j0001", "j0002"])
def test_ledger_append_dedupes_by_job(self):
"""M1: a second finaliser (kill racing the supervisor) never writes a second line."""
st = os.path.join(self.tmp, "st")
gremlins.ensure_state(st)
with gremlins.RegistryLock(st):
self.assertTrue(gremlins.ledger_append(st, {"job": "j0001", "status": "ok"}))
self.assertFalse(gremlins.ledger_append(st, {"job": "j0001", "status": "killed"}))
self.assertTrue(gremlins.ledger_append(st, {"job": "j0002", "status": "ok"}))
# a torn last line (writer killed mid-append) does not swallow the next line
with open(gremlins.ledger_path(st), "ab") as f:
f.write(b'{"job": "j0003", "sta')
with gremlins.RegistryLock(st):
self.assertTrue(gremlins.ledger_append(st, {"job": "j0004", "status": "ok"}))
self.assertFalse(gremlins.ledger_append(st, {"job": "j0002", "status": "killed"}))
with open(gremlins.ledger_path(st), "rb") as f:
lines = f.read().decode().splitlines()
parsed = []
for x in lines:
with contextlib.suppress(ValueError):
parsed.append(json.loads(x))
self.assertEqual([(p["job"], p["status"]) for p in parsed],
[("j0001", "ok"), ("j0002", "ok"), ("j0004", "ok")])
# --------------------------------------------------------------------------------------
class PidTests(Base):
def test_own_pid(self):
rec = gremlins.pid_record(os.getpid())
self.assertTrue(gremlins.record_alive(rec))
if rec["ctime"] is not None:
self.assertFalse(gremlins.pid_alive(os.getpid(), rec["ctime"] + 1))
def test_dead_pid(self):
p = subprocess.Popen([sys.executable, "-c", "pass"])
rec = gremlins.pid_record(p.pid)
p.wait()
self.assertFalse(gremlins.record_alive(rec))
self.assertFalse(gremlins.pid_alive(0))
def test_supervisor_interpreter(self):
self.assertTrue(os.path.isfile(gremlins.supervisor_interpreter()))
# --------------------------------------------------------------------------------------
class ProcessTreeTests(Base):
"""I2: walking a dead root's tree on a fake process table (nothing real is touched or killed)."""
ROOT, A, A_KID, B, C, C_KID, OTHER = 900001, 900011, 900012, 900021, 900031, 900032, 900041
def setUp(self):
super().setUp()
self.t0 = time.time() - 5000 # the job (its root) started here
self.hb = self.t0 + 500 # last heartbeat: the job was known alive until here
e2c = gremlins.epoch_to_ctime
if e2c(self.t0) is None:
self.skipTest("no epoch -> creation-time conversion on this host")
self.root_ct = e2c(self.t0)
self.ctimes = {
self.A: e2c(self.t0 + 600), # (a) reused pid's child: created after the last heartbeat
self.A_KID: e2c(self.t0 + 601),
self.B: e2c(self.t0 - 100), # (b) created before the job started
self.C: e2c(self.t0 + 100), # (c) the job's own child, created while it ran
self.C_KID: e2c(self.t0 + 700), # its child (parent verified alive): ours
self.OTHER: e2c(self.t0 + 50),
}
self.table = [(self.A, self.ROOT, "a.exe"), (self.A_KID, self.A, "a-kid.exe"),
(self.B, self.ROOT, "dwm.exe"), (self.C, self.ROOT, "bash.exe"),
(self.C_KID, self.C, "sleep.exe"), (self.OTHER, 4, "other.exe")]
self.live = set(self.ctimes)
for name, fn in (("process_table", lambda: list(self.table)),
("proc_ctime", lambda pid: self.ctimes.get(int(pid)) if int(pid) in self.live else None),
("pid_alive", self.fake_alive)):
p = mock.patch.object(gremlins, name, fn)
p.start()
self.addCleanup(p.stop)
def fake_alive(self, pid, ctime=None):
pid = int(pid)
return pid in self.live and (ctime is None or self.ctimes.get(pid) == ctime)
def rec(self):
return {"pid": self.ROOT, "ctime": self.root_ct}
def test_dead_root_claims_only_children_from_its_lifetime(self):
got = {p["pid"] for p in gremlins.process_tree(self.rec(), alive_until=self.hb)}
self.assertEqual(got, {self.C, self.C_KID})
def test_dead_root_without_bounds_claims_nothing(self):
self.assertEqual(gremlins.process_tree(self.rec()), [])
self.assertEqual(gremlins.process_tree({"pid": self.ROOT, "ctime": None}, alive_until=self.hb), [])
def test_live_root_unchanged(self):
self.live.add(self.ROOT)
self.ctimes[self.ROOT] = self.root_ct
got = {p["pid"] for p in gremlins.process_tree(self.rec(), alive_until=None)}
self.assertEqual(got, {self.ROOT, self.A, self.A_KID, self.C, self.C_KID}) # B is older
def test_reap_lost_bounds_by_heartbeat(self):
st = os.path.join(self.tmp, "st")
gremlins.ensure_state(st)
old = time.time() - 1000
reg = gremlins.new_registry()
reg["config"] = {"launch": {}, "ttl_s": 3600}
dead_sup = {"pid": 900091, "ctime": 1}
reg["jobs"]["j0001"] = gremlins._new_entry("j0001", "spawn", status="running", supervisor=dead_sup,
launched_at=gremlins._iso(old - 10))
with gremlins.RegistryLock(st):
gremlins.write_registry(st, reg)
gremlins.write_json_atomic(gremlins.job_file(st, "j0001"), {
"supervisor": dead_sup, "claude": self.rec(), "started_at": gremlins._iso(old - 10),
"heartbeat_at": gremlins._iso(old)})
seen = []
with mock.patch.object(gremlins, "kill_tree", lambda rec, extra=(), alive_until=None:
seen.append((rec, alive_until)) or {"snapshot": [], "survivors": [],
"tree_kill_rc": None}):
self.assertEqual(gremlins.reap_lost(st), ["j0001"])
self.assertEqual(seen, [(self.rec(), gremlins._ts(gremlins._iso(old)))])
# --- 3.3 / A2: kill verification and pid reuse (fake table; nothing real is killed) ---
SNAP = 900051
def verify(self, *, table, snap_name="sleep.exe", ctime_readable=True):
"""`kill_tree` of a dead root whose snapshot holds one pid, SNAP, that `pid_alive` reports
alive the way the Windows platform does on access denied (running; the creation time is
compared only when it can be read). Returns the survivors' pids and the terminate calls."""
ct = self.ctimes[self.C]
snap = [{"pid": self.SNAP, "ctime": ct, "ppid": self.ROOT, "name": snap_name}]
self.table = list(table)
self.live = {self.SNAP}
self.ctimes = {self.SNAP: ct if ctime_readable else None}
def alive(pid, ctime=None):
pid = int(pid)
if pid not in self.live:
return False
now = self.ctimes.get(pid)
return not (ctime is not None and now is not None and now != ctime)
killed = []
with mock.patch.object(gremlins, "pid_alive", alive), \
mock.patch.object(gremlins, "process_tree", lambda rec, alive_until=None: list(snap)), \
mock.patch.object(gremlins, "terminate", lambda rec: killed.append(rec["pid"]) or False), \
mock.patch.object(gremlins.PLATFORM, "tree_kill", lambda pid: self.fail("root is dead")), \
mock.patch.object(gremlins, "KILL_VERIFY_S", 0.3):
k = gremlins.kill_tree(self.rec(), alive_until=self.hb)
self.assertEqual(k["snapshot"], snap)
self.assertIsNone(k["tree_kill_rc"])
return [p["pid"] for p in k["survivors"]], killed
def test_kill_verify_pid_reused_by_unopenable_process_is_dead(self):
"""Class (a): the pid now carries another name and its creation time cannot be read."""
survivors, killed = self.verify(table=[(self.SNAP, 4, "svchost.exe"), (self.OTHER, 4, "other.exe")],
ctime_readable=False)
self.assertEqual((survivors, killed), ([], []))
def test_kill_verify_pid_gone_from_a_nonempty_table_is_dead(self):
survivors, killed = self.verify(table=[(self.OTHER, 4, "other.exe")], ctime_readable=False)
self.assertEqual((survivors, killed), ([], []))
def test_kill_verify_same_name_unverifiable_survives(self):
survivors, killed = self.verify(table=[(self.SNAP, self.ROOT, "sleep.exe")], ctime_readable=False)
self.assertEqual(survivors, [self.SNAP])
self.assertIn(self.SNAP, killed)
def test_kill_verify_empty_table_keeps_the_survivor(self):
survivors, killed = self.verify(table=[], ctime_readable=False)
self.assertEqual(survivors, [self.SNAP])
self.assertIn(self.SNAP, killed)
def test_kill_verify_unknown_snapshot_name_keeps_the_survivor(self):
survivors, _ = self.verify(table=[(self.SNAP, 4, "svchost.exe")], snap_name=None,
ctime_readable=False)
self.assertEqual(survivors, [self.SNAP])
def test_kill_verify_other_name_with_readable_ctime_is_record_alive(self):
"""A differing name with a readable creation time leaves the decision to `record_alive`:
the same creation time is still the same process (a survivor)."""
survivors, _ = self.verify(table=[(self.SNAP, 4, "svchost.exe")], ctime_readable=True)
self.assertEqual(survivors, [self.SNAP])
# --------------------------------------------------------------------------------------
class InitTests(Base):
def setUp(self):
super().setUp()
self.cwd = os.path.join(self.tmp, "proj")
os.makedirs(os.path.join(self.cwd, ".claude", "agents"))
os.makedirs(os.path.join(self.tmp, "reports"))
self.run_log = os.path.join(self.tmp, "reports", "2026-10-16-08-t-run.md")
self.state = gremlins.state_dir_for(self.run_log)
self.user_settings = os.path.join(self.tmp, "home", ".claude", "settings.json")
p = mock.patch.object(gremlins, "USER_SETTINGS_PATH", self.user_settings)
p.start()
self.addCleanup(p.stop)
self.stub = make_stub_binary(self.tmp)
def init(self, *extra, model="claude-opus-5-5[1m]"):
argv = ["init", "--run", self.run_log, "--model", model, "--cwd", self.cwd,
"--binary", self.stub, *extra]
return call(argv)
def write_settings(self, path, models):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
json.dump({"availableModels": models}, f)
def add_fake_base(self):
with gremlins.RegistryLock(self.state):
reg = gremlins.read_registry(self.state)
reg["bases"]["B1"] = {"status": "active", "session": "00000000-0000-0000-0000-000000000000"}
gremlins.write_registry(self.state, reg)
def reg_bytes(self):
with open(gremlins.registry_path(self.state), "rb") as f:
return f.read()
# --- basic ---
def test_init_creates_state_and_config(self):
code, out, raw = self.init()
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertTrue(out["created"])
st = self.state
for d in gremlins.STATE_SUBDIRS:
self.assertTrue(os.path.isdir(os.path.join(st, d)), d)
with open(os.path.join(st, ".gitignore"), "rb") as f:
self.assertEqual(f.read(), b"raw/\n*.lock\n")
self.assertEqual(gremlins.read_json(gremlins.mcp_empty_path(st)), {"mcpServers": {}})
self.assertFalse(os.path.exists(gremlins.lock_path(st)))
reg = gremlins.read_registry(st)
self.assertEqual(reg["schema"], 1)
self.assertEqual(reg["bases"], {})
self.assertEqual(reg["jobs"], {})
cfg = reg["config"]
la = cfg["launch"]
self.assertEqual(la["model"], "claude-opus-5-5[1m]")
self.assertEqual(la["effort"], "high")
self.assertEqual(la["permission_mode"], "dontAsk")
self.assertEqual(la["tools"], gremlins.DEFAULT_TOOLS.split(","))
self.assertEqual(la["allowed_tools"], la["tools"])
self.assertEqual(la["disallowed_tools"], gremlins.DEFAULT_DISALLOWED)
self.assertEqual(la["mcp"], {"strict": True, "config": gremlins.mcp_empty_path(st)})
self.assertEqual(la["cwd"], gremlins.norm_path(self.cwd))
self.assertIsNone(la["agent"])
self.assertEqual(la["env"]["pinned"], {"CLAUDE_CODE_PROMPT_CACHE_TTL": "1h", "DISABLE_AUTOUPDATER": "1"})
b = la["binary"]
self.assertEqual(b["path"], gremlins.norm_path(self.stub))
self.assertTrue(os.path.isabs(b["path"]))
self.assertEqual(b["version"], STUB_VERSION)
self.assertEqual(b["size"], os.path.getsize(self.stub))
pre = la["append_system_prompt"]
self.assertEqual(pre["path"], gremlins.preamble_path())
self.assertEqual(pre["sha256"], gremlins._sha256_file(gremlins.preamble_path()))
self.assertEqual(cfg["fingerprint"], gremlins.fingerprint_hash(la))
self.assertEqual(len(cfg["fingerprint"]), 64)
self.assertEqual(cfg["parallel_cap"], 2)
self.assertEqual(cfg["orphan_after"], 1200)
self.assertEqual(cfg["pings"], "on")
self.assertEqual(cfg["ttl_s"], 3600)
self.assertEqual(cfg["max_minutes"], 90)
self.assertEqual(cfg["stall_after_s"], 900)
self.assertEqual(cfg["thresholds"], gremlins.THRESHOLD_DEFAULTS)
self.assertEqual(out["config"], cfg)
self.assertEqual(out["warnings"], []) # N3: always present
def test_fingerprint_canonical(self):
a = {"b": [1, 2], "a": {"y": 1, "x": None}}
b = {"a": {"x": None, "y": 1}, "b": [1, 2]}
self.assertEqual(gremlins.fingerprint_hash(a), gremlins.fingerprint_hash(b))
self.assertEqual(gremlins.canonical_json(a), '{"a":{"x":null,"y":1},"b":[1,2]}')
def test_list_flags(self):
code, out, _ = self.init("--tools", "Read,Bash", "--allowed-tools", "Read", "Bash(git status:*)",
"--disallowed-tools", "Bash(rm:*),Bash(a,b)")
self.assertEqual(code, 0, out)
la = out["config"]["launch"]
self.assertEqual(la["tools"], ["Read", "Bash"])
self.assertEqual(la["allowed_tools"], ["Read", "Bash(git status:*)"])
self.assertEqual(la["disallowed_tools"], ["Bash(rm:*)", "Bash(a,b)"])
# --- idempotency ---
def test_idempotent(self):
self.assertEqual(self.init()[0], 0)
before = self.reg_bytes()
code, out, _ = self.init()
self.assertEqual(code, 0, out)
self.assertFalse(out["created"])
self.assertFalse(out["changed"])
self.assertEqual(self.reg_bytes(), before)
def test_changed_values_before_base(self):
_, first, _ = self.init()
code, out, _ = self.init("--effort", "medium", "--orphan-after", "1500")
self.assertEqual(code, 0, out)
self.assertTrue(out["changed"])
self.assertIn("launch.effort", out["changed_fields"])
self.assertIn("orphan_after", out["changed_fields"])
self.assertIn("fingerprint", out["changed_fields"])
cfg = gremlins.read_registry(self.state)["config"]
self.assertEqual(cfg["launch"]["effort"], "medium")
self.assertEqual(cfg["orphan_after"], 1500)
self.assertNotEqual(cfg["fingerprint"], first["config"]["fingerprint"])
def test_changed_values_after_base_allowed(self):
"""Block 2.2 replaces Phase 1's refusal: `init` may change values after a base exists (the
input path of `retrain_due: defining:`). The base keeps its own launch record; a base from
before per-base records gets the config it was trained with as its record."""
self.assertEqual(self.init()[0], 0)
self.add_fake_base()
code, out, raw = self.init("--effort", "medium")
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertTrue(out["changed"])
self.assertIn("launch.effort", out["changed_fields"])
self.assertEqual(out["bases"], ["B1"])
reg = gremlins.read_registry(self.state)
self.assertEqual(reg["config"]["launch"]["effort"], "medium")
self.assertEqual(reg["bases"]["B1"]["launch"]["effort"], "high")
self.assertEqual(reg["bases"]["B1"]["overrides"], [])
self.assertFalse(os.path.exists(gremlins.lock_path(self.state)))
# identical values stay unchanged with a base
before = self.reg_bytes()
code, out, _ = self.init("--effort", "medium")
self.assertEqual(code, 0, out)
self.assertFalse(out["changed"])
self.assertEqual(self.reg_bytes(), before)
def test_rerun_keeps_omitted_settings(self):
"""Minor 8: on a re-run, flags that are not given keep their current values (launch fields
and run-wide settings); defaults apply only on the first init; `changed_fields` lists only
what the given flags changed."""
code, first, _ = self.init("--pings", "off", "--parallel-cap", "3", "--rebase-commits", "9",
"--orphan-after", "1500", "--effort", "low", "--tools", "Read,Bash",
model="claude-sonnet-5-5")
self.assertEqual(code, 0, first)
self.add_fake_base()
before = gremlins.read_registry(self.state)["config"]
code, out, raw = call(["init", "--run", self.run_log, "--warm-margin-s", "100"])
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertTrue(out["changed"])
self.assertEqual(out["changed_fields"], ["warm_margin_s"])
cfg = gremlins.read_registry(self.state)["config"]
self.assertEqual(cfg["warm_margin_s"], 100)
self.assertEqual((cfg["pings"], cfg["parallel_cap"], cfg["orphan_after"]), ("off", 3, 1500))
self.assertEqual(cfg["thresholds"], dict(gremlins.THRESHOLD_DEFAULTS, rebase_commits=9))
self.assertEqual(cfg["launch"], before["launch"])
self.assertEqual((cfg["launch"]["model"], cfg["launch"]["effort"], cfg["launch"]["tools"]),
("claude-sonnet-5-5", "low", ["Read", "Bash"]))
self.assertEqual(cfg["fingerprint"], before["fingerprint"])
# a given flag still changes its value, and only it is listed
code, out, _ = call(["init", "--run", self.run_log, "--pings", "on"])
self.assertEqual(code, 0, out)
self.assertEqual(out["changed_fields"], ["pings"])
cfg = gremlins.read_registry(self.state)["config"]
self.assertEqual((cfg["pings"], cfg["warm_margin_s"], cfg["parallel_cap"]), ("on", 100, 3))
# the first init still needs --model (a usage error, nothing written)
other = os.path.join(self.tmp, "reports", "2026-10-16-09-u-run.md")
code, out, raw = call(["init", "--run", other, "--cwd", self.cwd, "--binary", self.stub])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertEqual(out["error"], "usage")
self.assertIn("--model", out["detail"])
self.assertFalse(os.path.exists(gremlins.state_dir_for(other)))
# --- N3: a re-run and the binary (3.3; digest §5.1, amendment A3, A4) ---
def named_stub(self, sub):
"""A stub named `claude` (`claude.cmd` on Windows) in its own dir, found by a PATH lookup."""
d = os.path.join(self.tmp, sub)
os.makedirs(d)
dst = os.path.join(d, "claude.cmd" if IS_WINDOWS else "claude")
os.replace(make_stub_binary(d), dst)
return dst
def bare_init(self, *extra):
"""A first init without --binary: the binary comes from $CLAUDE_CODE_EXECPATH or PATH."""
return call(["init", "--run", self.run_log, "--model", "claude-opus-5-5[1m]", "--cwd", self.cwd,
*extra])
def rerun(self, *extra):
return call(["init", "--run", self.run_log, *extra])
def path_only(self, d):
"""PATH is exactly `d` (an empty dir when it should hold no claude)."""
os.makedirs(d, exist_ok=True)
os.environ["PATH"] = d
def stored_binary(self):
cfg = gremlins.read_registry(self.state)["config"]
return cfg["launch"]["binary"]["path"], cfg["binary_resolved_by"]
def test_rerun_reresolves_execpath(self):
"""C14: a re-run without --binary follows $CLAUDE_CODE_EXECPATH to a new binary, warns, and
lists the change; an existing base keeps its own binary record."""
a, b = self.named_stub("a"), self.named_stub("b")
os.environ["CLAUDE_CODE_EXECPATH"] = a
code, first, _ = self.bare_init()
self.assertEqual(code, 0, first)
self.assertEqual(self.stored_binary(), (gremlins.norm_path(a), "CLAUDE_CODE_EXECPATH"))
self.assertEqual(first["warnings"], [])
self.add_fake_base()
os.environ["CLAUDE_CODE_EXECPATH"] = b
code, out, raw = self.rerun("--pings", "off")
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertEqual(self.stored_binary(), (gremlins.norm_path(b), "CLAUDE_CODE_EXECPATH"))
self.assertEqual(out["config"]["launch"]["binary"]["path"], gremlins.norm_path(b))
self.assertIn("launch.binary.path", out["changed_fields"])
self.assertIn("pings", out["changed_fields"])
self.assertEqual(len(out["warnings"]), 1, out["warnings"])
w = out["warnings"][0]
self.assertTrue(w.startswith("binary changed: "), w)
for s in (gremlins.norm_path(a), gremlins.norm_path(b), "(resolved by CLAUDE_CODE_EXECPATH)",
"existing bases keep their own binary record, new trainings use the new one"):
self.assertIn(s, w)
reg = gremlins.read_registry(self.state)
self.assertEqual(reg["bases"]["B1"]["launch"]["binary"]["path"], gremlins.norm_path(a))
# the same source again: nothing changes, no warning
code, out, _ = self.rerun()
self.assertEqual((code, out["changed"], out["changed_fields"], out["warnings"]), (0, False, [], []))
def test_rerun_keeps_a_binary_pin(self):
"""A `--binary` pin is kept on a re-run without --binary, whatever $CLAUDE_CODE_EXECPATH says."""
os.environ["CLAUDE_CODE_EXECPATH"] = self.named_stub("b")
code, first, _ = self.init()
self.assertEqual(code, 0, first)
self.assertEqual(self.stored_binary(), (gremlins.norm_path(self.stub), "--binary"))
code, out, _ = self.rerun("--pings", "off")
self.assertEqual(code, 0, out)
self.assertEqual(self.stored_binary(), (gremlins.norm_path(self.stub), "--binary"))
self.assertEqual(out["changed_fields"], ["pings"])
self.assertEqual(out["warnings"], [])
def test_rerun_without_a_source_keeps_the_stored_path(self):
"""A3 (first case): stored file present, $CLAUDE_CODE_EXECPATH unset, no claude on PATH: the
stored path is kept with the "could not re-resolve" warning, exit 0."""
a = self.named_stub("a")
os.environ["CLAUDE_CODE_EXECPATH"] = a
self.assertEqual(self.bare_init()[0], 0)
os.environ.pop("CLAUDE_CODE_EXECPATH")
self.path_only(os.path.join(self.tmp, "empty-path"))
code, out, raw = self.rerun("--pings", "off")
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertEqual(self.stored_binary(), (gremlins.norm_path(a), "CLAUDE_CODE_EXECPATH"))
self.assertEqual(out["changed_fields"], ["pings"])
self.assertEqual(out["warnings"], [
"could not re-resolve the claude binary through CLAUDE_CODE_EXECPATH; kept the stored path %s"
% gremlins.norm_path(a)])
def test_rerun_with_the_stored_file_gone_fails_loud(self):
"""A3 (second case): the stored file is deleted and nothing re-resolves it: exit 2, `claude
binary not found` (today's loud failure), nothing written."""
a = self.named_stub("a")
os.environ["CLAUDE_CODE_EXECPATH"] = a
self.assertEqual(self.bare_init()[0], 0)
before = self.reg_bytes()
os.remove(a)
os.environ.pop("CLAUDE_CODE_EXECPATH")
self.path_only(os.path.join(self.tmp, "empty-path"))
code, out, raw = self.rerun("--pings", "off")
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertEqual((out["error"], out["binary"]), ("claude binary not found", gremlins.norm_path(a)))
self.assertEqual(self.reg_bytes(), before)
def test_rerun_never_crosses_sources(self):
"""A4: a path stored from $CLAUDE_CODE_EXECPATH is not replaced by a PATH lookup (kept, with
the warning), and a path stored from PATH is re-resolved by PATH, not by the env."""
a, p1, p2 = self.named_stub("a"), self.named_stub("p1"), self.named_stub("p2")
os.environ["CLAUDE_CODE_EXECPATH"] = a
self.assertEqual(self.bare_init()[0], 0)
os.environ.pop("CLAUDE_CODE_EXECPATH")
self.path_only(os.path.dirname(p1))
code, out, _ = self.rerun("--pings", "off")
self.assertEqual(code, 0, out)
self.assertEqual(self.stored_binary(), (gremlins.norm_path(a), "CLAUDE_CODE_EXECPATH"))
self.assertEqual(len(out["warnings"]), 1, out["warnings"])
self.assertTrue(out["warnings"][0].startswith(
"could not re-resolve the claude binary through CLAUDE_CODE_EXECPATH"), out["warnings"])
# stored from PATH: the env does not win on a re-run
other = os.path.join(self.tmp, "reports", "2026-10-16-09-p-run.md")
# (a PATH lookup on Windows spells the extension from PATHEXT, e.g. `claude.CMD`: compare
# the paths case-folded, as the file system does)
nc = os.path.normcase
code, out, _ = call(["init", "--run", other, "--model", "claude-opus-5-5[1m]", "--cwd", self.cwd])
self.assertEqual(code, 0, out)
self.assertEqual((nc(out["config"]["launch"]["binary"]["path"]), out["config"]["binary_resolved_by"]),
(nc(gremlins.norm_path(p1)), "PATH"))
os.environ["CLAUDE_CODE_EXECPATH"] = a
self.path_only(os.path.dirname(p2))
code, out, _ = call(["init", "--run", other, "--pings", "off"])
self.assertEqual(code, 0, out)
self.assertEqual((nc(out["config"]["launch"]["binary"]["path"]), out["config"]["binary_resolved_by"]),
(nc(gremlins.norm_path(p2)), "PATH"))
self.assertIn("launch.binary.path", out["changed_fields"])
self.assertEqual(len(out["warnings"]), 1, out["warnings"])
self.assertIn("(resolved by PATH)", out["warnings"][0])
def test_warm_margin_must_be_below_ttl(self):
"""Minor 2: `init` refuses a warm_margin_s at or above the configured ttl_s (exit 2), on the
first init and on a re-run that lowers ttl_s under the current margin."""
out = self.assert_refused_no_state("--warm-margin-s", "3600")
self.assertEqual((out["warm_margin_s"], out["ttl_s"]), (3600, 3600))
self.assert_refused_no_state("--ttl-s", "300", "--warm-margin-s", "300")
code, out, _ = self.init("--warm-margin-s", "3599")
self.assertEqual(code, 0, out)
before = self.reg_bytes()
code, out, raw = self.init("--ttl-s", "3000")
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertIn("below ttl_s", out["error"])
self.assertEqual(self.reg_bytes(), before)
code, out, _ = self.init("--ttl-s", "300", "--warm-margin-s", "200")
self.assertEqual(code, 0, out)
self.assertEqual(sorted(out["changed_fields"]), ["ttl_s", "warm_margin_s"])
def test_fingerprint_hash_stable_across_runs(self):
"""M7: the hash covers the mcp config's content, not its per-run path: two runs with the
same launch fields hash the same; the path stays in the config, outside the hash."""
code, a, _ = self.init()
self.assertEqual(code, 0, a)
other = os.path.join(self.tmp, "elsewhere", "2026-10-16-09-u-run.md")
os.makedirs(os.path.dirname(other))
code, b, _ = call(["init", "--run", other, "--model", "claude-opus-5-5[1m]", "--cwd", self.cwd,
"--binary", self.stub])
self.assertEqual(code, 0, b)
la, lb = a["config"]["launch"], b["config"]["launch"]
self.assertNotEqual(la["mcp"]["config"], lb["mcp"]["config"])
self.assertEqual(la["mcp"]["config"], gremlins.mcp_empty_path(self.state))
self.assertEqual(a["config"]["fingerprint"], b["config"]["fingerprint"])
self.assertEqual(gremlins.fingerprint_view(la)["mcp"], {"strict": True, "servers": {}})
self.assertNotIn("config", gremlins.fingerprint_view(la)["mcp"])
# a second process computes the same hash (no per-process input)
r = subprocess.run([sys.executable, "-I", "-c",
"import importlib.util,json,sys;"
"s=importlib.util.spec_from_file_location('f',sys.argv[1]);m=importlib.util.module_from_spec(s);"
"s.loader.exec_module(m);print(m.fingerprint_hash(json.loads(sys.argv[2])))",
GREMLINS_PY, json.dumps(lb)], capture_output=True, timeout=60)
self.assertEqual(r.stdout.decode().strip(), a["config"]["fingerprint"], r.stderr)
# the content does count
self.assertNotEqual(gremlins.fingerprint_hash(dict(la, mcp=dict(la["mcp"], strict=False))),
a["config"]["fingerprint"])
# --- refusals ---
def assert_refused_no_state(self, *extra, model="claude-opus-5-5[1m]"):
code, out, raw = self.init(*extra, model=model)
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertIn("error", out)
self.assertFalse(os.path.exists(self.state), "a refused init must not write")
return out
def test_bad_model_ids(self):
for m in ("opus", "sonnet", "claude-opus-5-5[200k]", "Claude-Opus-5-5", "claude-opus 5"):
with self.subTest(model=m):
self.assert_refused_no_state(model=m)
def test_plan_mode_refused(self):
out = self.assert_refused_no_state("--permission-mode", "plan")
self.assertEqual(out["error"], "permission mode refused")
def test_unknown_mode_and_effort_refused(self):
self.assert_refused_no_state("--permission-mode", "yolo")
self.assert_refused_no_state("--effort", "turbo")
def test_permission_modes_match_2_1_299(self):
"""M5: `claude --help` 2.1.299 offers acceptEdits, auto, bypassPermissions, manual, dontAsk,
plan. `default` is not one of them; `plan` is refused."""
out = self.assert_refused_no_state("--permission-mode", "default")
self.assertEqual(out["error"], "unknown permission mode")
self.assertEqual(sorted(out["allowed"]),
sorted(["acceptEdits", "auto", "bypassPermissions", "manual", "dontAsk"]))
for mode in ("acceptEdits", "auto", "bypassPermissions", "manual", "dontAsk"):
with self.subTest(mode=mode):
code, out, _ = self.init("--permission-mode", mode)
self.assertEqual(code, 0, out)
self.assertEqual(out["config"]["launch"]["permission_mode"], mode)
def test_missing_binary_refused(self):
out = self.assert_refused_no_state("--binary", os.path.join(self.tmp, "nope.exe"))
self.assertEqual(out["error"], "claude binary not found")
def test_agent(self):
self.assert_refused_no_state("--agent", "ghost")
self.assert_refused_no_state("--agent", "../x")
ap = os.path.join(self.cwd, ".claude", "agents", "impl.md")
with open(ap, "w") as f:
f.write("---\nname: impl\n---\nbody\n")
code, out, _ = self.init("--agent", "impl")
self.assertEqual(code, 0, out)
ag = out["config"]["launch"]["agent"]
self.assertEqual(ag["name"], "impl")
self.assertEqual(ag["sha256"], gremlins._sha256_file(ap))
# --- C8 admission ---
def proj(self):
return os.path.join(self.cwd, ".claude", "settings.json")
def local(self):
return os.path.join(self.cwd, ".claude", "settings.local.json")
def test_admission_no_list_admits(self):
self.assertTrue(gremlins.model_admission("claude-whatever-9", self.cwd)["admitted"])
def test_admission_exact(self):
self.write_settings(self.user_settings, ["claude-opus-5-5[1m]"])
self.assertTrue(gremlins.model_admission("claude-opus-5-5[1m]", self.cwd)["admitted"])
# the bare id is not the [1m] entry and `opus` is not listed
self.assertFalse(gremlins.model_admission("claude-opus-5-5", self.cwd)["admitted"])
def test_admission_strips_1m(self):
self.write_settings(self.proj(), ["claude-opus-5-5"])
adm = gremlins.model_admission("claude-opus-5-5[1m]", self.cwd)
self.assertTrue(adm["admitted"])
self.assertIn("without [1m]", adm["via"])
def test_admission_family_alias(self):
self.write_settings(self.local(), ["sonnet"])
adm = gremlins.model_admission("claude-sonnet-5-5", self.cwd)
self.assertTrue(adm["admitted"])
self.assertIn("family alias", adm["via"])
def test_admission_any_list_admits(self):
self.write_settings(self.user_settings, ["claude-opus-5-5"])
self.write_settings(self.local(), ["haiku"])
self.assertTrue(gremlins.model_admission("claude-haiku-5-5", self.cwd)["admitted"])
def test_admission_unlisted_refused_by_init(self):
self.write_settings(self.user_settings, ["claude-opus-5-5"])
self.write_settings(self.local(), ["opus"])
with open(self.local(), "rb") as f:
local_before = f.read()
out = self.assert_refused_no_state(model="claude-sonnet-5-5")
self.assertEqual(out["error"], "model not admitted by availableModels")
self.assertEqual(out["model"], "claude-sonnet-5-5")
self.assertEqual(sorted(out["files"]), sorted([self.user_settings, self.local()]))
self.assertIn("must add", out["message"])
self.assertIn(self.local(), out["message"])
with open(self.local(), "rb") as f:
self.assertEqual(f.read(), local_before, "settings must never be written")
self.assertFalse(os.path.exists(self.proj()))
def test_admission_admitted_by_init(self):
self.write_settings(self.proj(), ["opus", "sonnet"])
code, out, _ = self.init(model="claude-sonnet-5-5")
self.assertEqual(code, 0, out)
self.assertIn("family alias", out["config"]["model_admission"])
# --------------------------------------------------------------------------------------
class CliTests(Base):
def test_commands_need_init(self):
run_log = os.path.join(self.tmp, "r-run.md")
brief = os.path.join(self.tmp, "b.md")
with open(brief, "w") as f:
f.write("x\n")
cases = (("train", "--base", "B1", "--brief", brief),
("spawn", "--base", "B1", "--block", "1.1", "--brief", brief),
("continue", "--job", "j0001", "--brief", brief),
("wait", "--all"), ("list",), ("expire", "--all"), ("kill", "--job", "j0001"),
("_supervise", "--job", "j0001"), ("tick",),
("rebase", "--base", "B1", "--as", "B2", "--brief", brief),
("usage",), ("usage", "--phase", "1"), ("usage", "--block", "1.1", "--format", "md"))
for argv in cases:
with self.subTest(cmd=argv[0]):
code, out, raw = call([argv[0], "--run", run_log, *argv[1:]])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertIn("error", out)
self.assertNotEqual(out["error"], "not implemented")
self.assertFalse(os.path.exists(gremlins.state_dir_for(run_log)))
def test_unknown_command_and_missing_run(self):
for argv in (["bogus"], [], ["list"], ["init", "--run", "x.md"],
["list", "--run", "x.md", "--bogus"]):
with self.subTest(argv=argv):
code, out, raw = call(argv)
self.assertEqual(code, 2)
self.assert_one_json_line(raw)
self.assertEqual(out["error"], "usage")
def test_help_is_json(self):
code, out, raw = call(["--help"])
self.assertEqual(code, 0)
self.assert_one_json_line(raw)
self.assertIn("where", out["commands"])
self.assertNotIn("_supervise", out["commands"])
code, out, _ = call(["init", "--help"])
self.assertEqual(code, 0)
self.assertIn("--model", out["help"])
def test_child_refuses_launching(self):
os.environ[gremlins.CHILD_MARKER] = "1"
run_log = os.path.join(self.tmp, "r-run.md")
code, out, _ = call(["spawn", "--run", run_log, "--base", "B1", "--block", "1", "--brief", "b"])
self.assertEqual(code, 2)
self.assertIn("inside a gremlin", out["error"])
code, out, _ = call(["list", "--run", run_log])
self.assertNotIn("inside a gremlin", out["error"]) # not refused as a launch (no init yet)
stub = make_stub_binary(self.tmp)
self.assertEqual(call(["where", "--binary", stub])[0], 0)
def test_internal_error_keeps_invariant(self):
def boom(args):
raise RuntimeError("kaboom")
with mock.patch.dict(gremlins.HANDLERS, {"list": boom}):
code, out, raw = call(["list", "--run", os.path.join(self.tmp, "r-run.md")])
self.assertEqual(code, 2)
self.assert_one_json_line(raw)
self.assertEqual(out["error"], "internal error")
def test_where_warns_on_base_url(self):
"""M9: `where` warns (exit 0) when ANTHROPIC_BASE_URL is set to anything but the default."""
stub = make_stub_binary(self.tmp)
os.environ.pop("ANTHROPIC_BASE_URL", None)
for value in (None, "https://api.anthropic.com", "https://api.anthropic.com/"):
with self.subTest(value=value):
if value is not None:
os.environ["ANTHROPIC_BASE_URL"] = value
code, out, raw = call(["where", "--binary", stub])
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertEqual(out["warnings"], gate_warning(out))
os.environ["ANTHROPIC_BASE_URL"] = "https://llm-gateway.example.internal/anthropic"
code, out, raw = call(["where", "--binary", stub])
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertTrue(out["ok"])
self.assertEqual(len(out["warnings"]), 1 + len(gate_warning(out)), out["warnings"])
self.assertIn("ANTHROPIC_BASE_URL", out["warnings"][0])
self.assertIn("llm-gateway.example.internal", out["warnings"][0])
self.assertEqual(out["warnings"][1:], gate_warning(out))
def test_where_platform(self):
"""Digest §3.6/§4, A12: `where.platform` is structured; `override` is false on a supported
platform even though the suite sets the override env (C13)."""
stub = make_stub_binary(self.tmp)
self.assertEqual(os.environ.get(gremlins.UNSUPPORTED_OK_ENV), "1")
code, out, raw = call(["where", "--binary", stub])
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertNotIn("spawn", out)
pf = out["platform"]
self.assertEqual(set(pf), {"name", "sys_platform", "wsl", "supported", "detach", "override"})
self.assertEqual(pf["sys_platform"], sys.platform)
self.assertEqual(pf["override"], not gremlins.PLATFORM.supported)
if WIN_PLATFORM:
self.assertEqual(pf, {"name": "windows", "sys_platform": sys.platform, "wsl": False,
"supported": True, "detach": "cim-hidden", "override": False})
self.assertEqual(out["warnings"], [])
def test_where_refused_binary_path(self):
"""A11: a `--binary` path form this platform refuses lands in `binary.error` and the
reasons (exit 2); the rest of the report is still filled."""
bad = "/mnt/c/x/claude.exe" if WIN_PLATFORM else "C:\\x\\claude.exe"
code, out, raw = call(["where", "--binary", bad])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertFalse(out["ok"])
self.assertIsNone(out["binary"]["path"])
self.assertEqual(out["binary"]["how"], "--binary")
self.assertEqual(out["binary"]["given"], bad)
self.assertIn("path form", out["binary"]["error"])
self.assertIn(out["binary"]["error"], out["reason"])
self.assertEqual(out["error"], out["reason"])
for k in ("script", "python", "uv", "git", "scrub", "platform", "warnings"):
self.assertIn(k, out)
def test_child_refuses_kill_expire_init(self):
"""M4: a gremlin may not run kill, expire or init."""
os.environ[gremlins.CHILD_MARKER] = "1"
stub = make_stub_binary(self.tmp)
run_log = os.path.join(self.tmp, "r-run.md")
for argv in (["kill", "--run", run_log, "--job", "j0001"],
["expire", "--run", run_log, "--all"],
["init", "--run", run_log, "--model", "claude-opus-5-5", "--cwd", self.tmp,
"--binary", stub]):
with self.subTest(cmd=argv[0]):
code, out, raw = call(argv)
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertIn("inside a gremlin", out["error"])
self.assertEqual(out["command"], argv[0])
self.assertFalse(os.path.exists(gremlins.state_dir_for(run_log)))
def test_child_does_not_stamp_orchestrator(self):
"""M4: list and wait from inside a gremlin leave run.last_orchestrator_call_at alone."""
os.makedirs(os.path.join(self.tmp, "reports"))
run_log = os.path.join(self.tmp, "reports", "r-run.md")
state = gremlins.state_dir_for(run_log)
gremlins.ensure_state(state)
old = gremlins._iso(time.time() - 5000)
reg = gremlins.new_registry()
reg["config"] = {"launch": {}, "ttl_s": 3600}
reg["run"]["last_orchestrator_call_at"] = old
with gremlins.RegistryLock(state):
gremlins.write_registry(state, reg)
os.environ[gremlins.CHILD_MARKER] = "1"
self.assertEqual(call(["list", "--run", run_log])[0], 0)
self.assertEqual(call(["wait", "--run", run_log, "--all", "--timeout", "1"])[0], 0)
self.assertEqual(gremlins.read_registry(state)["run"]["last_orchestrator_call_at"], old)
os.environ.pop(gremlins.CHILD_MARKER)
self.assertEqual(call(["list", "--run", run_log])[0], 0) # the orchestrator does stamp
self.assertNotEqual(gremlins.read_registry(state)["run"]["last_orchestrator_call_at"], old)
def test_where_in_process_missing_binary(self):
code, out, raw = call(["where", "--binary", os.path.join(self.tmp, "nope.exe")])
self.assertEqual(code, 2)
self.assert_one_json_line(raw)
self.assertFalse(out["ok"])
self.assertIn("reason", out)
self.assertIn("error", out)
# --------------------------------------------------------------------------------------
def closed_platform():
"""B1: a fake closed platform. It subclasses the host's own platform class (on Windows a
`WindowsPlatform`), so path handling stays the host's and `--binary`, `--run` and
`script_path()` work; it reports itself as WSL Linux and unsupported."""
host = type(gremlins.PLATFORM)
class ClosedPlatform(host):
supported = False
def is_wsl(self):
return True
def describe(self):
return {"name": "posix", "sys_platform": "linux", "wsl": True, "supported": False,
"detach": "setsid"}
return ClosedPlatform()
class GateTests(Base):
"""Digest §4 with amendments B1, A1, A7, A12: the Windows-only gate."""
def setUp(self):
super().setUp()
os.environ.pop(gremlins.UNSUPPORTED_OK_ENV, None) # C13: the gate tests run without it
self.cwd = os.path.join(self.tmp, "proj")
os.makedirs(self.cwd)
os.makedirs(os.path.join(self.tmp, "reports"))
self.run_log = os.path.join(self.tmp, "reports", "2026-10-16-20-g-run.md")
self.state = gremlins.state_dir_for(self.run_log)
p = mock.patch.object(gremlins, "USER_SETTINGS_PATH", os.path.join(self.tmp, "home", "settings.json"))
p.start()
self.addCleanup(p.stop)
self.stub = make_stub_binary(self.tmp)
def close_gate(self):
p = mock.patch.object(gremlins, "PLATFORM", closed_platform())
p.start()
self.addCleanup(p.stop)
def init(self):
return call(["init", "--run", self.run_log, "--model", "claude-opus-5-5[1m]", "--cwd", self.cwd,
"--binary", self.stub])
def test_supported_flags(self):
self.assertIs(gremlins.WindowsPlatform().supported, True)
for sp in ("linux", "darwin"):
with self.subTest(sys_platform=sp), mock.patch.object(sys, "platform", sp):
pp = gremlins.PosixPlatform()
self.assertIs(pp.supported, False)
self.assertIs(pp.describe()["supported"], False) # A12: read from `supported`
self.assertEqual(pp.describe()["sys_platform"], sp)
def test_is_wsl(self):
pp = gremlins.PosixPlatform()
os.environ["WSL_DISTRO_NAME"] = "Ubuntu-20.04"
self.assertTrue(pp.is_wsl())
os.environ.pop("WSL_DISTRO_NAME")
with mock.patch("builtins.open", mock.mock_open(read_data=b"5.15.90.1-microsoft-standard-WSL2")):
self.assertTrue(pp.is_wsl())
with mock.patch("builtins.open", mock.mock_open(read_data=b"6.8.0-45-generic")):
self.assertFalse(pp.is_wsl())
with mock.patch("builtins.open", side_effect=OSError("no /proc")):
self.assertFalse(pp.is_wsl())
def test_gate_reason(self):
"""A7: no version number; names the platform and WSL; A12: read from describe()."""
if gremlins.PLATFORM.supported:
self.assertIsNone(gremlins.platform_gate())
self.close_gate()
self.assertEqual(gremlins.platform_gate(),
"spire:gremlins is not tested on linux, WSL yet; use fresh agents for every "
"block (see SKILL.md, Platforms).")
os.environ[gremlins.UNSUPPORTED_OK_ENV] = "1"
self.assertIsNone(gremlins.platform_gate())
self.assertTrue(gremlins.gate_overridden())
def test_host_where(self):
"""On the real host without the override env: Windows is supported, no override."""
code, out, raw = call(["where", "--binary", self.stub])
self.assert_one_json_line(raw)
if WIN_PLATFORM:
self.assertEqual(code, 0, out)
self.assertTrue(out["platform"]["supported"])
self.assertFalse(out["platform"]["override"])
self.assertEqual(out["warnings"], [])
def test_closed_where(self):
self.close_gate()
code, out, raw = call(["where", "--binary", self.stub])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertFalse(out["ok"])
self.assertIn("not tested on linux, WSL", out["reason"])
self.assertIn("fresh agents", out["reason"])
self.assertEqual(out["error"], out["reason"])
self.assertEqual(out["platform"], {"name": "posix", "sys_platform": "linux", "wsl": True,
"supported": False, "detach": "setsid", "override": False})
# everything else is still filled (the porting hosts need it)
self.assertEqual(out["binary"]["version"], STUB_VERSION)
self.assertNotIn("error", out["binary"])
for k in ("script", "python", "uv", "git", "scrub", "warnings"):
self.assertIn(k, out)
self.assertEqual(out["warnings"], [])
def test_closed_init(self):
self.close_gate()
code, out, raw = self.init()
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertEqual(out["error"], "platform not supported")
self.assertIn("not tested on linux, WSL", out["reason"])
self.assertEqual(out["platform"]["supported"], False)
self.assertEqual(out["platform"]["override"], False)
self.assertFalse(os.path.exists(self.state))
# B1: the gate comes before any path handling (a bad --run is not even looked at)
code, out, _ = call(["init", "--run", os.path.join(self.tmp, "not-a-run.txt"), "--model",
"claude-opus-5-5[1m]", "--cwd", self.cwd, "--binary", self.stub])
self.assertEqual((code, out["error"]), (2, "platform not supported"))
def test_override(self):
self.close_gate()
os.environ[gremlins.UNSUPPORTED_OK_ENV] = "1"
code, out, raw = call(["where", "--binary", self.stub])
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertTrue(out["ok"])
self.assertTrue(out["platform"]["override"])
self.assertFalse(out["platform"]["supported"])
self.assertEqual(out["warnings"], [gremlins.GATE_OVERRIDE_WARNING])
self.assertIn(gremlins.UNSUPPORTED_OK_ENV, out["warnings"][0])
code, out, _ = self.init()
self.assertEqual(code, 0, out)
self.assertTrue(out["created"])
self.assertTrue(os.path.isfile(gremlins.registry_path(self.state)))
def test_closed_gate_on_existing_registry(self):
"""A1: a registry made with the gate open; with it closed, nothing launches and no job
entry is written; the tick launches no ping (it reads nothing).
The registry is made with the override env set, so the gate is open on any host (on a
supported one the env is ignored); the env is gone again before the gate is closed."""
with mock.patch.dict(os.environ, {gremlins.UNSUPPORTED_OK_ENV: "1"}):
self.assertIsNone(gremlins.platform_gate())
code, out, _ = self.init()
self.assertEqual(code, 0, out)
self.assertNotIn(gremlins.UNSUPPORTED_OK_ENV, os.environ)
before = gremlins.read_registry(self.state)
brief = os.path.join(self.tmp, "b.md")
with open(brief, "w") as f:
f.write("x\n")
self.close_gate()
for argv in (["spawn", "--base", "B1", "--block", "1.1", "--brief", brief],
["train", "--base", "B1", "--brief", brief],
["continue", "--job", "j0001", "--brief", brief],
["rebase", "--base", "B1", "--as", "B2", "--brief", brief],
["tick"]):
with self.subTest(cmd=argv[0]):
code, out, raw = call([argv[0], "--run", self.run_log, *argv[1:]])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertEqual(out["error"], "platform not supported")
self.assertIn("not tested on linux, WSL", out["reason"])
self.assertEqual(out["platform"]["supported"], False)
after = gremlins.read_registry(self.state)
self.assertEqual(after["jobs"], {})
self.assertEqual(after, before)
with mock.patch.object(gremlins, "read_registry", side_effect=AssertionError("tick read the registry")):
self.assertEqual(gremlins.tick_pass(self.state), {"pinged": [], "bases": []})
# list and wait still work on a closed gate
self.assertEqual(call(["list", "--run", self.run_log])[0], 0)
code, out, _ = call(["wait", "--run", self.run_log, "--all", "--timeout", "1"])
self.assertEqual((code, out["pings"]), (0, []))
# --------------------------------------------------------------------------------------
@unittest.skipUnless(shutil.which("uv"), "uv not on PATH")
class SubprocessTests(Base):
def run_uv(self, *argv):
return subprocess.run(["uv", "run", GREMLINS_PY, *argv], cwd=self.tmp, env=child_env(),
capture_output=True, timeout=180)
def test_where_via_uv_stub(self):
stub = make_stub_binary(self.tmp)
r = self.run_uv("where", "--binary", stub)
self.assertEqual(r.returncode, 0, r.stderr)
out = self.assert_one_json_line(r.stdout.decode("utf-8"))
self.assertTrue(out["ok"])
self.assertEqual(out["platform"]["detach"], "cim-hidden" if WIN_PLATFORM else "setsid")
self.assertNotIn("spawn", out)
self.assertEqual(out["binary"]["version"], STUB_VERSION)
self.assertEqual(out["binary"]["how"], "--binary")
self.assertEqual(os.path.normcase(out["script"]), os.path.normcase(os.path.abspath(GREMLINS_PY)))
for k in ("python", "uv", "git", "scrub"):
self.assertIn(k, out)
for k in ("drop", "keep", "pinned", "unknown"):
self.assertIn(k, out["scrub"])
self.assertIn("base_executable", out["python"])
self.assertIsNotNone(out["git"]["path"])
def test_where_via_uv_host_binary(self):
if not (os.environ.get("CLAUDE_CODE_EXECPATH") or shutil.which("claude")):
self.skipTest("no claude binary on this host")
r = self.run_uv("where")
self.assertEqual(r.returncode, 0, r.stderr)
self.assertTrue(self.assert_one_json_line(r.stdout.decode("utf-8"))["ok"])
def test_error_case_stdout_invariant(self):
run_log = os.path.join(self.tmp, "x-run.md")
r = self.run_uv("init", "--run", run_log, "--model", "opus", "--cwd", self.tmp)
self.assertEqual(r.returncode, 2, r.stderr)
out = self.assert_one_json_line(r.stdout.decode("utf-8"))
self.assertIn("error", out)
self.assertFalse(os.path.exists(gremlins.state_dir_for(run_log)))
def test_unknown_command_exit_2(self):
r = self.run_uv("frobnicate")
self.assertEqual(r.returncode, 2, r.stderr)
self.assertEqual(self.assert_one_json_line(r.stdout.decode("utf-8"))["error"], "usage")
# --------------------------------------------------------------------------------------
# Launch and supervise (block 1.2): a stream-json stub `claude`, the real detached spawn.
# --------------------------------------------------------------------------------------
TEST_MODEL = "claude-opus-5-5[1m]"
PRICE = {"in": 5e-6, "out": 25e-6, "cw": 10e-6, "cr": 0.5e-6}
# The stub `claude`. Behaviour is chosen per call by `@stub key[=value] ...` lines in the prompt
# (the brief reaches the stub on stdin). Sessions are kept next to the stub so that resume and
# fork see the cumulative modelUsage / total_cost_usd of the session, like the real CLI.
STUB_PY = r'''
import json, os, subprocess, sys, time, uuid
HERE = os.path.dirname(os.path.abspath(__file__))
SESS = os.path.join(HERE, "sessions")
CALLS = os.path.join(HERE, "calls")
VERSION = "%(version)s"
PRICE = %(price)r
ENV_KEYS = ("CLAUDE_CODE_EXECPATH", "SPIRE_GREMLIN_CHILD", "CLAUDE_CODE_PROMPT_CACHE_TTL",
"DISABLE_AUTOUPDATER", "VIRTUAL_ENV", "CLAUDE_EFFORT")
def emit(o):
sys.stdout.write(json.dumps(o) + "\n")
sys.stdout.flush()
def opt(a, name):
if name in a and a.index(name) + 1 < len(a):
return a[a.index(name) + 1]
return None
def load(sid):
try:
with open(os.path.join(SESS, sid + ".json")) as f:
return json.load(f)
except OSError:
return {"modelUsage": {}, "cost": 0.0, "context": 0}
def save(path, obj):
os.makedirs(os.path.dirname(path), exist_ok=True)
tmp = path + ".%%d.tmp" %% os.getpid()
with open(tmp, "w") as f:
json.dump(obj, f)
os.replace(tmp, path)
def main():
a = sys.argv[1:]
if "--version" in a:
print(VERSION + " (Claude Code stub)")
return 0
prompt = sys.stdin.read()
d = {}
for line in prompt.splitlines():
s = line.strip()
if s.startswith("@stub "):
for tok in s[6:].split():
k, _, v = tok.partition("=")
d[k] = v if v else "1"
model = opt(a, "--model") or "claude-stub"
resume, fork, sid = opt(a, "--resume"), "--fork-session" in a, opt(a, "--session-id")
session = resume if (resume and not fork) else (sid or str(uuid.uuid4()))
prior = load(resume) if resume else {"modelUsage": {}, "cost": 0.0, "context": 0}
rec = {"argv": a, "cwd": os.getcwd(), "pid": os.getpid(), "session": session, "directives": d,
"env": {k: os.environ.get(k) for k in ENV_KEYS}, "prompt": prompt}
if "child" in d:
c = subprocess.Popen([sys.executable, "-I", "-c", "import time; time.sleep(600)"],
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
rec["child_pid"] = c.pid
save(os.path.join(CALLS, "%%d.json" %% os.getpid()), rec)
tools = [t for t in (opt(a, "--tools") or "").split(",") if t]
emit({"type": "system", "subtype": "init", "cwd": os.getcwd(), "session_id": session, "model": model,
"permissionMode": opt(a, "--permission-mode"), "tools": tools, "mcp_servers": [],
"apiKeySource": d.get("apikey", "none"), "claude_code_version": VERSION,
"plugins": [{"name": "spire", "path": "x", "source": "spire@reliquary",
"version": d.get("plugin", "0.2.2")},
{"name": "skill-creator", "source": "skill-creator@official",
"path": "C:\\plugins\\cache\\skill-creator\\" + d.get("nover", "abc123")}],
"skills": sorted(["implementation", "workpad"] + ([d["skill"]] if "skill" in d else [])),
"agents": ["claude", "general-purpose"]})
used = model
if "restricted" in d:
used = "claude-opus-5-5" if d["restricted"] == "1" else d["restricted"]
emit({"type": "system", "subtype": "informational", "level": "warning", "session_id": session,
"content": "Model \"%%s\" is restricted by your organization's settings. Using %%s instead."
%% (model, used)})
if "model" in d:
used = d["model"] # silent substitution
if "silent" in d:
time.sleep(float(d["silent"]))
turns = int(d.get("turns", "3"))
if opt(a, "--max-turns"):
turns = min(turns, int(opt(a, "--max-turns")))
api = d.get("apierr") # api_error_status of the result, e.g. 429
is_error = "error" in d or api is not None
if is_error and "billed" not in d:
turns = 0 # rejected before the model answered: nothing billed
five = d.get("ttl") == "5m" # ttl=5m: cache writes land in ephemeral_5m
ctx = int(prior.get("context") or 0)
tot = {"in": 0, "out": 0, "cw": 0, "cr": 0}
prev = ctx
for i in range(turns):
if i == 0:
mode = d.get("cache", "hit" if resume else "cold")
cr = {"hit": ctx, "partial": ctx // 2, "miss": 0, "cold": 0}[mode]
cw = (ctx - cr) + 1000
else:
cr, cw = prev, 300
inp, out = 2, 40
u = {"input_tokens": inp, "cache_creation_input_tokens": cw, "cache_read_input_tokens": cr,
"cache_creation": {"ephemeral_5m_input_tokens": cw if five else 0,
"ephemeral_1h_input_tokens": 0 if five else cw},
"output_tokens": 1, "service_tier": "standard"}
mid = "msg_stub_" + uuid.uuid4().hex[:12]
for block in ("thinking", "text"): # two events per message id, interim output_tokens
emit({"type": "assistant", "session_id": session,
"message": {"id": mid, "model": model.replace("[1m]", ""), "role": "assistant",
"content": [{"type": block}], "usage": u}})
emit({"type": "rate_limit_event", "rate_limit_info": {"status": "allowed"}})
tot["in"] += inp; tot["out"] += out; tot["cw"] += cw; tot["cr"] += cr
prev = inp + cw + cr
if i == 0 and "hang" in d:
time.sleep(600 if d["hang"] == "1" else float(d["hang"]))
if i == 0 and "gate" in d: # gate=NAME: wait (at most 120 s) for HERE/NAME to exist
t_gate = time.time() + 120
while not os.path.exists(os.path.join(HERE, d["gate"])) and time.time() < t_gate:
time.sleep(0.05)
if i == 0 and "die" in d:
os._exit(int(d.get("code", "3")))
if "sleep" in d:
time.sleep(float(d["sleep"]))
cost = (tot["in"] * PRICE["in"] + tot["out"] * PRICE["out"] + tot["cw"] * PRICE["cw"]
+ tot["cr"] * PRICE["cr"])
mu = json.loads(json.dumps(prior.get("modelUsage") or {}))
m = mu.setdefault(used, {"inputTokens": 0, "outputTokens": 0, "cacheReadInputTokens": 0,
"cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.0,
"contextWindow": 1000000})
m["inputTokens"] += tot["in"]; m["outputTokens"] += tot["out"]
m["cacheReadInputTokens"] += tot["cr"]; m["cacheCreationInputTokens"] += tot["cw"]
m["costUSD"] += cost
total = float(prior.get("cost") or 0.0) + cost
usage = {"input_tokens": tot["in"], "cache_creation_input_tokens": tot["cw"],
"cache_read_input_tokens": tot["cr"],
"output_tokens": tot["out"] + (7 if "mismatch" in d else 0),
"cache_creation": {"ephemeral_1h_input_tokens": 0 if five else tot["cw"],
"ephemeral_5m_input_tokens": tot["cw"] if five else 0}}
denials = ([{"tool_name": "Bash", "tool_use_id": "toolu_stub", "tool_input": {"command": "rm -rf build"}}]
if "deny" in d else [])
errtext = d.get("errtext", "").replace("_", " ")
if is_error and not errtext:
errtext = ("API Error: " + api) if api else "stub error"
save(os.path.join(SESS, session + ".json"), {"modelUsage": mu, "cost": total, "context": prev})
emit({"type": "result", "subtype": "error_during_execution" if "error" in d else "success",
"is_error": is_error, "api_error_status": int(api) if api else None, "num_turns": turns,
"total_cost_usd": total, "usage": usage, "modelUsage": mu, "permission_denials": denials,
"session_id": session, "result": errtext if is_error else "READY"})
return 1 if is_error else int(d.get("exit", "0"))
if __name__ == "__main__":
sys.exit(main())
'''
def stub_interpreter() -> str:
base = getattr(sys, "_base_executable", None)
return base if base and os.path.isfile(base) else sys.executable
def make_stream_stub(d: str) -> str:
"""Generate the stream stub under `d` (never committed). Windows: a `.cmd` wrapper that runs
the base interpreter with -I; POSIX: a sh wrapper."""
sd = os.path.join(d, "stub")
os.makedirs(sd, exist_ok=True)
py = os.path.join(sd, "stub.py")
with open(py, "w", newline="\n") as f:
f.write(STUB_PY % {"version": STUB_VERSION, "price": PRICE})
if IS_WINDOWS:
p = os.path.join(sd, "claude.cmd")
with open(p, "w", newline="\r\n") as f:
f.write('@echo off\n"%s" -I "%s" %%*\n' % (stub_interpreter(), py))
else:
p = os.path.join(sd, "claude")
with open(p, "w", newline="\n") as f:
f.write('#!/bin/sh\nexec "%s" -I "%s" "$@"\n' % (stub_interpreter(), py))
os.chmod(p, 0o755)
return p
def stub_calls(stub: str) -> list[dict]:
d = os.path.join(os.path.dirname(stub), "calls")
out = []
for n in sorted(os.listdir(d)) if os.path.isdir(d) else []:
if n.endswith(".json"):
with open(os.path.join(d, n)) as f:
out.append(json.load(f))
return out
def stub_call_for(stub: str, job_prompt_marker: str) -> dict | None:
for c in stub_calls(stub):
if job_prompt_marker in c.get("prompt", ""):
return c
return None
def price(tokens: dict) -> float:
return (tokens["input"] * PRICE["in"] + tokens["output"] * PRICE["out"]
+ tokens["cache_creation"] * PRICE["cw"] + tokens["cache_read"] * PRICE["cr"])
def visible_windows() -> list[tuple]:
"""Visible top-level windows: [(hwnd, pid, class, title)] (`PLATFORM.visible_windows()`; []
where the platform has no such check)."""
return gremlins.PLATFORM.visible_windows() or []
# P0.1's detector: a visible console window (conhost) or Windows Terminal window. The latter belongs
# to WindowsTerminal.exe, not to the process tree that asked for it, so a tree-scoped check misses it.
CONSOLE_CLASSES = ("ConsoleWindowClass", "CASCADIA_HOSTING_WINDOW_CLASS")
def console_windows() -> list[tuple]:
return [w for w in visible_windows() if w[2] in CONSOLE_CLASSES]
def wait_for(pred, timeout=30.0, step=0.1):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
v = pred()
if v:
return v
time.sleep(step)
return pred()
STATE_DIRS: list[str] = [] # every launch test's state dir, for the module-end process check
class LaunchBase(Base):
"""One run per test: a git repo as cwd, a stream stub as --binary, the real detached spawn."""
def setUp(self):
super().setUp()
for name, val in (("WAIT_POLL_S", 0.2),):
p = mock.patch.object(gremlins, name, val)
p.start()
self.addCleanup(p.stop)
p = mock.patch.object(gremlins, "USER_SETTINGS_PATH", os.path.join(self.tmp, "home", "settings.json"))
p.start()
self.addCleanup(p.stop)
p = mock.patch.object(gremlins, "CLAUDE_PROJECTS_DIR", os.path.join(self.tmp, "home", "projects"))
p.start()
self.addCleanup(p.stop)
self.cwd = os.path.join(self.tmp, "proj")
os.makedirs(self.cwd)
self.git("init", "-q")
self.commit_file("README.md", "hello\n", "initial")
os.makedirs(os.path.join(self.tmp, "reports"))
self.run_log = os.path.join(self.tmp, "reports", "2026-10-16-12-t-run.md")
self.state = gremlins.state_dir_for(self.run_log)
STATE_DIRS.append(self.state)
self.stub = make_stream_stub(self.tmp)
self.addCleanup(self.cleanup_processes)
self.nbrief = 0
# --- helpers ---
def git(self, *args):
r = subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.cwd,
capture_output=True, timeout=60)
self.assertEqual(r.returncode, 0, r.stderr)
return r.stdout.decode().strip()
def commit_file(self, name, text, msg):
with open(os.path.join(self.cwd, name), "w", newline="\n") as f:
f.write(text)
self.git("add", name)
self.git("commit", "-q", "-m", msg)
def setup_run(self, *extra, orphan_after=600, stall=900, cap=2):
code, out, _ = call(["init", "--run", self.run_log, "--model", TEST_MODEL, "--cwd", self.cwd,
"--binary", self.stub, "--orphan-after", str(orphan_after),
"--stall-after-s", str(stall), "--parallel-cap", str(cap), *extra])
self.assertEqual(code, 0, out)
return out
def brief(self, *directives, text="Do the thing."):
self.nbrief += 1
p = os.path.join(self.tmp, "brief-%d.md" % self.nbrief)
marker = "BRIEF-MARKER-%d" % self.nbrief
with open(p, "w", newline="\n") as f:
f.write("%s\n%s\n" % (marker, text))
if directives:
f.write("@stub %s\n" % " ".join(directives))
return p
def cmd(self, *argv, code=0):
c, out, raw = call([argv[0], "--run", self.run_log, *argv[1:]])
self.assert_one_json_line(raw)
if code is not None:
self.assertEqual(c, code, out)
return out
def train(self, base="B1", *directives):
out = self.cmd("train", "--base", base, "--brief", self.brief(*directives))
self.assertEqual(set(out), {"job", "base", "session_id"})
return out
def spawn(self, *directives, base="B1", block="1.1", extra=(), code=0):
return self.cmd("spawn", "--base", base, "--block", block, "--brief", self.brief(*directives),
*extra, code=code)
def wait(self, *jobs, timeout=60, any_=False):
argv = ["wait", "--run", self.run_log, "--timeout", str(timeout)]
argv += list(jobs) if jobs else ["--all"]
if any_:
argv.append("--any")
code, out, raw = call(argv)
self.assert_one_json_line(raw)
return code, out
def finish(self, job, timeout=60):
"""Wait for one job and return its result (from `done`)."""
code, out = self.wait(job, timeout=timeout)
self.assertIn(code, (0, 4), out)
got = [r for r in out["done"] if r["job"] == job]
self.assertEqual(len(got), 1, out)
return got[0]
def reg(self):
return gremlins.read_registry(self.state)
def runtime(self, job):
return gremlins.read_json(gremlins.job_file(self.state, job)) or {}
def ledger(self):
p = gremlins.ledger_path(self.state)
if not os.path.exists(p):
return []
with open(p, "rb") as f:
return [json.loads(x) for x in f.read().decode("utf-8").splitlines() if x.strip()]
def until_running(self, job, need_child=False):
"""Wait until the job's claude runs and the stub (started by a `spawn_marked` brief) has
recorded itself, with its grandchild when `need_child`."""
def ready():
rt = self.runtime(job)
if not rt.get("claude"):
return None
c = stub_call_for(self.stub, "RUN-%s" % job)
if not c or (need_child and not c.get("child_pid")):
return None
return rt
rt = wait_for(ready, timeout=30)
self.assertTrue(rt, "job %s never got its claude running" % job)
return rt
def assert_tree_dead(self, recs, timeout=15):
wait_for(lambda: not any(gremlins.record_alive(r) for r in recs), timeout=timeout)
alive = [r for r in recs if gremlins.record_alive(r)]
self.assertEqual(alive, [], "processes still alive: %r" % alive)
def cleanup_processes(self):
reg = None
with contextlib.suppress(Exception):
reg = gremlins.read_registry(self.state)
for job, e in ((reg or {}).get("jobs") or {}).items():
rt = self.runtime(job)
recs = [r for r in (rt.get("supervisor"), e.get("supervisor"), rt.get("claude")) if r]
if recs:
gremlins.kill_tree(recs[0], extra=recs[1:])
def spawn_marked(self, *directives, **kw):
"""A spawn whose brief carries `RUN-<job>` so the stub call can be matched to the job."""
nxt = gremlins._next_job_id(self.reg())
p = self.brief(*directives, text="RUN-%s" % nxt)
out = self.cmd("spawn", "--base", kw.get("base", "B1"), "--block", kw.get("block", "1.1"),
"--brief", p, *kw.get("extra", ()), code=kw.get("code", 0))
if kw.get("code", 0) == 0:
self.assertEqual(out["job"], nxt)
return out
class RuleTests(Base):
"""Pure functions: the P0.3 usage rule (real numbers from the P0.2 seq-1 forked session), C5, C1."""
BASE_MU = {"claude-opus-5-5": {"inputTokens": 14, "outputTokens": 7250, "cacheReadInputTokens": 361690,
"cacheCreationInputTokens": 176553, "costUSD": 1.629818}}
SPAWN_RESULT = {
"usage": {"input_tokens": 14, "cache_creation_input_tokens": 3113, "cache_read_input_tokens": 1288193,
"output_tokens": 1448,
"cache_creation": {"ephemeral_1h_input_tokens": 3113, "ephemeral_5m_input_tokens": 0}},
"modelUsage": {"claude-opus-5-5": {"inputTokens": 28, "outputTokens": 8698,
"cacheReadInputTokens": 1649883,
"cacheCreationInputTokens": 179666, "costUSD": 1.9413766}},
"total_cost_usd": 1.9413766}
def test_usage_rule_real_spawn(self):
u = gremlins.usage_from_result(self.SPAWN_RESULT, self.BASE_MU, 1.629818)
self.assertEqual(u["tokens"], {"input": 14, "output": 1448, "cache_creation": 3113,
"cache_creation_1h": 3113, "cache_creation_5m": 0,
"cache_read": 1288193})
self.assertEqual(u["models_used"], ["claude-opus-5-5"])
self.assertFalse(u["mismatch"])
self.assertAlmostEqual(u["cost_delta"], 0.311559, places=6)
def test_usage_rule_mismatch_and_new_model(self):
res = json.loads(json.dumps(self.SPAWN_RESULT))
res["usage"]["output_tokens"] += 1
self.assertTrue(gremlins.usage_from_result(res, self.BASE_MU, 1.629818)["mismatch"])
# a model that did not grow is not "used"; a train has no prior totals
res = json.loads(json.dumps(self.SPAWN_RESULT))
res["modelUsage"]["claude-haiku-5-5"] = {"inputTokens": 0, "outputTokens": 0}
self.assertEqual(gremlins.usage_from_result(res, self.BASE_MU, 0)["models_used"], ["claude-opus-5-5"])
self.assertTrue(gremlins.usage_from_result(self.SPAWN_RESULT, {}, 0)["mismatch"])
def test_cache_verdict(self):
u = lambda cr: {"cache_read_input_tokens": cr}
self.assertEqual(gremlins.cache_verdict(u(182202), 182204), "hit")
self.assertEqual(gremlins.cache_verdict(u(900), 1000), "hit")
self.assertEqual(gremlins.cache_verdict(u(899), 1000), "partial")
self.assertEqual(gremlins.cache_verdict(u(7469), 92877), "partial") # P0.6 base B
self.assertEqual(gremlins.cache_verdict(u(0), 1000), "miss")
self.assertIsNone(gremlins.cache_verdict(None, 1000))
def test_warmth(self):
cfg = {"ttl_s": 3600}
t0 = 1_800_000_000.0
base = {"last_hit": gremlins._iso(t0)}
self.assertEqual(gremlins.base_warmth(base, cfg, t0 + 10)["base_state"], "warm")
self.assertEqual(gremlins.base_warmth(base, cfg, t0 + 3600 - 361)["base_state"], "warm")
self.assertEqual(gremlins.base_warmth(base, cfg, t0 + 3600 - 359)["base_state"], "expiring")
self.assertEqual(gremlins.base_warmth(base, cfg, t0 + 3601)["base_state"], "cold")
self.assertEqual(gremlins._ts(gremlins.base_warmth(base, cfg, t0)["warm_until"]), t0 + 3600)
short = {"ttl_s": 300}
self.assertEqual(gremlins.base_warmth(base, short, t0 + 300 - 121)["base_state"], "warm")
self.assertEqual(gremlins.base_warmth(base, short, t0 + 300 - 119)["base_state"], "expiring")
self.assertEqual(gremlins.base_warmth({}, cfg, t0)["base_state"], "cold")
def test_api_error_reason(self):
"""I1: API errors get their own reason, from api_error_status or the result text."""
f = gremlins.api_error_reason
self.assertIsNone(f(None))
self.assertIsNone(f({"is_error": False, "result": "we talked about the rate limit"}))
self.assertIsNone(f({"is_error": True, "subtype": "error_during_execution", "result": "boom"}))
cases = [
({"api_error_status": 429, "result": "x"}, "usage_limit"),
({"api_error_status": "429", "result": "x"}, "usage_limit"),
({"result": "Claude AI usage limit reached|1760000000"}, "usage_limit"),
({"result": "API Error: 429 rate_limit_error"}, "usage_limit"),
({"api_error_status": 529, "result": "x"}, "overloaded"),
({"result": 'API Error: 529 {"type":"error","error":{"type":"overloaded_error"}}'}, "overloaded"),
({"api_error_status": 400, "result": "Prompt is too long"}, "prompt_too_long"),
({"api_error_status": 500, "result": "Internal server error"}, "api_error"),
({"result": "API Error: 502 bad gateway"}, "api_error"),
]
for res, want in cases:
with self.subTest(res=res):
got = f(dict(res, is_error=True))
self.assertIsNotNone(got)
self.assertEqual(got[0], want)
self.assertIn("429", f({"is_error": True, "api_error_status": 429, "result": "x"})[1])
def test_sup_clock_jump(self):
"""M3: a suspend between supervisor ticks is not run time and re-arms the orphan watchdog."""
c = gremlins.SupClock(1000.0, tick_s=0.25, jump_s=60.0)
for t in (1000.25, 1000.5, 1000.75, 1030.75): # a 30 s stall is not a jump
self.assertIsNone(c.tick(t))
self.assertAlmostEqual(c.elapsed(1030.75), 30.75)
self.assertAlmostEqual(c.tick(8230.75), 7199.75) # 2 h suspend
self.assertAlmostEqual(c.elapsed(8231.0), 31.25) # 30.75 + one tick of the gap + 0.25
last = 990.0 # the last orchestrator call, before it
self.assertAlmostEqual(c.orphan_age(8231.0, last), 0.25) # re-armed at the jump
self.assertGreater(c.orphan_age(8230.75 + 1201, last), 1200) # one more orphan_after
self.assertAlmostEqual(c.orphan_age(9000.0, 8900.0), 100.0) # later calls count as usual
def test_stream_state(self):
ss = gremlins.StreamState()
u1 = {"input_tokens": 2, "cache_creation_input_tokens": 10, "cache_read_input_tokens": 100}
u2 = {"input_tokens": 2, "cache_creation_input_tokens": 5, "cache_read_input_tokens": 112}
for ev in ({"type": "system", "subtype": "init", "session_id": "s1", "tools": ["Read", "Bash"],
"permissionMode": "dontAsk", "apiKeySource": "none",
"plugins": [{"name": "spire", "version": "0.2.2"}], "agents": ["claude"]},
{"type": "assistant", "message": {"id": "m1", "usage": u1}},
{"type": "assistant", "message": {"id": "m1", "usage": u1}},
{"type": "system", "subtype": "informational", "level": "warning",
"content": "Model \"x\" is restricted by your organization's settings."},
{"type": "assistant", "message": {"id": "m2", "usage": u2}},
{"type": "result", "num_turns": 2}):
ss.feed(ev, 5.0)
self.assertEqual((ss.session_id, ss.turns), ("s1", 2))
self.assertEqual(ss.first_usage, u1)
self.assertEqual(gremlins.context_tokens(ss.last_usage), 119)
self.assertIn("is restricted", ss.refused)
rt = gremlins.runtime_fingerprint(ss.init)
self.assertEqual(rt["plugins"], ["spire@0.2.2"])
other = dict(rt, plugins=["spire@0.3.0"], apiKeySource="oauth")
self.assertEqual(gremlins.runtime_drift(other, rt, {}), ["apiKeySource", "plugins"])
self.assertEqual(gremlins.runtime_drift(rt, None, {"permission_mode": "default"}), ["permissionMode"])
def test_runtime_fingerprint_skills_and_versionless_plugins(self):
"""M6: sorted `skills` are compared; a plugin without a version is identified by the last
segment of its path. Both drift."""
init = {"tools": ["Read"], "permissionMode": "dontAsk", "apiKeySource": "none",
"agents": ["claude"], "skills": ["workpad", "implementation"],
"plugins": [{"name": "spire", "version": "0.2.2", "path": "C:\\c\\spire\\0.2.2"},
{"name": "skill-creator",
"path": "C:\\Users\\u\\.claude\\plugins\\cache\\skill-creator\\abc123\\"},
{"name": "builtin"}]}
rt = gremlins.runtime_fingerprint(init)
self.assertEqual(rt["plugins"], ["builtin@", "skill-creator@path:abc123", "spire@0.2.2"])
self.assertEqual(rt["skills"], ["implementation", "workpad"])
self.assertEqual(sorted(rt), sorted(gremlins.RUNTIME_FIELDS))
upd = json.loads(json.dumps(init))
upd["plugins"][1]["path"] = "/home/u/.claude/plugins/cache/skill-creator/def456"
self.assertEqual(gremlins.runtime_drift(gremlins.runtime_fingerprint(upd), rt, {}), ["plugins"])
more = dict(init, skills=["workpad", "implementation", "new-skill"])
self.assertEqual(gremlins.runtime_drift(gremlins.runtime_fingerprint(more), rt, {}), ["skills"])
same = dict(init, skills=["implementation", "workpad"]) # order does not count
self.assertEqual(gremlins.runtime_drift(gremlins.runtime_fingerprint(same), rt, {}), [])
old_ref = {k: v for k, v in rt.items() if k != "skills"} # a base from before M6
self.assertEqual(gremlins.runtime_drift(gremlins.runtime_fingerprint(more), old_ref, {}), [])
def test_note_written_by(self):
"""M11: a note counts only when its mtime is at or after the job's creation (less the
file-time slack)."""
p = os.path.join(self.tmp, "n.md")
start = time.time() - 100
entry = {"created_ts": start, "created_at": gremlins._iso(start)}
self.assertFalse(gremlins.note_written_by(p, entry)) # no file
with open(p, "w") as f:
f.write("note\n")
for delta, want in ((-gremlins.NOTE_MTIME_SLACK_S - 1, False), (-gremlins.NOTE_MTIME_SLACK_S + 0.5, True),
(0, True), (50, True)):
with self.subTest(delta=delta):
os.utime(p, (start + delta, start + delta))
self.assertEqual(gremlins.note_written_by(p, entry), want)
os.utime(p, (start - 60, start - 60))
self.assertFalse(gremlins.note_written_by(p, {"created_at": gremlins._iso(start)})) # ISO fallback
def test_late_rebase_of_a_retired_base(self):
"""N4 (C15), the `expire` race: the old base was retired when the rebase ended ok. The new
base is created `retired` as of the end time; the old base is left as it was."""
ended = gremlins._iso(time.time())
launch = {"cwd": self.tmp, "binary": {"path": "claude", "version": STUB_VERSION}}
old = {"name": "B1", "status": "retired", "retired_at": gremlins._iso(time.time() - 60),
"session": SESS_TRAIN, "ttl_s": 3600}
reg = {"config": {"launch": launch}, "bases": {"B1": old}, "jobs": {}}
entry = {"job": "j0002", "kind": "rebase", "base": "B1", "rebase_as": "B1r", "session": SESS_SPAWN,
"new_launch": launch, "phase": 2}
before = dict(old)
self.assertTrue(gremlins._rebase_target_free(reg, entry))
gremlins._rebase_succeeded(reg, entry, old, None, None, None, "abc1234", ended)
new = reg["bases"]["B1r"]
self.assertEqual((new["status"], new["retired_at"], new["parent"], new["session"]),
("retired", ended, "B1", SESS_SPAWN))
self.assertNotIn("superseded_by", new)
self.assertEqual(old, before)
self.assertEqual(entry["rebase_inactive_parent"], "B1 was retired when the rebase ended")
# the name is now taken: a second success would be a conflict, not a replacement
self.assertFalse(gremlins._rebase_target_free(reg, dict(entry, job="j0003")))
class WarmthTests(Base):
"""Design 5 under an injected clock (block 2.1): warmth boundaries, TTL inference, the ping rule."""
T0 = 1_800_000_000.0 # the base's last hit
E1H = T0 + 3600 - 100 # expiring at 1 h (margin 360)
E5M = T0 + 300 - 60 # expiring at 5 min (margin 120)
def make_reg(self, now, *, pings="on", margin=None, base_ttl=None, status="active", wait_ago=60.0):
reg = gremlins.new_registry()
reg["config"] = {"launch": {}, "ttl_s": 3600, "pings": pings, "warm_margin_s": margin}
reg["bases"]["B1"] = {"name": "B1", "status": status, "session": "s-b1",
"last_hit": gremlins._iso(self.T0), "ttl_s": base_ttl, "ping_in_flight": None}
reg["jobs"]["j0001"] = gremlins._new_entry("j0001", "train", base="B1", status="ok")
if wait_ago is not None:
reg["run"]["last_wait_at"] = gremlins._iso(now - wait_ago)
return reg
def add_job(self, reg, kind, base="B1", **kw):
j = gremlins._next_job_id(reg)
reg["jobs"][j] = gremlins._new_entry(j, kind, base=base, **kw)
return j
def decide(self, reg, now):
return gremlins.ping_decision(reg, "B1", now)
def test_boundaries_1h_and_5m(self):
T0 = self.T0
b = {"last_hit": gremlins._iso(T0)}
def st(base, cfg, t):
return gremlins.base_warmth(base, cfg, t)["base_state"]
cfg = {"ttl_s": 3600}
for t, want in ((T0 + 3239, "warm"), (T0 + 3240, "expiring"), (T0 + 3599, "expiring"),
(T0 + 3600, "cold")):
with self.subTest(ttl=3600, t=t - T0):
self.assertEqual(st(b, cfg, t), want)
w = gremlins.base_warmth(b, cfg, T0)
self.assertEqual((w["ttl_s"], w["margin_s"], gremlins._ts(w["warm_until"])), (3600, 360, T0 + 3600))
# 5 min: inferred on the base (config still 3600), and configured
for base, c in (({**b, "ttl_s": 300}, cfg), (b, {"ttl_s": 300})):
for t, want in ((T0 + 179, "warm"), (T0 + 180, "expiring"), (T0 + 299, "expiring"),
(T0 + 300, "cold")):
with self.subTest(ttl=300, base_ttl=base.get("ttl_s"), t=t - T0):
self.assertEqual(st(base, c, t), want)
w = gremlins.base_warmth(base, c, T0)
self.assertEqual((w["ttl_s"], w["margin_s"], gremlins._ts(w["warm_until"])), (300, 120, T0 + 300))
# init --warm-margin-s overrides the formula, at either TTL
m = {"ttl_s": 3600, "warm_margin_s": 900}
for t, want in ((T0 + 2699, "warm"), (T0 + 2700, "expiring"), (T0 + 3600, "cold")):
with self.subTest(margin=900, t=t - T0):
self.assertEqual(st(b, m, t), want)
self.assertEqual(gremlins.base_warmth(b, m, T0)["margin_s"], 900)
# Minor 2: at a 5-minute TTL the 900 s margin is capped at ttl/2 = 150 s
for t, want in ((T0, "warm"), (T0 + 149, "warm"), (T0 + 150, "expiring"), (T0 + 299, "expiring"),
(T0 + 300, "cold")):
with self.subTest(margin=900, base_ttl=300, t=t - T0):
self.assertEqual(st({**b, "ttl_s": 300}, m, t), want)
self.assertEqual(gremlins.base_warmth({**b, "ttl_s": 300}, m, T0)["margin_s"], 150)
self.assertEqual(st({}, cfg, T0), "cold")
def test_margin_capped_at_half_the_ttl(self):
"""Minor 2: a configured margin above ttl/2 is capped there, so a 5-minute downgrade does not
leave a base `expiring` (and pinged every tick) right after every hit."""
T0 = self.T0
b = {"last_hit": gremlins._iso(T0), "ttl_s": 300}
w = gremlins.base_warmth(b, {"ttl_s": 3600, "warm_margin_s": 600}, T0 + 1)
self.assertEqual((w["base_state"], w["margin_s"], w["ttl_s"]), ("warm", 150, 300))
reg = self.make_reg(T0 + 1, margin=600, base_ttl=300)
self.assertEqual(self.decide(reg, T0 + 1)["why"], "state:warm")
self.assertTrue(self.decide(reg, T0 + 150)["ping"])
# a margin below ttl/2 is used as given; the formula applies when none is set
self.assertEqual(gremlins.base_warmth(b, {"ttl_s": 3600, "warm_margin_s": 100}, T0)["margin_s"], 100)
self.assertEqual(gremlins.base_warmth(b, {"ttl_s": 3600, "warm_margin_s": None}, T0)["margin_s"], 120)
def test_infer_ttl_and_downgrade(self):
def t(h=0, m=0, r=0):
return {"cache_creation_1h": h, "cache_creation_5m": m, "cache_read": r}
f = gremlins.infer_ttl
self.assertEqual(f(t(h=100, r=5), None), 3600) # 1h-only
self.assertEqual(f(t(h=100), 300), 3600)
self.assertEqual(f(t(m=100, r=5), 3600), 300) # 5m-only
self.assertEqual(f(t(h=1, m=1), 300), 3600) # both written: 1 h
self.assertEqual(f(t(r=500), 3600), 3600) # read-only keeps the known TTL
self.assertEqual(f(t(r=500), 300), 300)
self.assertIsNone(f(t(r=500), None)) # none known: the config's
self.assertEqual(f(t(r=500), None, 3600), 3600)
self.assertEqual(f(t(), 300), 300)
self.assertEqual(f(None, 300), 300)
self.assertTrue(gremlins.wrote_only_5m(t(m=100)))
self.assertFalse(gremlins.wrote_only_5m(t(h=1, m=1)))
self.assertFalse(gremlins.wrote_only_5m(t(h=100)))
self.assertFalse(gremlins.wrote_only_5m(t(r=5)))
self.assertFalse(gremlins.wrote_only_5m(None))
# warmth with no inferred TTL falls back to the configured one
b = {"last_hit": gremlins._iso(self.T0), "ttl_s": None}
self.assertEqual(gremlins.base_warmth(b, {"ttl_s": 300}, self.T0)["ttl_s"], 300)
def test_break_even(self):
self.assertEqual(gremlins.break_even(3600), 19)
self.assertEqual(gremlins.break_even(300), 11)
def test_ping_rule_states_status_and_pings_off(self):
now = self.E1H
d = self.decide(self.make_reg(now), now)
self.assertTrue(d["ping"], d)
self.assertIsNone(d["why"])
self.assertEqual((d["base_state"], d["break_even"], d["pings_since_real_call"]), ("expiring", 19, 0))
for t, why in ((self.T0 + 100, "state:warm"), (self.T0 + 3601, "state:cold")):
d = self.decide(self.make_reg(t), t)
self.assertEqual((d["ping"], d["why"]), (False, why))
for status in ("training", "retired", "superseded", "failed"):
d = self.decide(self.make_reg(now, status=status), now)
self.assertEqual((d["ping"], d["why"]), (False, "status:%s" % status))
d = self.decide(self.make_reg(now, pings="off"), now)
self.assertEqual((d["ping"], d["why"]), (False, "pings_off"))
# 5 min TTL inferred on the base, margin from init
d = self.decide(self.make_reg(self.E5M, base_ttl=300), self.E5M)
self.assertEqual((d["ping"], d["ttl_s"], d["break_even"]), (True, 300, 11))
# init's margin, capped at ttl/2 (Minor 2): 3600 acts as 1800 at a 1 h TTL
t = self.T0 + 1800
self.assertTrue(self.decide(self.make_reg(t, margin=3600), t)["ping"])
t = self.T0 + 1799
self.assertEqual(self.decide(self.make_reg(t, margin=3600), t)["why"], "state:warm")
def test_ping_rule_activity_window(self):
now = self.E1H
self.assertTrue(self.decide(self.make_reg(now, wait_ago=599), now)["ping"])
for ago in (601, None):
d = self.decide(self.make_reg(now, wait_ago=ago), now)
self.assertEqual((d["ping"], d["why"]), (False, "no_activity"))
reg = self.make_reg(now, wait_ago=None)
self.add_job(reg, "train", base="B2", status="running") # not a spawn/continue
self.add_job(reg, "ping", base="B2", status="running")
self.assertEqual(self.decide(reg, now)["why"], "no_activity")
for kind in ("spawn", "continue"):
with self.subTest(kind=kind):
reg = self.make_reg(now, wait_ago=None)
j = self.add_job(reg, kind, base="B2", status="running")
self.assertTrue(self.decide(reg, now)["ping"]) # running now
reg["jobs"][j].update(status="ok", ended_at=gremlins._iso(now - 590))
self.assertTrue(self.decide(reg, now)["ping"]) # ended 590 s ago
reg["jobs"][j]["ended_at"] = gremlins._iso(now - 610)
self.assertEqual(self.decide(reg, now)["why"], "no_activity")
def test_ping_cap_19_and_11(self):
for now, base_ttl, cap in ((self.E1H, None, 19), (self.E5M, 300, 11)):
with self.subTest(cap=cap):
reg = self.make_reg(now, base_ttl=base_ttl)
for _ in range(cap - 1):
self.add_job(reg, "ping", status="ok")
d = self.decide(reg, now)
self.assertEqual((d["ping"], d["pings_since_real_call"], d["break_even"]), (True, cap - 1, cap))
self.add_job(reg, "ping", status="failed") # failed pings count too
d = self.decide(reg, now)
self.assertEqual((d["ping"], d["why"]), (False, "cap:%d/%d" % (cap, cap)))
self.add_job(reg, "ping", base="B2", status="ok") # another base's pings do not
self.assertEqual(self.decide(reg, now)["pings_since_real_call"], cap)
# a real call on the base starts the count again
self.add_job(reg, "spawn", status="ok", ended_at=gremlins._iso(now - 5000))
d = self.decide(reg, now)
self.assertEqual((d["ping"], d["pings_since_real_call"]), (True, 0))
self.add_job(reg, "ping", status="ok")
self.assertEqual(self.decide(reg, now)["pings_since_real_call"], 1)
def test_ping_in_flight(self):
now = self.E1H
reg = self.make_reg(now)
j = self.add_job(reg, "ping", status="running")
reg["bases"]["B1"]["ping_in_flight"] = j
d = self.decide(reg, now)
self.assertEqual((d["ping"], d["why"], d["ping_in_flight"]), (False, "in_flight:%s" % j, j))
reg["jobs"][j]["status"] = "lost" # a stale mark (its job ended) does not block
d = self.decide(reg, now)
self.assertEqual((d["ping"], d["ping_in_flight"]), (True, None))
def test_pings_not_in_cap(self):
reg = self.make_reg(self.E1H)
self.add_job(reg, "ping", status="running")
self.assertEqual(gremlins._running_capped(reg), [])
f = self.add_job(reg, "spawn", status="running")
self.assertEqual(gremlins._running_capped(reg), [f])
class TickTests(Base):
"""`tick_pass` on a real state dir and lock; the detach and the binary check are stubbed, so no
process is started (C3d dedupe, the `tick` command, N3)."""
def setUp(self):
super().setUp()
os.makedirs(os.path.join(self.tmp, "reports"))
self.run_log = os.path.join(self.tmp, "reports", "2026-10-16-16-t-run.md")
self.state = gremlins.state_dir_for(self.run_log)
gremlins.ensure_state(self.state)
now = time.time()
self.stub = make_stub_binary(self.tmp)
self.binfo = {"path": self.stub, "version": STUB_VERSION, "size": 0, "mtime": 0}
launch = {"model": TEST_MODEL, "effort": "high", "permission_mode": "dontAsk", "tools": ["Read"],
"allowed_tools": ["Read"], "disallowed_tools": [],
"mcp": {"strict": True, "config": gremlins.mcp_empty_path(self.state)},
"append_system_prompt": {"path": gremlins.preamble_path(), "sha256": "x"}, "agent": None,
"cwd": self.tmp, "binary": self.binfo, "env": {"pinned": {}}}
reg = gremlins.new_registry()
reg["config"] = {"launch": launch, "ttl_s": 3600, "pings": "on", "warm_margin_s": None,
"max_minutes": 90}
reg["bases"]["B1"] = {"name": "B1", "status": "active", "session": "s-b1",
"last_hit": gremlins._iso(now - 3500), "cost_cum_usd": 1.25,
"model_usage_cum": {TEST_MODEL: {"inputTokens": 1}}, "context_tokens": 1000}
reg["jobs"]["j0001"] = gremlins._new_entry("j0001", "train", base="B1", status="ok")
reg["run"]["last_wait_at"] = gremlins._iso(now - 60)
with gremlins.RegistryLock(self.state):
gremlins.write_registry(self.state, reg)
p = subprocess.Popen([sys.executable, "-c", "pass"])
dead = gremlins.pid_record(p.pid)
p.wait()
self.spawned = []
def fake_detach(state, run, job):
self.spawned.append((run, job))
return {"ok": True, "method": "test", "pid": dead["pid"], "ctime": dead["ctime"]}
for name, fn in (("detach_supervisor", fake_detach), ("_check_binary", lambda ref, la: (self.binfo, []))):
pt = mock.patch.object(gremlins, name, fn)
pt.start()
self.addCleanup(pt.stop)
def reg(self):
return gremlins.read_registry(self.state)
def test_two_tickers_one_ping(self):
"""C3(d): two tickers that both judged the base eligible (outside the lock) launch one ping;
the second skips the base as in flight."""
barrier = threading.Barrier(2, timeout=10)
def check(ref, la):
barrier.wait() # both are past the unlocked first look
return self.binfo, []
results, errors = [], []
def ticker():
try:
results.append(gremlins.tick_pass(self.state))
except Exception as e: # surfaced below
errors.append(e)
with mock.patch.object(gremlins, "_check_binary", check):
ths = [threading.Thread(target=ticker) for _ in range(2)]
for th in ths:
th.start()
for th in ths:
th.join(30)
self.assertEqual(errors, [])
self.assertEqual(len(results), 2)
pinged = [j for r in results for j in r["pinged"]]
self.assertEqual(len(pinged), 1, results)
job = pinged[0]
loser = [r for r in results if not r["pinged"]][0]
self.assertEqual(loser["bases"][0]["why"], "in_flight:%s" % job)
reg = self.reg()
self.assertEqual([j for j, e in reg["jobs"].items() if e["kind"] == "ping"], [job])
self.assertEqual(reg["bases"]["B1"]["ping_in_flight"], job)
self.assertEqual(self.spawned, [(gremlins.norm_path(self.run_log), job)])
e = reg["jobs"][job]
self.assertEqual((e["status"], e["base"], e["parent_session"], e["base_state_at_launch"]),
("running", "B1", "s-b1", "expiring"))
self.assertEqual((e["cost_before"], e["base_context"], e["max_s"]), (1.25, 1000, gremlins.PING_MAX_S))
a = e["argv"]
self.assertEqual(a[a.index("--resume") + 1], "s-b1")
self.assertIn("--fork-session", a)
self.assertEqual(a[a.index("--session-id") + 1], e["session"])
self.assertEqual(a[a.index("--max-turns") + 1], "1")
self.assertEqual(a[a.index("--model") + 1], TEST_MODEL)
with open(os.path.join(self.state, "briefs", job + ".md"), encoding="utf-8") as f:
self.assertEqual(f.read(), gremlins.PING_PROMPT)
self.assertEqual(gremlins._running_capped(reg), []) # pings are not in the cap
# a third ticker: still in flight
out = gremlins.tick_pass(self.state)
self.assertEqual((out["pinged"], out["bases"][0]["why"]), ([], "in_flight:%s" % job))
# the ping ends (any status): the mark clears, the result says it missed
res = gremlins.finalize_job(self.state, job, ss=None, hint="killed", reason="kill")
self.assertIn("ping_missed", res["flags"])
self.assertEqual(res["kind"], "ping")
self.assertIsNone(self.reg()["bases"]["B1"]["ping_in_flight"])
out = gremlins.tick_pass(self.state)
self.assertEqual(len(out["pinged"]), 1)
self.assertEqual(out["bases"][0]["pings_since_real_call"], 1) # counted before this one
def test_ping_carries_the_phase_and_counts_in_usage_phase(self):
"""I2: a ping records the phase of the latest train/spawn/continue/rebase job on its base
(the base's own phase if none has one), so `usage --phase N` (md and json) lists it and its
cost is in the priced total."""
reg = self.reg()
self.assertIsNone(gremlins.ping_phase(reg, "B1")) # nothing has a phase
reg["bases"]["B1"]["phase"] = 1
self.assertEqual(gremlins.ping_phase(reg, "B1"), 1) # the base's own
reg["jobs"]["j0002"] = gremlins._new_entry("j0002", "spawn", base="B1", phase=2, status="ok",
ended_at=gremlins._iso(time.time() - 4000))
reg["jobs"]["j0003"] = gremlins._new_entry("j0003", "spawn", base="B9", phase=7, status="ok")
reg["jobs"]["j0004"] = gremlins._new_entry("j0004", "continue", base="B1", phase=None, status="ok")
reg["jobs"]["j0005"] = gremlins._new_entry("j0005", "ping", base="B1", phase=5, status="ok")
self.assertEqual(gremlins.ping_phase(reg, "B1"), 2) # latest real job with a phase
with gremlins.RegistryLock(self.state):
gremlins.write_registry(self.state, reg)
out = gremlins.tick_pass(self.state)
self.assertEqual(len(out["pinged"]), 1, out)
job = out["pinged"][0]
self.assertEqual(self.reg()["jobs"][job]["phase"], 2)
# the ping ends ok with a known cost (cost basis: the base's 1.25)
ss = gremlins.StreamState()
t = time.time()
u = {"input_tokens": 1, "cache_read_input_tokens": 1000, "cache_creation_input_tokens": 0}
ss.feed({"type": "assistant", "message": {"id": "m1", "usage": u}}, t)
ss.feed({"type": "result", "is_error": False, "num_turns": 1, "total_cost_usd": 1.2623,
"usage": dict(u, output_tokens=2, cache_creation={}),
"modelUsage": {TEST_MODEL: {"inputTokens": 2, "outputTokens": 2,
"cacheReadInputTokens": 1000, "cacheCreationInputTokens": 0}}},
t)
res = gremlins.finalize_job(self.state, job, ss=ss, hint="exited", exit_code=0, wall_s=1.0)
self.assertEqual((res["status"], res["phase"], res["cost_delta_usd"]), ("ok", 2, 0.0123), res)
line = [x for x in gremlins.read_ledger(self.state) if x["job"] == job][0]
self.assertEqual(line["phase"], 2)
code, js, _ = call(["usage", "--run", self.run_log, "--phase", "2"])
self.assertEqual(code, 0, js)
self.assertEqual([x["job"] for x in js["rows"]], [job])
self.assertEqual(js["kinds"]["ping"]["calls"], 1)
self.assertAlmostEqual(js["kinds"]["ping"]["cost_usd"], 0.0123, places=6)
self.assertAlmostEqual(js["total"]["cost_usd"], 0.0123, places=6)
code, md = call_raw(["usage", "--run", self.run_log, "--phase", "2", "--format", "md"])
self.assertEqual(code, 0, md)
self.assertIn("| ping | 1 | 1 | 2 | 0 | 1,000 | $0.0123 |\n", md)
self.assertIn("| total | 1 | 1 | 2 | 0 | 1,000 | $0.0123 |\n", md)
self.assertTrue(md.endswith("- Priced total: $0.0123\n"), md)
self.assertEqual(call(["usage", "--run", self.run_log, "--phase", "1"])[1]["rows"], [])
def test_tick_command_and_n3(self):
code, out, raw = call(["tick", "--run", self.run_log])
self.assertEqual(code, 0, out)
self.assert_one_json_line(raw)
self.assertEqual(len(out["pinged"]), 1)
self.assertEqual(out["state"], self.state)
self.assertEqual(out["bases"][0]["job"], out["pinged"][0])
os.environ[gremlins.CHILD_MARKER] = "1"
code, out, raw = call(["tick", "--run", self.run_log])
self.assertEqual(code, 2, out)
self.assert_one_json_line(raw)
self.assertIn("inside a gremlin", out["error"])
self.assertEqual(len([e for e in self.reg()["jobs"].values() if e["kind"] == "ping"]), 1)
def test_tick_nothing_to_do_takes_no_lock_and_binary_missing(self):
with gremlins.RegistryLock(self.state):
reg = self.reg()
reg["config"]["pings"] = "off"
gremlins.write_registry(self.state, reg)
with gremlins.RegistryLock(self.state): # held: a pass with nothing to ping must not wait
out = gremlins.tick_pass(self.state)
self.assertEqual((out["pinged"], out["bases"][0]["why"]), ([], "pings_off"))
with gremlins.RegistryLock(self.state):
reg = self.reg()
reg["config"]["pings"] = "on"
gremlins.write_registry(self.state, reg)
def missing(ref, la):
raise gremlins.GremlinsError("claude binary missing", reason="binary_missing")
with mock.patch.object(gremlins, "_check_binary", missing):
out = gremlins.tick_pass(self.state)
self.assertEqual((out["pinged"], out["bases"][0]["why"]), ([], "binary_missing"))
self.assertEqual(self.spawned, [])
self.assertEqual(list(self.reg()["jobs"]), ["j0001"])
def call_raw(argv):
"""Like `call`, but for `usage --format md`: stdout is not JSON. Returns (exit code, raw stdout)."""
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = gremlins.main(list(argv))
return code, out.getvalue()
SONNET = "claude-sonnet-4-5"
SESS_TRAIN = "aaaaaaaa-0000-4000-8000-000000000001"
SESS_SPAWN = "bbbbbbbb-0000-4000-8000-000000000002"
SESS_KILLED = "cccccccc-0000-4000-8000-000000000005"
SESS_PAR = "dddddddd-0000-4000-8000-000000000004"
SESS_PING = "eeeeeeee-0000-4000-8000-000000000006"
BLOCK_HEADER = ("| Block | Owner (model id from modelUsage) | Session | Commit | in | out | cache write "
"| cache read | cost Δ |")
class UsageTests(Base):
"""`usage` against a fixed ledger fixture in a temp state dir (no `claude` call): the md text
exactly, the json totals, null usage, parallel Commit, R1 reconciliation, N3."""
def setUp(self):
super().setUp()
self.cwd = os.path.join(self.tmp, "proj")
os.makedirs(self.cwd)
self.git("init", "-q")
self.sha = []
for i in range(3):
with open(os.path.join(self.cwd, "f.txt"), "w", newline="\n") as f:
f.write("%d\n" % i)
self.git("add", "f.txt")
self.git("commit", "-q", "-m", "c%d" % (i + 1))
self.sha.append(self.git("rev-parse", "HEAD"))
os.makedirs(os.path.join(self.tmp, "reports"))
self.run_log = os.path.join(self.tmp, "reports", "2026-10-16-18-t-run.md")
self.state = gremlins.state_dir_for(self.run_log)
gremlins.ensure_state(self.state)
reg = gremlins.new_registry()
reg["config"] = {"launch": {"cwd": self.cwd}, "ttl_s": 3600, "pings": "on", "max_minutes": 90}
s = self.sha
specs = [ # job, kind, block, phase, mode, session, launch, end, models, tokens, cost, cache, flags
("j0001", "train", None, 1, None, SESS_TRAIN, s[0], s[0], [TEST_MODEL],
(10, 20, 1000, 1000, 0, 0), 0.5, None, []),
("j0002", "spawn", "1.1", 1, "sequential", SESS_SPAWN, s[0], s[2], [TEST_MODEL],
(2, 1500, 3000, 3000, 0, 1234567), 1.2345, "hit", []),
("j0003", "continue", "1.1", 1, "sequential", SESS_SPAWN, s[2], s[2], [TEST_MODEL, SONNET],
(3, 700, 200, 0, 200, 1300000), 0.25, "partial", ["cold_start"]),
("j0004", "spawn", "1.2", 1, "parallel", SESS_PAR, s[0], s[2], [TEST_MODEL],
(1, 100, 0, 0, 0, 50000), 0.1, "hit", []),
("j0005", "spawn", "1.3", 2, "sequential", SESS_KILLED, s[2], None, [],
None, None, None, []),
("j0006", "ping", None, None, None, SESS_PING, None, None, [TEST_MODEL],
(1, 1, 50, 50, 0, 5000), 0.01, "partial", []),
]
self.lines = []
for job, kind, block, phase, mode, sess, head0, head1, models, tok, cost, cache, flags in specs:
reg["jobs"][job] = gremlins._new_entry(job, kind, base="B1", block=block, phase=phase, mode=mode,
cwd=self.cwd, session=sess, status="ok",
head_at_launch=head0)
if head1:
reg["jobs"][job]["head_at_end"] = head1
tokens = ({k: None for k in gremlins.TOKEN_KEYS} if tok is None else
dict(zip(gremlins.TOKEN_KEYS, tok)))
self.lines.append({"job": job, "kind": kind, "phase": phase, "block": block, "base": "B1",
"session": sess, "models_used": models, "tokens": tokens,
"cost_delta_usd": cost, "cache": cache, "flags": flags,
"status": "killed" if tok is None else "ok"})
reg["bases"]["B1"] = {"name": "B1", "status": "active", "session": "s-b1"}
with gremlins.RegistryLock(self.state):
gremlins.write_registry(self.state, reg)
for ln in self.lines:
gremlins.ledger_append(self.state, ln)
def git(self, *args):
r = subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=self.cwd,
capture_output=True, timeout=60)
self.assertEqual(r.returncode, 0, r.stderr)
return r.stdout.decode().strip()
def usage(self, *argv, code=0):
c, out, raw = call(["usage", "--run", self.run_log, *argv])
self.assert_one_json_line(raw)
self.assertEqual(c, code, out)
return out
def usage_md(self, *argv, code=0):
c, raw = call_raw(["usage", "--run", self.run_log, *argv, "--format", "md"])
self.assertEqual(c, code, raw)
return raw
def ledger_lines(self):
with open(gremlins.ledger_path(self.state), "rb") as f:
return [json.loads(x) for x in f.read().decode("utf-8").splitlines() if x.strip()]
# --- md ---
def test_md_block_exact(self):
c2, c3 = self.sha[1][:7], self.sha[2][:7]
self.assertEqual(self.usage_md("--block", "1.1"), "\n".join([
BLOCK_HEADER,
"|---|---|---|---|---|---|---|---|---|",
"| 1.1 | %s | bbbbbbbb | %s, %s | 2 | 1,500 | 3,000 | 1,234,567 | $1.2345 |"
% (TEST_MODEL, c2, c3),
"| 1.1 | %s, %s | bbbbbbbb | — | 3 | 700 | 200 | 1,300,000 | $0.2500 |"
% (TEST_MODEL, SONNET),
""]))
def test_md_block_parallel_commit_is_dash(self):
"""A parallel spawn shows an em dash even though HEAD moved during it (the Athanor commits)."""
self.assertNotEqual(self.sha[0], self.sha[2])
self.assertEqual(self.usage_md("--block", "1.2"), "\n".join([
BLOCK_HEADER, "|---|---|---|---|---|---|---|---|---|",
"| 1.2 | %s | dddddddd | — | 1 | 100 | 0 | 50,000 | $0.1000 |" % TEST_MODEL, ""]))
def test_md_block_null_usage_renders_dash(self):
self.assertEqual(self.usage_md("--block", "1.3"), "\n".join([
BLOCK_HEADER, "|---|---|---|---|---|---|---|---|---|",
"| 1.3 | — | cccccccc | — | — | — | — | — | — |", ""]))
def test_md_block_unknown_is_header_only(self):
self.assertEqual(self.usage_md("--block", "9.9"),
BLOCK_HEADER + "\n|---|---|---|---|---|---|---|---|---|\n")
def test_md_phase_exact(self):
self.assertEqual(self.usage_md("--phase", "1"), "\n".join([
"| Kind | Calls | in | out | cache write | cache read | cost |",
"|---|---|---|---|---|---|---|",
"| train | 1 | 10 | 20 | 1,000 | 0 | $0.5000 |",
"| spawn | 2 | 3 | 1,600 | 3,000 | 1,284,567 | $1.3345 |",
"| continue | 1 | 3 | 700 | 200 | 1,300,000 | $0.2500 |",
"| total | 4 | 16 | 2,320 | 4,200 | 2,584,567 | $2.0845 |",
"",
"- Cold starts: 1",
"- Cache-hit ratio: 2/3 (66.7%)",
"- Priced total: $2.0845",
""]))
def test_md_phase_with_null_usage_and_no_verdicts(self):
self.assertEqual(self.usage_md("--phase", "2"), "\n".join([
"| Kind | Calls | in | out | cache write | cache read | cost |",
"|---|---|---|---|---|---|---|",
"| spawn | 1 | 0 | 0 | 0 | 0 | $0.0000 |",
"| total | 1 | 0 | 0 | 0 | 0 | $0.0000 |",
"",
"- Cold starts: 0",
"- Cache-hit ratio: n/a (no spawn, continue or rebase call with a verdict)",
"- Priced total: $0.0000",
""]))
def test_md_whole_run_is_a_phase_table_with_the_ping(self):
raw = self.usage_md()
self.assertTrue(raw.startswith("| Kind | Calls |"), raw)
self.assertIn("| ping | 1 | 1 | 1 | 50 | 5,000 | $0.0100 |\n", raw)
self.assertIn("| total | 6 | 17 | 2,321 | 4,250 | 2,589,567 | $2.0945 |\n", raw)
self.assertTrue(raw.endswith("- Priced total: $2.0945\n"), raw)
def test_md_stdout_is_one_fragment_not_json(self):
raw = self.usage_md("--phase", "1")
self.assertFalse(raw.lstrip().startswith("{"), raw)
with self.assertRaises(ValueError):
json.loads(raw)
self.assertTrue(raw.endswith("\n"))
self.assertNotIn("\n\n\n", raw)
def test_md_bytes_are_utf8_whatever_the_console(self):
"""Delta and em dash are not in cp1252/ascii: the bytes must still be UTF-8."""
env = child_env()
env["PYTHONIOENCODING"] = "ascii:strict"
r = subprocess.run([sys.executable, GREMLINS_PY, "usage", "--run", self.run_log, "--block", "1.3",
"--format", "md"], env=env, capture_output=True, timeout=120)
self.assertEqual(r.returncode, 0, r.stderr)
text = r.stdout.decode("utf-8")
self.assertEqual(text.splitlines()[0], BLOCK_HEADER)
self.assertIn("—", text)
self.assertNotIn(b"\r", r.stdout)
# --- json ---
def test_json_totals_per_kind_and_overall(self):
out = self.usage()
self.assertEqual(out["scope"], None)
self.assertEqual(out["reconciled"], [])
self.assertEqual(list(out["kinds"]), ["train", "rebase", "spawn", "continue", "ping"])
k = out["kinds"]
self.assertEqual(k["rebase"], {"calls": 0, "input": 0, "output": 0, "cache_creation": 0,
"cache_creation_1h": 0, "cache_creation_5m": 0, "cache_read": 0,
"cost_usd": 0.0})
self.assertEqual({x: k["spawn"][x] for x in ("calls", "input", "output", "cache_creation",
"cache_creation_1h", "cache_creation_5m",
"cache_read")},
{"calls": 3, "input": 3, "output": 1600, "cache_creation": 3000,
"cache_creation_1h": 3000, "cache_creation_5m": 0, "cache_read": 1284567})
self.assertAlmostEqual(k["spawn"]["cost_usd"], 1.3345, places=6)
self.assertEqual((k["continue"]["calls"], k["continue"]["cache_creation_5m"],
k["continue"]["cache_creation_1h"]), (1, 200, 0))
self.assertEqual((k["ping"]["calls"], k["ping"]["cache_read"]), (1, 5000))
t = out["total"]
self.assertEqual({x: t[x] for x in ("calls", "input", "output", "cache_creation",
"cache_creation_1h", "cache_creation_5m", "cache_read")},
{"calls": 6, "input": 17, "output": 2321, "cache_creation": 4250,
"cache_creation_1h": 4050, "cache_creation_5m": 200, "cache_read": 2589567})
self.assertAlmostEqual(t["cost_usd"], 2.0945, places=6)
self.assertAlmostEqual(t["cost_usd"], sum(x["cost_usd"] for x in k.values()), places=6)
self.assertEqual(out["cold_starts"], 1)
self.assertEqual((out["cache_hits"], out["cache_verdicts"]), (2, 3)) # ping excluded
self.assertAlmostEqual(out["cache_hit_ratio"], 2 / 3, places=3)
self.assertEqual([x["job"] for x in out["rows"]], ["j000%d" % i for i in range(1, 7)])
def test_json_scopes(self):
out = self.usage("--phase", "1")
self.assertEqual(out["scope"], {"phase": 1})
self.assertEqual([x["job"] for x in out["rows"]], ["j0001", "j0002", "j0003", "j0004"])
self.assertEqual(out["total"]["calls"], 4)
self.assertAlmostEqual(out["total"]["cost_usd"], 2.0845, places=6)
out = self.usage("--block", "1.1")
self.assertEqual(out["scope"], {"block": "1.1"})
self.assertEqual([x["job"] for x in out["rows"]], ["j0002", "j0003"])
self.assertEqual(out["kinds"]["spawn"]["calls"], 1)
out = self.usage("--block", "1.3") # null usage: counted as a call, adds nothing
self.assertEqual((out["total"]["calls"], out["total"]["input"], out["total"]["cost_usd"]),
(1, 0, 0.0))
self.assertIsNone(out["cache_hit_ratio"])
self.assertEqual(self.usage("--block", "nope")["rows"], [])
def test_format_json_explicit_and_errors_stay_json(self):
self.assertIn("kinds", self.usage("--format", "json"))
for argv in (["--block", "1.1", "--phase", "1"], ["--format", "yaml"]):
with self.subTest(argv=argv):
c, out, raw = call(["usage", "--run", self.run_log, *argv])
self.assertEqual(c, 2, out)
self.assert_one_json_line(raw)
self.assertEqual(out["error"], "usage")
c, raw = call_raw(["usage", "--run", os.path.join(self.tmp, "none-run.md"), "--format", "md"])
self.assertEqual(c, 2)
self.assertEqual(json.loads(raw)["error"], "no launch config for this run; call init first")
def test_allowed_inside_a_gremlin(self):
os.environ[gremlins.CHILD_MARKER] = "1"
out = self.usage("--phase", "1")
self.assertEqual(out["total"]["calls"], 4)
self.assertIn("| spawn | 2 |", self.usage_md("--phase", "1"))
# --- R1 ---
def result_for(self, job, **kw):
res = {"job": job, "kind": "spawn", "method": "fork", "base": "B1", "block": "2.1", "phase": 2,
"status": "ok", "flags": [], "fingerprint": "ok", "session": "ffffffff-0000-4000-8000-000000000007",
"parent_session": "s-b1", "models_used": [TEST_MODEL],
"tokens": dict(zip(gremlins.TOKEN_KEYS, (4, 50, 600, 600, 0, 7000))),
"cost_delta_usd": 0.75, "cache": "hit", "base_state": "warm", "wall_s": 12.5, "turns": 3,
"denials": 0, "note": None, "raw": "raw/%s.jsonl" % job, "paths": None, "reason": None,
"model_requested": TEST_MODEL}
res.update(kw)
gremlins.write_json_atomic(gremlins.result_file(self.state, job), res)
return res
def test_r1_result_without_ledger_line_gets_exactly_one(self):
# a finished result whose ledger line was lost (killed between registry write and append)
with gremlins.RegistryLock(self.state):
reg = gremlins.read_registry(self.state)
reg["jobs"]["j0007"] = gremlins._new_entry(
"j0007", "spawn", method="fork", base="B1", block="2.1", phase=2, status="ok",
cwd=self.cwd,
ended_at="2026-10-16T17:00:00+00:00", cost_cum_usd=3.25, binary_version="2.1.999",
started_at="2026-10-16T16:59:00+00:00", head_at_launch=self.sha[0])
gremlins.write_registry(self.state, reg)
self.result_for("j0007")
self.result_for("j0002") # has a line already: must stay at one
n0 = len(self.ledger_lines())
out = self.usage("--phase", "2")
self.assertEqual(out["reconciled"], ["j0007"])
lines = [x for x in self.ledger_lines() if x["job"] == "j0007"]
self.assertEqual(len(lines), 1)
self.assertEqual(len(self.ledger_lines()), n0 + 1)
ln = lines[0]
self.assertTrue(ln["reconciled"])
self.assertEqual((ln["kind"], ln["block"], ln["phase"], ln["status"], ln["cache"], ln["turns"]),
("spawn", "2.1", 2, "ok", "hit", 3))
self.assertEqual(ln["method"], "fork")
self.assertEqual(ln["tokens"]["cache_read"], 7000)
self.assertEqual((ln["binary"], ln["cost_cumulative_usd"], ln["ended"], ln["started"]),
("2.1.999", 3.25, "2026-10-16T17:00:00+00:00", "2026-10-16T16:59:00+00:00"))
self.assertEqual(ln["models_used"], [TEST_MODEL])
# it is counted: phase 2 = the killed spawn (null) + the reconciled one
self.assertEqual(out["total"]["calls"], 2)
self.assertEqual(out["total"]["cache_read"], 7000)
self.assertAlmostEqual(out["total"]["cost_usd"], 0.75, places=6)
# a second usage (either format) adds nothing
out = self.usage("--phase", "2")
self.assertEqual(out["reconciled"], [])
self.usage_md("--phase", "2")
self.assertEqual(len(self.ledger_lines()), n0 + 1)
self.assertEqual(len([x for x in self.ledger_lines() if x["job"] == "j0007"]), 1)
self.assertEqual(len([x for x in self.ledger_lines() if x["job"] == "j0002"]), 1)
def test_r1_without_registry_entry_and_with_torn_ledger_tail(self):
self.result_for("j0009", block="3.1", phase=3) # no registry entry at all
with open(gremlins.ledger_path(self.state), "ab") as f: # a torn last line (no newline)
f.write(b'{"job": "j00')
out = self.usage("--phase", "3")
self.assertEqual(out["reconciled"], ["j0009"])
self.assertEqual(out["total"]["calls"], 1)
self.assertIsNone(out["rows"][0]["cost_cumulative_usd"])
self.assertEqual(self.usage("--phase", "3")["reconciled"], [])
def test_r1_waits_for_the_registry_lock(self):
"""With the lock held and a line missing, the reconciliation itself blocks until the lock is
released, then appends. Minor 9: `reconcile_ledger` is called directly (`usage` would wait
on its orchestrator stamp first, so a reconciliation without the lock would pass too)."""
self.result_for("j0010", block="4.1", phase=4)
done = []
lock = gremlins.RegistryLock(self.state)
lock.acquire()
try:
th = threading.Thread(target=lambda: done.append(gremlins.reconcile_ledger(self.state)))
th.start()
time.sleep(1.0)
self.assertEqual(done, []) # blocked on the lock
self.assertNotIn("j0010", [x["job"] for x in self.ledger_lines()])
finally:
lock.release()
th.join(30)
self.assertEqual(done, [["j0010"]])
self.assertEqual(len([x for x in self.ledger_lines() if x["job"] == "j0010"]), 1)
def test_status_from_result_when_it_changed_after_the_ledger_line(self):
"""Minor 11: the ledger stays append-only; a job finalised again after its line was written
(R1 line, then `reap_lost`/`kill` rewrote the result) reads its status, flags and reason
from the result in `usage`; tokens and cost stay the ledger's."""
self.result_for("j0002", status="lost", flags=["cold_start"], reason="supervisor_dead",
block="1.1", phase=1, tokens=dict(zip(gremlins.TOKEN_KEYS, (9, 9, 9, 9, 0, 9))),
cost_delta_usd=9.0)
self.result_for("j0004", block="1.2", phase=1, flags=[]) # agrees with its line
n0 = len(self.ledger_lines())
out = self.usage("--phase", "1")
self.assertEqual(out["status_from_result"], ["j0002"])
row = {x["job"]: x for x in out["rows"]}["j0002"]
self.assertEqual((row["status"], row["flags"], row["reason"]), ("lost", ["cold_start"], "supervisor_dead"))
self.assertEqual(row["tokens"]["cache_read"], 1234567) # the ledger's, not the result's
self.assertAlmostEqual(row["cost_delta_usd"], 1.2345, places=6)
self.assertAlmostEqual(out["total"]["cost_usd"], 2.0845, places=6)
self.assertEqual(out["cold_starts"], 2) # the result's flags count
self.assertEqual({x["job"]: x["status"] for x in out["rows"]}["j0004"], "ok")
# the ledger itself is unchanged (append-only)
led = self.ledger_lines()
self.assertEqual(len(led), n0)
self.assertEqual([x["status"] for x in led if x["job"] == "j0002"], ["ok"])
self.assertEqual(self.usage("--phase", "2")["status_from_result"], [])
def test_r1_nothing_missing_takes_no_lock(self):
self.result_for("j0002") # a result that has its ledger line
with gremlins.RegistryLock(self.state): # held: a pass with nothing to add must not wait
self.assertEqual(gremlins.reconcile_ledger(self.state), [])
self.assertEqual(len([x for x in self.ledger_lines() if x["job"] == "j0002"]), 1)
def test_r1_corrupt_result_is_skipped(self):
with open(gremlins.result_file(self.state, "j0011"), "w") as f:
f.write("{not json")
self.assertEqual(self.usage("--phase", "1")["reconciled"], [])
self.assertNotIn("j0011", [x["job"] for x in self.ledger_lines()])
class LaunchTests(LaunchBase):
def test_train_spawn_continue_end_to_end(self):
self.setup_run()
t = self.train("B1", "turns=4")
self.assertEqual((t["job"], t["base"]), ("j0001", "B1"))
r = self.finish("j0001")
self.assertEqual(r["status"], "ok", r)
self.assertEqual(set(r), {"job", "kind", "method", "base", "block", "phase", "status", "flags",
"fingerprint", "session", "parent_session", "models_used", "tokens",
"cost_delta_usd", "cache", "base_state", "wall_s", "turns", "denials",
"note", "raw", "paths", "reason", "model_requested"})
self.assertEqual(r["kind"], "train")
self.assertIsNone(r["method"]) # only a spawn has a method
self.assertIsNone(r["cache"])
self.assertEqual(r["session"], t["session_id"])
self.assertIsNone(r["parent_session"])
self.assertEqual(r["models_used"], [TEST_MODEL])
self.assertEqual(r["model_requested"], TEST_MODEL)
self.assertEqual(r["fingerprint"], "ok")
self.assertEqual(r["turns"], 4)
self.assertEqual(r["flags"], [])
self.assertIsNone(r["reason"])
self.assertAlmostEqual(r["cost_delta_usd"], price(r["tokens"]), places=6)
self.assertEqual(r["raw"], "raw/j0001.jsonl")
self.assertTrue(os.path.isfile(os.path.join(self.state, r["raw"])))
base = self.reg()["bases"]["B1"]
self.assertEqual(base["status"], "active")
self.assertEqual(base["session"], t["session_id"])
self.assertEqual(base["base_commit"], self.git("rev-parse", "HEAD"))
# base context = input + creation + read of the last assistant message: 2 + 300 + (prev)
self.assertEqual(base["context_tokens"], 2 + 1000 + 3 * (2 + 300) + 0)
self.assertIsNotNone(base["last_hit"])
self.assertAlmostEqual(base["training_cost_usd"], r["cost_delta_usd"], places=6)
# the command line: absolute stored binary, full fingerprint, pre-assigned session id
call0 = stub_calls(self.stub)[0]
argv = call0["argv"]
self.assertEqual(argv[argv.index("--session-id") + 1], t["session_id"])
self.assertEqual(argv[argv.index("--model") + 1], TEST_MODEL)
self.assertEqual(argv[argv.index("--permission-mode") + 1], "dontAsk")
self.assertIn("--strict-mcp-config", argv)
self.assertEqual(argv[argv.index("--append-system-prompt-file") + 1], gremlins.preamble_path())
self.assertNotIn("--resume", argv)
self.assertEqual(self.reg()["jobs"]["j0001"]["argv"][0], gremlins.norm_path(self.stub))
self.assertEqual(os.path.normcase(call0["cwd"]), os.path.normcase(self.cwd))
self.assertEqual(call0["env"]["CLAUDE_CODE_EXECPATH"], gremlins.norm_path(self.stub))
self.assertEqual(call0["env"]["SPIRE_GREMLIN_CHILD"], "1")
self.assertEqual(call0["env"]["CLAUDE_CODE_PROMPT_CACHE_TTL"], "1h")
self.assertIsNone(call0["env"]["VIRTUAL_ENV"])
self.assertIn("READY", call0["prompt"]) # templates/train.md frame
# --- spawn (method fork), after a commit on the trunk ---
self.commit_file("src.py", "x = 1\n", "add src after training")
os.makedirs(os.path.join(self.state, "notes"), exist_ok=True)
f_out = self.spawn("turns=2", "sleep=1", extra=("--phase", "1", "--paths", "src.py", "docs/a.md"))
with open(os.path.join(self.state, "notes", "1.1.md"), "w") as f: # M11: written during the job
f.write("note\n")
self.assertEqual(set(f_out), {"job", "base", "method", "base_state", "warm_until", "rebase_due",
"retrain_due", "reasons"})
self.assertEqual(f_out["method"], "fork")
self.assertEqual(f_out["base_state"], "warm")
self.assertFalse(f_out["rebase_due"])
self.assertFalse(f_out["retrain_due"])
self.assertEqual(f_out["reasons"], [])
fj = f_out["job"]
r2 = self.finish(fj)
self.assertEqual(r2["status"], "ok", r2)
self.assertEqual(r2["kind"], "spawn")
self.assertEqual(r2["method"], "fork")
self.assertEqual(r2["cache"], "hit")
self.assertEqual(r2["phase"], 1)
self.assertEqual(r2["base_state"], "warm")
self.assertEqual(r2["flags"], [])
self.assertEqual(r2["fingerprint"], "ok")
self.assertEqual(r2["parent_session"], t["session_id"])
self.assertNotEqual(r2["session"], t["session_id"])
self.assertEqual(r2["paths"], ["src.py", "docs/a.md"])
self.assertEqual(r2["note"], "notes/1.1.md")
self.assertEqual(r2["models_used"], [TEST_MODEL])
# first request reads the base context, the second the base context plus the first turn
self.assertEqual(r2["tokens"]["cache_read"], 2 * base["context_tokens"] + 1002)
self.assertAlmostEqual(r2["cost_delta_usd"], price(r2["tokens"]), places=6)
fcall = [c for c in stub_calls(self.stub) if c["session"] == r2["session"]][0]
a = fcall["argv"]
self.assertEqual(a[a.index("--resume") + 1], t["session_id"])
self.assertIn("--fork-session", a)
self.assertEqual(a[a.index("--session-id") + 1], r2["session"])
self.assertIn("--append-system-prompt-file", a) # flags passed again
with open(os.path.join(self.state, "briefs", fj + ".md"), encoding="utf-8") as fh:
brief_txt = fh.read()
self.assertIn("add src after training", brief_txt) # delta header: git log
self.assertIn("1 file changed", brief_txt) # delta header: shortstat
self.assertIn("stale", brief_txt) # R13 sentence
self.assertIn(os.path.join(self.state, "notes", "1.1.md"), brief_txt)
# --- continue the gremlin: cost delta against the gremlin's own last total ---
c_out = self.cmd("continue", "--job", fj, "--brief", self.brief("turns=1"))
self.assertEqual(set(c_out), {"job"})
r3 = self.finish(c_out["job"])
self.assertEqual(r3["status"], "ok", r3)
self.assertEqual(r3["kind"], "continue")
self.assertIsNone(r3["method"]) # a continue resumes, it does not fork
self.assertEqual(r3["session"], r2["session"])
self.assertEqual(r3["block"], "1.1")
self.assertAlmostEqual(r3["cost_delta_usd"], price(r3["tokens"]), places=6)
self.assertEqual(r3["cache"], "hit")
ccall = [c for c in stub_calls(self.stub) if c["session"] == r2["session"] and c is not fcall
and "--fork-session" not in c["argv"]][0]
self.assertEqual(ccall["argv"][ccall["argv"].index("--resume") + 1], r2["session"])
self.assertNotIn("--session-id", ccall["argv"])
# ledger lines match the results
led = {x["job"]: x for x in self.ledger()}
self.assertEqual(sorted(led), ["j0001", fj, c_out["job"]])
for res in (r, r2, r3):
line = led[res["job"]]
for k in ("kind", "method", "status", "flags", "tokens", "cost_delta_usd", "cache", "models_used",
"session", "parent_session", "fingerprint", "turns", "reason", "note"):
self.assertEqual(line[k], res[k], k)
self.assertEqual(line["binary"], STUB_VERSION)
self.assertIsNotNone(line["cost_cumulative_usd"])
self.assertEqual([led[j]["method"] for j in ("j0001", fj, c_out["job"])], [None, "fork", None])
# list shows every base and job; nothing running
lst = self.cmd("list")
self.assertEqual([b["name"] for b in lst["bases"]], ["B1"])
self.assertEqual(lst["bases"][0]["base_state"], "warm")
self.assertEqual(len(lst["jobs"]), 3)
self.assertEqual([(x["kind"], x["method"]) for x in lst["jobs"]],
[("train", None), ("spawn", "fork"), ("continue", None)])
self.assertEqual(lst["bases"][0]["gremlins"], [fj])
self.assertEqual(lst["running"], [])
self.assertIsNotNone(self.reg()["run"]["last_orchestrator_call_at"])
# refusals on continue
out = self.cmd("continue", "--job", "j0001", "--brief", self.brief(), code=2) # not a spawn
self.assertEqual(out["error"], "continue takes a spawn job")
self.cmd("continue", "--job", "j0999", "--brief", self.brief(), code=2) # unknown
self.cmd("train", "--base", "B1", "--brief", self.brief(), code=2) # base exists
def test_cache_verdicts_and_cold_start(self):
self.setup_run()
self.train("B1")
self.finish("j0001")
a = self.spawn("cache=partial", extra=("--max-budget-usd", "2.5"))["job"]
b = self.spawn("cache=miss")["job"]
argv_a = self.reg()["jobs"][a]["argv"]
self.assertEqual(argv_a[argv_a.index("--max-budget-usd") + 1], "2.5")
self.assertNotIn("--max-budget-usd", self.reg()["jobs"][b]["argv"])
code, out = self.wait(a, b)
self.assertEqual(code, 0, out)
res = {x["job"]: x for x in out["done"]}
self.assertEqual(res[a]["cache"], "partial")
self.assertEqual(res[b]["cache"], "miss")
for j in (a, b):
self.assertIn("cold_start", res[j]["flags"])
self.assertEqual(res[j]["status"], "ok")
self.assertEqual(res[j]["base_state"], "warm")
# a cold base: hit, but cold at launch -> cold_start
with gremlins.RegistryLock(self.state):
reg = gremlins.read_registry(self.state)
reg["bases"]["B1"]["last_hit"] = gremlins._iso(time.time() - 4000)
gremlins.write_registry(self.state, reg)
out = self.spawn()
self.assertEqual(out["base_state"], "cold")
r = self.finish(out["job"])
self.assertEqual(r["cache"], "hit")
self.assertEqual(r["base_state"], "cold")
self.assertIn("cold_start", r["flags"])
# the hit refreshed last_hit: the base is warm again
self.assertEqual(self.cmd("list")["bases"][0]["base_state"], "warm")
def test_failures_flags_and_drift(self):
self.setup_run()
self.train("B1")
self.finish("j0001")
cases = [("die=1",), ("error",), ("restricted=1",), ("model=claude-other-9",), ("deny",),
("mismatch",), ("plugin=0.3.0",)]
results = {}
for i in range(0, len(cases), 2):
jobs = [self.spawn(*c)["job"] for c in cases[i:i + 2]]
code, out = self.wait(*jobs)
for x in out["done"]:
results[x["job"]] = x
r = [results["j%04d" % (i + 2)] for i in range(len(cases))]
die, err, restricted, substituted, deny, mismatch, plugin = r
self.assertEqual((die["status"], die["reason"]), ("failed", "exit_code"))
self.assertEqual(die["turns"], 1)
self.assertIsNone(die["cost_delta_usd"])
self.assertEqual((err["status"], err["reason"]), ("failed", "is_error"))
self.assertEqual((restricted["status"], restricted["reason"]), ("failed", "model_refused"))
self.assertEqual((substituted["status"], substituted["reason"]), ("failed", "model_refused"))
self.assertEqual(substituted["models_used"], ["claude-other-9"])
self.assertEqual(deny["status"], "ok")
self.assertIn("needs_attention", deny["flags"])
self.assertEqual(deny["denials"], 1)
self.assertEqual(mismatch["status"], "ok")
self.assertIn("usage_mismatch", mismatch["flags"])
self.assertEqual(plugin["fingerprint"], "drift:plugins")
self.assertNotIn("cold_start", plugin["flags"])
# model refused while training: the base fails
t = self.train("B2", "restricted=1")
r = self.finish(t["job"])
self.assertEqual((r["status"], r["reason"]), ("failed", "model_refused"))
self.assertEqual(self.reg()["bases"]["B2"]["status"], "failed")
self.spawn(base="B2", code=2) # not active
# binary drift: the stored binary changed size and mtime since training
with open(self.stub, "a", newline="\r\n" if IS_WINDOWS else "\n") as f:
f.write("rem drift\n" if IS_WINDOWS else "# drift\n")
r = self.finish(self.spawn()["job"])
self.assertEqual(r["status"], "ok")
self.assertEqual(r["fingerprint"], "drift:binary")
def test_api_errors_are_not_model_refused(self):
"""I1: an error before the model answers (nothing billed, models_used []) is not a refusal;
API errors get their own reason; a real substitution still reads model_refused."""
self.setup_run(cap=8)
self.train("B1")
self.finish("j0001")
cases = [(("error",), "is_error", None),
(("apierr=429",), "usage_limit", "429"),
(("error", "errtext=Claude_AI_usage_limit_reached"), "usage_limit", "usage limit"),
(("apierr=529",), "overloaded", "529"),
(("apierr=400", "errtext=Prompt_is_too_long"), "prompt_too_long", "400"),
(("apierr=500",), "api_error", "500"),
(("model=claude-other-9",), "model_refused", "claude-other-9")]
jobs = [self.spawn(*c)["job"] for c, _, _ in cases]
code, out = self.wait(*jobs)
self.assertEqual(code, 4, out)
res = {x["job"]: x for x in out["done"]}
reg = self.reg()
for job, (directives, reason, detail) in zip(jobs, cases):
with self.subTest(directives=directives):
r = res[job]
self.assertEqual((r["status"], r["reason"]), ("failed", reason), r)
if reason != "model_refused":
self.assertEqual(r["models_used"], [])
self.assertEqual(r["cost_delta_usd"], 0.0)
if detail:
self.assertIn(detail, reg["jobs"][job]["reason_detail"])
def test_continue_leaves_base_last_hit(self):
"""I3: a continue reads the gremlin's own longer prefix; only a spawn refreshes the base."""
self.setup_run()
self.train("B1")
self.finish("j0001")
fj = self.spawn("turns=1")["job"]
self.assertEqual(self.finish(fj)["status"], "ok")
old = gremlins._iso(time.time() - 1000)
with gremlins.RegistryLock(self.state):
reg = gremlins.read_registry(self.state)
reg["bases"]["B1"]["last_hit"] = old
gremlins.write_registry(self.state, reg)
r = self.finish(self.cmd("continue", "--job", fj, "--brief", self.brief("turns=2"))["job"])
self.assertEqual((r["status"], r["kind"]), ("ok", "continue"))
self.assertEqual(self.reg()["bases"]["B1"]["last_hit"], old)
r = self.finish(self.spawn("turns=1")["job"]) # a spawn still refreshes it
self.assertEqual(r["status"], "ok")
self.assertGreater(gremlins._ts(self.reg()["bases"]["B1"]["last_hit"]), gremlins._ts(old))
def test_clock_jump_not_orphaned_or_timed_out(self):
"""M3: a supervisor (run in-process under an injected clock) sees the clock jump 2 h mid-run,
as after a host suspend. Neither orphan_after (600 s) nor max_minutes (1) may fire; the
result carries `clock_jump` and the jump is logged."""
self.setup_run() # orphan_after 600 s
self.train("B1")
self.finish("j0001")
p = subprocess.Popen([sys.executable, "-c", "pass"])
dead = gremlins.pid_record(p.pid) # recorded as the supervisor: never us
p.wait()
fake = {"ok": True, "method": "test", "pid": dead["pid"], "ctime": dead["ctime"]}
with mock.patch.object(gremlins, "detach_supervisor", lambda *a: fake):
job = self.spawn("sleep=1", "turns=3", extra=("--max-minutes", "1"))["job"]
sup = gremlins.Supervisor(self.state, job)
sup.rt["supervisor"] = dead
offset = [0.0]
real = time.time
jump = threading.Timer(1.0, offset.__setitem__, args=(0, 7200.0))
err = io.StringIO()
try:
with mock.patch.dict(os.environ), mock.patch.object(gremlins, "_now", lambda: real() + offset[0]), \
contextlib.redirect_stderr(err):
jump.start()
out, code = sup.run()
finally:
jump.cancel()
self.assertEqual(offset[0], 7200.0, "the clock never jumped during the run")
self.assertEqual(code, 0, out)
e = self.reg()["jobs"][job]
self.assertEqual((e["status"], e["reason"]), ("ok", None), e)
self.assertIn("clock_jump", e["flags"])
self.assertIn("clock_jump", gremlins.read_json(gremlins.result_file(self.state, job))["flags"])
self.assertIn("clock jump", err.getvalue())
def test_timeout_and_stalled(self):
self.setup_run(stall=1)
self.train("B1")
self.finish("j0001")
slow = self.spawn("silent=2.5")["job"]
hang = self.spawn_marked("hang=1", extra=("--max-minutes", "0.06"))["job"]
rt = self.until_running(hang)
tree = gremlins.process_tree(rt["supervisor"])
self.assertGreaterEqual(len(tree), 3)
code, out = self.wait(slow, hang, timeout=60)
self.assertEqual(code, 4, out)
res = {x["job"]: x for x in out["done"]}
self.assertEqual(res[slow]["status"], "ok")
self.assertIn("stalled", res[slow]["flags"])
self.assertEqual(res[hang]["status"], "timeout")
self.assertIsNone(res[hang]["reason"])
self.assertIn("stalled", res[hang]["flags"])
self.assert_tree_dead(tree) # claude tree killed; the supervisor exited
def test_kill_tree(self):
"""The visible-window part moved to `test_no_visible_window` (M8), so that a host without a
desktop skips it instead of failing the kill test."""
self.setup_run()
self.train("B1")
self.finish("j0001")
job = self.spawn_marked("hang=1", "child")["job"]
rt = self.until_running(job, need_child=True)
sup = rt["supervisor"]
tree = gremlins.process_tree(sup)
pids = {p["pid"] for p in tree}
stub = stub_call_for(self.stub, "RUN-%s" % job)
self.assertIn(sup["pid"], pids)
self.assertIn(rt["claude"]["pid"], pids)
self.assertIn(stub["pid"], pids)
self.assertIn(stub["child_pid"], pids)
if WIN_PLATFORM:
# spawned through CIM: the supervisor's parent is the WMI provider host, not us
table = {pid: (ppid, name) for pid, ppid, name in gremlins.process_table()}
parent = table.get(table[sup["pid"]][0], (None, None))[1]
self.assertEqual((parent or "").lower(), "wmiprvse.exe")
self.assertIn("python.exe", [p["name"].lower() for p in tree])
self.assertIn("cmd.exe", [p["name"].lower() for p in tree])
out = self.cmd("kill", "--job", job)
self.assertTrue(out["killed"])
self.assertTrue(out["ok"], out)
self.assertEqual(out["survivors"], [])
self.assertEqual(out["status"], "killed")
snap_pids = {p["pid"] for p in out["snapshot"]}
for pid in (sup["pid"], rt["claude"]["pid"], stub["pid"], stub["child_pid"]):
self.assertIn(pid, snap_pids)
self.assert_tree_dead(tree, timeout=1)
r = self.finish(job)
self.assertEqual((r["status"], r["reason"]), ("killed", "kill"))
self.assertEqual(r["turns"], 1)
self.assertEqual([x["status"] for x in self.ledger() if x["job"] == job], ["killed"])
again = self.cmd("kill", "--job", job)
self.assertFalse(again["killed"])
self.cmd("kill", "--job", "j0999", code=2)
def test_lost(self):
self.setup_run()
self.train("B1")
self.finish("j0001")
job = self.spawn_marked("hang=1")["job"]
rt = self.until_running(job)
tree = gremlins.process_tree(rt["supervisor"])
self.assertTrue(gremlins.terminate(rt["supervisor"]))
self.assertTrue(wait_for(lambda: not gremlins.record_alive(rt["supervisor"]), 10))
lst = self.cmd("list") # heartbeat is fresh: not lost yet
self.assertEqual(lst["running"], [job])
later = time.time() + gremlins.LOST_AFTER_S + 30
with mock.patch.object(gremlins, "_now", lambda: later):
lst = self.cmd("list")
self.assertEqual(lst["lost_now"], [job])
self.assertEqual(lst["running"], [])
self.assert_tree_dead(tree) # its orphaned claude tree went with it
code, out = self.wait(job)
self.assertEqual(code, 4)
self.assertEqual((out["done"][0]["status"], out["done"][0]["reason"]), ("lost", "supervisor_dead"))
self.assertEqual([x["status"] for x in self.ledger() if x["job"] == job], ["lost"])
def test_orphan_watchdog(self):
self.setup_run() # orphan_after 600 s
self.train("B1")
self.finish("j0001")
job = self.spawn_marked("hang=1", "child")["job"]
rt = self.until_running(job, need_child=True)
tree = gremlins.process_tree(rt["claude"])
self.assertGreaterEqual(len(tree), 2)
self.assertEqual(self.reg()["jobs"][job]["status"], "running")
# injected clock: the last orchestrator call is recorded as 2000 s ago
past = time.time() - 2000
with mock.patch.object(gremlins, "_now", lambda: past):
gremlins.touch_orchestrator(self.state)
# no orchestrator call from here on: poll the registry directly (no touch)
final = wait_for(lambda: self.reg()["jobs"][job]["status"] not in gremlins.RUNNING, timeout=30, step=0.2)
self.assertTrue(final)
e = self.reg()["jobs"][job]
self.assertEqual((e["status"], e["reason"]), ("killed", "orphaned"))
self.assertIn("orphaned", e["flags"])
self.assert_tree_dead(tree)
self.assertTrue(wait_for(lambda: not gremlins.record_alive(rt["supervisor"]), 15))
self.assertEqual([x["flags"] for x in self.ledger() if x["job"] == job], [e["flags"]])
def test_expire_kills_running_jobs(self):
self.setup_run()
self.train("B1")
self.finish("j0001")
job = self.spawn_marked("hang=1")["job"]
rt = self.until_running(job)
tree = gremlins.process_tree(rt["supervisor"])
out = self.cmd("expire", "--base", "B1")
self.assertEqual(out["retired"], ["B1"])
self.assertEqual([k["job"] for k in out["killed"]], [job])
self.assertTrue(out["ok"])
self.assert_tree_dead(tree)
r = self.finish(job)
self.assertEqual((r["status"], r["reason"]), ("killed", "expire"))
self.assertIn("expired", r["flags"])
self.assertEqual(self.reg()["bases"]["B1"]["status"], "retired")
self.spawn(code=2) # retired: refused without --force
r = self.finish(self.spawn(extra=("--force",))["job"])
self.assertEqual(r["status"], "ok")
self.assertEqual(self.cmd("expire", "--all")["retired"], [])
def test_wait_exit_codes(self):
self.setup_run()
self.train("B1")
self.assertEqual(self.wait()[0], 0) # settles the training job
a = self.spawn("die=1")["job"]
b = self.spawn("sleep=2", "turns=5")["job"]
wait_for(lambda: self.reg()["jobs"][a]["status"] == "failed", timeout=30)
code, out = self.wait(timeout=1) # mixed: one failed, one running
self.assertEqual(code, 3, out)
self.assertEqual([x["job"] for x in out["done"]], [a])
self.assertEqual(out["done"][0]["status"], "failed")
self.assertEqual([x["job"] for x in out["running"]], [b])
# M16: the progress counter is `messages` (distinct assistant message ids), not `turns`
self.assertEqual(set(out["running"][0]), {"job", "block", "elapsed_s", "last_event_s_ago", "messages"})
self.assertEqual(out["pings"], [])
code, out = self.wait(timeout=60) # condition met, one failed -> 4
self.assertEqual(code, 4, out)
self.assertEqual([x["job"] for x in out["done"]], [b]) # a is not returned twice
self.assertEqual(out["done"][0]["status"], "ok")
code, out = self.wait(timeout=5) # nothing left
self.assertEqual((code, out["done"], out["running"]), (0, [], []))
self.assertEqual(self.wait(b)[0], 0)
self.assertEqual(self.wait(a)[0], 4)
# --any
c = self.spawn("die=1")["job"]
d = self.spawn("sleep=1", "turns=3")["job"]
code, out = self.wait(timeout=60, any_=True)
self.assertEqual(code, 4, out)
self.assertEqual([x["job"] for x in out["done"]], [c])
code, out = self.wait(timeout=60)
self.assertEqual(code, 0, out)
self.assertEqual([x["job"] for x in out["done"]], [d])
# usage errors
for argv in (["wait", "--run", self.run_log, "--all", "--timeout", "571"],
["wait", "--run", self.run_log],
["wait", "--run", self.run_log, "j0999"],
["wait", "--run", self.run_log, "--all", "j0001"]):
self.assertEqual(call(argv)[0], 2, argv)
def test_detach_failed(self):
self.setup_run()
failed = {"ok": False, "method": gremlins.PLATFORM.detach_method,
"detail": "Win32_Process.Create returned 9"}
with mock.patch.object(gremlins, "detach_supervisor", lambda *a: failed):
t = self.train("B1")
code, out = self.wait(t["job"], timeout=5)
self.assertEqual(code, 4, out)
r = out["done"][0]
self.assertEqual((r["status"], r["reason"]), ("failed", "detach_failed"))
self.assertEqual(r["tokens"], {k: None for k in gremlins.TOKEN_KEYS})
self.assertEqual(self.reg()["bases"]["B1"]["status"], "failed")
self.assertEqual(self.reg()["jobs"][t["job"]]["reason_detail"], failed["detail"])
self.assertEqual([x["reason"] for x in self.ledger()], ["detach_failed"])
self.assertEqual(stub_calls(self.stub), [])
def test_refusals(self):
# launch-field overrides on spawn and continue (refused before anything is read)
values = {"model": "claude-x-1", "effort": "low", "permission_mode": "default", "tools": "Read",
"allowed_tools": "Read", "disallowed_tools": "Bash", "agent": "a", "cwd": self.tmp,
"binary": self.stub}
for name in gremlins.LAUNCH_FIELDS:
flag = "--" + name.replace("_", "-")
for argv in (["spawn", "--base", "B1", "--block", "1", "--brief", "b.md"],
["continue", "--job", "j0001", "--brief", "b.md"],
["rebase", "--base", "B1", "--as", "B2", "--brief", "b.md"]):
with self.subTest(field=name, cmd=argv[0]):
out = self.cmd(*argv, flag, values[name], code=2)
self.assertEqual(out["fields"], [name])
self.assertFalse(os.path.exists(self.state))
self.setup_run()
# N2: parallel needs --paths
out = self.cmd("spawn", "--base", "B1", "--block", "1", "--brief", self.brief(), "--mode",
"parallel", code=2)
self.assertIn("--paths", out["error"])
self.spawn(base="B9", code=2) # unknown base
self.train("B1")
self.finish("j0001")
# parallel cap (2): two hanging parallel gremlins, a third launch is refused
j1 = self.spawn_marked("hang=1", extra=("--mode", "parallel", "--paths", "a.py"))["job"]
j2 = self.spawn_marked("hang=1", extra=("--mode", "parallel", "--paths", "b.py", "c.py"))["job"]
out = self.spawn(code=2)
self.assertEqual(out["error"], "parallel cap reached")
self.assertEqual(out["running"], [j1, j2])
out = self.cmd("continue", "--job", j1, "--brief", self.brief(), code=2) # still running
self.assertIn("running", out["error"])
# N3: launching refused inside a gremlin; list and wait stay allowed
os.environ[gremlins.CHILD_MARKER] = "1"
try:
for argv in (["train", "--base", "B3", "--brief", self.brief()],
["spawn", "--base", "B1", "--block", "2", "--brief", self.brief()],
["continue", "--job", j1, "--brief", self.brief()]):
out = self.cmd(*argv, code=2)
self.assertIn("inside a gremlin", out["error"])
self.assertEqual(self.cmd("list")["running"], [j1, j2])
code, out = self.wait(j1, timeout=1)
self.assertEqual(code, 3)
finally:
os.environ.pop(gremlins.CHILD_MARKER, None)
for j in (j1, j2):
self.until_running(j)
out = self.cmd("kill", "--job", j)
self.assertTrue(out["ok"], out) # 3.3: a failure prints the survivors
code, out = self.wait(j1, j2)
self.assertEqual(code, 4)
self.assertEqual({x["job"]: x["paths"] for x in out["done"]}, {j1: ["a.py"], j2: ["b.py", "c.py"]})
# missing binary: the stored path is gone
moved = self.stub + ".moved"
os.replace(self.stub, moved)
try:
out = self.spawn(code=2)
self.assertEqual(out["reason"], "binary_missing")
out = self.cmd("train", "--base", "B4", "--brief", self.brief(), code=2)
self.assertEqual(out["reason"], "binary_missing")
finally:
os.replace(moved, self.stub)
self.cmd("spawn", "--base", "B1", "--block", "1", "--brief", os.path.join(self.tmp, "nope.md"), code=2)
# --- block 2.1 -----------------------------------------------------------------------
@unittest.skipUnless(WIN_PLATFORM, "visible windows are a Windows check")
def test_no_visible_window(self):
"""M8: while a job runs, no visible window belongs to its process tree, and no visible
console or Windows Terminal window appears anywhere on the desktop (a Terminal window
belongs to WindowsTerminal.exe, not to the tree). A positive control, a deliberately visible
console (minimised, not activated) closed at once, shows the check sees one. Skipped, not
failed, when there is no desktop (or the platform has no window check: None)."""
if not gremlins.PLATFORM.visible_windows():
self.skipTest("no desktop: window enumeration sees no visible window")
# positive control
before = {w[0] for w in console_windows()}
marker = "gremlins-test-poscontrol-%d" % os.getpid() # matched by the module-end sweep
si = subprocess.STARTUPINFO()
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
si.wShowWindow = 7 # SW_SHOWMINNOACTIVE
ctl = subprocess.Popen(
[stub_interpreter(), "-I", "-c", "import ctypes, time; "
"ctypes.windll.kernel32.SetConsoleTitleW(%r); time.sleep(120)" % marker],
creationflags=subprocess.CREATE_NEW_CONSOLE, startupinfo=si)
ctl_rec = gremlins.pid_record(ctl.pid)
try:
seen = wait_for(lambda: [w for w in console_windows() if w[0] not in before or marker in w[3]],
timeout=15)
finally:
gremlins.kill_tree(ctl_rec, alive_until=time.time())
with contextlib.suppress(subprocess.TimeoutExpired):
ctl.wait(10)
self.assertTrue(seen, "positive control: a deliberately visible console was not detected")
hwnds = {w[0] for w in seen}
self.assertTrue(wait_for(lambda: not [w for w in console_windows() if w[0] in hwnds], timeout=15),
"positive control: its console window outlived the process")
# the job
self.setup_run()
self.train("B1")
self.finish("j0001")
before = {w[0] for w in console_windows()}
job = self.spawn_marked("hang=1", "child")["job"]
rt = self.until_running(job, need_child=True)
try:
for _ in range(5):
pids = {p["pid"] for p in gremlins.process_tree(rt["supervisor"])}
self.assertIn(rt["claude"]["pid"], pids)
mine = [w for w in visible_windows() if w[1] in pids]
self.assertEqual(mine, [], "visible window of the job's process tree")
new = [w for w in console_windows() if w[0] not in before]
self.assertEqual(new, [], "a console or Terminal window appeared while the job ran")
time.sleep(0.2)
finally:
self.assertTrue(self.cmd("kill", "--job", job)["ok"])
def test_error_after_requested_model_billed(self):
"""The call billed the requested model, then its result is an error: `failed` with an error
reason (`is_error`, or the API-error reason), never `model_refused` (stub `billed`)."""
self.setup_run(cap=4)
self.train("B1")
self.finish("j0001")
a = self.spawn("error", "billed", "turns=2")["job"]
b = self.spawn("apierr=500", "billed", "turns=2")["job"]
code, out = self.wait(a, b)
self.assertEqual(code, 4, out)
res = {x["job"]: x for x in out["done"]}
for job, reason in ((a, "is_error"), (b, "api_error")):
with self.subTest(reason=reason):
r = res[job]
self.assertEqual((r["status"], r["reason"]), ("failed", reason), r)
self.assertEqual(r["models_used"], [TEST_MODEL])
self.assertEqual(r["tokens"]["output"], 2 * 40)
self.assertGreater(r["cost_delta_usd"], 0)
self.assertAlmostEqual(r["cost_delta_usd"], price(r["tokens"]), places=6)
def set_registry(self, fn):
with gremlins.RegistryLock(self.state):
reg = gremlins.read_registry(self.state)
fn(reg)
gremlins.write_registry(self.state, reg)
def set_last_hit(self, iso):
self.set_registry(lambda r: r["bases"]["B1"].__setitem__("last_hit", iso))
def set_pings(self, on):
self.set_registry(lambda r: r["config"].__setitem__("pings", "on" if on else "off"))
def test_ttl_inference_downgrade_and_cold_start(self):
"""Design 5 / C5: a call that wrote only 5-minute entries carries `ttl_downgrade` and sets the
base's TTL to 300 s; warmth, and so `cold_start`, follow the inferred TTL; a later 1 h write
brings it back to 3600 s."""
self.setup_run()
self.train("B1", "ttl=5m")
r = self.finish("j0001")
self.assertEqual(r["status"], "ok", r)
self.assertIn("ttl_downgrade", r["flags"])
self.assertEqual(r["tokens"]["cache_creation_1h"], 0)
self.assertGreater(r["tokens"]["cache_creation_5m"], 0)
b = self.reg()["bases"]["B1"]
self.assertEqual((b["ttl_s"], b["ttl_from"]), (300, "j0001"))
row = self.cmd("list")["bases"][0]
self.assertEqual((row["ttl_s"], row["margin_s"], row["ttl_from"]), (300, 120, "j0001"))
# 400 s after the last hit: warm under the configured 3600 s, cold under the inferred 300 s
self.set_last_hit(gremlins._iso(time.time() - 400))
self.assertEqual(self.cmd("list")["bases"][0]["base_state"], "cold")
out = self.spawn("turns=1")
self.assertEqual(out["base_state"], "cold")
r2 = self.finish(out["job"])
self.assertEqual((r2["status"], r2["cache"], r2["base_state"]), ("ok", "hit", "cold"), r2)
self.assertIn("cold_start", r2["flags"])
self.assertNotIn("ttl_downgrade", r2["flags"])
b = self.reg()["bases"]["B1"]
self.assertEqual((b["ttl_s"], b["ttl_from"]), (3600, out["job"]))
row = self.cmd("list")["bases"][0]
self.assertEqual((row["base_state"], row["ttl_s"]), ("warm", 3600))
def ping_jobs(self):
return sorted(j for j, e in self.reg()["jobs"].items() if e["kind"] == "ping")
def collect_ping(self, job):
"""Wait (pings off) until the ping ended; return it from `wait`'s `pings` list."""
self.assertTrue(wait_for(lambda: self.reg()["jobs"][job]["status"] in gremlins.FINAL, timeout=60),
self.reg()["jobs"][job])
code, out = self.wait(timeout=5)
self.assertEqual((code, out["done"]), (0, []), out)
self.assertEqual([x["job"] for x in out["pings"]], [job], out)
return out["pings"][0]
def test_ping_end_to_end_through_wait(self):
"""A stubbed ping: launched by `wait`'s tick, returned once through `wait`'s `pings` (never
`done`); a `partial` ping refreshes last_hit, a missed or failed one does not and carries
`ping_missed`; the in-flight mark clears; each ping has its ledger line."""
# expiring from 1800 s after the last hit on (a margin must be below ttl_s; Minor 2)
self.setup_run("--warm-margin-s", "1800")
self.train("B1")
self.finish("j0001")
base_session = self.reg()["bases"]["B1"]["session"]
old = gremlins._iso(time.time() - 2000)
self.set_last_hit(old)
# 1. partial, launched by wait's tick (the wait itself is the activity)
with mock.patch.object(gremlins, "PING_PROMPT", "Reply OK\n@stub cache=partial"):
code, out = self.wait(timeout=5)
self.assertEqual((code, out["done"], out["pings"]), (0, [], []), out)
pings = self.ping_jobs()
self.assertEqual(len(pings), 1, self.reg()["jobs"])
p1 = pings[0]
self.assertEqual(self.reg()["bases"]["B1"]["ping_in_flight"], p1)
self.set_pings(False) # no further pings from the waits below
r1 = self.collect_ping(p1)
self.assertEqual((r1["kind"], r1["status"], r1["cache"], r1["base"], r1["base_state"]),
("ping", "ok", "partial", "B1", "expiring"), r1)
self.assertNotIn("ping_missed", r1["flags"])
self.assertEqual((r1["parent_session"], r1["turns"], r1["block"]), (base_session, 1, None))
self.assertEqual(r1["models_used"], [TEST_MODEL])
b = self.reg()["bases"]["B1"]
self.assertGreater(gremlins._ts(b["last_hit"]), gremlins._ts(old)) # partial refreshes
self.assertIsNone(b["ping_in_flight"])
self.assertEqual(self.wait(timeout=5)[1]["pings"], []) # returned once
c = stub_call_for(self.stub, "Reply OK")
a = c["argv"]
self.assertEqual(a[a.index("--resume") + 1], base_session)
self.assertIn("--fork-session", a)
self.assertEqual(a[a.index("--session-id") + 1], r1["session"])
self.assertEqual(a[a.index("--max-turns") + 1], "1")
self.assertEqual(c["env"]["SPIRE_GREMLIN_CHILD"], "1")
# 2. a miss, launched by `tick` by hand: last_hit stays, flag ping_missed
self.set_last_hit(old)
self.set_pings(True)
with mock.patch.object(gremlins, "PING_PROMPT", "Reply OK\n@stub cache=miss"):
out = self.cmd("tick")
self.assertEqual(len(out["pinged"]), 1, out)
p2 = out["pinged"][0]
d = [x for x in out["bases"] if x["base"] == "B1"][0]
self.assertEqual((d["pings_since_real_call"], d["break_even"], d["job"]), (1, 19, p2))
self.set_pings(False)
r2 = self.collect_ping(p2)
self.assertEqual((r2["status"], r2["cache"]), ("ok", "miss"), r2)
self.assertIn("ping_missed", r2["flags"])
self.assertEqual(self.reg()["bases"]["B1"]["last_hit"], old)
# 3. a failed ping: last_hit stays, flag ping_missed
self.set_pings(True)
with mock.patch.object(gremlins, "PING_PROMPT", "Reply OK\n@stub error"):
p3 = self.cmd("tick")["pinged"][0]
self.set_pings(False)
r3 = self.collect_ping(p3)
self.assertEqual((r3["status"], r3["reason"]), ("failed", "is_error"), r3)
self.assertIn("ping_missed", r3["flags"])
self.assertEqual(self.reg()["bases"]["B1"]["last_hit"], old)
self.assertIsNone(self.reg()["bases"]["B1"]["ping_in_flight"])
led = [x for x in self.ledger() if x["kind"] == "ping"]
self.assertEqual([x["job"] for x in led], [p1, p2, p3])
self.assertEqual([x["status"] for x in led], ["ok", "ok", "failed"])
def test_tick_runs_in_wait_and_supervisor(self):
"""The tick runs inside `wait` (at its start, then every TICK_WAIT_S; never in a gremlin) and
inside every supervisor (every TICK_SUP_S)."""
self.setup_run()
self.train("B1")
self.finish("j0001")
calls = []
def rec(state):
calls.append(state)
return {"pinged": [], "bases": []}
with mock.patch.object(gremlins, "TICK_WAIT_S", 0.5), mock.patch.object(gremlins, "tick_pass", rec):
job = self.spawn("sleep=1", "turns=3")["job"]
t0 = time.monotonic()
self.finish(job)
took = time.monotonic() - t0
self.assertGreaterEqual(took, 1.5)
self.assertGreaterEqual(len(calls), 3, (len(calls), took))
self.assertEqual(set(calls), {self.state})
calls.clear()
os.environ[gremlins.CHILD_MARKER] = "1"
try:
with mock.patch.object(gremlins, "tick_pass", rec):
self.wait(job, timeout=1)
finally:
os.environ.pop(gremlins.CHILD_MARKER, None)
self.assertEqual(calls, [])
# a supervisor, run in-process
p = subprocess.Popen([sys.executable, "-c", "pass"])
dead = gremlins.pid_record(p.pid)
p.wait()
fake = {"ok": True, "method": "test", "pid": dead["pid"], "ctime": dead["ctime"]}
with mock.patch.object(gremlins, "detach_supervisor", lambda *a: fake):
job = self.spawn("sleep=1", "turns=3")["job"]
sup = gremlins.Supervisor(self.state, job)
sup.rt["supervisor"] = dead
with mock.patch.dict(os.environ), mock.patch.object(gremlins, "TICK_SUP_S", 0.5), \
mock.patch.object(gremlins, "tick_pass", rec):
out, code = sup.run()
self.assertEqual(code, 0, out)
self.assertEqual(self.reg()["jobs"][job]["status"], "ok")
self.assertGreaterEqual(len(calls), 3, len(calls))
self.assertIn("messages", self.runtime(job)) # M16: the job file's counter
self.assertNotIn("turns", self.runtime(job))
# --- block 2.2 -----------------------------------------------------------------------
REBASE_REASONS = ("phase_boundary", "commits", "lines", "delta_tokens", "binary_drift", "runtime_drift")
def dead_detach(self) -> dict:
p = subprocess.Popen([sys.executable, "-c", "pass"])
dead = gremlins.pid_record(p.pid)
p.wait()
return {"ok": True, "method": "test", "pid": dead["pid"], "ctime": dead["ctime"]}
def fake_spawn(self, base, *extra, brief=None, code=0):
"""A spawn whose supervisor is never started (detach stubbed with a dead pid), finalised at once
as killed: only its launch return is looked at."""
fake = self.dead_detach()
with mock.patch.object(gremlins, "detach_supervisor", lambda *a: fake):
out = self.cmd("spawn", "--base", base, "--block", "d", "--brief", brief or self.brief(),
*extra, code=code)
if code == 0:
gremlins.finalize_job(self.state, out["job"], ss=None, hint="killed", reason="kill")
return out
def clone_base(self, name, src="B1", **changes):
"""A copy of base `src` under a new name, at the current HEAD unless `base_commit` is given."""
head = self.git("rev-parse", "HEAD")
def f(reg):
b = json.loads(json.dumps(reg["bases"][src]))
b.update({"name": name, "gremlins": [], "base_commit": head})
b.update(changes)
reg["bases"][name] = b
self.set_registry(f)
def assert_due(self, out, *reasons):
self.assertEqual(out["reasons"], list(reasons), out)
rebase = [r for r in reasons if r.split(":")[0] in self.REBASE_REASONS]
self.assertEqual(out["rebase_due"], bool(rebase), out)
self.assertEqual(out["retrain_due"], len(rebase) < len(reasons), out)
def test_due_reasons_each_alone(self):
"""Design 5 / digest 2.2: every rebase_due and retrain_due reason triggered on its own; the
spawn's launch return lists exactly it (and `list` the base-level ones)."""
self.setup_run(cap=4)
self.train("B1")
self.finish("j0001")
c0 = self.git("rev-parse", "HEAD")
self.assert_due(self.fake_spawn("B1")) # nothing due
# phase_boundary: above the highest phase of the base's spawns, or of the base itself
self.clone_base("Bp")
self.assert_due(self.fake_spawn("Bp", "--phase", "1")) # nothing known yet
self.assert_due(self.fake_spawn("Bp", "--phase", "1"))
self.assert_due(self.fake_spawn("Bp", "--phase", "2"), "phase_boundary")
self.assert_due(self.fake_spawn("Bp")) # no --phase
self.clone_base("Bq", phase=3)
self.assert_due(self.fake_spawn("Bq", "--phase", "3"))
self.assert_due(self.fake_spawn("Bq", "--phase", "4"), "phase_boundary")
self.assert_due(self.fake_spawn("Bq", "--phase", "4"), "phase_boundary") # still above the base's 3
self.assert_due(self.fake_spawn("Bq", "--phase", "2"))
# commits: 7 > rebase_commits 6 (seven one-line files)
for i in range(7):
self.commit_file("c%d.txt" % i, "x\n", "commit %d" % i)
self.clone_base("Bc", base_commit=c0)
self.assert_due(self.fake_spawn("Bc"), "commits:7")
# lines: 1600 > rebase_lines 1500, in one commit
c7 = self.git("rev-parse", "HEAD")
self.commit_file("big.txt", "line\n" * 1600, "big change")
self.clone_base("Bl", base_commit=c7)
self.assert_due(self.fake_spawn("Bl"), "lines:1600")
# every clone from here on starts at HEAD
# delta_tokens: chars / 4 of the delta header plus the attached delta notes > 12000 (I1)
big_note = os.path.join(self.state, "notes", "big-note.md")
with open(big_note, "w") as f:
f.write("n" * 60000)
self.clone_base("Bd")
self.assert_due(self.fake_spawn("Bd")) # no note attached
self.assert_due(self.fake_spawn("Bd", brief=self.brief(text="Read notes/big-note.md first.")))
out = self.fake_spawn("Bd", "--delta-note", big_note)
self.assertEqual(len(out["reasons"]), 1, out)
self.assertRegex(out["reasons"][0], r"^delta_tokens:\d+$")
self.assertGreater(int(out["reasons"][0].split(":")[1]), 15000)
self.assertEqual((out["rebase_due"], out["retrain_due"]), (True, False))
# binary_drift: a clone with its own copy of the binary, changed after it was recorded
ext = os.path.splitext(self.stub)[1]
b2 = os.path.join(os.path.dirname(self.stub), "claude2" + ext)
shutil.copy2(self.stub, b2)
info = gremlins.binary_info(gremlins.norm_path(b2))
b1_launch = self.reg()["bases"]["B1"]["launch"]
self.clone_base("Bb", launch=dict(b1_launch, binary=info), binary=info)
self.assert_due(self.fake_spawn("Bb"))
with open(b2, "a", newline="\r\n" if IS_WINDOWS else "\n") as f:
f.write("rem drift\n" if IS_WINDOWS else "# drift\n")
self.assert_due(self.fake_spawn("Bb"), "binary_drift")
# runtime_drift (M6): the base's last call saw a versionless plugin and the skills change
self.clone_base("Br")
r = self.finish(self.spawn("turns=1", "nover=def456", "skill=extra", base="Br")["job"])
self.assertEqual((r["status"], r["fingerprint"]), ("ok", "drift:plugins,skills"), r)
self.assert_due(self.fake_spawn("Br"), "runtime_drift:plugins,skills")
# defining: the base's record differs from the config in a field it did not override
self.clone_base("Bm", launch=dict(b1_launch, model="claude-opus-4-1"))
self.assert_due(self.fake_spawn("Bm"), "defining:model")
# max_base_tokens: from the stored context, and from the transcript when it exists (P0.7)
self.clone_base("Bx", context_tokens=300000)
self.assert_due(self.fake_spawn("Bx"), "max_base_tokens:300000")
sid = "11111111-2222-3333-4444-555555555555"
self.clone_base("Bt", session=sid, context_tokens=1000)
tp = gremlins.transcript_path(b1_launch["cwd"], sid)
self.assertTrue(tp.startswith(os.path.join(self.tmp, "home", "projects")), tp)
os.makedirs(os.path.dirname(tp))
lines = [{"type": "user", "message": {"role": "user", "content": "x" * 300}},
{"type": "assistant", "message": {"id": "m1", "usage": {
"input_tokens": 2, "cache_creation_input_tokens": 500, "cache_read_input_tokens": 1000}}},
{"type": "assistant", "message": {"id": "m2", "usage": {
"input_tokens": 2, "cache_creation_input_tokens": 1000, "cache_read_input_tokens": 260000}}},
{"type": "system", "subtype": "informational", "content": "y" * 300}]
with open(tp, "w", newline="\n") as f:
f.write("".join(json.dumps(x) + "\n" for x in lines))
with mock.patch.object(gremlins, "TRANSCRIPT_TAIL_BYTES", 64): # the tail grows
self.assert_due(self.fake_spawn("Bt"), "max_base_tokens:261002")
# rebases: lineage_rebases >= max_rebases 4
self.clone_base("Bn3", lineage_rebases=3)
self.assert_due(self.fake_spawn("Bn3"))
self.clone_base("Bn", lineage_rebases=4)
self.assert_due(self.fake_spawn("Bn"), "rebases:4")
# binary_missing: the stored path is gone; launches refuse (exit 2)
gone = dict(info, path=os.path.join(self.tmp, "gone", "claude" + ext))
self.clone_base("Bz", launch=dict(b1_launch, binary=gone), binary=gone)
out = self.fake_spawn("Bz", code=2)
self.assertEqual((out["reason"], out["retrain_due"], out["reasons"]),
("binary_missing", True, ["binary_missing"]), out)
# list: the base-level reasons, per active base
rows = {b["name"]: b for b in self.cmd("list")["bases"]}
for name, want in (("Bl", ["lines:1600"]), ("Bd", []), ("Bb", ["binary_drift"]),
("Br", ["runtime_drift:plugins,skills"]), ("Bm", ["defining:model"]),
("Bx", ["max_base_tokens:300000"]), ("Bt", ["max_base_tokens:261002"]),
("Bn", ["rebases:4"]), ("Bz", ["binary_missing"])):
with self.subTest(base=name):
self.assert_due(rows[name], *want)
def test_delta_tokens_counts_only_what_the_base_does_not_know(self):
"""I1: delta_tokens = the delta header + `--delta-note` files newer than the base's training
+ a `## Changes since base` section of the brief. Large reference docs a brief merely names
never count; an attached note is in the gremlin's prompt with its path; a base trained or
rebased after the note's mtime does not count it, so a rebase clears the reason."""
self.setup_run(cap=4)
self.train("B1")
self.finish("j0001")
os.makedirs(os.path.join(self.cwd, "docs"))
for rel in ("docs/build-plan.md", "docs/run-log.md"):
with open(os.path.join(self.cwd, *rel.split("/")), "w") as f:
f.write("r" * 80000)
with open(os.path.join(self.state, "notes", "1.1.md"), "w") as f:
f.write("h" * 80000)
named = self.brief(text="Read docs/build-plan.md, docs/run-log.md and notes/1.1.md; edit "
"docs/run-log.md and %s." % os.path.join(self.cwd, "docs", "build-plan.md"))
self.assert_due(self.fake_spawn("B1", brief=named)) # named only: nothing
# a large attached note trips it, and the gremlin's prompt carries it with its path
note = os.path.join(self.tmp, "delta-1.md")
with open(note, "w", newline="\n") as f:
f.write("NOTE-MARKER\n" + "d" * 60000 + "\n")
# `trained_at` has whole seconds, so a note written in the same second as a training's or
# a rebase's end would sit on either side of it by chance: B1's training is moved a minute
# back and the note half a minute back, so B1 predates the note and the rebase below
# (ending now or later) postdates it, on any host speed.
t_note = time.time() - 30
os.utime(note, (t_note, t_note))
self.set_registry(lambda reg: reg["bases"]["B1"].__setitem__("trained_at", gremlins._iso(t_note - 30)))
self.assertLess(gremlins._ts(self.reg()["bases"]["B1"]["trained_at"]), os.path.getmtime(note))
out = self.fake_spawn("B1", "--delta-note", note, brief=named)
self.assertEqual(len(out["reasons"]), 1, out)
self.assertRegex(out["reasons"][0], r"^delta_tokens:\d+$")
self.assertGreater(int(out["reasons"][0].split(":")[1]), 15000)
with open(os.path.join(self.state, "briefs", out["job"] + ".md"), encoding="utf-8") as fh:
prompt = fh.read()
delta_at = prompt.index("## Delta since training")
self.assertLess(delta_at, prompt.index("`%s`" % gremlins.norm_path(note)))
self.assertLess(prompt.index("NOTE-MARKER"), prompt.index("## Rules for this gremlin"))
self.assertEqual(self.reg()["jobs"][out["job"]]["delta_notes"], [gremlins.norm_path(note)])
# a small attached note does not; a missing one is refused
small = os.path.join(self.tmp, "delta-small.md")
with open(small, "w") as f:
f.write("small\n")
self.assert_due(self.fake_spawn("B1", "--delta-note", small))
self.assertEqual(self.fake_spawn("B1", "--delta-note", os.path.join(self.tmp, "nope.md"),
code=2)["error"], "cannot read delta note")
# a base trained after the note's mtime already knows it
mtime = os.path.getmtime(note)
self.clone_base("Bk", trained_at=gremlins._iso(mtime + 60))
self.assert_due(self.fake_spawn("Bk", "--delta-note", note))
# a large `## Changes since base` section trips it; the same text under another heading not
body = "c" * 60000
out = self.fake_spawn("Bk", brief=self.brief(text="## Changes since base\n\n%s\n\n## Gate\n\nrun it" % body))
self.assertEqual(len(out["reasons"]), 1, out)
self.assertRegex(out["reasons"][0], r"^delta_tokens:\d+$")
self.assert_due(self.fake_spawn("Bk", brief=self.brief(text="## Background\n\n%s\n" % body)))
# a rebase after the note clears the reason; the rebase's prompt carries the note too
r = self.cmd("rebase", "--base", "B1", "--as", "B2", "--delta-note", note,
"--brief", self.brief("turns=1", text="Absorb."))
with open(os.path.join(self.state, "briefs", r["job"] + ".md"), encoding="utf-8") as fh:
rprompt = fh.read()
self.assertIn("NOTE-MARKER", rprompt)
self.assertLess(rprompt.index("## Delta since the base commit"), rprompt.index("NOTE-MARKER"))
self.assertLess(rprompt.index("NOTE-MARKER"), rprompt.index("## Brief"))
self.assertEqual(self.finish(r["job"])["status"], "ok")
self.assertGreater(gremlins._ts(self.reg()["bases"]["B2"]["trained_at"]), mtime)
self.assert_due(self.fake_spawn("B2", "--delta-note", note))
# list counts the delta header only
rows = {b["name"]: b for b in self.cmd("list")["bases"]}
self.assert_due(rows["B2"])
def test_train_supersedes(self):
"""Minor 1: `train --supersedes B` (repeatable) marks each named active base superseded by
the new one when its training ends ok: no more pings, and a spawn needs --force. Unknown
names are refused at launch; a failed training changes nothing."""
self.setup_run(cap=4)
self.train("B1")
self.finish("j0001")
self.train("B0")
self.finish("j0002")
out = self.cmd("train", "--base", "B2", "--brief", self.brief(), "--supersedes", "nope", code=2)
self.assertEqual((out["error"], out["bases"]), ("unknown base", ["nope"]))
out = self.cmd("train", "--base", "B2", "--brief", self.brief(), "--supersedes", "B2", code=2)
self.assertIn("itself", out["error"])
self.assertNotIn("B2", self.reg()["bases"])
# a failed training: nothing changes
t = self.cmd("train", "--base", "B2", "--brief", self.brief("error"), "--supersedes", "B1")
self.assertEqual(self.finish(t["job"])["status"], "failed")
reg = self.reg()
self.assertEqual((reg["bases"]["B2"]["status"], reg["bases"]["B1"]["status"]), ("failed", "active"))
self.assertNotIn("superseded_by", reg["bases"]["B1"])
# an ok training supersedes every named active base
t = self.cmd("train", "--base", "B3", "--brief", self.brief(), "--supersedes", "B1",
"--supersedes", "B0", "--supersedes", "B2")
self.assertEqual(self.finish(t["job"])["status"], "ok")
reg = self.reg()
for name in ("B1", "B0"):
with self.subTest(base=name):
b = reg["bases"][name]
self.assertEqual((b["status"], b["superseded_by"]), ("superseded", "B3"))
self.assertIsNotNone(b["superseded_at"])
self.assertEqual(reg["bases"]["B2"]["status"], "failed") # not active: untouched
self.assertEqual(reg["bases"]["B3"]["status"], "active")
self.assertEqual(gremlins.ping_decision(reg, "B1")["why"], "status:superseded")
self.assertIn("not active", self.fake_spawn("B1", code=2)["error"])
self.fake_spawn("B1", "--force")
def test_train_overrides_and_init_model_change(self):
"""M10 (plan 4.3): `train` takes init's launch flags as overrides; the base records its
launch fields and their hash, and every later launch on it uses that record. `init`
changing the model afterwards: a base that did not override it reports `retrain_due:
defining:model`; one that did reports nothing."""
self.setup_run(cap=4)
t = self.cmd("train", "--base", "B1", "--brief", self.brief(), "--model", "claude-sonnet-5-5",
"--effort", "low", "--tools", "Read,Bash")
r = self.finish(t["job"])
self.assertEqual((r["status"], r["model_requested"], r["models_used"]),
("ok", "claude-sonnet-5-5", ["claude-sonnet-5-5"]), r)
reg = self.reg()
cfg, b1 = reg["config"], reg["bases"]["B1"]
rec = b1["launch"]
self.assertEqual((rec["model"], rec["effort"], rec["tools"]), ("claude-sonnet-5-5", "low", ["Read", "Bash"]))
self.assertEqual(rec["allowed_tools"], cfg["launch"]["allowed_tools"]) # not overridden
self.assertEqual(rec["cwd"], cfg["launch"]["cwd"])
self.assertEqual(b1["overrides"], ["effort", "model", "tools"])
self.assertEqual(b1["fingerprint"], gremlins.fingerprint_hash(rec))
self.assertNotEqual(b1["fingerprint"], cfg["fingerprint"])
a = stub_calls(self.stub)[0]["argv"]
self.assertEqual((a[a.index("--model") + 1], a[a.index("--effort") + 1], a[a.index("--tools") + 1]),
("claude-sonnet-5-5", "low", "Read,Bash"))
# a base trained without overrides records the config's fields
self.train("B2")
self.assertEqual(self.finish("j0002")["status"], "ok")
b2 = self.reg()["bases"]["B2"]
self.assertEqual((b2["launch"], b2["overrides"], b2["fingerprint"]),
(cfg["launch"], [], cfg["fingerprint"]))
# spawn and continue of B1 build their command line from B1's record
f = self.spawn("turns=1", base="B1")
self.assert_due(f)
rf = self.finish(f["job"])
self.assertEqual((rf["status"], rf["model_requested"], rf["models_used"]),
("ok", "claude-sonnet-5-5", ["claude-sonnet-5-5"]), rf)
c = self.cmd("continue", "--job", f["job"], "--brief", self.brief("turns=1"))
rc = self.finish(c["job"])
self.assertEqual((rc["status"], rc["model_requested"]), ("ok", "claude-sonnet-5-5"), rc)
for job in (f["job"], c["job"]):
with self.subTest(job=job):
argv = self.reg()["jobs"][job]["argv"]
self.assertEqual((argv[argv.index("--model") + 1], argv[argv.index("--effort") + 1],
argv[argv.index("--tools") + 1]), ("claude-sonnet-5-5", "low", "Read,Bash"))
# init changes the model after the bases exist
out = self.setup_run("--model", "claude-opus-4-1", cap=4)
self.assertTrue(out["changed"], out)
self.assertIn("launch.model", out["changed_fields"])
rows = {b["name"]: b for b in self.cmd("list")["bases"]}
self.assert_due(rows["B2"], "defining:model")
self.assert_due(rows["B1"])
f2 = self.spawn("turns=1", base="B2")
self.assert_due(f2, "defining:model") # advisory: the gremlin still launches
r2 = self.finish(f2["job"])
self.assertEqual((r2["status"], r2["model_requested"]), ("ok", TEST_MODEL), r2) # B2's record
f1 = self.spawn("turns=1", base="B1")
self.assert_due(f1)
self.assertEqual(self.finish(f1["job"])["model_requested"], "claude-sonnet-5-5")
# overrides pass init's checks: aliases, plan mode, availableModels
self.cmd("train", "--base", "B3", "--brief", self.brief(), "--model", "sonnet", code=2)
self.cmd("train", "--base", "B3", "--brief", self.brief(), "--permission-mode", "plan", code=2)
settings = os.path.join(self.tmp, "home", "settings.json")
os.makedirs(os.path.dirname(settings), exist_ok=True)
with open(settings, "w") as fh:
json.dump({"availableModels": ["opus"]}, fh)
out = self.cmd("train", "--base", "B3", "--brief", self.brief(), "--model", "claude-sonnet-5-5", code=2)
self.assertEqual(out["error"], "model not admitted by availableModels")
self.assertNotIn("B3", self.reg()["bases"])
def test_note_only_from_this_job(self):
"""M11: `note` is set only when the note file was written during this job; a note left by an
earlier gremlin of the block, or by the gremlin a continue resumes, does not count."""
self.setup_run()
self.train("B1")
self.finish("j0001")
note = os.path.join(self.state, "notes", "1.1.md")
with open(note, "w") as f:
f.write("old note\n")
old = time.time() - 600
os.utime(note, (old, old))
r = self.finish(self.spawn("turns=1")["job"])
self.assertEqual((r["status"], r["note"]), ("ok", None), r)
j = self.spawn("sleep=1", "turns=2")["job"]
with open(note, "w") as f: # written while the job runs
f.write("new note\n")
r = self.finish(j)
self.assertEqual((r["status"], r["note"]), ("ok", "notes/1.1.md"), r)
os.utime(note, (old, old))
rc = self.finish(self.cmd("continue", "--job", j, "--brief", self.brief("turns=1"))["job"])
self.assertEqual((rc["status"], rc["note"]), ("ok", None), rc)
led = {x["job"]: x for x in self.ledger()}
self.assertEqual((led[j]["note"], led[rc["job"]]["note"]), ("notes/1.1.md", None))
def test_rebase_end_to_end(self):
"""`rebase` with the stub: the new base is active (parent, lineage, HEAD, context, launch
record), the old one superseded; a spawn from the new base hits the new context; a failed
rebase leaves the base active and creates nothing; a binary drift is recorded by the new
base."""
self.setup_run()
self.train("B1")
self.finish("j0001")
b1 = self.reg()["bases"]["B1"]
self.commit_file("src.py", "x = 1\n", "work of phase 1")
f1 = self.spawn("turns=1", extra=("--phase", "1"))
self.assert_due(f1)
self.finish(f1["job"])
f2 = self.spawn("turns=1", extra=("--phase", "2"))
self.assert_due(f2, "phase_boundary")
self.finish(f2["job"])
out = self.cmd("rebase", "--base", "B1", "--as", "B2", "--phase", "2",
"--brief", self.brief("turns=2", text="Absorb phase 1."))
self.assertEqual(set(out), {"job", "base", "parent", "session_id"})
self.assertEqual((out["base"], out["parent"]), ("B2", "B1"))
rr = self.finish(out["job"])
self.assertEqual((rr["kind"], rr["status"], rr["base"], rr["cache"], rr["phase"], rr["fingerprint"]),
("rebase", "ok", "B1", "hit", 2, "ok"), rr)
self.assertEqual((rr["session"], rr["parent_session"]), (out["session_id"], b1["session"]))
reg = self.reg()
old, new = reg["bases"]["B1"], reg["bases"]["B2"]
self.assertEqual((old["status"], old["superseded_by"]), ("superseded", "B2"))
self.assertEqual((new["status"], new["parent"], new["lineage_rebases"], new["session"], new["phase"],
new["rebase_job"]), ("active", "B1", 1, out["session_id"], 2, out["job"]))
self.assertEqual(new["base_commit"], self.git("rev-parse", "HEAD"))
self.assertEqual((new["launch"], new["fingerprint"], new["overrides"]),
(old["launch"], old["fingerprint"], []))
self.assertGreater(new["context_tokens"], old["context_tokens"])
self.assertIsNotNone(new["last_hit"])
line = {x["job"]: x for x in self.ledger()}[out["job"]]
self.assertEqual((line["kind"], line["status"]), ("rebase", "ok"))
self.assertAlmostEqual(new["cost_cum_usd"], line["cost_cumulative_usd"], places=6)
a = [c for c in stub_calls(self.stub) if c["session"] == out["session_id"]][0]["argv"]
self.assertEqual(a[a.index("--resume") + 1], b1["session"])
self.assertIn("--fork-session", a)
self.assertEqual(a[a.index("--session-id") + 1], out["session_id"])
with open(os.path.join(self.state, "briefs", out["job"] + ".md"), encoding="utf-8") as fh:
txt = fh.read()
for s in ("READY", "Do not change code", "work of phase 1", "`B2`", "Absorb phase 1.", "stale"):
self.assertIn(s, txt)
# a spawn from the new base: a hit against the new base's context
f3 = self.spawn("turns=1", base="B2", extra=("--phase", "2"))
self.assert_due(f3)
r3 = self.finish(f3["job"])
self.assertEqual((r3["status"], r3["cache"], r3["parent_session"]), ("ok", "hit", new["session"]), r3)
self.assertEqual(self.reg()["jobs"][f3["job"]]["base_context"], new["context_tokens"])
self.assertGreaterEqual(r3["tokens"]["cache_read"], new["context_tokens"])
a = [c for c in stub_calls(self.stub) if c["session"] == r3["session"]][0]["argv"]
self.assertEqual(a[a.index("--resume") + 1], new["session"])
# the superseded base: no spawn, no rebase
self.spawn(base="B1", code=2)
self.cmd("rebase", "--base", "B1", "--as", "B9", "--brief", self.brief(), code=2)
# a failed rebase: the base stays active, the new base is never created
rf = self.finish(self.cmd("rebase", "--base", "B2", "--as", "B3", "--brief", self.brief("error"))["job"])
self.assertEqual((rf["status"], rf["reason"]), ("failed", "is_error"), rf)
reg = self.reg()
self.assertEqual(reg["bases"]["B2"]["status"], "active")
self.assertNotIn("B3", reg["bases"])
rows = {b["name"]: b for b in self.cmd("list")["bases"]}
self.assertEqual((rows["B1"]["status"], rows["B1"]["superseded_by"], rows["B2"]["parent"]),
("superseded", "B2", "B1"))
# refusals
for argv, why in ((["--base", "B2", "--as", "B2"], "new base"),
(["--base", "B2", "--as", "B1"], "base exists"),
(["--base", "B9", "--as", "B4"], "unknown base"),
(["--base", "B2", "--as", "a/b"], "bad base name")):
with self.subTest(argv=argv):
self.assertIn(why, self.cmd("rebase", *argv, "--brief", self.brief(), code=2)["error"])
os.environ[gremlins.CHILD_MARKER] = "1"
try:
out = self.cmd("rebase", "--base", "B2", "--as", "B4", "--brief", self.brief(), code=2)
self.assertIn("inside a gremlin", out["error"])
finally:
os.environ.pop(gremlins.CHILD_MARKER, None)
# binary drift: the rebase runs under the current binary, and the new base records it
with open(self.stub, "a", newline="\r\n" if IS_WINDOWS else "\n") as f:
f.write("rem drift\n" if IS_WINDOWS else "# drift\n")
fd = self.spawn("turns=1", base="B2")
self.assert_due(fd, "binary_drift")
self.finish(fd["job"])
out = self.cmd("rebase", "--base", "B2", "--as", "B4", "--brief", self.brief("turns=1"))
rb = self.finish(out["job"])
self.assertEqual((rb["status"], rb["fingerprint"]), ("ok", "drift:binary"), rb)
b2, b4 = self.reg()["bases"]["B2"], self.reg()["bases"]["B4"]
cur = gremlins.binary_info(gremlins.norm_path(self.stub))
for k in ("version", "size", "mtime"):
self.assertEqual((b4["binary"][k], b4["launch"]["binary"][k]), (cur[k], cur[k]), k)
self.assertEqual((b4["parent"], b4["lineage_rebases"], b2["status"]), ("B2", 2, "superseded"))
self.assertEqual(b4["fingerprint"], gremlins.fingerprint_hash(b4["launch"]))
self.assertNotEqual(b4["fingerprint"], b2["fingerprint"])
f4 = self.spawn("turns=1", base="B4")
self.assert_due(f4)
r4 = self.finish(f4["job"])
self.assertEqual((r4["status"], r4["fingerprint"], r4["cache"]), ("ok", "ok", "hit"), r4)
def test_rebase_counts_toward_cap(self):
"""A running rebase counts toward parallel_cap, blocks a second rebase of its base and a
train of its new name; killed, it leaves the base active and creates nothing."""
self.setup_run(cap=1)
self.train("B1")
self.finish("j0001")
nxt = gremlins._next_job_id(self.reg())
out = self.cmd("rebase", "--base", "B1", "--as", "B2",
"--brief", self.brief("hang=1", text="RUN-%s" % nxt))
job = out["job"]
self.assertEqual(job, nxt)
self.until_running(job)
refused = self.spawn(code=2)
self.assertEqual((refused["error"], refused["running"]), ("parallel cap reached", [job]))
self.assertIn("rebased", self.cmd("rebase", "--base", "B1", "--as", "B3", "--brief", self.brief(),
code=2)["error"])
self.assertIn("rebase creates", self.cmd("train", "--base", "B2", "--brief", self.brief(),
code=2)["error"])
self.assertTrue(self.cmd("kill", "--job", job)["ok"])
r = self.finish(job)
self.assertEqual((r["status"], r["kind"]), ("killed", "rebase"))
reg = self.reg()
self.assertEqual(reg["bases"]["B1"]["status"], "active")
self.assertNotIn("B2", reg["bases"])
def test_late_rebase_of_a_superseded_base(self):
"""N4 (C15): `rebase --base B1 --as B1r` is still running when `train --base B3 --supersedes
B1` ends ok. When the rebase then ends ok, its result is `ok` with `parent_inactive`; B1r is
created superseded by B3 (no pings, a spawn needs --force); B1 keeps `superseded_by == B3`.
A second rebase whose new name is taken by then (`rebase_conflict`) creates nothing and is
not flagged."""
self.setup_run(cap=4)
self.train("B1")
self.finish("j0001")
self.train("B2")
self.finish("j0002")
gates = os.path.dirname(self.stub) # the stub's `gate=NAME` waits for this file
r1 = self.cmd("rebase", "--base", "B1", "--as", "B1r", "--phase", "2",
"--brief", self.brief("gate=g1", "turns=2"))
r2 = self.cmd("rebase", "--base", "B2", "--as", "B2r", "--brief", self.brief("gate=g2", "turns=2"))
t = self.cmd("train", "--base", "B3", "--brief", self.brief("turns=1"), "--supersedes", "B1",
"--supersedes", "B2")
self.assertEqual(self.finish(t["job"])["status"], "ok")
reg = self.reg()
self.assertEqual([reg["jobs"][j]["status"] in gremlins.RUNNING for j in (r1["job"], r2["job"])],
[True, True], "the rebases must still run when the training ends")
b1_before = dict(reg["bases"]["B1"])
self.assertEqual((b1_before["status"], b1_before["superseded_by"]), ("superseded", "B3"))
self.clone_base("B2r", src="B3") # B2r's name is taken before its rebase ends
b2r_before = self.reg()["bases"]["B2r"]
for g in ("g1", "g2"):
with open(os.path.join(gates, g), "w"):
pass
rr1, rr2 = self.finish(r1["job"]), self.finish(r2["job"])
# the late rebase: ok, flagged, the new base superseded by the same base as its parent
self.assertEqual((rr1["kind"], rr1["status"], rr1["base"]), ("rebase", "ok", "B1"), rr1)
self.assertIn("parent_inactive", rr1["flags"])
reg = self.reg()
new, b1 = reg["bases"]["B1r"], reg["bases"]["B1"]
e1 = reg["jobs"][r1["job"]]
self.assertEqual((new["status"], new["superseded_by"], new["superseded_at"]),
("superseded", "B3", e1["ended_at"]))
self.assertEqual((new["parent"], new["session"], new["lineage_rebases"]), ("B1", r1["session_id"], 1))
lineage = ("status", "superseded_by", "superseded_at") # B1's lineage untouched
self.assertEqual([b1[k] for k in lineage], [b1_before[k] for k in lineage])
self.assertEqual(e1["rebase_inactive_parent"], "B1 was superseded when the rebase ended")
self.assertIn("parent_inactive", e1["flags"])
self.assertIn("parent_inactive", {x["job"]: x for x in self.ledger()}[r1["job"]]["flags"])
self.assertEqual(gremlins.ping_decision(reg, "B1r")["why"], "status:superseded")
self.assertIn("not active", self.spawn(base="B1r", code=2)["error"]) # needs --force
self.fake_spawn("B1r", "--force")
# the conflict: ok, not flagged, nothing created or replaced
self.assertEqual(rr2["status"], "ok", rr2)
self.assertNotIn("parent_inactive", rr2["flags"])
reg = self.reg()
self.assertIn("rebase_conflict", reg["jobs"][r2["job"]])
self.assertNotIn("rebase_inactive_parent", reg["jobs"][r2["job"]])
self.assertEqual(reg["bases"]["B2r"], b2r_before)
self.assertEqual((reg["bases"]["B2"]["status"], reg["bases"]["B2"]["superseded_by"]), ("superseded", "B3"))
def test_head_at_end_and_usage_commit_column(self):
"""2.3: the supervisor records `head_at_end`; `usage --block` lists the commits made during a
sequential spawn, and the ledger/usage figures agree with the result."""
self.setup_run()
self.train("B1")
self.finish("j0001")
out = self.spawn_marked("turns=3", "sleep=1", extra=("--phase", "1"))
job = out["job"]
self.until_running(job)
before = self.git("rev-parse", "HEAD")
self.commit_file("made.py", "x = 1\n", "commit made during the spawn")
after = self.git("rev-parse", "HEAD")
res = self.finish(job)
self.assertEqual(res["status"], "ok", res)
e = self.reg()["jobs"][job]
self.assertEqual(e["head_at_launch"], before)
self.assertEqual(e["head_at_end"], after)
self.assertEqual(self.reg()["jobs"]["j0001"]["head_at_end"], before) # the train too
c, raw = call_raw(["usage", "--run", self.run_log, "--block", "1.1", "--format", "md"])
self.assertEqual(c, 0, raw)
rows = raw.splitlines()
self.assertEqual(rows[0], BLOCK_HEADER)
self.assertEqual(len(rows), 3, raw)
cells = [x.strip() for x in rows[2].strip("|").split("|")]
self.assertEqual(cells[0], "1.1")
self.assertEqual(cells[1], TEST_MODEL)
self.assertEqual(cells[2], res["session"][:8])
self.assertEqual(cells[3], after[:7])
t = res["tokens"]
self.assertEqual(cells[4:8], ["{:,}".format(t[k]) for k in ("input", "output", "cache_creation",
"cache_read")])
self.assertEqual(cells[8], "${:.4f}".format(res["cost_delta_usd"]))
# the json view agrees with the ledger
c, js, raw = call(["usage", "--run", self.run_log, "--phase", "1"])
self.assertEqual((c, js["reconciled"], js["total"]["calls"]), (0, [], 1))
self.assertEqual(js["kinds"]["spawn"]["cache_read"], res["tokens"]["cache_read"])
def tearDownModule():
"""No supervisor or stub of this suite may survive it (checked by command line, never by name)."""
if not IS_WINDOWS or not STATE_DIRS:
return
ps = ("$p = 'gremlins' + '-test-'; Get-CimInstance Win32_Process | "
"Where-Object { $_.CommandLine -and $_.CommandLine.Contains($p) } | "
"ForEach-Object { '{0}|{1}' -f $_.ProcessId, $_.CommandLine }")
r = subprocess.run(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
capture_output=True, timeout=120, creationflags=0x08000000)
lines = [x for x in r.stdout.decode("utf-8", "replace").splitlines() if x.strip()]
for x in lines:
with contextlib.suppress(ValueError, OSError):
gremlins.terminate(gremlins.pid_record(int(x.split("|", 1)[0])))
if lines:
raise AssertionError("leftover test processes (killed now): %s" % lines)
if __name__ == "__main__":
unittest.main(verbosity=2)
Reference in New Issue View Git Blame Copy Permalink