🐛 gremlins: init re-resolves the binary, late rebase of an inactive base, kill verification
This commit is contained in:
@@ -1179,7 +1179,8 @@ def kill_tree(rec: dict | None, extra=(), alive_until: float | None = None) -> d
|
|||||||
break # one tree kill of the first live root; the sweep covers the rest
|
break # one tree kill of the first live root; the sweep covers the rest
|
||||||
deadline = time.monotonic() + KILL_VERIFY_S
|
deadline = time.monotonic() + KILL_VERIFY_S
|
||||||
while True:
|
while True:
|
||||||
survivors = [p for p in snap if record_alive(p)]
|
table = {pid: name for pid, _, name in process_table()} # one table per sweep
|
||||||
|
survivors = [p for p in snap if snapshot_survivor(p, table)]
|
||||||
if not survivors or time.monotonic() >= deadline:
|
if not survivors or time.monotonic() >= deadline:
|
||||||
break
|
break
|
||||||
for p in survivors:
|
for p in survivors:
|
||||||
@@ -1188,6 +1189,22 @@ def kill_tree(rec: dict | None, extra=(), alive_until: float | None = None) -> d
|
|||||||
return {"snapshot": snap, "tree_kill_rc": rc, "survivors": survivors}
|
return {"snapshot": snap, "tree_kill_rc": rc, "survivors": survivors}
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_survivor(p: dict, table: dict) -> bool:
|
||||||
|
"""Is snapshot entry `p` ({pid, ctime, name}) still alive? (C16 as narrowed by A2.)
|
||||||
|
`table` is {pid: name} from one `process_table()`. The pid is dead when the table is
|
||||||
|
non-empty and lacks it, or when its snapshot name is known, the table now gives the pid another
|
||||||
|
name and its creation time cannot be read (`proc_ctime` None): pid reuse by a process that
|
||||||
|
refuses `OpenProcess`, which `pid_alive` reports alive (access denied) without being able to
|
||||||
|
compare creation times (class (a)). In every other case `record_alive` decides."""
|
||||||
|
pid = int(p["pid"])
|
||||||
|
if table and pid not in table:
|
||||||
|
return False
|
||||||
|
name = p.get("name")
|
||||||
|
if name is not None and pid in table and table[pid] != name and proc_ctime(pid) is None:
|
||||||
|
return False
|
||||||
|
return record_alive(p)
|
||||||
|
|
||||||
|
|
||||||
def _run(args, cwd=None, timeout=60, env=None) -> subprocess.CompletedProcess:
|
def _run(args, cwd=None, timeout=60, env=None) -> subprocess.CompletedProcess:
|
||||||
"""Run a short helper process: stdout captured (never leaks to ours), no window."""
|
"""Run a short helper process: stdout captured (never leaks to ours), no window."""
|
||||||
return subprocess.run(args, cwd=cwd, env=env, stdin=subprocess.DEVNULL, capture_output=True,
|
return subprocess.run(args, cwd=cwd, env=env, stdin=subprocess.DEVNULL, capture_output=True,
|
||||||
@@ -1295,15 +1312,29 @@ def resolve_binary(explicit: str | None = None, environ=None) -> tuple[str | Non
|
|||||||
environ = os.environ
|
environ = os.environ
|
||||||
if explicit:
|
if explicit:
|
||||||
return norm_path(explicit), "--binary"
|
return norm_path(explicit), "--binary"
|
||||||
ep = environ.get("CLAUDE_CODE_EXECPATH")
|
for how in BINARY_SOURCES:
|
||||||
if ep:
|
path = resolve_binary_via(how, environ)
|
||||||
return norm_path(ep), "CLAUDE_CODE_EXECPATH"
|
if path:
|
||||||
w = shutil.which("claude")
|
return path, how
|
||||||
if w:
|
|
||||||
return norm_path(w), "PATH"
|
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
BINARY_SOURCES = ("CLAUDE_CODE_EXECPATH", "PATH") # the lookup order without --binary
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_binary_via(how: str, environ=None) -> str | None:
|
||||||
|
"""The binary through one source only (N3, A4): `CLAUDE_CODE_EXECPATH` or a `PATH` lookup."""
|
||||||
|
if environ is None:
|
||||||
|
environ = os.environ
|
||||||
|
if how == "CLAUDE_CODE_EXECPATH":
|
||||||
|
ep = environ.get("CLAUDE_CODE_EXECPATH")
|
||||||
|
return norm_path(ep) if ep else None
|
||||||
|
if how == "PATH":
|
||||||
|
w = shutil.which("claude")
|
||||||
|
return norm_path(w) if w else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _binary_version(path: str) -> str | None:
|
def _binary_version(path: str) -> str | None:
|
||||||
try:
|
try:
|
||||||
r = _run([path, "--version"], timeout=60)
|
r = _run([path, "--version"], timeout=60)
|
||||||
@@ -1785,6 +1816,12 @@ def _conclude(state: str, reg: dict, entry: dict, ss: StreamState | None, hint:
|
|||||||
and bool(ss and ss.first_at))
|
and bool(ss and ss.first_at))
|
||||||
if kind == "ping" and not ping_refreshed:
|
if kind == "ping" and not ping_refreshed:
|
||||||
flags.append("ping_missed")
|
flags.append("ping_missed")
|
||||||
|
# N4 (C15): a rebase that ends ok after its base stopped being active (a `train --supersedes`
|
||||||
|
# ended first, or an `expire` raced the kill) still creates its new base, with the old base's
|
||||||
|
# status. Flagged only when the new base is actually created (not on a `rebase_conflict`).
|
||||||
|
if (kind == "rebase" and status == "ok" and base is not None and base.get("status") != "active"
|
||||||
|
and _rebase_target_free(reg, entry)):
|
||||||
|
flags.append("parent_inactive")
|
||||||
flags = _uniq(flags)
|
flags = _uniq(flags)
|
||||||
turns = int(res.get("num_turns") or 0) if res else (ss.turns if ss else 0)
|
turns = int(res.get("num_turns") or 0) if res else (ss.turns if ss else 0)
|
||||||
note = note_rel(entry.get("block")) if kind != "train" else None
|
note = note_rel(entry.get("block")) if kind != "train" else None
|
||||||
@@ -1867,20 +1904,35 @@ def _conclude(state: str, reg: dict, entry: dict, ss: StreamState | None, hint:
|
|||||||
return result, ledger
|
return result, ledger
|
||||||
|
|
||||||
|
|
||||||
|
def _rebase_target_free(reg: dict, entry: dict) -> bool:
|
||||||
|
"""True if a successful rebase `entry` may create its new base: it names one, and no base of
|
||||||
|
that name exists other than a failed one (which it replaces)."""
|
||||||
|
name = entry.get("rebase_as")
|
||||||
|
if not name:
|
||||||
|
return False
|
||||||
|
existing = reg["bases"].get(name)
|
||||||
|
return not (existing and existing.get("status") != "failed")
|
||||||
|
|
||||||
|
|
||||||
def _rebase_succeeded(reg: dict, entry: dict, old: dict, ss, u, rt, base_commit, ended: str) -> None:
|
def _rebase_succeeded(reg: dict, entry: dict, old: dict, ss, u, rt, base_commit, ended: str) -> None:
|
||||||
"""A rebase ended `ok`: its session becomes base `entry.rebase_as` (active, parent = the old
|
"""A rebase ended `ok`: its session becomes base `entry.rebase_as` (active, parent = the old
|
||||||
base, one more rebase in the lineage, HEAD at the end, context from its last assistant
|
base, one more rebase in the lineage, HEAD at the end, context from its last assistant
|
||||||
message, the launch record computed at launch: the old base's, with the binary it ran
|
message, the launch record computed at launch: the old base's, with the binary it ran
|
||||||
under); the old base becomes `superseded`. On any other status nothing here runs: the old
|
under); the old base becomes `superseded`. On any other status nothing here runs: the old
|
||||||
base stays active and the new one is never created."""
|
base stays active and the new one is never created.
|
||||||
|
N4 (C15): if the old base is no longer active when the rebase ends (superseded by a training
|
||||||
|
that ended first, or retired by an `expire` that raced the kill), the new base takes the old
|
||||||
|
base's status (`superseded` by the same base, or `retired`, as of this end time) and the old
|
||||||
|
base is left untouched; the job records `rebase_inactive_parent`."""
|
||||||
name = entry.get("rebase_as")
|
name = entry.get("rebase_as")
|
||||||
if not name:
|
if not name:
|
||||||
return
|
return
|
||||||
existing = reg["bases"].get(name)
|
if not _rebase_target_free(reg, entry):
|
||||||
if existing and existing.get("status") != "failed":
|
existing = reg["bases"][name]
|
||||||
entry["rebase_conflict"] = "base %s exists (%s); not replaced" % (name, existing.get("status"))
|
entry["rebase_conflict"] = "base %s exists (%s); not replaced" % (name, existing.get("status"))
|
||||||
return
|
return
|
||||||
launch = entry.get("new_launch") or launch_record(old, reg["config"])
|
launch = entry.get("new_launch") or launch_record(old, reg["config"])
|
||||||
|
old_status = old.get("status")
|
||||||
reg["bases"][name] = {
|
reg["bases"][name] = {
|
||||||
"name": name, "status": "active", "session": entry.get("session"), "cwd": launch["cwd"],
|
"name": name, "status": "active", "session": entry.get("session"), "cwd": launch["cwd"],
|
||||||
"launch": launch, "fingerprint": fingerprint_hash(launch),
|
"launch": launch, "fingerprint": fingerprint_hash(launch),
|
||||||
@@ -1896,6 +1948,16 @@ def _rebase_succeeded(reg: dict, entry: dict, old: dict, ss, u, rt, base_commit,
|
|||||||
"ttl_s": infer_ttl(u["tokens"] if u else None, old.get("ttl_s")), "ttl_from": entry["job"],
|
"ttl_s": infer_ttl(u["tokens"] if u else None, old.get("ttl_s")), "ttl_from": entry["job"],
|
||||||
"ping_in_flight": None,
|
"ping_in_flight": None,
|
||||||
}
|
}
|
||||||
|
if old_status != "active":
|
||||||
|
new = reg["bases"][name]
|
||||||
|
new["status"] = old_status
|
||||||
|
if old_status == "superseded":
|
||||||
|
new.update({"superseded_by": old.get("superseded_by"), "superseded_at": ended})
|
||||||
|
elif old_status == "retired":
|
||||||
|
new["retired_at"] = ended
|
||||||
|
entry["rebase_inactive_parent"] = "%s was %s when the rebase ended" % (
|
||||||
|
old.get("name") or entry.get("base"), old_status)
|
||||||
|
return
|
||||||
old["status"] = "superseded"
|
old["status"] = "superseded"
|
||||||
old["superseded_by"] = name
|
old["superseded_by"] = name
|
||||||
old["superseded_at"] = ended
|
old["superseded_at"] = ended
|
||||||
@@ -2393,12 +2455,34 @@ def build_launch(state: str, *, model, effort, permission_mode, tools, allowed,
|
|||||||
return launch, adm["via"]
|
return launch, adm["via"]
|
||||||
|
|
||||||
|
|
||||||
def build_init_config(args, state: str, old: dict | None = None) -> dict:
|
def rerun_binary(stored: str, how: str | None, warnings: list) -> tuple[str, str | None]:
|
||||||
|
"""N3 (C14, A4): the binary of an `init` re-run without `--binary`. A `--binary` pin keeps the
|
||||||
|
stored path; otherwise the path is looked up again through the source it came from
|
||||||
|
(`CLAUDE_CODE_EXECPATH` or `PATH`, never the other one). Nothing found there keeps the stored
|
||||||
|
path. A change and a failed lookup each add one entry to `warnings`."""
|
||||||
|
if how not in BINARY_SOURCES: # "--binary" (an explicit pin), or no recorded source
|
||||||
|
return stored, how
|
||||||
|
found = resolve_binary_via(how)
|
||||||
|
if found is None:
|
||||||
|
warnings.append("could not re-resolve the claude binary through %s; kept the stored path %s"
|
||||||
|
% (how, stored))
|
||||||
|
return stored, how
|
||||||
|
if os.path.normcase(found) == os.path.normcase(stored):
|
||||||
|
return stored, how
|
||||||
|
warnings.append("binary changed: %s → %s (resolved by %s); existing bases keep their own binary "
|
||||||
|
"record, new trainings use the new one" % (stored, found, how))
|
||||||
|
return found, how
|
||||||
|
|
||||||
|
|
||||||
|
def build_init_config(args, state: str, old: dict | None = None, warnings: list | None = None) -> dict:
|
||||||
"""Validate the launch fields and build the config block. Writes nothing.
|
"""Validate the launch fields and build the config block. Writes nothing.
|
||||||
Minor 8: `old` is the current config (None on the first init). A flag that is not given keeps
|
Minor 8: `old` is the current config (None on the first init). A flag that is not given keeps
|
||||||
its current value (launch fields and run-wide settings alike); defaults apply only on the
|
its current value (launch fields and run-wide settings alike); defaults apply only on the
|
||||||
first init. What is read from disk (the binary's version, size and mtime, the preamble and
|
first init. What is read from disk (the binary's version, size and mtime, the preamble and
|
||||||
agent hashes) is re-read either way."""
|
agent hashes) is re-read either way. N3: the binary is the exception (`rerun_binary`); its
|
||||||
|
warnings go to `warnings`."""
|
||||||
|
if warnings is None:
|
||||||
|
warnings = []
|
||||||
ol = (old or {}).get("launch") or {}
|
ol = (old or {}).get("launch") or {}
|
||||||
|
|
||||||
def keep(given, key, default):
|
def keep(given, key, default):
|
||||||
@@ -2428,7 +2512,7 @@ def build_init_config(args, state: str, old: dict | None = None) -> dict:
|
|||||||
raise GremlinsError("model must be a full id (aliases are refused)", model=model,
|
raise GremlinsError("model must be a full id (aliases are refused)", model=model,
|
||||||
pattern=MODEL_RE.pattern)
|
pattern=MODEL_RE.pattern)
|
||||||
if args.binary is None and (ol.get("binary") or {}).get("path"):
|
if args.binary is None and (ol.get("binary") or {}).get("path"):
|
||||||
bpath, how = ol["binary"]["path"], (old or {}).get("binary_resolved_by")
|
bpath, how = rerun_binary(ol["binary"]["path"], (old or {}).get("binary_resolved_by"), warnings)
|
||||||
else:
|
else:
|
||||||
bpath, how = resolve_binary(args.binary)
|
bpath, how = resolve_binary(args.binary)
|
||||||
if bpath is None:
|
if bpath is None:
|
||||||
@@ -2477,7 +2561,8 @@ def cmd_init(args) -> tuple[dict, int]:
|
|||||||
state = state_dir_for(args.run)
|
state = state_dir_for(args.run)
|
||||||
snap = read_registry(state) if os.path.isdir(state) else None
|
snap = read_registry(state) if os.path.isdir(state) else None
|
||||||
old_cfg = (snap or {}).get("config")
|
old_cfg = (snap or {}).get("config")
|
||||||
cfg = build_init_config(args, state, old_cfg) # all validation before any write
|
warnings: list[str] = []
|
||||||
|
cfg = build_init_config(args, state, old_cfg, warnings) # all validation before any write
|
||||||
ensure_state(state)
|
ensure_state(state)
|
||||||
with RegistryLock(state):
|
with RegistryLock(state):
|
||||||
reg = read_registry(state)
|
reg = read_registry(state)
|
||||||
@@ -2508,7 +2593,7 @@ def cmd_init(args) -> tuple[dict, int]:
|
|||||||
cfg = old
|
cfg = old
|
||||||
return {"ok": True, "state": state, "registry": registry_path(state), "created": created,
|
return {"ok": True, "state": state, "registry": registry_path(state), "created": created,
|
||||||
"changed": bool(old is not None and changed_fields), "changed_fields": changed_fields,
|
"changed": bool(old is not None and changed_fields), "changed_fields": changed_fields,
|
||||||
"bases": sorted(reg.get("bases", {})), "config": cfg}, EXIT_OK
|
"bases": sorted(reg.get("bases", {})), "config": cfg, "warnings": warnings}, EXIT_OK
|
||||||
|
|
||||||
|
|
||||||
# --- launching: train, spawn, continue ------------------------------------------------
|
# --- launching: train, spawn, continue ------------------------------------------------
|
||||||
|
|||||||
@@ -687,6 +687,67 @@ class ProcessTreeTests(Base):
|
|||||||
self.assertEqual(gremlins.reap_lost(st), ["j0001"])
|
self.assertEqual(gremlins.reap_lost(st), ["j0001"])
|
||||||
self.assertEqual(seen, [(self.rec(), gremlins._ts(gremlins._iso(old)))])
|
self.assertEqual(seen, [(self.rec(), gremlins._ts(gremlins._iso(old)))])
|
||||||
|
|
||||||
|
# --- 3.3 / A2: kill verification and pid reuse (fake table; nothing real is killed) ---
|
||||||
|
SNAP = 900051
|
||||||
|
|
||||||
|
def verify(self, *, table, snap_name="sleep.exe", ctime_readable=True):
|
||||||
|
"""`kill_tree` of a dead root whose snapshot holds one pid, SNAP, that `pid_alive` reports
|
||||||
|
alive the way the Windows platform does on access denied (running; the creation time is
|
||||||
|
compared only when it can be read). Returns the survivors' pids and the terminate calls."""
|
||||||
|
ct = self.ctimes[self.C]
|
||||||
|
snap = [{"pid": self.SNAP, "ctime": ct, "ppid": self.ROOT, "name": snap_name}]
|
||||||
|
self.table = list(table)
|
||||||
|
self.live = {self.SNAP}
|
||||||
|
self.ctimes = {self.SNAP: ct if ctime_readable else None}
|
||||||
|
|
||||||
|
def alive(pid, ctime=None):
|
||||||
|
pid = int(pid)
|
||||||
|
if pid not in self.live:
|
||||||
|
return False
|
||||||
|
now = self.ctimes.get(pid)
|
||||||
|
return not (ctime is not None and now is not None and now != ctime)
|
||||||
|
killed = []
|
||||||
|
with mock.patch.object(gremlins, "pid_alive", alive), \
|
||||||
|
mock.patch.object(gremlins, "process_tree", lambda rec, alive_until=None: list(snap)), \
|
||||||
|
mock.patch.object(gremlins, "terminate", lambda rec: killed.append(rec["pid"]) or False), \
|
||||||
|
mock.patch.object(gremlins.PLATFORM, "tree_kill", lambda pid: self.fail("root is dead")), \
|
||||||
|
mock.patch.object(gremlins, "KILL_VERIFY_S", 0.3):
|
||||||
|
k = gremlins.kill_tree(self.rec(), alive_until=self.hb)
|
||||||
|
self.assertEqual(k["snapshot"], snap)
|
||||||
|
self.assertIsNone(k["tree_kill_rc"])
|
||||||
|
return [p["pid"] for p in k["survivors"]], killed
|
||||||
|
|
||||||
|
def test_kill_verify_pid_reused_by_unopenable_process_is_dead(self):
|
||||||
|
"""Class (a): the pid now carries another name and its creation time cannot be read."""
|
||||||
|
survivors, killed = self.verify(table=[(self.SNAP, 4, "svchost.exe"), (self.OTHER, 4, "other.exe")],
|
||||||
|
ctime_readable=False)
|
||||||
|
self.assertEqual((survivors, killed), ([], []))
|
||||||
|
|
||||||
|
def test_kill_verify_pid_gone_from_a_nonempty_table_is_dead(self):
|
||||||
|
survivors, killed = self.verify(table=[(self.OTHER, 4, "other.exe")], ctime_readable=False)
|
||||||
|
self.assertEqual((survivors, killed), ([], []))
|
||||||
|
|
||||||
|
def test_kill_verify_same_name_unverifiable_survives(self):
|
||||||
|
survivors, killed = self.verify(table=[(self.SNAP, self.ROOT, "sleep.exe")], ctime_readable=False)
|
||||||
|
self.assertEqual(survivors, [self.SNAP])
|
||||||
|
self.assertIn(self.SNAP, killed)
|
||||||
|
|
||||||
|
def test_kill_verify_empty_table_keeps_the_survivor(self):
|
||||||
|
survivors, killed = self.verify(table=[], ctime_readable=False)
|
||||||
|
self.assertEqual(survivors, [self.SNAP])
|
||||||
|
self.assertIn(self.SNAP, killed)
|
||||||
|
|
||||||
|
def test_kill_verify_unknown_snapshot_name_keeps_the_survivor(self):
|
||||||
|
survivors, _ = self.verify(table=[(self.SNAP, 4, "svchost.exe")], snap_name=None,
|
||||||
|
ctime_readable=False)
|
||||||
|
self.assertEqual(survivors, [self.SNAP])
|
||||||
|
|
||||||
|
def test_kill_verify_other_name_with_readable_ctime_is_record_alive(self):
|
||||||
|
"""A differing name with a readable creation time leaves the decision to `record_alive`:
|
||||||
|
the same creation time is still the same process (a survivor)."""
|
||||||
|
survivors, _ = self.verify(table=[(self.SNAP, 4, "svchost.exe")], ctime_readable=True)
|
||||||
|
self.assertEqual(survivors, [self.SNAP])
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------------------
|
# --------------------------------------------------------------------------------------
|
||||||
class InitTests(Base):
|
class InitTests(Base):
|
||||||
@@ -770,6 +831,7 @@ class InitTests(Base):
|
|||||||
self.assertEqual(cfg["stall_after_s"], 900)
|
self.assertEqual(cfg["stall_after_s"], 900)
|
||||||
self.assertEqual(cfg["thresholds"], gremlins.THRESHOLD_DEFAULTS)
|
self.assertEqual(cfg["thresholds"], gremlins.THRESHOLD_DEFAULTS)
|
||||||
self.assertEqual(out["config"], cfg)
|
self.assertEqual(out["config"], cfg)
|
||||||
|
self.assertEqual(out["warnings"], []) # N3: always present
|
||||||
|
|
||||||
def test_fingerprint_canonical(self):
|
def test_fingerprint_canonical(self):
|
||||||
a = {"b": [1, 2], "a": {"y": 1, "x": None}}
|
a = {"b": [1, 2], "a": {"y": 1, "x": None}}
|
||||||
@@ -871,6 +933,140 @@ class InitTests(Base):
|
|||||||
self.assertIn("--model", out["detail"])
|
self.assertIn("--model", out["detail"])
|
||||||
self.assertFalse(os.path.exists(gremlins.state_dir_for(other)))
|
self.assertFalse(os.path.exists(gremlins.state_dir_for(other)))
|
||||||
|
|
||||||
|
# --- N3: a re-run and the binary (3.3; digest §5.1, amendment A3, A4) ---
|
||||||
|
def named_stub(self, sub):
|
||||||
|
"""A stub named `claude` (`claude.cmd` on Windows) in its own dir, found by a PATH lookup."""
|
||||||
|
d = os.path.join(self.tmp, sub)
|
||||||
|
os.makedirs(d)
|
||||||
|
dst = os.path.join(d, "claude.cmd" if IS_WINDOWS else "claude")
|
||||||
|
os.replace(make_stub_binary(d), dst)
|
||||||
|
return dst
|
||||||
|
|
||||||
|
def bare_init(self, *extra):
|
||||||
|
"""A first init without --binary: the binary comes from $CLAUDE_CODE_EXECPATH or PATH."""
|
||||||
|
return call(["init", "--run", self.run_log, "--model", "claude-opus-5-5[1m]", "--cwd", self.cwd,
|
||||||
|
*extra])
|
||||||
|
|
||||||
|
def rerun(self, *extra):
|
||||||
|
return call(["init", "--run", self.run_log, *extra])
|
||||||
|
|
||||||
|
def path_only(self, d):
|
||||||
|
"""PATH is exactly `d` (an empty dir when it should hold no claude)."""
|
||||||
|
os.makedirs(d, exist_ok=True)
|
||||||
|
os.environ["PATH"] = d
|
||||||
|
|
||||||
|
def stored_binary(self):
|
||||||
|
cfg = gremlins.read_registry(self.state)["config"]
|
||||||
|
return cfg["launch"]["binary"]["path"], cfg["binary_resolved_by"]
|
||||||
|
|
||||||
|
def test_rerun_reresolves_execpath(self):
|
||||||
|
"""C14: a re-run without --binary follows $CLAUDE_CODE_EXECPATH to a new binary, warns, and
|
||||||
|
lists the change; an existing base keeps its own binary record."""
|
||||||
|
a, b = self.named_stub("a"), self.named_stub("b")
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = a
|
||||||
|
code, first, _ = self.bare_init()
|
||||||
|
self.assertEqual(code, 0, first)
|
||||||
|
self.assertEqual(self.stored_binary(), (gremlins.norm_path(a), "CLAUDE_CODE_EXECPATH"))
|
||||||
|
self.assertEqual(first["warnings"], [])
|
||||||
|
self.add_fake_base()
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = b
|
||||||
|
code, out, raw = self.rerun("--pings", "off")
|
||||||
|
self.assertEqual(code, 0, out)
|
||||||
|
self.assert_one_json_line(raw)
|
||||||
|
self.assertEqual(self.stored_binary(), (gremlins.norm_path(b), "CLAUDE_CODE_EXECPATH"))
|
||||||
|
self.assertEqual(out["config"]["launch"]["binary"]["path"], gremlins.norm_path(b))
|
||||||
|
self.assertIn("launch.binary.path", out["changed_fields"])
|
||||||
|
self.assertIn("pings", out["changed_fields"])
|
||||||
|
self.assertEqual(len(out["warnings"]), 1, out["warnings"])
|
||||||
|
w = out["warnings"][0]
|
||||||
|
self.assertTrue(w.startswith("binary changed: "), w)
|
||||||
|
for s in (gremlins.norm_path(a), gremlins.norm_path(b), "(resolved by CLAUDE_CODE_EXECPATH)",
|
||||||
|
"existing bases keep their own binary record, new trainings use the new one"):
|
||||||
|
self.assertIn(s, w)
|
||||||
|
reg = gremlins.read_registry(self.state)
|
||||||
|
self.assertEqual(reg["bases"]["B1"]["launch"]["binary"]["path"], gremlins.norm_path(a))
|
||||||
|
# the same source again: nothing changes, no warning
|
||||||
|
code, out, _ = self.rerun()
|
||||||
|
self.assertEqual((code, out["changed"], out["changed_fields"], out["warnings"]), (0, False, [], []))
|
||||||
|
|
||||||
|
def test_rerun_keeps_a_binary_pin(self):
|
||||||
|
"""A `--binary` pin is kept on a re-run without --binary, whatever $CLAUDE_CODE_EXECPATH says."""
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = self.named_stub("b")
|
||||||
|
code, first, _ = self.init()
|
||||||
|
self.assertEqual(code, 0, first)
|
||||||
|
self.assertEqual(self.stored_binary(), (gremlins.norm_path(self.stub), "--binary"))
|
||||||
|
code, out, _ = self.rerun("--pings", "off")
|
||||||
|
self.assertEqual(code, 0, out)
|
||||||
|
self.assertEqual(self.stored_binary(), (gremlins.norm_path(self.stub), "--binary"))
|
||||||
|
self.assertEqual(out["changed_fields"], ["pings"])
|
||||||
|
self.assertEqual(out["warnings"], [])
|
||||||
|
|
||||||
|
def test_rerun_without_a_source_keeps_the_stored_path(self):
|
||||||
|
"""A3 (first case): stored file present, $CLAUDE_CODE_EXECPATH unset, no claude on PATH: the
|
||||||
|
stored path is kept with the "could not re-resolve" warning, exit 0."""
|
||||||
|
a = self.named_stub("a")
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = a
|
||||||
|
self.assertEqual(self.bare_init()[0], 0)
|
||||||
|
os.environ.pop("CLAUDE_CODE_EXECPATH")
|
||||||
|
self.path_only(os.path.join(self.tmp, "empty-path"))
|
||||||
|
code, out, raw = self.rerun("--pings", "off")
|
||||||
|
self.assertEqual(code, 0, out)
|
||||||
|
self.assert_one_json_line(raw)
|
||||||
|
self.assertEqual(self.stored_binary(), (gremlins.norm_path(a), "CLAUDE_CODE_EXECPATH"))
|
||||||
|
self.assertEqual(out["changed_fields"], ["pings"])
|
||||||
|
self.assertEqual(out["warnings"], [
|
||||||
|
"could not re-resolve the claude binary through CLAUDE_CODE_EXECPATH; kept the stored path %s"
|
||||||
|
% gremlins.norm_path(a)])
|
||||||
|
|
||||||
|
def test_rerun_with_the_stored_file_gone_fails_loud(self):
|
||||||
|
"""A3 (second case): the stored file is deleted and nothing re-resolves it: exit 2, `claude
|
||||||
|
binary not found` (today's loud failure), nothing written."""
|
||||||
|
a = self.named_stub("a")
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = a
|
||||||
|
self.assertEqual(self.bare_init()[0], 0)
|
||||||
|
before = self.reg_bytes()
|
||||||
|
os.remove(a)
|
||||||
|
os.environ.pop("CLAUDE_CODE_EXECPATH")
|
||||||
|
self.path_only(os.path.join(self.tmp, "empty-path"))
|
||||||
|
code, out, raw = self.rerun("--pings", "off")
|
||||||
|
self.assertEqual(code, 2, out)
|
||||||
|
self.assert_one_json_line(raw)
|
||||||
|
self.assertEqual((out["error"], out["binary"]), ("claude binary not found", gremlins.norm_path(a)))
|
||||||
|
self.assertEqual(self.reg_bytes(), before)
|
||||||
|
|
||||||
|
def test_rerun_never_crosses_sources(self):
|
||||||
|
"""A4: a path stored from $CLAUDE_CODE_EXECPATH is not replaced by a PATH lookup (kept, with
|
||||||
|
the warning), and a path stored from PATH is re-resolved by PATH, not by the env."""
|
||||||
|
a, p1, p2 = self.named_stub("a"), self.named_stub("p1"), self.named_stub("p2")
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = a
|
||||||
|
self.assertEqual(self.bare_init()[0], 0)
|
||||||
|
os.environ.pop("CLAUDE_CODE_EXECPATH")
|
||||||
|
self.path_only(os.path.dirname(p1))
|
||||||
|
code, out, _ = self.rerun("--pings", "off")
|
||||||
|
self.assertEqual(code, 0, out)
|
||||||
|
self.assertEqual(self.stored_binary(), (gremlins.norm_path(a), "CLAUDE_CODE_EXECPATH"))
|
||||||
|
self.assertEqual(len(out["warnings"]), 1, out["warnings"])
|
||||||
|
self.assertTrue(out["warnings"][0].startswith(
|
||||||
|
"could not re-resolve the claude binary through CLAUDE_CODE_EXECPATH"), out["warnings"])
|
||||||
|
# stored from PATH: the env does not win on a re-run
|
||||||
|
other = os.path.join(self.tmp, "reports", "2026-10-16-09-p-run.md")
|
||||||
|
# (a PATH lookup on Windows spells the extension from PATHEXT, e.g. `claude.CMD`: compare
|
||||||
|
# the paths case-folded, as the file system does)
|
||||||
|
nc = os.path.normcase
|
||||||
|
code, out, _ = call(["init", "--run", other, "--model", "claude-opus-5-5[1m]", "--cwd", self.cwd])
|
||||||
|
self.assertEqual(code, 0, out)
|
||||||
|
self.assertEqual((nc(out["config"]["launch"]["binary"]["path"]), out["config"]["binary_resolved_by"]),
|
||||||
|
(nc(gremlins.norm_path(p1)), "PATH"))
|
||||||
|
os.environ["CLAUDE_CODE_EXECPATH"] = a
|
||||||
|
self.path_only(os.path.dirname(p2))
|
||||||
|
code, out, _ = call(["init", "--run", other, "--pings", "off"])
|
||||||
|
self.assertEqual(code, 0, out)
|
||||||
|
self.assertEqual((nc(out["config"]["launch"]["binary"]["path"]), out["config"]["binary_resolved_by"]),
|
||||||
|
(nc(gremlins.norm_path(p2)), "PATH"))
|
||||||
|
self.assertIn("launch.binary.path", out["changed_fields"])
|
||||||
|
self.assertEqual(len(out["warnings"]), 1, out["warnings"])
|
||||||
|
self.assertIn("(resolved by PATH)", out["warnings"][0])
|
||||||
|
|
||||||
def test_warm_margin_must_be_below_ttl(self):
|
def test_warm_margin_must_be_below_ttl(self):
|
||||||
"""Minor 2: `init` refuses a warm_margin_s at or above the configured ttl_s (exit 2), on the
|
"""Minor 2: `init` refuses a warm_margin_s at or above the configured ttl_s (exit 2), on the
|
||||||
first init and on a re-run that lowers ttl_s under the current margin."""
|
first init and on a re-run that lowers ttl_s under the current margin."""
|
||||||
@@ -1527,6 +1723,10 @@ def main():
|
|||||||
prev = inp + cw + cr
|
prev = inp + cw + cr
|
||||||
if i == 0 and "hang" in d:
|
if i == 0 and "hang" in d:
|
||||||
time.sleep(600 if d["hang"] == "1" else float(d["hang"]))
|
time.sleep(600 if d["hang"] == "1" else float(d["hang"]))
|
||||||
|
if i == 0 and "gate" in d: # gate=NAME: wait (at most 120 s) for HERE/NAME to exist
|
||||||
|
t_gate = time.time() + 120
|
||||||
|
while not os.path.exists(os.path.join(HERE, d["gate"])) and time.time() < t_gate:
|
||||||
|
time.sleep(0.05)
|
||||||
if i == 0 and "die" in d:
|
if i == 0 and "die" in d:
|
||||||
os._exit(int(d.get("code", "3")))
|
os._exit(int(d.get("code", "3")))
|
||||||
if "sleep" in d:
|
if "sleep" in d:
|
||||||
@@ -1940,6 +2140,28 @@ class RuleTests(Base):
|
|||||||
os.utime(p, (start - 60, start - 60))
|
os.utime(p, (start - 60, start - 60))
|
||||||
self.assertFalse(gremlins.note_written_by(p, {"created_at": gremlins._iso(start)})) # ISO fallback
|
self.assertFalse(gremlins.note_written_by(p, {"created_at": gremlins._iso(start)})) # ISO fallback
|
||||||
|
|
||||||
|
def test_late_rebase_of_a_retired_base(self):
|
||||||
|
"""N4 (C15), the `expire` race: the old base was retired when the rebase ended ok. The new
|
||||||
|
base is created `retired` as of the end time; the old base is left as it was."""
|
||||||
|
ended = gremlins._iso(time.time())
|
||||||
|
launch = {"cwd": self.tmp, "binary": {"path": "claude", "version": STUB_VERSION}}
|
||||||
|
old = {"name": "B1", "status": "retired", "retired_at": gremlins._iso(time.time() - 60),
|
||||||
|
"session": SESS_TRAIN, "ttl_s": 3600}
|
||||||
|
reg = {"config": {"launch": launch}, "bases": {"B1": old}, "jobs": {}}
|
||||||
|
entry = {"job": "j0002", "kind": "rebase", "base": "B1", "rebase_as": "B1r", "session": SESS_SPAWN,
|
||||||
|
"new_launch": launch, "phase": 2}
|
||||||
|
before = dict(old)
|
||||||
|
self.assertTrue(gremlins._rebase_target_free(reg, entry))
|
||||||
|
gremlins._rebase_succeeded(reg, entry, old, None, None, None, "abc1234", ended)
|
||||||
|
new = reg["bases"]["B1r"]
|
||||||
|
self.assertEqual((new["status"], new["retired_at"], new["parent"], new["session"]),
|
||||||
|
("retired", ended, "B1", SESS_SPAWN))
|
||||||
|
self.assertNotIn("superseded_by", new)
|
||||||
|
self.assertEqual(old, before)
|
||||||
|
self.assertEqual(entry["rebase_inactive_parent"], "B1 was retired when the rebase ended")
|
||||||
|
# the name is now taken: a second success would be a conflict, not a replacement
|
||||||
|
self.assertFalse(gremlins._rebase_target_free(reg, dict(entry, job="j0003")))
|
||||||
|
|
||||||
|
|
||||||
class WarmthTests(Base):
|
class WarmthTests(Base):
|
||||||
"""Design 5 under an injected clock (block 2.1): warmth boundaries, TTL inference, the ping rule."""
|
"""Design 5 under an injected clock (block 2.1): warmth boundaries, TTL inference, the ping rule."""
|
||||||
@@ -3181,7 +3403,8 @@ class LaunchTests(LaunchBase):
|
|||||||
os.environ.pop(gremlins.CHILD_MARKER, None)
|
os.environ.pop(gremlins.CHILD_MARKER, None)
|
||||||
for j in (j1, j2):
|
for j in (j1, j2):
|
||||||
self.until_running(j)
|
self.until_running(j)
|
||||||
self.assertTrue(self.cmd("kill", "--job", j)["ok"])
|
out = self.cmd("kill", "--job", j)
|
||||||
|
self.assertTrue(out["ok"], out) # 3.3: a failure prints the survivors
|
||||||
code, out = self.wait(j1, j2)
|
code, out = self.wait(j1, j2)
|
||||||
self.assertEqual(code, 4)
|
self.assertEqual(code, 4)
|
||||||
self.assertEqual({x["job"]: x["paths"] for x in out["done"]}, {j1: ["a.py"], j2: ["b.py", "c.py"]})
|
self.assertEqual({x["job"]: x["paths"] for x in out["done"]}, {j1: ["a.py"], j2: ["b.py", "c.py"]})
|
||||||
@@ -3885,6 +4108,61 @@ class LaunchTests(LaunchBase):
|
|||||||
self.assertEqual(reg["bases"]["B1"]["status"], "active")
|
self.assertEqual(reg["bases"]["B1"]["status"], "active")
|
||||||
self.assertNotIn("B2", reg["bases"])
|
self.assertNotIn("B2", reg["bases"])
|
||||||
|
|
||||||
|
def test_late_rebase_of_a_superseded_base(self):
|
||||||
|
"""N4 (C15): `rebase --base B1 --as B1r` is still running when `train --base B3 --supersedes
|
||||||
|
B1` ends ok. When the rebase then ends ok, its result is `ok` with `parent_inactive`; B1r is
|
||||||
|
created superseded by B3 (no pings, a spawn needs --force); B1 keeps `superseded_by == B3`.
|
||||||
|
A second rebase whose new name is taken by then (`rebase_conflict`) creates nothing and is
|
||||||
|
not flagged."""
|
||||||
|
self.setup_run(cap=4)
|
||||||
|
self.train("B1")
|
||||||
|
self.finish("j0001")
|
||||||
|
self.train("B2")
|
||||||
|
self.finish("j0002")
|
||||||
|
gates = os.path.dirname(self.stub) # the stub's `gate=NAME` waits for this file
|
||||||
|
r1 = self.cmd("rebase", "--base", "B1", "--as", "B1r", "--phase", "2",
|
||||||
|
"--brief", self.brief("gate=g1", "turns=2"))
|
||||||
|
r2 = self.cmd("rebase", "--base", "B2", "--as", "B2r", "--brief", self.brief("gate=g2", "turns=2"))
|
||||||
|
t = self.cmd("train", "--base", "B3", "--brief", self.brief("turns=1"), "--supersedes", "B1",
|
||||||
|
"--supersedes", "B2")
|
||||||
|
self.assertEqual(self.finish(t["job"])["status"], "ok")
|
||||||
|
reg = self.reg()
|
||||||
|
self.assertEqual([reg["jobs"][j]["status"] in gremlins.RUNNING for j in (r1["job"], r2["job"])],
|
||||||
|
[True, True], "the rebases must still run when the training ends")
|
||||||
|
b1_before = dict(reg["bases"]["B1"])
|
||||||
|
self.assertEqual((b1_before["status"], b1_before["superseded_by"]), ("superseded", "B3"))
|
||||||
|
self.clone_base("B2r", src="B3") # B2r's name is taken before its rebase ends
|
||||||
|
b2r_before = self.reg()["bases"]["B2r"]
|
||||||
|
for g in ("g1", "g2"):
|
||||||
|
with open(os.path.join(gates, g), "w"):
|
||||||
|
pass
|
||||||
|
rr1, rr2 = self.finish(r1["job"]), self.finish(r2["job"])
|
||||||
|
# the late rebase: ok, flagged, the new base superseded by the same base as its parent
|
||||||
|
self.assertEqual((rr1["kind"], rr1["status"], rr1["base"]), ("rebase", "ok", "B1"), rr1)
|
||||||
|
self.assertIn("parent_inactive", rr1["flags"])
|
||||||
|
reg = self.reg()
|
||||||
|
new, b1 = reg["bases"]["B1r"], reg["bases"]["B1"]
|
||||||
|
e1 = reg["jobs"][r1["job"]]
|
||||||
|
self.assertEqual((new["status"], new["superseded_by"], new["superseded_at"]),
|
||||||
|
("superseded", "B3", e1["ended_at"]))
|
||||||
|
self.assertEqual((new["parent"], new["session"], new["lineage_rebases"]), ("B1", r1["session_id"], 1))
|
||||||
|
lineage = ("status", "superseded_by", "superseded_at") # B1's lineage untouched
|
||||||
|
self.assertEqual([b1[k] for k in lineage], [b1_before[k] for k in lineage])
|
||||||
|
self.assertEqual(e1["rebase_inactive_parent"], "B1 was superseded when the rebase ended")
|
||||||
|
self.assertIn("parent_inactive", e1["flags"])
|
||||||
|
self.assertIn("parent_inactive", {x["job"]: x for x in self.ledger()}[r1["job"]]["flags"])
|
||||||
|
self.assertEqual(gremlins.ping_decision(reg, "B1r")["why"], "status:superseded")
|
||||||
|
self.assertIn("not active", self.spawn(base="B1r", code=2)["error"]) # needs --force
|
||||||
|
self.fake_spawn("B1r", "--force")
|
||||||
|
# the conflict: ok, not flagged, nothing created or replaced
|
||||||
|
self.assertEqual(rr2["status"], "ok", rr2)
|
||||||
|
self.assertNotIn("parent_inactive", rr2["flags"])
|
||||||
|
reg = self.reg()
|
||||||
|
self.assertIn("rebase_conflict", reg["jobs"][r2["job"]])
|
||||||
|
self.assertNotIn("rebase_inactive_parent", reg["jobs"][r2["job"]])
|
||||||
|
self.assertEqual(reg["bases"]["B2r"], b2r_before)
|
||||||
|
self.assertEqual((reg["bases"]["B2"]["status"], reg["bases"]["B2"]["superseded_by"]), ("superseded", "B3"))
|
||||||
|
|
||||||
def test_head_at_end_and_usage_commit_column(self):
|
def test_head_at_end_and_usage_commit_column(self):
|
||||||
"""2.3: the supervisor records `head_at_end`; `usage --block` lists the commits made during a
|
"""2.3: the supervisor records `head_at_end`; `usage --block` lists the commits made during a
|
||||||
sequential spawn, and the ledger/usage figures agree with the result."""
|
sequential spawn, and the ledger/usage figures agree with the result."""
|
||||||
|
|||||||
Reference in New Issue
Block a user