g4borg
f1de89e869
Standalone plugin with one skill: when to use orbit and when grep, a mandatory preflight for stale indexes and blind regions (cfg(test), features, macros, dyn dispatch), a scripted review pass over isolated worktree DBs, and a JSONL feedback log for mining real usage later. Registered in the marketplace as a prototype.
238 lines
12 KiB
Python
Executable File
238 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Recipe 1: map `git diff base..tip` onto definitions of both indexed commits.
|
|
|
|
Usage: defdiff.py <env-file> [--json out.json] [--all-kinds]
|
|
env-file: output of review-setup.sh (DB, BASE_WT, TIP_WT, BASE_SHA, TIP_SHA, REPO)
|
|
|
|
Prints a compact per-file report:
|
|
+ added - removed > moved (same fqn, other file)
|
|
S signature changed (declaration text differs)
|
|
~ body touched (a diff hunk hits the definition; size shown)
|
|
? relocated candidate (removed+added pair with same name+kind, different fqn)
|
|
and an 'unattributed' tally of changed lines no definition covers (the blindness input).
|
|
"""
|
|
import collections, json, re, sys
|
|
sys.path.insert(0, __file__.rsplit('/', 1)[0])
|
|
from ob import load_env, q, git
|
|
from regions import cfg_regions, region_of
|
|
|
|
env = load_env(sys.argv[1])
|
|
DB, BW, TW, REPO = env['DB'], env['BASE_WT'], env['TIP_WT'], env['REPO']
|
|
B, T = env['BASE_SHA'], env['TIP_SHA']
|
|
JSON_OUT = sys.argv[sys.argv.index('--json') + 1] if '--json' in sys.argv else None
|
|
|
|
SQL = """SELECT id, file_path, fqn, name, definition_type k, start_line s, end_line e
|
|
FROM gl_definition WHERE file_path LIKE '%.rs'"""
|
|
ROLLUP = {'Field', 'Variant'} # reported through their parent
|
|
CONTAINER = {'Module'} # only used if nothing smaller covers a line
|
|
|
|
def load(wt):
|
|
defs = q(SQL, repo=wt, db=DB)
|
|
byfile = collections.defaultdict(list)
|
|
for d in defs: byfile[d['file_path']].append(d)
|
|
return defs, byfile
|
|
|
|
bdefs, bfile = load(BW)
|
|
tdefs, tfile = load(TW)
|
|
|
|
# ---- hunks (-U0, rename detection so moved files map old->new path)
|
|
diff = git(REPO, 'diff', '-U0', '-M', '--no-color', B, T, '--', '*.rs')
|
|
old_lines = collections.defaultdict(set); new_lines = collections.defaultdict(set)
|
|
old_text = collections.defaultdict(dict); new_text = collections.defaultdict(dict)
|
|
renames = {}
|
|
of = nf = None
|
|
for line in diff.split('\n'):
|
|
if line.startswith('--- '):
|
|
of = None if line[4:] == '/dev/null' else line[6:]
|
|
elif line.startswith('+++ '):
|
|
nf = None if line[4:] == '/dev/null' else line[6:]
|
|
if of and nf and of != nf: renames[of] = nf
|
|
elif line.startswith('@@'):
|
|
m = re.match(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@', line)
|
|
os_, oc, ns, nc = int(m[1]), int(m[2] or 1), int(m[3]), int(m[4] or 1)
|
|
oi, ni = os_, ns
|
|
elif line.startswith('-') and of:
|
|
old_lines[of].add(oi); old_text[of][oi] = line[1:]; oi += 1
|
|
elif line.startswith('+') and nf:
|
|
new_lines[nf].add(ni); new_text[nf][ni] = line[1:]; ni += 1
|
|
|
|
def innermost(defs, ln):
|
|
cov = [d for d in defs if d['s'] <= ln <= d['e']]
|
|
if not cov: return None
|
|
non_container = [d for d in cov if d['k'] not in CONTAINER]
|
|
cov = non_container or cov
|
|
cov.sort(key=lambda d: d['e'] - d['s'])
|
|
d = cov[0]
|
|
if d['k'] in ROLLUP: # roll fields/variants up to their parent type
|
|
parents = [p for p in cov if p['k'] not in ROLLUP]
|
|
if parents: d = parents[0]
|
|
return d
|
|
|
|
def imports(wt):
|
|
r = collections.defaultdict(list)
|
|
for d in q("SELECT DISTINCT file_path, start_line s, end_line e FROM gl_imported_symbol WHERE file_path LIKE '%.rs'", repo=wt, db=DB):
|
|
r[d['file_path']].append((d['s'], d['e']))
|
|
return r
|
|
bimp, timp = imports(BW), imports(TW)
|
|
|
|
def attribute(lines, byfile, texts, wt, imp):
|
|
hit = collections.defaultdict(set); unattr = collections.Counter(); samples = collections.defaultdict(list)
|
|
for f, ls in lines.items():
|
|
defs = byfile.get(f, [])
|
|
real = [d for d in defs if d['k'] != 'Macro'] # macro_rules! are indexed even in unreachable files
|
|
for ln in ls:
|
|
d = innermost(defs, ln)
|
|
if d: hit[d['id']].add(ln); continue
|
|
t = texts[f].get(ln, '').strip()
|
|
reg = region_of(cfg_regions(src(wt, f)), ln)
|
|
cat = (f"cfg-{reg[2]}" if reg else
|
|
'file-has-no-defs' if not real else
|
|
'import' if any(a <= ln <= b for a, b in imp.get(f, [])) else
|
|
'impl-header/brace' if re.match(r'(unsafe\s+)?impl\b', t) or t in ('}', '};', '});') else
|
|
'blank/comment' if not t or t.startswith('//') else
|
|
'use/mod-decl' if re.match(r'(pub(\(.*?\))?\s+)?(use|mod)\s', t) else
|
|
'attribute' if t.startswith('#') else 'other')
|
|
unattr[(f, cat)] += 1
|
|
if len(samples[(f, cat)]) < 2: samples[(f, cat)].append(f"L{ln}: {t[:70]}")
|
|
return hit, unattr, samples
|
|
|
|
|
|
# ---- declaration text (signature) from the worktree sources
|
|
_src = {}
|
|
def src(wt, f):
|
|
if (wt, f) not in _src:
|
|
try: _src[(wt, f)] = open(f"{wt}/{f}", encoding='utf-8', errors='replace').read().split('\n')
|
|
except FileNotFoundError: _src[(wt, f)] = []
|
|
return _src[(wt, f)]
|
|
|
|
def decl(wt, d):
|
|
lines = src(wt, d['file_path'])[d['s'] - 1:d['e']]
|
|
out = []
|
|
for l in lines:
|
|
t = l.strip()
|
|
if not out and (t.startswith(('///', '//', '#[', '#!')) or not t): continue
|
|
out.append(t)
|
|
joined = ' '.join(out)
|
|
if d['k'] in ROLLUP: return joined.rstrip(',')
|
|
if '{' in t or t.endswith(';') or len(out) > 12: break
|
|
s = ' '.join(out)
|
|
s = s.split('{')[0] if '{' in s else s
|
|
return re.sub(r'\s+', ' ', s).strip()
|
|
|
|
bhit, bun, bsam = attribute(old_lines, bfile, old_text, BW, bimp)
|
|
thit, tun, tsam = attribute(new_lines, tfile, new_text, TW, timp)
|
|
|
|
# ---- match definitions across commits on (fqn, kind), tie-break by file then order
|
|
def key(d): return (d['fqn'], d['k'])
|
|
bidx = collections.defaultdict(list); tidx = collections.defaultdict(list)
|
|
for d in bdefs: bidx[key(d)].append(d)
|
|
for d in tdefs: tidx[key(d)].append(d)
|
|
pairs, added, removed = [], [], []
|
|
for k in set(bidx) | set(tidx):
|
|
bl, tl = sorted(bidx.get(k, []), key=lambda d: (d['file_path'], d['s'])), sorted(tidx.get(k, []), key=lambda d: (d['file_path'], d['s']))
|
|
# pair same-file (after rename mapping) first
|
|
rest_t = list(tl)
|
|
for bd in list(bl):
|
|
nfp = renames.get(bd['file_path'], bd['file_path'])
|
|
m = next((td for td in rest_t if td['file_path'] == nfp), None)
|
|
if m: pairs.append((bd, m)); rest_t.remove(m); bl.remove(bd)
|
|
for bd, td in zip(bl, rest_t): pairs.append((bd, td))
|
|
removed += bl[len(rest_t):]; added += rest_t[len(bl):]
|
|
|
|
# fields/variants: roll into parent change notes
|
|
def parent_fqn(d): return d['fqn'].rsplit('::', 1)[0]
|
|
member_notes = collections.defaultdict(list)
|
|
variants = {d['fqn'] for d in bdefs + tdefs if d['k'] == 'Variant'}
|
|
def owner(d): # Enum::Variant::field -> (Enum, 'Variant.field')
|
|
p = parent_fqn(d)
|
|
if p in variants: return parent_fqn({'fqn': p}), p.rsplit('::', 1)[1] + '.' + d['name']
|
|
return p, d['name']
|
|
for side, lst, sign in (('t', added, '+'), ('b', removed, '-')):
|
|
for d in lst:
|
|
if d['k'] in ROLLUP:
|
|
o, n = owner(d); member_notes[(side, o)].append(sign + n)
|
|
|
|
rows = []
|
|
for d in added:
|
|
if d['k'] not in ROLLUP: rows.append(dict(op='+', tid=d['id'], file=d['file_path'], k=d['k'], fqn=d['fqn'], name=d['name'], lines=f"L{d['s']}-{d['e']}", new_len=d['e']-d['s']+1))
|
|
for d in removed:
|
|
if d['k'] not in ROLLUP: rows.append(dict(op='-', bid=d['id'], file=d['file_path'], k=d['k'], fqn=d['fqn'], name=d['name'], lines=f"L{d['s']}-{d['e']}", old_len=d['e']-d['s']+1))
|
|
for bd, td in pairs:
|
|
if td['k'] in ROLLUP: continue
|
|
moved = renames.get(bd['file_path'], bd['file_path']) != td['file_path']
|
|
touched = td['id'] in thit or bd['id'] in bhit
|
|
mem = member_notes.get(('t', td['fqn']), []) + member_notes.get(('b', bd['fqn']), [])
|
|
ol, nl = bd['e'] - bd['s'] + 1, td['e'] - td['s'] + 1
|
|
if not (moved or touched or mem or ol != nl): continue
|
|
sb, st = decl(BW, bd), decl(TW, td)
|
|
op = '>' if moved else ('S' if sb != st else '~')
|
|
r = dict(op=op, bid=bd['id'], tid=td['id'], file=td['file_path'], k=td['k'], fqn=td['fqn'], name=td['name'], lines=f"L{td['s']}-{td['e']}", old_len=ol, new_len=nl)
|
|
if moved: r['from'] = bd['file_path']
|
|
if sb != st: r['sig_old'], r['sig_new'] = sb, st
|
|
if mem: r['members'] = mem
|
|
rows.append(r)
|
|
|
|
# relocation candidates: removed + added with same (name, kind), different fqn
|
|
rem_by = collections.defaultdict(list)
|
|
for r in rows:
|
|
if r['op'] == '-': rem_by[(r['name'], r['k'])].append(r)
|
|
GENERIC = {'new', 'default', 'build', 'from', 'fmt', 'run', 'plugin', 'tests', 'drop', 'clone', 'eq', 'deref'}
|
|
for r in rows:
|
|
if r['op'] == '+' and (r['name'], r['k']) in rem_by and r['name'] not in GENERIC:
|
|
r['relocated_from'] = sorted({x['file'] for x in rem_by[(r['name'], r['k'])]})[:3]
|
|
|
|
# ---- collapse children of added/removed types into their parent row (keeps JSON complete)
|
|
TYPES = {'Struct', 'Enum', 'Trait', 'Module'}
|
|
whole = {(r['op'], r['fqn']) for r in rows if r['op'] in '+-' and r['k'] in TYPES - {'Module'}}
|
|
for r in rows:
|
|
par = r['fqn'].rsplit('::', 1)[0]
|
|
if r['op'] in '+-' and (r['op'], par) in whole:
|
|
r['collapsed'] = True
|
|
nchild = collections.Counter((r['op'], r['fqn'].rsplit('::', 1)[0]) for r in rows if r.get('collapsed'))
|
|
for r in rows:
|
|
if (r['op'], r['fqn']) in nchild: r['children'] = nchild[(r['op'], r['fqn'])]
|
|
|
|
# ---- print
|
|
def sigdiff(a, b, ctx=25):
|
|
i = 0
|
|
while i < min(len(a), len(b)) and a[i] == b[i]: i += 1
|
|
j = 0
|
|
while j < min(len(a), len(b)) - i and a[-1 - j] == b[-1 - j]: j += 1
|
|
pre = ('…' + a[max(0, i - ctx):i]) if i > ctx else a[:i]
|
|
sa, sb = a[i:len(a) - j], b[i:len(b) - j]
|
|
post = a[len(a) - j:][:ctx] + ('…' if j > ctx else '')
|
|
return f" sig: {pre}[-{sa}-]{{+{sb}+}}{post}"
|
|
order = {'-': 0, '>': 1, 'S': 2, '+': 3, '~': 4}
|
|
byf = collections.defaultdict(list)
|
|
for r in rows:
|
|
if not r.get('collapsed'): byf[r['file']].append(r)
|
|
cnt = collections.Counter(r['op'] for r in rows)
|
|
print(f"# defdiff {B[:8]}..{T[:8]} files_with_changes={len(set(new_lines)|set(old_lines))} "
|
|
f"removed={cnt['-']} moved={cnt['>']} sig={cnt['S']} added={cnt['+']} body={cnt['~']}")
|
|
for f in sorted(byf):
|
|
print(f)
|
|
for r in sorted(byf[f], key=lambda r: (order[r['op']], r['lines'])):
|
|
extra = ''
|
|
if r['op'] == '~': extra = f" {r['old_len']}->{r['new_len']} lines"
|
|
if r['op'] == '>': extra = f" from {r['from']}"
|
|
if r.get('relocated_from'): extra += f" (same name removed in {', '.join(r['relocated_from'])})"
|
|
if r.get('children'): extra += f" (+{r['children']} members)" if r['op'] == '+' else f" (-{r['children']} members)"
|
|
if r.get('members'): extra += f" members {' '.join(r['members'][:8])}{' …' if len(r['members'])>8 else ''}"
|
|
print(f" {r['op']} {r['k']} {r['fqn'].split('::',1)[-1]} {r['lines']}{extra}")
|
|
if 'sig_old' in r and r['op'] in 'S>':
|
|
print(sigdiff(r['sig_old'], r['sig_new']))
|
|
un = collections.Counter()
|
|
for (f, c), n in list(bun.items()) + list(tun.items()): un[c] += n
|
|
tot = sum(len(v) for v in old_lines.values()) + sum(len(v) for v in new_lines.values())
|
|
print(f"# changed lines: {tot}; unattributed: {sum(un.values())} " + ' '.join(f"{c}={n}" for c, n in un.most_common()))
|
|
perfile = collections.Counter()
|
|
for (f, c), n in list(bun.items()) + list(tun.items()):
|
|
if c != 'blank/comment': perfile[(f, c)] += n
|
|
for (f, c), n in perfile.most_common(12): print(f"# {n:4d} {c:16s} {f}")
|
|
if JSON_OUT:
|
|
allun = collections.Counter()
|
|
for (f, c), n in list(bun.items()) + list(tun.items()): allun[f"{f}|{c}"] += n
|
|
json.dump(dict(touched_tip=sorted(thit), touched_base=sorted(bhit), rows=rows, unattributed=dict(allun),
|
|
samples={f"{f}|{c}": s for (f, c), s in list(bsam.items()) + list(tsam.items())},
|
|
changed_files=sorted(set(new_lines) | set(old_lines)), renames=renames), open(JSON_OUT, 'w'), indent=1)
|