1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
|
--- /dev/null
+++ b/src/foundry/repository_maintenance.py
@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+"""Prepare source-maintenance reviews; never edit or publish authoring inputs."""
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+from datetime import datetime, timezone
+import difflib
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tarfile
+import tempfile
+import tomllib
+from urllib.request import Request, urlopen
+import uuid
+
+from foundry.merge import _best_version, _compare_versions, rel_filtered
+from foundry.repositories import load_repository, overlay, state_for
+from foundry.repository_freshness import release_versions
+from foundry.transition import Runtime, atomic_json, atomic_text, lock, name, tree_digest
+
+
+ACTIONABLE = {"new-release", "upstream-changed", "new-commit", "needs-rule", "error"}
+
+
+def stable_version(tag: str, prefix: str = "v") -> str | None:
+ value = tag[len(prefix):] if prefix and tag.startswith(prefix) else tag
+ return value if re.fullmatch(r"\d+(?:\.\d+)*", value) else None
+
+
+def request_json(url: str):
+ request = Request(url, headers={"User-Agent": "crucible-maintain/1.0",
+ "Accept": "application/json"})
+ with urlopen(request, timeout=20) as response:
+ body = response.read(8 * 1024 * 1024 + 1)
+ if len(body) > 8 * 1024 * 1024:
+ raise ValueError("upstream response exceeds 8 MiB")
+ return json.loads(body)
+
+
+def remote_release(rule: dict, recipe: str, fetch=request_json) -> dict:
+ kind = rule["kind"]
+ if kind == "pypi":
+ project = rule["project"]
+ if not re.fullmatch(r"[A-Za-z0-9_.-]+", project):
+ raise ValueError("invalid PyPI project")
+ data = fetch(f"https://pypi.org/pypi/{project}/json")
+ versions = {v for v, files in data["releases"].items()
+ if stable_version(v, "") and any(not f.get("yanked") for f in files)}
+ if not versions:
+ raise ValueError("no stable, non-yanked PyPI release")
+ version = _best_version(versions)
+ return {"version": version, "url": f"https://pypi.org/project/{project}/{version}/"}
+ repo = rule["repository"]
+ if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo):
+ raise ValueError("invalid GitHub repository")
+ api = f"https://api.github.com/repos/{repo}"
+ if kind == "github-commit":
+ data = fetch(api + "/commits?per_page=1")
+ if not data or not re.fullmatch(r"[0-9a-f]{40}", data[0]["sha"]):
+ raise ValueError("no upstream commit")
+ pinned = re.search(rule["pin_pattern"], recipe) if rule.get("pin_pattern") else None
+ return {"commit": data[0]["sha"], "local_commit": pinned.group(1) if pinned else None,
+ "url": data[0]["html_url"], "date": data[0]["commit"]["committer"]["date"]}
+ if kind != "github":
+ raise ValueError(f"unknown remote provider: {kind}")
+ release = fetch(api + "/releases/latest")
+ version = stable_version(release["tag_name"], rule.get("tag_prefix", "v"))
+ if release.get("draft") or release.get("prerelease") or not version:
+ raise ValueError("no matching stable GitHub release; configure commit tracking if needed")
+ return {"version": version, "url": release["html_url"], "tag": release["tag_name"]}
+
+
+def snapshot(source: Path, dest: Path) -> str:
+ before = tree_digest(source)
+ overlay(source, dest)
+ if tree_digest(source) != before or tree_digest(dest) != before:
+ raise ValueError(f"source changed during capture: {source}")
+ return before
+
+
+def recipe_digest(root: Path) -> str:
+ """Track package bytes and link targets independently of extraction umask."""
+ digest = hashlib.sha256()
+ for path in sorted(root.rglob("*")):
+ if ".git" in path.relative_to(root).parts:
+ continue
+ if path.is_symlink():
+ payload = b"link:" + os.fsencode(os.readlink(path))
+ elif path.is_file():
+ payload = b"file:" + path.read_bytes()
+ else:
+ continue
+ digest.update(path.relative_to(root).as_posix().encode() + b"\0")
+ digest.update(hashlib.sha256(payload).digest())
+ return "content-v1:" + digest.hexdigest()
+
+
+def published_base(runtime: Runtime, repo: str, data: dict, packages: list[str], dest: Path) -> dict:
+ """Export only selected packages from the exact published Gentoo input."""
+ home = runtime.repositories_root / repo
+ candidate = state_for(home).get("published")
+ if not candidate:
+ raise ValueError("no published candidate; run crucible update before maintenance")
+ name(candidate)
+ manifest = json.loads((home / "candidates" / candidate / "manifest.json").read_text())
+ base_name = data.get("composition", {}).get("base")
+ entry = next((x for x in manifest["inputs"] if x["name"] == base_name), None)
+ if not entry or entry.get("kind") != "git" or not entry.get("source_id"):
+ raise ValueError("published base is not a cached Git input")
+ configured = next(x for x in data["sources"] if x["name"] == base_name)
+ if configured.get("url") != entry.get("url") or configured.get("ref", "HEAD") != entry.get("ref"):
+ raise ValueError("base configuration changed since publication; update first")
+ bare = runtime.cache_root / "upstreams" / name(entry["source_id"])
+ commit = entry["commit"]
+ if not re.fullmatch(r"[0-9a-f]{40,64}", commit):
+ raise ValueError("invalid published input commit")
+ paths = subprocess.run(["git", "--git-dir", str(bare), "ls-tree", "-r", "--name-only",
+ commit, "--", *packages], check=True, text=True,
+ capture_output=True, timeout=120).stdout.splitlines() if packages else []
+ present = sorted({"/".join(p.split("/")[:2]) for p in paths if p.endswith(".ebuild")})
+ dest.mkdir(parents=True)
+ if present:
+ archive = dest.parent / "base.tar"
+ subprocess.run(["git", "--git-dir", str(bare), "archive", "--format=tar",
+ f"--output={archive}", commit, "--", *present], check=True,
+ capture_output=True, timeout=120)
+ with tarfile.open(archive) as stream:
+ stream.extractall(dest, filter="data")
+ archive.unlink()
+ return {"candidate": candidate, "commit": commit, "url": entry["url"],
+ "filters": configured.get("filter", []),
+ "published_at": datetime.fromtimestamp(
+ (home / "candidates" / candidate / "manifest.json").stat().st_mtime,
+ timezone.utc).isoformat()}
+
+
+def package_comparison(local: Path, upstream: Path, package: str, previous: dict,
+ filters: list[str], source_identity: str) -> dict:
+ local_version = _best_version(release_versions(local.parent.parent, package, []))
+ upstream_version = _best_version(release_versions(upstream.parent.parent, package, filters))
+ result = {"local_version": local_version or "live-only", "upstream_version": upstream_version,
+ "upstream_digest": recipe_digest(upstream), "upstream_identity": source_identity}
+ if not upstream_version:
+ result.update(status="needs-rule", detail="No eligible non-live upstream ebuild.")
+ elif not local_version or _compare_versions(upstream_version, local_version) > 0:
+ result.update(status="new-release", detail="Newer upstream recipe available; preserve local changes when updating.")
+ elif (previous.get("upstream_identity") == source_identity and previous.get("upstream_digest", "").startswith("content-v1:")
+ and (previous["upstream_digest"] != result["upstream_digest"] or (
+ previous.get("status") == "upstream-changed" and previous.get("local_digest") == tree_digest(local)))):
+ result.update(status="upstream-changed", detail="Upstream package content changed since the previous report.")
+ else:
+ result.update(status="no-newer-release", detail="No newer release detected; local customizations remain in place.")
+ return result
+
+
+def prepare_review(local: Path, upstream: Path | None, output: Path, item: dict) -> None:
+ output.mkdir(parents=True)
+ overlay(local, output / "local")
+ if upstream:
+ overlay(upstream, output / "upstream")
+ paths = sorted({p.relative_to(root).as_posix() for root in (local, upstream)
+ for p in root.rglob("*") if p.is_file() and not p.is_symlink()})
+ diffs = []
+ for path in paths:
+ a, b = local / path, upstream / path
+ before = a.read_bytes() if a.is_file() else b""
+ after = b.read_bytes() if b.is_file() else b""
+ if before == after:
+ continue
+ try:
+ diffs.extend(difflib.unified_diff(before.decode().splitlines(True), after.decode().splitlines(True),
+ fromfile=f"local/{path}", tofile=f"upstream/{path}"))
+ except UnicodeDecodeError:
+ diffs.append(f"Binary file differs: {path}\n")
+ atomic_text(output / "upstream.diff", "".join(diffs))
+ atomic_json(output / "review.json", item)
+
+
+def render_report(report: dict) -> str:
+ lines = [f"# {report['repository']} package maintenance review", "",
+ f"Generated: {report['created_at']}", "",
+ "Review material only. No authoring files were changed, no ebuilds were executed, and nothing was published.",
+ "Upstream snapshots and diffs are reference material, not drop-in replacements for customized recipes.", "",
+ f"Published input: {report['base'].get('candidate', 'unavailable')} "
+ f"({report['base'].get('commit', 'unavailable')}).", "",
+ f"Counts: {', '.join(f'{k}={v}' for k, v in sorted(report['counts'].items()))}.", "",
+ "## Needs attention", ""]
+ for item in report["packages"]:
+ if item["status"] not in ACTIONABLE:
+ continue
+ lines += [f"### {item['package']}", "", f"{item['status']}: {item['detail']}", "",
+ f"Local: {item.get('local_version', '?')}; upstream: "
+ f"{item.get('upstream_version') or item.get('commit') or 'unknown'}.", ""]
+ if item.get("url"):
+ lines += [f"Upstream: {item['url']}", ""]
+ if item.get("review"):
+ lines += [f"Files: [review bundle]({item['review']}/review.json)", ""]
+ lines += ["## Complete inventory", "", "| Package | Provider | Status |", "| --- | --- | --- |"]
+ lines += [f"| {p['package']} | {p['provider']} | {p['status']} |" for p in report["packages"]]
+ lines += ["", "## Review workflow", "",
+ "Use the local and upstream snapshots to update the authoring recipe, retaining required patches and dependencies. "
+ "Regenerate its Manifest and run appropriate package checks/builds. "
+ "Then use `crucible update REPO --no-publish`, inspect the candidate, and publish its explicit candidate ID.",
+ "The existing automatic publisher captures edits inside configured source directories. "
+ "Keep unfinished work outside those directories.", ""]
+ return "\n".join(lines)
+
+
+def run(config_path: Path, runtime: Runtime, *, fetch=request_json) -> dict:
+ config = tomllib.loads(config_path.read_text())
+ repo, source_name = name(config["repository"]), name(config.get("source", "distribution"))
+ data = load_repository(runtime, repo)
+ source = next(x for x in data["sources"] if x["name"] == source_name)
+ if not source.get("path"):
+ raise ValueError("maintenance source must be local")
+ root = Path(source["path"]).resolve(strict=True)
+ home = runtime.repositories_root / repo / "maintenance" / source_name
+ if any(home.resolve().is_relative_to(Path(s["path"]).resolve()) for s in data["sources"] if s.get("path")):
+ raise ValueError("review storage must be outside every active source")
+ with lock(home / "maintenance.lock"):
+ latest = home / "latest.json"
+ previous = json.loads(latest.read_text()) if latest.exists() else {}
+ by_package = {p["package"]: p for p in previous.get("packages", [])}
+ stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]
+ output = home / "runs" / stamp
+ output.mkdir(parents=True)
+ packages = sorted({p.parent.relative_to(root).as_posix() for p in root.glob("*/*/*.ebuild")})
+ rules = config.get("packages", {})
+ report = {"schema_version": 1, "repository": repo, "source": source_name,
+ "created_at": datetime.now(timezone.utc).isoformat(), "output": str(output),
+ "config_sha256": hashlib.sha256(config_path.read_bytes()).hexdigest(), "packages": []}
+ with tempfile.TemporaryDirectory(prefix=".capture-", dir=home) as temporary:
+ work = Path(temporary)
+ try:
+ report["base"] = published_base(runtime, repo, data, packages, work / "gentoo")
+ except (ValueError, OSError, subprocess.SubprocessError, KeyError) as error:
+ report["base"] = {"error": str(error)}
+ for package in packages:
+ item = {"package": package, "provider": "unassigned"}
+ local = work / "local" / package
+ upstream = None
+ try:
+ item["local_digest"] = snapshot(root / package, local)
+ versions = release_versions(work / "local", package, [])
+ version = _best_version(versions)
+ item["local_version"] = version or "live-only"
+ recipe = (local / f"{local.name}-{version}.ebuild").read_text() if version else ""
+ rule = rules.get(package)
+ if not source.get("enabled", True) or rel_filtered(package, source.get("filter", [])) or (
+ source.get("select") and package not in source["select"]):
+ item.update(provider="source-policy", status="excluded", detail="Excluded from this Crucible source.")
+ elif rule and rule["kind"] in {"manual", "fixed", "live"}:
+ item.update(provider=rule["kind"], status=rule["kind"], detail=rule["reason"])
+ elif not rule and package.startswith(("acct-user/", "acct-group/")):
+ item.update(provider="account", status="fixed", detail="Account definition; no software release to track.")
+ elif not rule and not version:
+ item.update(provider="live-ebuild", status="live", detail="Live recipe follows its source when rebuilt; no release bump inferred.")
+ elif rule and rule["kind"] in {"github", "pypi", "github-commit"}:
+ item["provider"] = rule["kind"]
+ release = remote_release(rule, recipe, fetch)
+ item.update({k: v for k, v in release.items() if k != "version"})
+ if release.get("version"):
+ item["upstream_version"] = release["version"]
+ newer = not version or _compare_versions(release["version"], version) > 0
+ item.update(status="new-release" if newer else "no-newer-release",
+ detail="Release tracking only; adapting this recipe and its dependencies requires review.")
+ elif not release["local_commit"]:
+ item.update(status="needs-rule", detail="Upstream commit found, but the local source commit is not identified.")
+ else:
+ item.update(status="new-commit" if release["commit"] != release["local_commit"] else "no-newer-release",
+ detail="Compare the pinned source commit with upstream before changing snapshot packaging.")
+ else:
+ if rule and rule["kind"] == "local-ebuilds":
+ item["provider"] = "project-packaging"
+ source_package = Path(rule["path"]) / package
+ upstream = work / "project" / package
+ snapshot(source_package, upstream)
+ identity = str(source_package)
+ filters = []
+ elif rule and rule["kind"] != "gentoo":
+ raise ValueError(f"unknown provider {rule['kind']}")
+ else:
+ item["provider"] = "gentoo"
+ if report["base"].get("error"):
+ raise ValueError(report["base"]["error"])
+ upstream = work / "gentoo" / package
+ identity = report["base"]["url"]
+ filters = report["base"]["filters"]
+ if not upstream.is_dir():
+ item.update(status="needs-rule", detail="Not present in the published Gentoo input; configure a release source.")
+ upstream = None
+ else:
+ item.update(package_comparison(local, upstream, package, by_package.get(package, {}), filters, identity))
+ if item["status"] in ACTIONABLE:
+ item["review"] = f"reviews/{package}"
+ prepare_review(local, upstream, output / item["review"], item)
+ except (ValueError, OSError, KeyError, IndexError, TypeError, AttributeError, subprocess.SubprocessError) as error:
+ item.update(status="error", detail=str(error))
+ # Preserve the comparison baseline across transient lookup failures.
+ for key in ("upstream_digest", "upstream_identity"):
+ if key in by_package.get(package, {}):
+ item.setdefault(key, by_package[package][key])
+ report["packages"].append(item)
+ report["counts"] = dict(Counter(p["status"] for p in report["packages"]))
+ report["upstream_errors"] = bool(report["base"].get("error") or report["counts"].get("error"))
+ atomic_json(output / "report.json", report)
+ atomic_text(output / "report.md", render_report(report))
+ atomic_json(latest, report)
+ atomic_text(home / "latest.md", render_report(report).replace(
+ "](reviews/", f"](runs/{stamp}/reviews/"))
+ return report
+
+
+def main(argv=None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("repository")
+ parser.add_argument("--config", type=Path)
+ parser.add_argument("--engine-config", type=Path, default=Path("/etc/foundry/engine.toml"))
+ args = parser.parse_args(argv)
+ try:
+ repo = name(args.repository)
+ config = args.config or Path("/etc/crucible/maintenance") / f"{repo}.toml"
+ if tomllib.loads(config.read_text())["repository"] != repo:
+ raise ValueError("configuration repository does not match the requested repository")
+ report = run(config, Runtime.load(args.engine_config))
+ print(f"Review report: {report['output']}/report.md")
+ print(json.dumps(report["counts"], sort_keys=True))
+ return 1 if report["upstream_errors"] else 0
+ except (ValueError, OSError, KeyError, StopIteration, subprocess.SubprocessError) as error:
+ print(f"crucible-maintain: {error}", file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
--- /dev/null
+++ b/tests/test_crucible_maintenance.py
@@ -0,0 +1,176 @@
+import json
+from pathlib import Path
+import subprocess
+from tempfile import TemporaryDirectory
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from foundry.transition import tree_digest
+from foundry import repository_maintenance as maintenance
+
+
+class MaintenanceTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.local = self.root / "source"
+ self.base = self.root / "base"
+ self.package = "sci-libs/example"
+ self.put(self.local, "example-1.9-r1.ebuild", 'EAPI=8\nPATCHES=("${FILESDIR}/local.patch")\n')
+ self.put(self.local, "files/local.patch", "local compiler fix\n")
+ self.put(self.base, "example-1.10.ebuild", "EAPI=8\n")
+
+ def put(self, root, filename, text):
+ path = root / self.package / filename
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text)
+ return path
+
+ def compare(self, previous=None, filters=None):
+ return maintenance.package_comparison(self.local / self.package, self.base / self.package,
+ self.package, previous or {}, filters or [], "gentoo")
+
+ def test_numeric_version_order_and_local_revision(self):
+ self.assertEqual(self.compare()["status"], "new-release")
+ self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n")
+ self.assertEqual(self.compare()["status"], "no-newer-release")
+
+ def test_same_version_upstream_edits_are_reported_after_baseline(self):
+ self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n")
+ baseline = self.compare()
+ self.put(self.base, "example-1.10.ebuild", "EAPI=8\n# upstream repair\n")
+ self.assertEqual(self.compare(baseline)["status"], "upstream-changed")
+ self.assertEqual(self.compare(dict(baseline, upstream_identity="other"))["status"], "no-newer-release")
+
+ def test_filtered_base_version_is_not_proposed(self):
+ result = self.compare(filters=[self.package + "/example-1.10.ebuild"])
+ self.assertEqual(result["status"], "needs-rule")
+
+ def test_extraction_permissions_do_not_create_false_upstream_changes(self):
+ self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n")
+ support = self.put(self.base, "files/upstream.patch", "unchanged patch\n")
+ baseline = self.compare()
+ support.parent.chmod(0o700)
+ support.chmod(0o600)
+ self.assertEqual(self.compare(baseline)["status"], "no-newer-release")
+
+ def test_review_preserves_both_originals_and_local_patch(self):
+ before = tree_digest(self.local)
+ output = self.root / "review"
+ maintenance.prepare_review(self.local / self.package, self.base / self.package,
+ output, self.compare())
+ self.assertEqual(tree_digest(self.local), before)
+ self.assertEqual((output / "local/files/local.patch").read_text(), "local compiler fix\n")
+ self.assertTrue((output / "upstream/example-1.10.ebuild").exists())
+ self.assertIn("local.patch", (output / "upstream.diff").read_text())
+ self.assertFalse((self.local / self.package / "example-1.10.ebuild").exists())
+
+ def test_pypi_ignores_prereleases_yanked_and_empty_uploads(self):
+ data = {"releases": {"1.9": [{"yanked": False}], "1.10": [{"yanked": False}],
+ "2.0": [{"yanked": True}], "3.0": [], "4.0rc1": [{"yanked": False}]}}
+ result = maintenance.remote_release({"kind": "pypi", "project": "example"}, "", lambda _: data)
+ self.assertEqual(result["version"], "1.10")
+
+ def test_github_honors_prefix_and_excludes_draft_prerelease(self):
+ rule = {"kind": "github", "repository": "org/project", "tag_prefix": "rust-v"}
+ data = {"tag_name": "rust-v1.10", "html_url": "https://example/1.10"}
+ result = maintenance.remote_release(rule, "", lambda _: data)
+ self.assertEqual(result["version"], "1.10")
+ for invalid in [{"tag_name": "rust-v2.0", "draft": True},
+ {"tag_name": "rust-v3.0", "prerelease": True},
+ {"tag_name": "rust-v4.0-alpha.1"}]:
+ with self.assertRaisesRegex(ValueError, "no matching stable"):
+ maintenance.remote_release(rule, "", lambda _: invalid)
+
+ def test_unresolved_upstream_change_stays_visible_until_local_edit(self):
+ self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n")
+ previous = dict(self.compare(), status="upstream-changed", local_digest=tree_digest(self.local / self.package))
+ self.assertEqual(self.compare(previous)["status"], "upstream-changed")
+ self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n# reviewed\n")
+ self.assertEqual(self.compare(previous)["status"], "no-newer-release")
+
+ def test_pinned_commit_extracted_without_executing_ebuild(self):
+ old, new = "a" * 40, "b" * 40
+ result = maintenance.remote_release(
+ {"kind": "github-commit", "repository": "org/project", "pin_pattern": r'COMMIT="([0-9a-f]{40})"'},
+ f'COMMIT="{old}"\ntouch /should-not-run\n',
+ lambda _: [{"sha": new, "html_url": "https://example/commit", "commit": {"committer": {"date": "now"}}}])
+ self.assertEqual(result["local_commit"], old)
+ self.assertEqual(result["commit"], new)
+
+ def test_capture_refuses_concurrent_local_edits(self):
+ original = maintenance.overlay
+ def change(source, dest):
+ original(source, dest)
+ self.put(self.local, "example-1.9-r1.ebuild", "new author edit\n")
+ with patch.object(maintenance, "overlay", side_effect=change):
+ with self.assertRaisesRegex(ValueError, "changed during capture"):
+ maintenance.snapshot(self.local / self.package, self.root / "copy")
+
+ def test_complete_run_reports_errors_and_exclusions_without_source_changes(self):
+ runtime = SimpleNamespace(repositories_root=self.root / "repos", cache_root=self.root / "cache")
+ config = self.root / "maintenance.toml"
+ config.write_text('repository="test"\nsource="distribution"\n'
+ '[packages."sci-libs/remote"]\nkind="github"\nrepository="org/remote"\n')
+ remote = self.local / "sci-libs/remote/remote-1.0.ebuild"
+ remote.parent.mkdir(parents=True)
+ marker = self.root / "executed"
+ remote.write_text(f'EAPI=8\ntouch "{marker}"\n')
+ excluded = self.local / "sci-libs/excluded/excluded-1.0.ebuild"
+ excluded.parent.mkdir(parents=True)
+ excluded.write_text("EAPI=8\n")
+ data = {"sources": [{"name": "distribution", "path": str(self.local),
+ "filter": ["sci-libs/excluded"]}]}
+ def base(runtime, repo, data, packages, dest):
+ maintenance.overlay(self.base, dest)
+ return {"url": "gentoo", "filters": [], "candidate": "test", "commit": "abc"}
+ before = tree_digest(self.local)
+ with patch.object(maintenance, "load_repository", return_value=data), \
+ patch.object(maintenance, "published_base", side_effect=base):
+ report = maintenance.run(config, runtime, fetch=lambda _: (_ for _ in ()).throw(OSError("offline")))
+ statuses = {x["package"]: x["status"] for x in report["packages"]}
+ self.assertEqual(statuses, {self.package: "new-release", "sci-libs/remote": "error",
+ "sci-libs/excluded": "excluded"})
+ self.assertTrue(report["upstream_errors"])
+ self.assertEqual(tree_digest(self.local), before)
+ self.assertFalse(marker.exists())
+ home = runtime.repositories_root / "test/maintenance/distribution"
+ self.assertTrue((home / "latest.md").is_file())
+ self.assertIn("runs/", (home / "latest.md").read_text())
+ self.assertEqual(json.loads((home / "latest.json").read_text())["counts"]["new-release"], 1)
+
+ def test_published_base_exports_pinned_commit_not_newest_git_head(self):
+ upstream = self.root / "git-upstream"
+ upstream.mkdir()
+ git = lambda *args: subprocess.check_output(["git", "-C", str(upstream), *args], text=True).strip()
+ git("init", "-q")
+ git("config", "user.email", "test@example.invalid")
+ git("config", "user.name", "Test")
+ self.put(upstream, "example-1.10.ebuild", "EAPI=8\n")
+ git("add", ".")
+ git("commit", "-qm", "pinned")
+ pinned = git("rev-parse", "HEAD")
+ self.put(upstream, "example-2.0.ebuild", "EAPI=8\n")
+ git("add", ".")
+ git("commit", "-qm", "later")
+ runtime = SimpleNamespace(repositories_root=self.root / "repos", cache_root=self.root / "cache")
+ bare = runtime.cache_root / "upstreams/base-id"
+ bare.parent.mkdir(parents=True)
+ subprocess.run(["git", "clone", "-q", "--bare", str(upstream), str(bare)], check=True)
+ home = runtime.repositories_root / "test"
+ candidate = home / "candidates/published"
+ candidate.mkdir(parents=True)
+ (home / "state.json").write_text(json.dumps({"published": "published"}))
+ (candidate / "manifest.json").write_text(json.dumps({"inputs": [{"name": "gentoo", "kind": "git",
+ "source_id": "base-id", "commit": pinned, "url": "example", "ref": "HEAD"}]}))
+ result = maintenance.published_base(runtime, "test", {"composition": {"base": "gentoo"},
+ "sources": [{"name": "gentoo", "url": "example"}]}, [self.package], self.root / "export")
+ self.assertEqual(result["commit"], pinned)
+ self.assertTrue((self.root / "export" / self.package / "example-1.10.ebuild").exists())
+ self.assertFalse((self.root / "export" / self.package / "example-2.0.ebuild").exists())
+
+
+if __name__ == "__main__":
+ unittest.main()
--- /dev/null
+++ b/docs/crucible-maintenance.md
@@ -0,0 +1,122 @@
+# Automatic package maintenance reviews
+
+`crucible-maintain srcux` prepares a review queue for local package maintenance.
+It does not bump ebuilds, merge patches, regenerate Manifests, execute ebuilds,
+change source exclusions, or publish. Packages in a first-party source may be
+customized; an upstream directory is reference material, not a replacement.
+
+The helper is maintained in `src/foundry/repository_maintenance.py` and installed
+as `/usr/bin/crucible-maintain` by Foundry's Python console entry point.
+`app-admin/crucible-0.6.0-r2` requires `app-admin/foundry-0.6.0-r2` or a compatible
+later revision. Portage owns the command and implementation. The source-tree
+`scripts/crucible-maintain.py` is only a compatibility launcher.
+
+Tracking rules and schedules are administrator configuration. Installing the
+package does not create an automatic job or overwrite existing rules. An
+example is installed with Foundry's documentation as
+`examples/crucible-maintenance.toml`; adapt its repository, source, and provider
+rules before saving it under `/etc/crucible/maintenance/REPO.toml`.
+
+## Alpha deployment
+
+- Rules: `/etc/crucible/maintenance/srcux.toml`.
+- Command: `/usr/bin/crucible-maintain srcux`.
+- Scheduled wrapper: `/usr/local/sbin/crucible-maintain-srcux`.
+- Schedule: `/etc/cron.d/srcux-maintenance`, daily at 03:20 in cron's timezone.
+- Latest report: `/var/lib/crucible/repos/srcux/maintenance/distribution/latest.md`.
+- Structured report: the adjacent `latest.json`.
+- Retained runs: the adjacent `runs/` directory.
+- Cron log: `/var/log/osforge/package-maintenance.cron.log`.
+
+The host's scheduled wrapper invokes the packaged command, shares the repository
+updater's flock and waits up to
+two hours if it is still running. The helper also serializes its own reports.
+Run the wrapper for the same locking behavior when starting a manual run.
+On this fcron host, reload system cron entries with
+`/usr/libexec/check_system_crontabs -i` after editing the schedule. Verify the
+active entry with `fcrondyn -x ls`; writing a cron.d file alone is insufficient.
+
+The Git comparison uses the exact base commit in the last published candidate's
+manifest. It exports only the relevant package directories from Crucible's
+existing upstream cache. It does not fetch Gentoo itself; the ordinary 03:00
+repository update provides refreshed input. Each report identifies its candidate
+and commit. If publication fails, the report continues using the previous
+published input; consult `crucible status srcux` and the report's input date.
+
+## Tracking and review material
+
+Unconfigured versioned packages are compared with the published Gentoo input.
+Explicit rules can track a local project's packaging directory, GitHub's latest
+stable release, PyPI's highest stable non-yanked release, or a pinned GitHub
+commit. Account definitions, live ebuilds, and packages excluded by Crucible
+policy are identified separately. Local applications without a configured
+release source remain explicitly marked `manual`. New unrecognized packages
+appear as `needs-rule`; they are not silently treated as current.
+
+Provider contracts are documented by [GitHub releases](https://docs.github.com/en/rest/releases/releases#get-the-latest-release)
+and [PyPI's JSON API](https://docs.pypi.org/api/json/). No GitHub credentials are
+read. API errors and missing repositories remain visible, and the command exits
+nonzero after saving the partial report. Commit tracking signals different
+upstream HEAD bytes; it does not prove ancestry, release status, or suitability.
+
+For a newer recipe, a review directory contains separate `local/` and
+`upstream/` snapshots, `upstream.diff`, and `review.json`. Release-only trackers
+save the local recipe and release information; no upstream ebuild is invented.
+Local patches and support files remain in the local snapshot. No proposal is
+registered as an input to Crucible, so the nightly publisher cannot select it.
+
+After the first baseline, package content changes under the same upstream
+version are also reported. Such findings remain visible on subsequent runs
+until the local package changes. This is a maintenance signal, not a statement
+that either recipe is buildable or that every keyword/slot is covered.
+
+To review:
+
+1. Open `latest.md` and the relevant review bundle.
+2. Adapt the ebuild in a separate working directory, preserving required local
+ patches and dependencies. Do not copy the upstream directory over the local
+ package without reviewing its customizations.
+3. Regenerate its Manifest and run appropriate metadata, preparation, and build
+ checks. These are maintainer actions, not checks performed by this helper.
+4. Move the completed change into the authoring source. Run
+ `crucible update srcux --no-publish`, inspect the candidate, then publish
+ with `crucible publish srcux --candidate CANDIDATE_ID`.
+
+The existing daily publisher captures all edits in configured source
+directories, including uncommitted files. Keep unfinished work outside those
+directories. The maintenance report does not change that publication policy.
+
+Reports are retained without automatic deletion. Inspect disk use periodically;
+old `runs/` directories are disposable review artifacts once no longer needed.
+Keep the latest run and any bundle under review.
+
+## Validation and removal
+
+Run the targeted tests from the Foundry checkout:
+
+```sh
+PYTHONPATH=src:tests python3 -m unittest test_crucible_maintenance -v
+```
+
+They cover Portage version ordering, preserved local patches, frozen upstream
+commit selection, exclusions, unexecuted ebuilds, capture races, release
+filtering, lookup errors, and persistent review findings.
+
+To disable scheduling, remove `/etc/cron.d/srcux-maintenance`, then run
+`/usr/libexec/check_system_crontabs -i` and verify its absence in `fcrondyn -x ls`.
+The host-specific scheduled wrapper and maintenance configuration can then be
+removed independently. Manage `/usr/bin/crucible-maintain` and its Python module
+through the Foundry package; do not remove package-owned files by hand.
+Existing review artifacts may be retained. The normal Crucible update schedule,
+authoring sources, and published releases are independent of this helper.
+
+## Migration from the initial local helper
+
+After installing the revised Crucible/Foundry packages, verify
+`/usr/bin/crucible-maintain --help`. Archive the old
+`/usr/local/bin/crucible-maintain` launcher and
+`/usr/local/libexec/crucible-maintain.py`, then remove them from those locations
+so the local command no longer shadows Portage's command. Change the scheduled
+wrapper to call `/usr/bin/crucible-maintain`. Preserve the existing rules,
+reports, cron entry, and lock path. This does not require a Foundry daemon
+restart because the new CLI runs in its own process.
--- /dev/null
+++ b/examples/crucible-maintenance.toml
@@ -0,0 +1,26 @@
+# Administrator example: customize repository/source before enabling a schedule.
+# All results are review-only and live outside the repository's source inputs.
+repository = "example"
+source = "local"
+
+# Unlisted release packages use the last published Gentoo base input.
+[packages."sci-libs/vtk"]
+kind = "gentoo"
+
+[packages."dev-util/codex"]
+kind = "github"
+repository = "openai/codex"
+tag_prefix = "rust-v"
+
+[packages."app-misc/graphify"]
+kind = "pypi"
+project = "graphifyy"
+
+# Replace the example path with the actual project's maintained packaging tree.
+[packages."app-admin/my-application"]
+kind = "local-ebuilds"
+path = "/path/to/project/packaging/gentoo"
+
+[packages."app-misc/custom-fork"]
+kind = "manual"
+reason = "Maintain the custom source archive, patches and recipe together."
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,6 +19,7 @@
foundryd = "foundry.daemon:main"
foundry-engine = "foundry.cli:main"
crucible = "foundry.crucible:main"
+crucible-maintain = "foundry.repository_maintenance:main"
# Compatibility entry points use the same implementation and state.
osforge = "foundry.cli:main"
osforged = "foundry.daemon:main"
|