--- /dev/null +++ b/src/foundry/repository_maintenance.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Prepare source-maintenance reviews; never edit or publish authoring inputs.""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timezone +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import tomllib +from urllib.request import Request, urlopen +import uuid + +from foundry.merge import _best_version, _compare_versions, rel_filtered +from foundry.repositories import load_repository, overlay, state_for +from foundry.repository_freshness import release_versions +from foundry.transition import Runtime, atomic_json, atomic_text, lock, name, tree_digest + + +ACTIONABLE = {"new-release", "upstream-changed", "new-commit", "needs-rule", "error"} + + +def stable_version(tag: str, prefix: str = "v") -> str | None: + value = tag[len(prefix):] if prefix and tag.startswith(prefix) else tag + return value if re.fullmatch(r"\d+(?:\.\d+)*", value) else None + + +def request_json(url: str): + request = Request(url, headers={"User-Agent": "crucible-maintain/1.0", + "Accept": "application/json"}) + with urlopen(request, timeout=20) as response: + body = response.read(8 * 1024 * 1024 + 1) + if len(body) > 8 * 1024 * 1024: + raise ValueError("upstream response exceeds 8 MiB") + return json.loads(body) + + +def remote_release(rule: dict, recipe: str, fetch=request_json) -> dict: + kind = rule["kind"] + if kind == "pypi": + project = rule["project"] + if not re.fullmatch(r"[A-Za-z0-9_.-]+", project): + raise ValueError("invalid PyPI project") + data = fetch(f"https://pypi.org/pypi/{project}/json") + versions = {v for v, files in data["releases"].items() + if stable_version(v, "") and any(not f.get("yanked") for f in files)} + if not versions: + raise ValueError("no stable, non-yanked PyPI release") + version = _best_version(versions) + return {"version": version, "url": f"https://pypi.org/project/{project}/{version}/"} + repo = rule["repository"] + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo): + raise ValueError("invalid GitHub repository") + api = f"https://api.github.com/repos/{repo}" + if kind == "github-commit": + data = fetch(api + "/commits?per_page=1") + if not data or not re.fullmatch(r"[0-9a-f]{40}", data[0]["sha"]): + raise ValueError("no upstream commit") + pinned = re.search(rule["pin_pattern"], recipe) if rule.get("pin_pattern") else None + return {"commit": data[0]["sha"], "local_commit": pinned.group(1) if pinned else None, + "url": data[0]["html_url"], "date": data[0]["commit"]["committer"]["date"]} + if kind != "github": + raise ValueError(f"unknown remote provider: {kind}") + release = fetch(api + "/releases/latest") + version = stable_version(release["tag_name"], rule.get("tag_prefix", "v")) + if release.get("draft") or release.get("prerelease") or not version: + raise ValueError("no matching stable GitHub release; configure commit tracking if needed") + return {"version": version, "url": release["html_url"], "tag": release["tag_name"]} + + +def snapshot(source: Path, dest: Path) -> str: + before = tree_digest(source) + overlay(source, dest) + if tree_digest(source) != before or tree_digest(dest) != before: + raise ValueError(f"source changed during capture: {source}") + return before + + +def recipe_digest(root: Path) -> str: + """Track package bytes and link targets independently of extraction umask.""" + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if ".git" in path.relative_to(root).parts: + continue + if path.is_symlink(): + payload = b"link:" + os.fsencode(os.readlink(path)) + elif path.is_file(): + payload = b"file:" + path.read_bytes() + else: + continue + digest.update(path.relative_to(root).as_posix().encode() + b"\0") + digest.update(hashlib.sha256(payload).digest()) + return "content-v1:" + digest.hexdigest() + + +def published_base(runtime: Runtime, repo: str, data: dict, packages: list[str], dest: Path) -> dict: + """Export only selected packages from the exact published Gentoo input.""" + home = runtime.repositories_root / repo + candidate = state_for(home).get("published") + if not candidate: + raise ValueError("no published candidate; run crucible update before maintenance") + name(candidate) + manifest = json.loads((home / "candidates" / candidate / "manifest.json").read_text()) + base_name = data.get("composition", {}).get("base") + entry = next((x for x in manifest["inputs"] if x["name"] == base_name), None) + if not entry or entry.get("kind") != "git" or not entry.get("source_id"): + raise ValueError("published base is not a cached Git input") + configured = next(x for x in data["sources"] if x["name"] == base_name) + if configured.get("url") != entry.get("url") or configured.get("ref", "HEAD") != entry.get("ref"): + raise ValueError("base configuration changed since publication; update first") + bare = runtime.cache_root / "upstreams" / name(entry["source_id"]) + commit = entry["commit"] + if not re.fullmatch(r"[0-9a-f]{40,64}", commit): + raise ValueError("invalid published input commit") + paths = subprocess.run(["git", "--git-dir", str(bare), "ls-tree", "-r", "--name-only", + commit, "--", *packages], check=True, text=True, + capture_output=True, timeout=120).stdout.splitlines() if packages else [] + present = sorted({"/".join(p.split("/")[:2]) for p in paths if p.endswith(".ebuild")}) + dest.mkdir(parents=True) + if present: + archive = dest.parent / "base.tar" + subprocess.run(["git", "--git-dir", str(bare), "archive", "--format=tar", + f"--output={archive}", commit, "--", *present], check=True, + capture_output=True, timeout=120) + with tarfile.open(archive) as stream: + stream.extractall(dest, filter="data") + archive.unlink() + return {"candidate": candidate, "commit": commit, "url": entry["url"], + "filters": configured.get("filter", []), + "published_at": datetime.fromtimestamp( + (home / "candidates" / candidate / "manifest.json").stat().st_mtime, + timezone.utc).isoformat()} + + +def package_comparison(local: Path, upstream: Path, package: str, previous: dict, + filters: list[str], source_identity: str) -> dict: + local_version = _best_version(release_versions(local.parent.parent, package, [])) + upstream_version = _best_version(release_versions(upstream.parent.parent, package, filters)) + result = {"local_version": local_version or "live-only", "upstream_version": upstream_version, + "upstream_digest": recipe_digest(upstream), "upstream_identity": source_identity} + if not upstream_version: + result.update(status="needs-rule", detail="No eligible non-live upstream ebuild.") + elif not local_version or _compare_versions(upstream_version, local_version) > 0: + result.update(status="new-release", detail="Newer upstream recipe available; preserve local changes when updating.") + elif (previous.get("upstream_identity") == source_identity and previous.get("upstream_digest", "").startswith("content-v1:") + and (previous["upstream_digest"] != result["upstream_digest"] or ( + previous.get("status") == "upstream-changed" and previous.get("local_digest") == tree_digest(local)))): + result.update(status="upstream-changed", detail="Upstream package content changed since the previous report.") + else: + result.update(status="no-newer-release", detail="No newer release detected; local customizations remain in place.") + return result + + +def prepare_review(local: Path, upstream: Path | None, output: Path, item: dict) -> None: + output.mkdir(parents=True) + overlay(local, output / "local") + if upstream: + overlay(upstream, output / "upstream") + paths = sorted({p.relative_to(root).as_posix() for root in (local, upstream) + for p in root.rglob("*") if p.is_file() and not p.is_symlink()}) + diffs = [] + for path in paths: + a, b = local / path, upstream / path + before = a.read_bytes() if a.is_file() else b"" + after = b.read_bytes() if b.is_file() else b"" + if before == after: + continue + try: + diffs.extend(difflib.unified_diff(before.decode().splitlines(True), after.decode().splitlines(True), + fromfile=f"local/{path}", tofile=f"upstream/{path}")) + except UnicodeDecodeError: + diffs.append(f"Binary file differs: {path}\n") + atomic_text(output / "upstream.diff", "".join(diffs)) + atomic_json(output / "review.json", item) + + +def render_report(report: dict) -> str: + lines = [f"# {report['repository']} package maintenance review", "", + f"Generated: {report['created_at']}", "", + "Review material only. No authoring files were changed, no ebuilds were executed, and nothing was published.", + "Upstream snapshots and diffs are reference material, not drop-in replacements for customized recipes.", "", + f"Published input: {report['base'].get('candidate', 'unavailable')} " + f"({report['base'].get('commit', 'unavailable')}).", "", + f"Counts: {', '.join(f'{k}={v}' for k, v in sorted(report['counts'].items()))}.", "", + "## Needs attention", ""] + for item in report["packages"]: + if item["status"] not in ACTIONABLE: + continue + lines += [f"### {item['package']}", "", f"{item['status']}: {item['detail']}", "", + f"Local: {item.get('local_version', '?')}; upstream: " + f"{item.get('upstream_version') or item.get('commit') or 'unknown'}.", ""] + if item.get("url"): + lines += [f"Upstream: {item['url']}", ""] + if item.get("review"): + lines += [f"Files: [review bundle]({item['review']}/review.json)", ""] + lines += ["## Complete inventory", "", "| Package | Provider | Status |", "| --- | --- | --- |"] + lines += [f"| {p['package']} | {p['provider']} | {p['status']} |" for p in report["packages"]] + lines += ["", "## Review workflow", "", + "Use the local and upstream snapshots to update the authoring recipe, retaining required patches and dependencies. " + "Regenerate its Manifest and run appropriate package checks/builds. " + "Then use `crucible update REPO --no-publish`, inspect the candidate, and publish its explicit candidate ID.", + "The existing automatic publisher captures edits inside configured source directories. " + "Keep unfinished work outside those directories.", ""] + return "\n".join(lines) + + +def run(config_path: Path, runtime: Runtime, *, fetch=request_json) -> dict: + config = tomllib.loads(config_path.read_text()) + repo, source_name = name(config["repository"]), name(config.get("source", "distribution")) + data = load_repository(runtime, repo) + source = next(x for x in data["sources"] if x["name"] == source_name) + if not source.get("path"): + raise ValueError("maintenance source must be local") + root = Path(source["path"]).resolve(strict=True) + home = runtime.repositories_root / repo / "maintenance" / source_name + if any(home.resolve().is_relative_to(Path(s["path"]).resolve()) for s in data["sources"] if s.get("path")): + raise ValueError("review storage must be outside every active source") + with lock(home / "maintenance.lock"): + latest = home / "latest.json" + previous = json.loads(latest.read_text()) if latest.exists() else {} + by_package = {p["package"]: p for p in previous.get("packages", [])} + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] + output = home / "runs" / stamp + output.mkdir(parents=True) + packages = sorted({p.parent.relative_to(root).as_posix() for p in root.glob("*/*/*.ebuild")}) + rules = config.get("packages", {}) + report = {"schema_version": 1, "repository": repo, "source": source_name, + "created_at": datetime.now(timezone.utc).isoformat(), "output": str(output), + "config_sha256": hashlib.sha256(config_path.read_bytes()).hexdigest(), "packages": []} + with tempfile.TemporaryDirectory(prefix=".capture-", dir=home) as temporary: + work = Path(temporary) + try: + report["base"] = published_base(runtime, repo, data, packages, work / "gentoo") + except (ValueError, OSError, subprocess.SubprocessError, KeyError) as error: + report["base"] = {"error": str(error)} + for package in packages: + item = {"package": package, "provider": "unassigned"} + local = work / "local" / package + upstream = None + try: + item["local_digest"] = snapshot(root / package, local) + versions = release_versions(work / "local", package, []) + version = _best_version(versions) + item["local_version"] = version or "live-only" + recipe = (local / f"{local.name}-{version}.ebuild").read_text() if version else "" + rule = rules.get(package) + if not source.get("enabled", True) or rel_filtered(package, source.get("filter", [])) or ( + source.get("select") and package not in source["select"]): + item.update(provider="source-policy", status="excluded", detail="Excluded from this Crucible source.") + elif rule and rule["kind"] in {"manual", "fixed", "live"}: + item.update(provider=rule["kind"], status=rule["kind"], detail=rule["reason"]) + elif not rule and package.startswith(("acct-user/", "acct-group/")): + item.update(provider="account", status="fixed", detail="Account definition; no software release to track.") + elif not rule and not version: + item.update(provider="live-ebuild", status="live", detail="Live recipe follows its source when rebuilt; no release bump inferred.") + elif rule and rule["kind"] in {"github", "pypi", "github-commit"}: + item["provider"] = rule["kind"] + release = remote_release(rule, recipe, fetch) + item.update({k: v for k, v in release.items() if k != "version"}) + if release.get("version"): + item["upstream_version"] = release["version"] + newer = not version or _compare_versions(release["version"], version) > 0 + item.update(status="new-release" if newer else "no-newer-release", + detail="Release tracking only; adapting this recipe and its dependencies requires review.") + elif not release["local_commit"]: + item.update(status="needs-rule", detail="Upstream commit found, but the local source commit is not identified.") + else: + item.update(status="new-commit" if release["commit"] != release["local_commit"] else "no-newer-release", + detail="Compare the pinned source commit with upstream before changing snapshot packaging.") + else: + if rule and rule["kind"] == "local-ebuilds": + item["provider"] = "project-packaging" + source_package = Path(rule["path"]) / package + upstream = work / "project" / package + snapshot(source_package, upstream) + identity = str(source_package) + filters = [] + elif rule and rule["kind"] != "gentoo": + raise ValueError(f"unknown provider {rule['kind']}") + else: + item["provider"] = "gentoo" + if report["base"].get("error"): + raise ValueError(report["base"]["error"]) + upstream = work / "gentoo" / package + identity = report["base"]["url"] + filters = report["base"]["filters"] + if not upstream.is_dir(): + item.update(status="needs-rule", detail="Not present in the published Gentoo input; configure a release source.") + upstream = None + else: + item.update(package_comparison(local, upstream, package, by_package.get(package, {}), filters, identity)) + if item["status"] in ACTIONABLE: + item["review"] = f"reviews/{package}" + prepare_review(local, upstream, output / item["review"], item) + except (ValueError, OSError, KeyError, IndexError, TypeError, AttributeError, subprocess.SubprocessError) as error: + item.update(status="error", detail=str(error)) + # Preserve the comparison baseline across transient lookup failures. + for key in ("upstream_digest", "upstream_identity"): + if key in by_package.get(package, {}): + item.setdefault(key, by_package[package][key]) + report["packages"].append(item) + report["counts"] = dict(Counter(p["status"] for p in report["packages"])) + report["upstream_errors"] = bool(report["base"].get("error") or report["counts"].get("error")) + atomic_json(output / "report.json", report) + atomic_text(output / "report.md", render_report(report)) + atomic_json(latest, report) + atomic_text(home / "latest.md", render_report(report).replace( + "](reviews/", f"](runs/{stamp}/reviews/")) + return report + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("repository") + parser.add_argument("--config", type=Path) + parser.add_argument("--engine-config", type=Path, default=Path("/etc/foundry/engine.toml")) + args = parser.parse_args(argv) + try: + repo = name(args.repository) + config = args.config or Path("/etc/crucible/maintenance") / f"{repo}.toml" + if tomllib.loads(config.read_text())["repository"] != repo: + raise ValueError("configuration repository does not match the requested repository") + report = run(config, Runtime.load(args.engine_config)) + print(f"Review report: {report['output']}/report.md") + print(json.dumps(report["counts"], sort_keys=True)) + return 1 if report["upstream_errors"] else 0 + except (ValueError, OSError, KeyError, StopIteration, subprocess.SubprocessError) as error: + print(f"crucible-maintain: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) --- /dev/null +++ b/tests/test_crucible_maintenance.py @@ -0,0 +1,176 @@ +import json +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from foundry.transition import tree_digest +from foundry import repository_maintenance as maintenance + + +class MaintenanceTests(unittest.TestCase): + def setUp(self): + self.temp = TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.local = self.root / "source" + self.base = self.root / "base" + self.package = "sci-libs/example" + self.put(self.local, "example-1.9-r1.ebuild", 'EAPI=8\nPATCHES=("${FILESDIR}/local.patch")\n') + self.put(self.local, "files/local.patch", "local compiler fix\n") + self.put(self.base, "example-1.10.ebuild", "EAPI=8\n") + + def put(self, root, filename, text): + path = root / self.package / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + def compare(self, previous=None, filters=None): + return maintenance.package_comparison(self.local / self.package, self.base / self.package, + self.package, previous or {}, filters or [], "gentoo") + + def test_numeric_version_order_and_local_revision(self): + self.assertEqual(self.compare()["status"], "new-release") + self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n") + self.assertEqual(self.compare()["status"], "no-newer-release") + + def test_same_version_upstream_edits_are_reported_after_baseline(self): + self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n") + baseline = self.compare() + self.put(self.base, "example-1.10.ebuild", "EAPI=8\n# upstream repair\n") + self.assertEqual(self.compare(baseline)["status"], "upstream-changed") + self.assertEqual(self.compare(dict(baseline, upstream_identity="other"))["status"], "no-newer-release") + + def test_filtered_base_version_is_not_proposed(self): + result = self.compare(filters=[self.package + "/example-1.10.ebuild"]) + self.assertEqual(result["status"], "needs-rule") + + def test_extraction_permissions_do_not_create_false_upstream_changes(self): + self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n") + support = self.put(self.base, "files/upstream.patch", "unchanged patch\n") + baseline = self.compare() + support.parent.chmod(0o700) + support.chmod(0o600) + self.assertEqual(self.compare(baseline)["status"], "no-newer-release") + + def test_review_preserves_both_originals_and_local_patch(self): + before = tree_digest(self.local) + output = self.root / "review" + maintenance.prepare_review(self.local / self.package, self.base / self.package, + output, self.compare()) + self.assertEqual(tree_digest(self.local), before) + self.assertEqual((output / "local/files/local.patch").read_text(), "local compiler fix\n") + self.assertTrue((output / "upstream/example-1.10.ebuild").exists()) + self.assertIn("local.patch", (output / "upstream.diff").read_text()) + self.assertFalse((self.local / self.package / "example-1.10.ebuild").exists()) + + def test_pypi_ignores_prereleases_yanked_and_empty_uploads(self): + data = {"releases": {"1.9": [{"yanked": False}], "1.10": [{"yanked": False}], + "2.0": [{"yanked": True}], "3.0": [], "4.0rc1": [{"yanked": False}]}} + result = maintenance.remote_release({"kind": "pypi", "project": "example"}, "", lambda _: data) + self.assertEqual(result["version"], "1.10") + + def test_github_honors_prefix_and_excludes_draft_prerelease(self): + rule = {"kind": "github", "repository": "org/project", "tag_prefix": "rust-v"} + data = {"tag_name": "rust-v1.10", "html_url": "https://example/1.10"} + result = maintenance.remote_release(rule, "", lambda _: data) + self.assertEqual(result["version"], "1.10") + for invalid in [{"tag_name": "rust-v2.0", "draft": True}, + {"tag_name": "rust-v3.0", "prerelease": True}, + {"tag_name": "rust-v4.0-alpha.1"}]: + with self.assertRaisesRegex(ValueError, "no matching stable"): + maintenance.remote_release(rule, "", lambda _: invalid) + + def test_unresolved_upstream_change_stays_visible_until_local_edit(self): + self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n") + previous = dict(self.compare(), status="upstream-changed", local_digest=tree_digest(self.local / self.package)) + self.assertEqual(self.compare(previous)["status"], "upstream-changed") + self.put(self.local, "example-1.10-r1.ebuild", "EAPI=8\n# reviewed\n") + self.assertEqual(self.compare(previous)["status"], "no-newer-release") + + def test_pinned_commit_extracted_without_executing_ebuild(self): + old, new = "a" * 40, "b" * 40 + result = maintenance.remote_release( + {"kind": "github-commit", "repository": "org/project", "pin_pattern": r'COMMIT="([0-9a-f]{40})"'}, + f'COMMIT="{old}"\ntouch /should-not-run\n', + lambda _: [{"sha": new, "html_url": "https://example/commit", "commit": {"committer": {"date": "now"}}}]) + self.assertEqual(result["local_commit"], old) + self.assertEqual(result["commit"], new) + + def test_capture_refuses_concurrent_local_edits(self): + original = maintenance.overlay + def change(source, dest): + original(source, dest) + self.put(self.local, "example-1.9-r1.ebuild", "new author edit\n") + with patch.object(maintenance, "overlay", side_effect=change): + with self.assertRaisesRegex(ValueError, "changed during capture"): + maintenance.snapshot(self.local / self.package, self.root / "copy") + + def test_complete_run_reports_errors_and_exclusions_without_source_changes(self): + runtime = SimpleNamespace(repositories_root=self.root / "repos", cache_root=self.root / "cache") + config = self.root / "maintenance.toml" + config.write_text('repository="test"\nsource="distribution"\n' + '[packages."sci-libs/remote"]\nkind="github"\nrepository="org/remote"\n') + remote = self.local / "sci-libs/remote/remote-1.0.ebuild" + remote.parent.mkdir(parents=True) + marker = self.root / "executed" + remote.write_text(f'EAPI=8\ntouch "{marker}"\n') + excluded = self.local / "sci-libs/excluded/excluded-1.0.ebuild" + excluded.parent.mkdir(parents=True) + excluded.write_text("EAPI=8\n") + data = {"sources": [{"name": "distribution", "path": str(self.local), + "filter": ["sci-libs/excluded"]}]} + def base(runtime, repo, data, packages, dest): + maintenance.overlay(self.base, dest) + return {"url": "gentoo", "filters": [], "candidate": "test", "commit": "abc"} + before = tree_digest(self.local) + with patch.object(maintenance, "load_repository", return_value=data), \ + patch.object(maintenance, "published_base", side_effect=base): + report = maintenance.run(config, runtime, fetch=lambda _: (_ for _ in ()).throw(OSError("offline"))) + statuses = {x["package"]: x["status"] for x in report["packages"]} + self.assertEqual(statuses, {self.package: "new-release", "sci-libs/remote": "error", + "sci-libs/excluded": "excluded"}) + self.assertTrue(report["upstream_errors"]) + self.assertEqual(tree_digest(self.local), before) + self.assertFalse(marker.exists()) + home = runtime.repositories_root / "test/maintenance/distribution" + self.assertTrue((home / "latest.md").is_file()) + self.assertIn("runs/", (home / "latest.md").read_text()) + self.assertEqual(json.loads((home / "latest.json").read_text())["counts"]["new-release"], 1) + + def test_published_base_exports_pinned_commit_not_newest_git_head(self): + upstream = self.root / "git-upstream" + upstream.mkdir() + git = lambda *args: subprocess.check_output(["git", "-C", str(upstream), *args], text=True).strip() + git("init", "-q") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "Test") + self.put(upstream, "example-1.10.ebuild", "EAPI=8\n") + git("add", ".") + git("commit", "-qm", "pinned") + pinned = git("rev-parse", "HEAD") + self.put(upstream, "example-2.0.ebuild", "EAPI=8\n") + git("add", ".") + git("commit", "-qm", "later") + runtime = SimpleNamespace(repositories_root=self.root / "repos", cache_root=self.root / "cache") + bare = runtime.cache_root / "upstreams/base-id" + bare.parent.mkdir(parents=True) + subprocess.run(["git", "clone", "-q", "--bare", str(upstream), str(bare)], check=True) + home = runtime.repositories_root / "test" + candidate = home / "candidates/published" + candidate.mkdir(parents=True) + (home / "state.json").write_text(json.dumps({"published": "published"})) + (candidate / "manifest.json").write_text(json.dumps({"inputs": [{"name": "gentoo", "kind": "git", + "source_id": "base-id", "commit": pinned, "url": "example", "ref": "HEAD"}]})) + result = maintenance.published_base(runtime, "test", {"composition": {"base": "gentoo"}, + "sources": [{"name": "gentoo", "url": "example"}]}, [self.package], self.root / "export") + self.assertEqual(result["commit"], pinned) + self.assertTrue((self.root / "export" / self.package / "example-1.10.ebuild").exists()) + self.assertFalse((self.root / "export" / self.package / "example-2.0.ebuild").exists()) + + +if __name__ == "__main__": + unittest.main() --- /dev/null +++ b/docs/crucible-maintenance.md @@ -0,0 +1,122 @@ +# Automatic package maintenance reviews + +`crucible-maintain srcux` prepares a review queue for local package maintenance. +It does not bump ebuilds, merge patches, regenerate Manifests, execute ebuilds, +change source exclusions, or publish. Packages in a first-party source may be +customized; an upstream directory is reference material, not a replacement. + +The helper is maintained in `src/foundry/repository_maintenance.py` and installed +as `/usr/bin/crucible-maintain` by Foundry's Python console entry point. +`app-admin/crucible-0.6.0-r2` requires `app-admin/foundry-0.6.0-r2` or a compatible +later revision. Portage owns the command and implementation. The source-tree +`scripts/crucible-maintain.py` is only a compatibility launcher. + +Tracking rules and schedules are administrator configuration. Installing the +package does not create an automatic job or overwrite existing rules. An +example is installed with Foundry's documentation as +`examples/crucible-maintenance.toml`; adapt its repository, source, and provider +rules before saving it under `/etc/crucible/maintenance/REPO.toml`. + +## Alpha deployment + +- Rules: `/etc/crucible/maintenance/srcux.toml`. +- Command: `/usr/bin/crucible-maintain srcux`. +- Scheduled wrapper: `/usr/local/sbin/crucible-maintain-srcux`. +- Schedule: `/etc/cron.d/srcux-maintenance`, daily at 03:20 in cron's timezone. +- Latest report: `/var/lib/crucible/repos/srcux/maintenance/distribution/latest.md`. +- Structured report: the adjacent `latest.json`. +- Retained runs: the adjacent `runs/` directory. +- Cron log: `/var/log/osforge/package-maintenance.cron.log`. + +The host's scheduled wrapper invokes the packaged command, shares the repository +updater's flock and waits up to +two hours if it is still running. The helper also serializes its own reports. +Run the wrapper for the same locking behavior when starting a manual run. +On this fcron host, reload system cron entries with +`/usr/libexec/check_system_crontabs -i` after editing the schedule. Verify the +active entry with `fcrondyn -x ls`; writing a cron.d file alone is insufficient. + +The Git comparison uses the exact base commit in the last published candidate's +manifest. It exports only the relevant package directories from Crucible's +existing upstream cache. It does not fetch Gentoo itself; the ordinary 03:00 +repository update provides refreshed input. Each report identifies its candidate +and commit. If publication fails, the report continues using the previous +published input; consult `crucible status srcux` and the report's input date. + +## Tracking and review material + +Unconfigured versioned packages are compared with the published Gentoo input. +Explicit rules can track a local project's packaging directory, GitHub's latest +stable release, PyPI's highest stable non-yanked release, or a pinned GitHub +commit. Account definitions, live ebuilds, and packages excluded by Crucible +policy are identified separately. Local applications without a configured +release source remain explicitly marked `manual`. New unrecognized packages +appear as `needs-rule`; they are not silently treated as current. + +Provider contracts are documented by [GitHub releases](https://docs.github.com/en/rest/releases/releases#get-the-latest-release) +and [PyPI's JSON API](https://docs.pypi.org/api/json/). No GitHub credentials are +read. API errors and missing repositories remain visible, and the command exits +nonzero after saving the partial report. Commit tracking signals different +upstream HEAD bytes; it does not prove ancestry, release status, or suitability. + +For a newer recipe, a review directory contains separate `local/` and +`upstream/` snapshots, `upstream.diff`, and `review.json`. Release-only trackers +save the local recipe and release information; no upstream ebuild is invented. +Local patches and support files remain in the local snapshot. No proposal is +registered as an input to Crucible, so the nightly publisher cannot select it. + +After the first baseline, package content changes under the same upstream +version are also reported. Such findings remain visible on subsequent runs +until the local package changes. This is a maintenance signal, not a statement +that either recipe is buildable or that every keyword/slot is covered. + +To review: + +1. Open `latest.md` and the relevant review bundle. +2. Adapt the ebuild in a separate working directory, preserving required local + patches and dependencies. Do not copy the upstream directory over the local + package without reviewing its customizations. +3. Regenerate its Manifest and run appropriate metadata, preparation, and build + checks. These are maintainer actions, not checks performed by this helper. +4. Move the completed change into the authoring source. Run + `crucible update srcux --no-publish`, inspect the candidate, then publish + with `crucible publish srcux --candidate CANDIDATE_ID`. + +The existing daily publisher captures all edits in configured source +directories, including uncommitted files. Keep unfinished work outside those +directories. The maintenance report does not change that publication policy. + +Reports are retained without automatic deletion. Inspect disk use periodically; +old `runs/` directories are disposable review artifacts once no longer needed. +Keep the latest run and any bundle under review. + +## Validation and removal + +Run the targeted tests from the Foundry checkout: + +```sh +PYTHONPATH=src:tests python3 -m unittest test_crucible_maintenance -v +``` + +They cover Portage version ordering, preserved local patches, frozen upstream +commit selection, exclusions, unexecuted ebuilds, capture races, release +filtering, lookup errors, and persistent review findings. + +To disable scheduling, remove `/etc/cron.d/srcux-maintenance`, then run +`/usr/libexec/check_system_crontabs -i` and verify its absence in `fcrondyn -x ls`. +The host-specific scheduled wrapper and maintenance configuration can then be +removed independently. Manage `/usr/bin/crucible-maintain` and its Python module +through the Foundry package; do not remove package-owned files by hand. +Existing review artifacts may be retained. The normal Crucible update schedule, +authoring sources, and published releases are independent of this helper. + +## Migration from the initial local helper + +After installing the revised Crucible/Foundry packages, verify +`/usr/bin/crucible-maintain --help`. Archive the old +`/usr/local/bin/crucible-maintain` launcher and +`/usr/local/libexec/crucible-maintain.py`, then remove them from those locations +so the local command no longer shadows Portage's command. Change the scheduled +wrapper to call `/usr/bin/crucible-maintain`. Preserve the existing rules, +reports, cron entry, and lock path. This does not require a Foundry daemon +restart because the new CLI runs in its own process. --- /dev/null +++ b/examples/crucible-maintenance.toml @@ -0,0 +1,26 @@ +# Administrator example: customize repository/source before enabling a schedule. +# All results are review-only and live outside the repository's source inputs. +repository = "example" +source = "local" + +# Unlisted release packages use the last published Gentoo base input. +[packages."sci-libs/vtk"] +kind = "gentoo" + +[packages."dev-util/codex"] +kind = "github" +repository = "openai/codex" +tag_prefix = "rust-v" + +[packages."app-misc/graphify"] +kind = "pypi" +project = "graphifyy" + +# Replace the example path with the actual project's maintained packaging tree. +[packages."app-admin/my-application"] +kind = "local-ebuilds" +path = "/path/to/project/packaging/gentoo" + +[packages."app-misc/custom-fork"] +kind = "manual" +reason = "Maintain the custom source archive, patches and recipe together." --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ foundryd = "foundry.daemon:main" foundry-engine = "foundry.cli:main" crucible = "foundry.crucible:main" +crucible-maintain = "foundry.repository_maintenance:main" # Compatibility entry points use the same implementation and state. osforge = "foundry.cli:main" osforged = "foundry.daemon:main"