blob: 654078f177bccb85b3bd65b995db9ca16e84e9c5 [file]
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright (C) Opinsys Oy 2026
from __future__ import annotations
import argparse
import bisect
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
from typing import Any
from state import BuildState
_EXCLUDES = ("__pycache__", "*.pyc", "dist", ".git")
_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9+_.-]*\Z")
_SHA256_RE = re.compile(r"[0-9a-f]{64}\Z")
_CONFIG_REF_RE = re.compile(r"\bBR2_[A-Za-z0-9_]+\b")
_CONFIG_ASSIGNMENT_RE = re.compile(r"(BR2_[A-Za-z0-9_]+)=(.*)\Z")
_NOT_SET_RE = re.compile(r"# (BR2_[A-Za-z0-9_]+) is not set\Z")
_CONFIG_METADATA = frozenset(
{
"BR2_DEFCONFIG",
"BR2_DL_DIR",
"BR2_JLEVEL",
"BR2_PRIMARY_SITE",
"BR2_PRIMARY_SITE_ONLY",
"BR2_BACKUP_SITE",
"BR2_KERNEL_MIRROR",
"BR2_GNU_MIRROR",
"BR2_LUAROCKS_MIRROR",
"BR2_CPAN_MIRROR",
"BR2_GOOGLE_MIRROR",
"BR2_EXTERNAL_NAMES",
"BR2_HAVE_DOT_CONFIG",
"BR2_TARGET_ROOTFS_OCI_LABELS",
}
)
_DATA_FIELDS = ("KCONFIG_FILE", "KCONFIG_FRAGMENT_FILES", "CONFIG_FILE")
_MAKE_FUNCTIONS = frozenset(
(
"subst patsubst strip findstring filter filter-out sort word wordlist words "
"firstword lastword dir notdir suffix basename addsuffix addprefix join "
"wildcard realpath abspath if or and intcmp foreach let call value eval "
"origin flavor shell file error warning info guile"
).split()
)
_EPHEMERAL_MAKE_VARS = frozenset(
{
"MAKEFLAGS",
"MFLAGS",
"MAKELEVEL",
"MAKE_TERMOUT",
"MAKE_TERMERR",
"DATE",
"BR2_LOCALVERSION",
"BR2_VERSION_FULL",
"BR2_VERSION_GIT_EPOCH",
"CUR_UMASK",
"BR_ORIG_UMASK",
"TPMDD_PACKAGE_STATE_DIR",
"-*-command-variables-*-",
"-*-eval-flags-*-",
}
)
_ACCUMULATOR_GLOBALS = frozenset(
{
"KEEP_PYTHON_PY_FILES",
"PACKAGES_BUSYBOX_CONFIG_FIXUPS",
"PACKAGES_LINUX_CONFIG_FIXUPS",
"PACKAGES_SELINUX_MODULES",
"PACKAGES_SELINUX_EXTRA_MODULES_DIRS",
}
)
@dataclass(frozen=True)
class ScanResult:
references: frozenset[str]
calls: frozenset[tuple[str, tuple[str, ...]]]
prefixes: frozenset[str]
suffixes: frozenset[str]
opaque: bool
class MakeScanner:
@staticmethod
def split_call_arguments(text: str) -> list[str]:
arguments: list[str] = []
start = 0
parens = 0
braces = 0
for index, char in enumerate(text):
if char == "(":
parens += 1
elif char == ")":
parens = max(0, parens - 1)
elif char == "{":
braces += 1
elif char == "}":
braces = max(0, braces - 1)
elif char == "," and parens == 0 and braces == 0:
arguments.append(text[start:index].strip())
start = index + 1
arguments.append(text[start:].strip())
return arguments
@staticmethod
def bind_context(text: str, prefix: str, params: Sequence[str] = ()) -> str:
def replace(match: re.Match[str]) -> str:
if match.group(0) == "$$":
return "$$"
token = match.group(1) or match.group(2)
if token == "PKG":
return prefix
position = int(token)
return params[position - 1] if position <= len(params) else ""
return re.sub(r"\$\$|\$\((PKG|[1-9])\)|\$\{(PKG|[1-9])\}", replace, text)
@classmethod
def scan_bound(cls, text: str) -> ScanResult:
references: set[str] = set()
calls: set[tuple[str, tuple[str, ...]]] = set()
prefixes: set[str] = set()
suffixes: set[str] = set()
opaque = False
index = 0
while index < len(text):
if text[index] != "$" or index + 1 == len(text):
index += 1
continue
opening = text[index + 1]
if opening == "$":
index += 2
continue
if opening not in "({":
if opening not in "@%<?^+*|0123456789":
references.add(opening)
index += 2
continue
closing = ")" if opening == "(" else "}"
end = index + 2
depth = 1
while end < len(text) and depth:
depth += (text[end] == opening) - (text[end] == closing)
end += 1
if depth:
return ScanResult(
frozenset(references),
frozenset(calls),
frozenset(prefixes),
frozenset(suffixes),
True,
)
body = text[index + 2 : end - 1].strip()
if not body or re.fullmatch(
r"(?:[@%<?^+*|][DF]?|[0-9]+|\$\(strip\s*\))", body
):
pass
elif re.fullmatch(r"[A-Za-z_.][A-Za-z0-9_.-]*", body):
references.add(body)
elif match := re.fullmatch(
r"\$\(strip\s+([A-Za-z_.][A-Za-z0-9_.-]*)\)", body
):
references.add(match[1])
else:
match = re.match(r"([A-Za-z-]+)(?:\s+|\Z)", body)
if match and match[1] in _MAKE_FUNCTIONS:
function = match[1]
rest = body[match.end() :]
arguments = cls.split_call_arguments(rest) if rest else [""]
if function == "call":
target = arguments[0]
if re.fullmatch(r"[A-Za-z_.][A-Za-z0-9_.-]*", target):
calls.add((target, tuple(arguments[1:])))
elif pfx_match := re.match(
r"([A-Za-z_.][A-Za-z0-9_.-]*)\$", target
):
prefixes.add(pfx_match[1])
else:
opaque = True
elif function in ("value", "origin", "flavor"):
if re.fullmatch(r"[A-Za-z_.][A-Za-z0-9_.-]*", arguments[0]):
references.add(arguments[0])
else:
opaque = True
elif function == "eval":
if not rest.lstrip().startswith("__tmp "):
opaque = True
elif function == "guile":
opaque = True
elif match := re.match(r"([A-Za-z_.][A-Za-z0-9_.-]*):", body):
references.add(match[1])
elif pfx_match := re.match(r"([A-Za-z_.][A-Za-z0-9_.-]*)\$", body):
prefixes.add(pfx_match[1])
elif sfx_match := re.fullmatch(
r"\$\(call\s+UPPERCASE,\$\([A-Za-z_][A-Za-z0-9_]*\)\)(_[A-Za-z0-9_]+)",
body,
):
suffixes.add(sfx_match[1])
else:
opaque = True
nested = cls.scan_bound(body)
references.update(nested.references)
calls.update(nested.calls)
prefixes.update(nested.prefixes)
suffixes.update(nested.suffixes)
opaque |= nested.opaque
index = end
return ScanResult(
frozenset(references),
frozenset(calls),
frozenset(prefixes),
frozenset(suffixes),
opaque,
)
@classmethod
def helper_inputs(
cls,
data: Mapping[str, Any],
variables: Mapping[str, Mapping[str, str]],
globals_data: Mapping[str, str],
universe: Sequence[str],
suffix_cache: dict[str, list[str]],
) -> tuple[dict[str, Mapping[str, str]], bool]:
prefix = data["prefix"]
resolved = {
**globals_data,
**{f"{prefix}_{field}": value for field, value in data["expanded"].items()},
}
helpers: dict[str, Mapping[str, str]] = {}
seen: set[tuple[str, tuple[str, ...], bool]] = set()
opaque = False
queue: list[tuple[str, tuple[str, ...], int, bool]] = [
(name, (), 0, False) for name in data["definitions"]
]
for hooks in data["hooks"].values():
queue.extend((hook, (), 0, True) for hook in hooks)
def enqueue_matches(
prefixes: frozenset[str], suffixes: frozenset[str]
) -> None:
for pfx in prefixes:
position = bisect.bisect_left(universe, pfx)
while position < len(universe) and universe[position].startswith(pfx):
queue.append((universe[position], (), 0, True))
position += 1
for sfx in suffixes:
matches = suffix_cache.get(sfx)
if matches is None:
matches = [item for item in universe if item.endswith(sfx)]
suffix_cache[sfx] = matches
queue.extend((item, (), 0, True) for item in matches)
for field, body in data["raw"].items():
scanned = cls.scan_bound(cls.bind_context(body, prefix))
queue.extend((name, (), 0, True) for name in scanned.references)
queue.extend((target, args, 1, True) for target, args in scanned.calls)
enqueue_matches(scanned.prefixes, scanned.suffixes)
opaque |= scanned.opaque
if field.endswith("_HOOKS"):
queue.extend(
(word, (), 0, True)
for word in re.findall(r"[A-Za-z_][A-Za-z0-9_.-]*", body)
if word in variables
)
while queue:
name, params, depth, called = queue.pop()
if name in _EPHEMERAL_MAKE_VARS or name.startswith("TPMDD_STATE_"):
continue
if (name, params, called) in seen:
continue
seen.add((name, params, called))
record: Mapping[str, str] = (
{"value": resolved[name], "flavor": "simple"}
if name in resolved
else variables.get(name, {"value": "", "flavor": "undefined"})
)
helpers.setdefault(name, record)
body = record["value"]
if name.endswith("_HOOKS"):
queue.extend(
(word, (), 0, True)
for word in re.findall(r"[A-Za-z_][A-Za-z0-9_.-]*", body)
if word in variables
)
if record["flavor"] != "recursive":
continue
if not called and re.search(r"\$(?:[1-9]|\([1-9]\)|\{[1-9]\})", body):
continue
if depth > 6:
opaque = True
continue
scanned = cls.scan_bound(cls.bind_context(body, prefix, params))
queue.extend((target, (), 0, True) for target in scanned.references)
queue.extend(
(target, args, depth + 1, True) for target, args in scanned.calls
)
enqueue_matches(scanned.prefixes, scanned.suffixes)
opaque |= scanned.opaque
return helpers, opaque
class PackageCollector:
@staticmethod
def digest(value: object) -> str:
return hashlib.sha256(
json.dumps(
value, sort_keys=True, ensure_ascii=True, separators=(",", ":")
).encode()
).hexdigest()
@staticmethod
def read_config(path: Path) -> dict[str, str]:
config: dict[str, str] = {}
for line in path.read_text(encoding="utf-8").splitlines():
if match := _CONFIG_ASSIGNMENT_RE.fullmatch(line):
name, value = match.groups()
elif match := _NOT_SET_RE.fullmatch(line):
name, value = match.group(1), "n"
elif not line or line.startswith("#"):
continue
else:
raise ValueError(f"unrecognized configuration line in {path}: {line!r}")
if name in config:
raise ValueError(f"duplicate configuration symbol: {name}")
config[name] = value
if config.get("BR2_HAVE_DOT_CONFIG") != "y":
raise ValueError(f"not a configured Buildroot output directory: {path.parent}")
return config
@staticmethod
def global_config(config: Mapping[str, str]) -> dict[str, str]:
return {
name: value
for name, value in config.items()
if not name.startswith(("BR2_PACKAGE_", "BR2_LINUX_KERNEL", "BR2_CCACHE"))
and not (
name.startswith("BR2_EXTERNAL_")
and name.endswith(("_PATH", "_VERSION"))
)
and name not in _CONFIG_METADATA
}
@staticmethod
def normalizer(
root: Path,
external: Path,
board: Path,
source: Path,
download: Path | None = None,
) -> Callable[[Any], Any]:
items: list[tuple[Path, str]] = [
(root, "<workspace>"),
(external, "<external>"),
(board, "<output>"),
(source, "<buildroot>"),
]
if download is not None:
items.append((download, "<download>"))
prefixes = sorted(
((str(path), label) for path, label in items),
key=lambda item: len(item[0]),
reverse=True,
)
pattern = re.compile(
"(?:"
+ "|".join(re.escape(prefix) for prefix, _ in prefixes)
+ r")(?![A-Za-z0-9_.+-])"
)
labels = dict(prefixes)
def normalize(value: Any) -> Any:
if isinstance(value, str):
return pattern.sub(lambda match: labels[match.group()], value)
if isinstance(value, list):
return [normalize(item) for item in value]
if isinstance(value, dict):
return {name: normalize(item) for name, item in value.items()}
return value
return normalize
@staticmethod
def content_fingerprint(path: Path) -> str:
previous = os.open(".", os.O_RDONLY | os.O_DIRECTORY)
try:
os.chdir(path.parent)
return BuildState.fingerprint([path.name], excludes=_EXCLUDES)
finally:
os.fchdir(previous)
os.close(previous)
@staticmethod
def input_path(
value: str,
external: Path,
board: Path,
source: Path,
*,
board_relative: bool = False,
allow_external: bool = False,
) -> Path:
path = Path(value)
if not path.is_absolute():
base = board if board_relative and (board / path).exists() else source
path = base / path
path = Path(os.path.normpath(path))
resolved = path.resolve()
if resolved.is_relative_to(board) and not resolved.is_relative_to(source):
raise ValueError(f"configured data input points into build output: {path}")
if not allow_external and not (
resolved.is_relative_to(external) or resolved.is_relative_to(source)
):
raise ValueError(
f"unclassified package input outside external/Buildroot trees: {path}"
)
if not path.exists():
raise FileNotFoundError(f"missing configured package input: {path}")
return path
@staticmethod
def build_dirs(info: Mapping[str, Any]) -> list[str]:
value = info.get("stamp_dir")
if not isinstance(value, str):
raise ValueError(f"missing or invalid stamp_dir: {info!r}")
if not value:
if info["virtual"]:
return []
raise ValueError(f"missing nonvirtual stamp_dir: {info!r}")
path = Path(value)
if (
path.is_absolute()
or len(value.split("/")) != 2
or len(path.parts) != 2
or path.parts[0] != "build"
or path.parts[1] in (".", "..")
):
raise ValueError(f"unsafe stamp_dir: {value!r}")
return [path.as_posix()]
@staticmethod
def make_arguments(
make_args: Sequence[str], external: Path, board: Path, source: Path
) -> tuple[list[str], dict[str, str]]:
index = 0
assignments: dict[str, str] = {}
while index < len(make_args):
arg = make_args[index]
if arg in ("-j", "-l", "-C"):
index += 1
if index == len(make_args):
raise ValueError(f"missing value after {arg}")
value = make_args[index]
if arg == "-C":
if Path(value).resolve() != source:
raise ValueError("make -C does not match source")
elif not re.fullmatch(r"[0-9]+(?:\.[0-9]+)?", value):
raise ValueError(f"invalid make option: {arg} {value}")
elif re.fullmatch(r"-[jl][0-9]+(?:\.[0-9]+)?", arg):
pass
elif arg.startswith("-C") and Path(arg[2:]).resolve() == source:
pass
elif match := re.fullmatch(r"([A-Za-z_][A-Za-z0-9_]*)=(.*)", arg, re.DOTALL):
name, value = match.groups()
assignments[name] = value
else:
raise ValueError(f"unsupported metadata make argument: {arg!r}")
index += 1
for name, expected in (("O", board), ("BR2_EXTERNAL", external)):
if name in assignments and Path(assignments[name]).resolve() != expected:
raise ValueError(f"make {name} does not match collector input")
overrides = {
name: value
for name, value in assignments.items()
if name
not in {
"O",
"BR2_EXTERNAL",
"BR2_CCACHE_DIR",
"BR2_DL_DIR",
"TPMDD_VERSION",
}
}
return [*make_args, "-C", str(source), f"O={board}", f"BR2_EXTERNAL={external}"], overrides
@classmethod
def collect(
cls, root: Path, board: Path, source: Path, make_args: Sequence[str]
) -> dict[str, Any]:
root = root.resolve(strict=True)
external = (root / "br2-external").resolve(strict=True)
board = board.resolve(strict=True)
source = source.resolve(strict=True)
config = cls.read_config(board / ".config")
if config.get("BR2_PER_PACKAGE_DIRECTORIES") != "y":
raise ValueError("safe invalidation requires BR2_PER_PACKAGE_DIRECTORIES=y")
arguments, overrides = cls.make_arguments(make_args, external, board, source)
collector = Path(__file__).resolve().with_name("package-state.mk")
with tempfile.TemporaryDirectory(prefix="tpmdd-package-state-") as temporary:
result = subprocess.run(
[
"make",
*arguments,
"--no-print-directory",
"-s",
"-f",
str(source / "Makefile"),
"-f",
str(collector),
f"TPMDD_PACKAGE_STATE_DIR={temporary}",
"show-info",
],
cwd=source,
text=True,
stdout=subprocess.PIPE,
check=True,
umask=0o022,
)
upstream = json.loads(result.stdout)
if not isinstance(upstream, dict) or not upstream:
raise ValueError("Buildroot show-info did not return a package graph")
globals_data = json.loads(
(Path(temporary) / "globals.json").read_text(encoding="utf-8")
)
if (
not isinstance(globals_data, dict)
or not all(isinstance(value, str) for value in globals_data.values())
or not globals_data.get("BR2_VERSION")
):
raise ValueError("invalid Buildroot global metadata")
download = (
Path(globals_data["DL_DIR"]).resolve()
if globals_data.get("DL_DIR")
else None
)
normalize = cls.normalizer(root, external, board, source, download)
raw_data = json.loads(
(Path(temporary) / "raw-definitions.json").read_text(encoding="utf-8")
)
variables = raw_data.get("variables")
if not isinstance(variables, dict) or not all(
isinstance(record, dict)
and isinstance(record.get("value"), str)
and record.get("flavor") in ("simple", "recursive", "undefined")
for record in variables.values()
):
raise ValueError("invalid raw Make helper metadata")
records: dict[str, dict[str, Any]] = {}
for name, info in upstream.items():
if not _NAME_RE.fullmatch(name) or not isinstance(info, dict):
raise ValueError(f"invalid upstream package metadata: {name!r}")
if info.get("type") == "rootfs":
continue
if info.get("type") not in ("host", "target") or type(
info.get("virtual")
) is not bool:
raise ValueError(f"unsupported package type/virtual metadata: {name}")
data = json.loads(
(Path(temporary) / f"{name}.json").read_text(encoding="utf-8")
)
if not isinstance(data, dict) or data.get("info") != {name: info}:
raise ValueError(f"inconsistent package metadata: {name}")
for field in ("expanded", "raw", "hooks", "definitions"):
if not isinstance(data.get(field), dict):
raise ValueError(f"missing bounded {field} metadata: {name}")
if not all(
isinstance(value, str)
for field in ("expanded", "raw", "definitions")
for value in data[field].values()
):
raise ValueError(f"invalid bounded inputs: {name}")
if not isinstance(data.get("prefix"), str) or not all(
isinstance(hooks, dict)
and all(isinstance(body, str) for body in hooks.values())
for hooks in data["hooks"].values()
):
raise ValueError(f"invalid hook metadata: {name}")
records[name] = data
packages: dict[str, dict[str, Any]] = {}
identity_inputs: dict[str, dict[str, Any]] = {}
content_cache: dict[Path, str] = {}
universe = sorted(set(variables) | set(config))
suffix_cache: dict[str, list[str]] = {}
def content(path: Path) -> list[str]:
if path not in content_cache:
content_cache[path] = cls.content_fingerprint(path)
return [normalize(str(path)), content_cache[path]]
for name, data in sorted(records.items()):
info = data["info"][name]
dependencies = info.get("dependencies")
if not isinstance(dependencies, list) or not all(
isinstance(dep, str) and dep in records for dep in dependencies
):
raise ValueError(f"incomplete package dependency graph: {name}")
pkgdir = info.get("package_dir")
if not isinstance(pkgdir, str) or not pkgdir:
raise ValueError(f"missing package_dir: {name}")
recipe = cls.input_path(pkgdir, external, board, source)
if not recipe.is_dir():
raise ValueError(f"package_dir is not a directory: {recipe}")
inputs = [content(recipe)]
rawname = info.get("name")
if not isinstance(rawname, str) or not _NAME_RE.fullmatch(rawname):
raise ValueError(f"invalid upstream raw package name: {name}")
patches = external / "patches" / rawname
if patches.exists():
inputs.append(content(patches))
for field in ("hashes", "patches"):
paths = info.get(field, [])
if not isinstance(paths, list) or not all(
isinstance(path, str) for path in paths
):
raise ValueError(f"invalid upstream {field}: {name}")
for value in paths:
if "://" in value:
continue
inputs.append(
content(
cls.input_path(
value,
external,
board,
source,
board_relative=field == "patches",
)
)
)
expanded = data["expanded"]
for field in _DATA_FIELDS:
for value in expanded.get(field, "").split():
inputs.append(content(cls.input_path(value, external, board, source)))
override_srcdir = expanded.get("OVERRIDE_SRCDIR", "")
if override_srcdir:
inputs.append(
content(
cls.input_path(
override_srcdir,
external,
board,
source,
allow_external=True,
)
)
)
elif expanded.get("SITE_METHOD") == "local" and expanded.get("SITE", ""):
inputs.append(
content(cls.input_path(expanded["SITE"], external, board, source))
)
helpers, opaque = MakeScanner.helper_inputs(
data, variables, globals_data, universe, suffix_cache
)
references = set(
_CONFIG_REF_RE.findall(
json.dumps([data["definitions"], data["raw"], data["hooks"], helpers])
)
)
if opaque:
references.update(
symbol for symbol in config if symbol.startswith("BR2_PACKAGE_")
)
package_config = {
symbol: config.get(symbol, "n")
for symbol in sorted(references)
if not (
symbol.startswith("BR2_EXTERNAL_")
and symbol.endswith(("_PATH", "_VERSION"))
)
}
if name == "linux":
inputs.append(content(external / "linux"))
package_config.update(
{
symbol: value
for symbol, value in config.items()
if symbol.startswith("BR2_LINUX_KERNEL")
}
)
metadata = {
key: val
for key, val in info.items()
if key not in ("dependencies", "reverse_dependencies")
}
identity_inputs[name] = normalize(
{
"metadata": metadata,
"expanded": expanded,
"raw": data["raw"],
"hooks": data["hooks"],
"definitions": data["definitions"],
"config": package_config,
"files": inputs,
"helpers": helpers,
}
)
packages[name] = {
"dependencies": sorted(set(dependencies)),
"build_dirs": cls.build_dirs(info),
"virtual": info["virtual"],
"conservative": opaque,
}
for name, package in packages.items():
pkg_inputs = identity_inputs[name]
if name == "linux":
pkg_inputs["dependency_metadata"] = {
dep: {
key: value
for key, value in identity_inputs[dep].items()
if key != "files"
}
for dep in package["dependencies"]
}
package["identity"] = cls.digest(pkg_inputs)
common = [
source / name
for name in (
"Makefile",
"Makefile.legacy",
"Config.in",
"Config.in.legacy",
"arch",
"toolchain",
"support",
"system",
"package/Makefile.in",
)
]
common.extend(sorted((source / "package").glob("pkg-*.mk")))
common.extend(root / name for name in ("buildroot.patch", "sdk"))
common.extend(external / name for name in ("Config.in", "external.desc"))
infrastructure = normalize(
{
"files": [content(path) for path in common],
"globals": {
name: value
for name, value in globals_data.items()
if name not in _ACCUMULATOR_GLOBALS
},
"config": cls.global_config(config),
"overrides": overrides,
"source_hash": (
source.parent.name
if _SHA256_RE.fullmatch(source.parent.name)
else None
),
}
)
return {
"version": 1,
"infrastructure": cls.digest(infrastructure),
"external_mk": (external / "external.mk").read_text(encoding="utf-8"),
"packages": packages,
}
class PackageInvalidator:
@staticmethod
def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
@staticmethod
def state_paths(board: Path) -> tuple[Path, Path, Path]:
directory = board / "stamp/orchestration"
return (
directory,
directory / "packages.json",
directory / "packages.pending.json",
)
@staticmethod
def validate_snapshot(snapshot: Any) -> None:
if (
not isinstance(snapshot, dict)
or snapshot.get("version") != 1
or not isinstance(snapshot.get("infrastructure"), str)
or not isinstance(snapshot.get("external_mk"), str)
or not isinstance(snapshot.get("packages"), dict)
):
raise ValueError("invalid package state snapshot")
for name, package in snapshot["packages"].items():
if (
not isinstance(name, str)
or not _NAME_RE.fullmatch(name)
or not isinstance(package, dict)
or not isinstance(package.get("identity"), str)
or not _SHA256_RE.fullmatch(package["identity"])
or not isinstance(package.get("dependencies"), list)
or not all(
isinstance(dep, str) and _NAME_RE.fullmatch(dep)
for dep in package["dependencies"]
)
or not isinstance(package.get("build_dirs"), list)
or not all(isinstance(path, str) for path in package["build_dirs"])
):
raise ValueError(f"invalid package state for {name}")
@staticmethod
def include_layout(text: str) -> bool:
if "\0" in text:
return False
includes = 0
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("include "):
expression = line.removeprefix("include ")
includes += 1
elif line.startswith("TPMDD_PYTHON_PACKAGE_MKS := "):
expression = line.removeprefix("TPMDD_PYTHON_PACKAGE_MKS := ")
else:
return False
for wildcard in ("*", "python-*"):
expression = expression.replace(
f"$(BR2_EXTERNAL_TPMDD_PATH)/package/{wildcard}/*.mk", "\0"
)
expression = expression.replace("$(TPMDD_PYTHON_PACKAGE_MKS)", "\0")
expression = re.sub(r"\$\((?:sort|wildcard|filter-out)\s+", "(", expression)
if "\0" not in expression or not re.fullmatch(r"[\x00(),\s]+", expression):
return False
return includes > 0
@staticmethod
def board_path(board: Path, relative: str) -> Path:
path = Path(relative)
if (
path.is_absolute()
or len(path.parts) != 2
or path.parts[0] not in ("build", "per-package")
):
raise ValueError(f"not a board-owned package directory: {relative}")
if not _NAME_RE.fullmatch(path.parts[1]) or path.parts[1] in (".", ".."):
raise ValueError(f"invalid package directory: {relative}")
result = board / path
if (board / path.parts[0]).is_symlink() or result.is_symlink():
raise ValueError(f"refusing a symlinked package directory: {result}")
return result
@staticmethod
def cached_packages(board: Path) -> bool:
packages = board / "per-package"
return (
(packages.is_dir() and any(packages.iterdir()))
or any(
path.parent.name != "buildroot-config"
for path in (board / "build").glob("*/.stamp_*")
)
or any(
path.is_file() or path.is_symlink()
for name in ("target", "host")
for path in (board / name).rglob("*")
)
)
@classmethod
def plan(
cls,
board: Path,
current: Mapping[str, Any],
baseline: Mapping[str, Any] | None,
journal: Mapping[str, Any] | None,
) -> tuple[set[str], set[str]]:
cls.validate_snapshot(current)
reference = baseline
if journal is not None:
if (
not isinstance(journal, dict)
or journal.get("version") != 1
or journal.get("phase") not in ("invalidating", "prepared")
or not isinstance(journal.get("dirty"), list)
or not all(
isinstance(name, str) and _NAME_RE.fullmatch(name)
for name in journal["dirty"]
)
or not isinstance(journal.get("paths"), list)
or not all(isinstance(path, str) for path in journal["paths"])
):
raise ValueError("invalid package invalidation journal")
cls.validate_snapshot(journal["snapshot"])
if journal["phase"] == "prepared":
reference = journal["snapshot"]
snapshots: list[Mapping[str, Any]] = [
snap for snap in (baseline, reference, current) if snap is not None
]
if journal is not None:
snapshots.append(journal["snapshot"])
for snap in snapshots:
cls.validate_snapshot(snap)
if snap["infrastructure"] != current["infrastructure"]:
raise ValueError(
"build infrastructure or toolchain changed; targeted invalidation is not sufficient"
)
if snap["external_mk"] != current["external_mk"] and not (
cls.include_layout(snap["external_mk"])
and cls.include_layout(current["external_mk"])
):
raise ValueError(
"unclassified external.mk change; refusing to reuse cached packages"
)
previous = reference["packages"] if reference is not None else {}
packages = current["packages"]
dirty = {
name
for name in previous.keys() | packages.keys()
if previous.get(name) != packages.get(name)
}
paths: set[str] = set()
if journal is not None and journal["phase"] == "invalidating":
dirty.update(journal["dirty"])
paths.update(journal["paths"])
reverse: dict[str, set[str]] = {}
for snap in snapshots:
for name, package in snap["packages"].items():
for dependency in package["dependencies"]:
reverse.setdefault(dependency, set()).add(name)
queue = list(dirty)
while queue:
for consumer in reverse.get(queue.pop(), ()):
if consumer not in dirty:
dirty.add(consumer)
queue.append(consumer)
for name in dirty:
paths.add(f"per-package/{name}")
for snap in snapshots:
package = snap["packages"].get(name)
if package is not None:
paths.update(package["build_dirs"])
for relative in paths:
cls.board_path(board, relative)
return dirty, paths
@classmethod
def prepare(
cls, board: Path, current: Mapping[str, Any], *, readonly: bool = False
) -> None:
directory, baseline_path, journal_path = cls.state_paths(board)
baseline = cls.read_json(baseline_path) if baseline_path.exists() else None
journal = cls.read_json(journal_path) if journal_path.exists() else None
if baseline is None and journal is None and cls.cached_packages(board):
raise ValueError(
"cached packages have no verified package metadata; run a clean build first"
)
dirty, paths = cls.plan(board, current, baseline, journal)
cached = sorted(
name
for name in dirty
if (board / "per-package" / name).exists()
or any(
cls.board_path(board, rel).exists()
for snap in (baseline, current)
if snap is not None
for rel in snap["packages"].get(name, {}).get("build_dirs", [])
)
)
if readonly:
print(
json.dumps(
{
"dirty": sorted(dirty),
"cached": cached,
"paths": sorted(paths),
},
sort_keys=True,
)
)
return
BuildState.write_json(directory / "packages.next.json", current)
if not dirty and journal is None:
return
if dirty:
if cached:
print(
f"{board.name}: invalidating {len(cached)} cached packages: {' '.join(cached)}",
flush=True,
)
pending = {
"version": 1,
"phase": "invalidating",
"snapshot": current,
"dirty": sorted(dirty),
"paths": sorted(paths),
}
BuildState.write_json(journal_path, pending)
(directory / "build.json").unlink(missing_ok=True)
for relative in sorted(paths):
path = cls.board_path(board, relative)
if path.exists():
shutil.rmtree(path)
for sync_target in (board / "build", board / "per-package", board):
if sync_target.exists():
BuildState.sync_directory(sync_target)
pending["phase"] = "prepared"
BuildState.write_json(journal_path, pending)
elif journal is not None:
updated_journal = dict(journal)
updated_journal["snapshot"] = current
BuildState.write_json(journal_path, updated_journal)
(directory / "build.json").unlink(missing_ok=True)
@classmethod
def assemble(cls, board: Path) -> None:
directory, _, journal_path = cls.state_paths(board)
current = cls.read_json(directory / "packages.next.json")
cls.validate_snapshot(current)
if journal_path.exists():
journal = cls.read_json(journal_path)
if journal.get("phase") != "prepared" or journal.get("snapshot") != current:
raise ValueError(
"package invalidation must finish before aggregate assembly"
)
else:
BuildState.write_json(
journal_path,
{
"version": 1,
"phase": "prepared",
"snapshot": current,
"dirty": [],
"paths": [],
},
)
(directory / "build.json").unlink(missing_ok=True)
for name in ("target", "host"):
path = board / name
if path.is_symlink():
raise ValueError(f"refusing a symlinked aggregate directory: {path}")
if path.exists():
shutil.rmtree(path)
BuildState.sync_directory(board)
@classmethod
def commit(cls, board: Path, current: Mapping[str, Any]) -> None:
directory, baseline_path, journal_path = cls.state_paths(board)
expected = cls.read_json(directory / "packages.next.json")
cls.validate_snapshot(current)
if current != expected:
raise ValueError(
"package inputs changed during the build; not recording success"
)
if journal_path.exists():
journal = cls.read_json(journal_path)
if journal.get("phase") != "prepared" or journal.get("snapshot") != current:
raise ValueError("package invalidation is not ready to commit")
BuildState.write_json(baseline_path, current)
(directory / "packages.next.json").unlink()
journal_path.unlink(missing_ok=True)
BuildState.sync_directory(directory)
class PackageStateCli:
@staticmethod
def run(argv: Sequence[str] | None = None) -> int:
raw_argv = list(sys.argv[1:] if argv is None else argv)
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command", required=True)
snapshot_parser = subparsers.add_parser("snapshot")
for name in ("root", "board", "source", "output"):
snapshot_parser.add_argument(f"--{name}", type=Path, required=True)
snapshot_parser.add_argument("make_args", nargs=argparse.REMAINDER)
for name in ("prepare", "plan", "commit"):
cmd_parser = subparsers.add_parser(name)
cmd_parser.add_argument("--root", required=True, type=Path)
cmd_parser.add_argument("--board", required=True, type=Path)
cmd_parser.add_argument("--source", required=True, type=Path)
cmd_parser.add_argument("make_args", nargs=argparse.REMAINDER)
assemble_parser = subparsers.add_parser("assemble")
assemble_parser.add_argument("--board", required=True, type=Path)
args = parser.parse_args(raw_argv)
board = args.board.resolve()
if not board.is_dir() or not (board / ".config").is_file():
parser.error("board must be an existing configured Buildroot output")
try:
if args.command == "assemble":
PackageInvalidator.assemble(board)
else:
make_args = (
args.make_args[1:]
if args.make_args[:1] == ["--"]
else args.make_args
)
snapshot = PackageCollector.collect(
args.root.resolve(), board, args.source.resolve(), make_args
)
if args.command == "snapshot":
BuildState.write_json(args.output, snapshot, indent=2)
elif args.command == "commit":
PackageInvalidator.commit(board, snapshot)
else:
PackageInvalidator.prepare(
board, snapshot, readonly=args.command == "plan"
)
except (
OSError,
ValueError,
KeyError,
subprocess.CalledProcessError,
) as error:
print(f"package-state: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(PackageStateCli.run())