mirror of
https://github.com/openglow-org/forgefirm.git
synced 2026-09-27 16:51:12 -07:00
248 lines
11 KiB
Python
248 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright 2020-2026 514 LLC d/b/a OpenGlow
|
|
# Written by Scott Wiederhold
|
|
# https://community.openglow.org
|
|
# SPDX-License-Identifier: MIT
|
|
#
|
|
# Build a ForgeFIRM image manifest from the source tree - the same file lists
|
|
# forgefirm-image-manifest.bbclass puts in the image, computed from the recipe
|
|
# pins with git instead of a Yocto build. For the coverage lint in CI and for
|
|
# checking a coverage map on a workstation; NOT a substitute for the image's
|
|
# manifest in the release gate (the platform section carries placeholders
|
|
# where only a build knows the answer: kernel config hash, modules dir, DTB).
|
|
#
|
|
# manifest-from-tree.py [--meta-openglow PATH] [--out manifest.json]
|
|
# [--cache DIR] [--kernel-srcrev REV]
|
|
#
|
|
# Component revisions come from the recipes in meta-forgefirm and the sibling
|
|
# meta-openglow checkout (default ../meta-openglow relative to this repo); a
|
|
# recipe's `require`d files in its own directory are read too, which is
|
|
# where the pins live (<recipe>-pin.inc). Each pinned commit is fetched
|
|
# shallowly into --cache (default .manifest-cache/, gitignored) and listed
|
|
# with `git ls-tree`; a submodule gitlink is followed through .gitmodules.
|
|
# The layer content hash mirrors forgefirm-image-manifest.bbclass: every
|
|
# file under the layer except *.md and *-pin.inc.
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, os.path.join(REPO, "forgetest"))
|
|
from forgetest import manifest as manifest_mod # noqa: E402
|
|
|
|
RECIPES = [
|
|
# (component, recipe path relative to the repo or meta-openglow, layer)
|
|
("forgectrl", "meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl.bb", "forgefirm"),
|
|
("grblhal-glowforge", "meta-forgefirm/recipes-forgefirm/grblhal-glowforge/grblhal-glowforge.bb", "forgefirm"),
|
|
("forgefirm-app", "meta-forgefirm/recipes-forgefirm/forgefirm-app/forgefirm-app.inc", "forgefirm"),
|
|
("kernel-module-glowforge", "meta-glowforge-bsp/recipes-kernel/kernel-modules/kernel-module-glowforge.bb", "meta-openglow"),
|
|
("python3-gfhardware", "meta-glowforge-bsp/recipes-devtools/python/python3-gfhardware.bb", "meta-openglow"),
|
|
("python3-gfutilities", "meta-openglow-core/recipes-devtools/python/python3-gfutilities_git.bb", "meta-openglow"),
|
|
]
|
|
# Components built from files in a layer (FORGEFIRM_MANIFEST_SRC in the
|
|
# recipe): (component, directory relative to the repo, layer, recipe).
|
|
FILE_COMPONENTS = [
|
|
("ffboot", "meta-forgefirm/recipes-forgefirm/ffboot/files", "forgefirm", "ffboot.bb"),
|
|
]
|
|
CONTENT_LAYERS = {"meta-forgefirm": ("forgefirm", "meta-forgefirm"),
|
|
"meta-glowforge-bsp": ("meta-openglow", "meta-glowforge-bsp"),
|
|
"meta-openglow-core": ("meta-openglow", "meta-openglow-core")}
|
|
# Left out of a layer's content, as in forgefirm-image-manifest.bbclass
|
|
# (FORGEFIRM_MANIFEST_PIN_SUFFIX): documentation and the component pin files.
|
|
LAYER_SKIP_SUFFIXES = (".md", "-pin.inc")
|
|
|
|
|
|
def git(args, cwd=None, input=None):
|
|
return subprocess.run(["git"] + args, cwd=cwd, input=input, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, check=True).stdout
|
|
|
|
|
|
def recipe_text(path, seen=None):
|
|
"""The recipe's text with its `require`/`include`d files from the same
|
|
directory appended (bitbake resolves a relative name against the
|
|
including file's directory first). Only local files are followed;
|
|
anything else is left to BBPATH and skipped here."""
|
|
seen = seen if seen is not None else set()
|
|
path = os.path.abspath(path)
|
|
if path in seen:
|
|
return ""
|
|
seen.add(path)
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
for m in re.finditer(r'^\s*(?:require|include)\s+(\S+)\s*$', text, re.M):
|
|
cand = os.path.join(os.path.dirname(path), m.group(1))
|
|
if os.path.isfile(cand):
|
|
text += "\n" + recipe_text(cand, seen)
|
|
return text
|
|
|
|
|
|
def parse_recipe(path):
|
|
text = recipe_text(path)
|
|
uri = re.search(r'^SRC_URI\s*\+?=\s*"([^"]+)"', text, re.M)
|
|
rev = re.search(r'^SRCREV\s*\??=\s*"([0-9a-fA-F]+)"', text, re.M)
|
|
if not uri or not rev:
|
|
raise SystemExit("cannot find SRC_URI/SRCREV in %s" % path)
|
|
first = uri.group(1).split()[0]
|
|
url = first.split(";")[0]
|
|
params = dict(p.split("=", 1) for p in first.split(";")[1:] if "=" in p)
|
|
proto = params.get("protocol", "https")
|
|
if url.startswith("git://") or url.startswith("gitsm://"):
|
|
url = proto + "://" + url.split("://", 1)[1]
|
|
return url, rev.group(1)
|
|
|
|
|
|
def fetch(url, rev, cache):
|
|
"""A bare cache repo containing rev (fetched shallowly)."""
|
|
name = hashlib.sha1(url.encode()).hexdigest()[:16]
|
|
repo = os.path.join(cache, name)
|
|
if not os.path.isdir(repo):
|
|
os.makedirs(repo)
|
|
git(["init", "-q", "--bare"], cwd=repo)
|
|
try:
|
|
git(["cat-file", "-e", rev + "^{commit}"], cwd=repo)
|
|
except subprocess.CalledProcessError:
|
|
git(["fetch", "-q", "--depth", "1", url, rev], cwd=repo)
|
|
return repo
|
|
|
|
|
|
def ls_tree(repo, rev, url, cache, prefix, files):
|
|
out = git(["ls-tree", "-r", "--full-tree", rev], cwd=repo).decode()
|
|
modules = None
|
|
for line in out.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
meta, path = line.split("\t", 1)
|
|
typ, obj = meta.split()[1], meta.split()[2]
|
|
files.append([prefix + path, obj])
|
|
if typ == "commit":
|
|
if modules is None:
|
|
modules = {}
|
|
try:
|
|
gm = git(["show", "%s:.gitmodules" % rev], cwd=repo).decode()
|
|
except subprocess.CalledProcessError:
|
|
gm = ""
|
|
cur = None
|
|
for l in gm.splitlines():
|
|
l = l.strip()
|
|
m = re.match(r'^path\s*=\s*(.+)$', l)
|
|
if m:
|
|
cur = m.group(1).strip()
|
|
m = re.match(r'^url\s*=\s*(.+)$', l)
|
|
if m and cur:
|
|
modules[cur] = m.group(1).strip()
|
|
sub_url = modules.get(path)
|
|
if sub_url:
|
|
if sub_url.startswith("../") or sub_url.startswith("./"):
|
|
base = url.rsplit("/", 1)[0]
|
|
sub_url = base + "/" + sub_url.lstrip("./")
|
|
sub_repo = fetch(sub_url, obj, cache)
|
|
ls_tree(sub_repo, obj, sub_url, cache, prefix + path + "/", files)
|
|
|
|
|
|
def dir_files(path):
|
|
"""[path, blob-id] per regular file under path, relative paths, the
|
|
same walk and hashing as forgefirm-manifest.bbclass (hash-object, so
|
|
the ids compare with tree ids)."""
|
|
paths = []
|
|
for root, dirs, fns in os.walk(path):
|
|
dirs[:] = sorted(x for x in dirs if x not in (".git", "__pycache__"))
|
|
for fn in fns:
|
|
if fn.endswith((".pyc", ".pyo")):
|
|
continue
|
|
p = os.path.join(root, fn)
|
|
if os.path.isfile(p) and not os.path.islink(p):
|
|
paths.append(os.path.relpath(p, path).replace(os.sep, "/"))
|
|
paths.sort()
|
|
if not paths:
|
|
return []
|
|
# absolute paths: hash-object --stdin-paths resolves relative ones against
|
|
# the enclosing repository's top level, not the cwd
|
|
ids = git(["hash-object", "--stdin-paths"], cwd=path,
|
|
input=(chr(10).join(os.path.join(path, p) for p in paths) + chr(10)).encode()).decode().split()
|
|
return [[p, i] for p, i in zip(paths, ids)]
|
|
|
|
|
|
def layer_content(path):
|
|
out = git(["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", "."], cwd=path)
|
|
paths = sorted(set(p.decode("utf-8", "replace") for p in out.split(b"\0") if p))
|
|
paths = [p for p in paths
|
|
if os.path.isfile(os.path.join(path, p)) and not p.endswith(LAYER_SKIP_SUFFIXES)]
|
|
if not paths:
|
|
return None
|
|
# hash-object --stdin-paths resolves against the repository top level
|
|
prefix = git(["rev-parse", "--show-prefix"], cwd=path).decode().strip()
|
|
ids = git(["hash-object", "--stdin-paths"], cwd=path,
|
|
input=("\n".join(prefix + p for p in paths) + "\n").encode()).decode().split()
|
|
h = hashlib.sha256()
|
|
for p, i in zip(paths, ids):
|
|
h.update(p.encode("utf-8") + b"\0" + i.encode("ascii") + b"\n")
|
|
return h.hexdigest()
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--meta-openglow", default=os.path.join(os.path.dirname(REPO), "meta-openglow"))
|
|
ap.add_argument("--out", default="-")
|
|
ap.add_argument("--cache", default=os.path.join(REPO, ".manifest-cache"))
|
|
ap.add_argument("--kernel-srcrev", default=None,
|
|
help="linux-fslc SRCREV (default: read from layers/meta-freescale if present)")
|
|
args = ap.parse_args(argv)
|
|
os.makedirs(args.cache, exist_ok=True)
|
|
|
|
components = {}
|
|
for name, rel, layer in RECIPES:
|
|
base = REPO if layer == "forgefirm" else args.meta_openglow
|
|
path = os.path.join(base, rel)
|
|
url, rev = parse_recipe(path)
|
|
repo = fetch(url, rev, args.cache)
|
|
files = []
|
|
ls_tree(repo, rev, url, args.cache, "", files)
|
|
files.sort()
|
|
components[name] = {"srcrev": rev, "source": url, "files": files, "recipes": [os.path.basename(rel)]}
|
|
print("%s: %s (%d files)" % (name, rev[:12], len(files)), file=sys.stderr)
|
|
for name, rel, layer, recipe in FILE_COMPONENTS:
|
|
base = REPO if layer == "forgefirm" else args.meta_openglow
|
|
files = dir_files(os.path.join(base, rel))
|
|
components[name] = {"srcrev": None, "source": "files", "files": files, "recipes": [recipe]}
|
|
print("%s: files (%d files)" % (name, len(files)), file=sys.stderr)
|
|
|
|
ksrc = args.kernel_srcrev
|
|
if not ksrc:
|
|
for cand in ("layers/meta-freescale/recipes-kernel/linux/linux-fslc_6.12.bb",):
|
|
p = os.path.join(REPO, cand)
|
|
if os.path.exists(p):
|
|
m = re.search(r'^SRCREV\s*=\s*"([0-9a-f]+)"', open(p, encoding="utf-8").read(), re.M)
|
|
if m:
|
|
ksrc = m.group(1)
|
|
components["linux-fslc"] = {"srcrev": ksrc, "source": "git://github.com/Freescale/linux-fslc.git",
|
|
"config_sha256": None, "recipes": ["linux-fslc"],
|
|
"files": [["@config", "unknown-without-a-build"], ["@srcrev", ksrc or "unknown"]]}
|
|
|
|
layers = {}
|
|
for lname, (repo_key, sub) in CONTENT_LAYERS.items():
|
|
base = REPO if repo_key == "forgefirm" else args.meta_openglow
|
|
lpath = os.path.join(base, sub)
|
|
if os.path.isdir(lpath):
|
|
layers[lname] = {"content_sha256": layer_content(lpath)}
|
|
platform = {"machine": "glowforge", "layers": layers, "kernel_modules": [], "dtb": {}}
|
|
canonical = manifest_mod.canonical({"components": components, "platform": platform})
|
|
out = {"format": 1, "image": {"name": "tree", "version": "tree (no build)"},
|
|
"content_sha256": hashlib.sha256(canonical.encode()).hexdigest(),
|
|
"components": components, "platform": platform}
|
|
text = json.dumps(out, sort_keys=True, indent=1) + "\n"
|
|
if args.out == "-":
|
|
sys.stdout.write(text)
|
|
else:
|
|
with open(args.out, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
print("wrote %s" % args.out, file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|