From 9b5558dfbcd972a918c24b494ba2f2fd0cb38a04 Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Sat, 15 Aug 2026 15:57:10 -0400 Subject: [PATCH] meta-forgefirm: the image manifest (/etc/forgefirm-manifest.json) Every image records the identity of its build inputs. forgefirm-manifest.bbclass gives a component recipe a per-file source fingerprint (git ls-tree blob ids, submodules recursed; file:// sources hashed with git hash-object); the kernel and glowforge.ko publish theirs through do_deploy because kernel-module-split leaves the module recipe's main package empty. forgefirm-image-manifest.bbclass assembles the entries with the platform identity - machine, modules directory, device tree hashes, layer content hashes (meta-forgefirm and the meta-openglow layers by content, the kas-managed layers by revision) - into /etc/forgefirm-manifest.json and deploys a copy next to the image. Layer revisions and dirty flags stay outside the identity, so a rebuild from an unchanged tree keeps its content hash and the release and dev images of one build share it. The acceptance tool and the release gate read this file. --- .../classes/forgefirm-image-manifest.bbclass | 166 ++++++++++++++++++ .../classes/forgefirm-manifest.bbclass | 132 ++++++++++++++ .../python/python3-gfhardware.bbappend | 3 + .../python/python3-gfutilities_%.bbappend | 3 + .../recipes-forgefirm/forgectrl/forgectrl.bb | 2 +- .../forgefirm-app/forgefirm-app.inc | 5 + .../grblhal-glowforge/grblhal-glowforge.bb | 2 +- .../images/forgefirm-image.bb | 4 + .../kernel-module-glowforge.bbappend | 18 ++ .../linux/linux-fslc_%.bbappend | 15 ++ 10 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 meta-forgefirm/classes/forgefirm-image-manifest.bbclass create mode 100644 meta-forgefirm/classes/forgefirm-manifest.bbclass create mode 100644 meta-forgefirm/recipes-devtools/python/python3-gfhardware.bbappend create mode 100644 meta-forgefirm/recipes-devtools/python/python3-gfutilities_%.bbappend create mode 100644 meta-forgefirm/recipes-kernel/kernel-modules/kernel-module-glowforge.bbappend create mode 100644 meta-forgefirm/recipes-kernel/linux/linux-fslc_%.bbappend diff --git a/meta-forgefirm/classes/forgefirm-image-manifest.bbclass b/meta-forgefirm/classes/forgefirm-image-manifest.bbclass new file mode 100644 index 0000000..511405c --- /dev/null +++ b/meta-forgefirm/classes/forgefirm-image-manifest.bbclass @@ -0,0 +1,166 @@ +# forgefirm-image-manifest.bbclass - assemble /etc/forgefirm-manifest.json +# +# Inherited by the ForgeFIRM images. At rootfs postprocess it collects the +# per-component entries (forgefirm-manifest.bbclass) from the rootfs +# (/etc/forgefirm-manifest.d/*.json) and from DEPLOY_DIR_IMAGE +# (forgefirm-manifest.d/*.json: the kernel and the kernel module, which +# cannot ship a rootfs file), adds the platform identity, and writes +# +# /etc/forgefirm-manifest.json (in the image) +# ${IMAGE_NAME}.forgefirm-manifest.json (deployed next to the image, +# with the usual link name) +# +# The platform section records what the components' fingerprints do not: +# the machine, the kernel modules directory (carries the kernel's local +# version), the device tree blob hashes, and the layers. Layers named in +# FORGEFIRM_MANIFEST_CONTENT_LAYERS are identified by content (tracked and +# untracked-unignored files under the layer directory, git blob ids), so +# a build from a dirty working tree and a later build from the identical +# committed tree agree; the other layers are identified by revision. +# Revisions and dirty flags of the content-hashed layers are informational +# and live in the "build" section, outside the identity. Nothing +# host-specific (paths, user, time) is recorded: the manifest travels in +# a public release artifact. +# +# content_sha256 is the identity of the build's inputs: sha256 over the +# canonical JSON (sorted keys, no whitespace) of {"components", "platform"}. +# The image name, version string and the build section are metadata +# outside that hash, so the release image and the dev image of one build +# share the same identity, and so does a rebuild from an unchanged tree. + +FORGEFIRM_MANIFEST_DIR ?= "${sysconfdir}/forgefirm-manifest.d" +FORGEFIRM_MANIFEST_CONTENT_LAYERS ?= "meta-forgefirm meta-glowforge-bsp meta-openglow-core" + +do_rootfs[depends] += "virtual/kernel:do_deploy kernel-module-glowforge:do_deploy" + +ROOTFS_POSTPROCESS_COMMAND += "forgefirm_manifest_assemble;" +forgefirm_manifest_assemble[vardepsexclude] += "DATETIME" + +def forgefirm_manifest_layer_content(path): + import hashlib, os, subprocess + out = subprocess.run(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', '.'], + cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout + paths = sorted(set(p.decode('utf-8', 'replace') for p in out.split(b'\0') if p)) + paths = [p for p in paths + if os.path.isfile(os.path.join(path, p)) and not p.endswith('.md')] + if not paths: + return None + # hash-object --stdin-paths resolves against the repository top level + prefix = subprocess.check_output(['git', 'rev-parse', '--show-prefix'], cwd=path, + stderr=subprocess.DEVNULL).decode().strip() + ids = subprocess.run(['git', 'hash-object', '--stdin-paths'], cwd=path, + input=('\n'.join(prefix + p for p in paths) + '\n').encode(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().split() + if len(ids) != len(paths): + bb.fatal("forgefirm-image-manifest: hash-object returned %d ids for %d paths under %s" + % (len(ids), len(paths), path)) + h = hashlib.sha256() + for p, i in zip(paths, ids): + h.update(p.encode('utf-8')) + h.update(b'\0') + h.update(i.encode('ascii')) + h.update(b'\n') + return h.hexdigest() + +def forgefirm_manifest_layers(d): + """Returns (identity, build): identity[name] = {"content_sha256"} for + content-hashed layers or {"rev"} for the rest; build[name] = revision + and dirty flag of every layer checkout (informational).""" + import os, subprocess + content_layers = (d.getVar('FORGEFIRM_MANIFEST_CONTENT_LAYERS') or '').split() + identity, build = {}, {} + for layer in (d.getVar('BBLAYERS') or '').split(): + name = os.path.basename(layer.rstrip('/')) + try: + rev = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=layer, + stderr=subprocess.DEVNULL).decode().strip() + status = subprocess.check_output(['git', 'status', '--porcelain', '--', '.'], cwd=layer, + stderr=subprocess.DEVNULL).decode() + build[name] = {'rev': rev, 'dirty': bool(status.strip())} + except (subprocess.CalledProcessError, OSError): + rev = None + build[name] = {'rev': None, 'dirty': None} + if name in content_layers: + if rev is None: + bb.fatal("forgefirm-image-manifest: layer %s must be a git checkout to be content-hashed" % name) + identity[name] = {'content_sha256': forgefirm_manifest_layer_content(layer)} + else: + identity[name] = {'rev': rev} + return identity, build + +def forgefirm_manifest_sha256_file(path): + import hashlib + h = hashlib.sha256() + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(1 << 20), b''): + h.update(chunk) + return h.hexdigest() + +python forgefirm_manifest_assemble() { + import glob, hashlib, json, os + + rootfs = d.getVar('IMAGE_ROOTFS') + components = {} + + def add(path): + with open(path) as f: + e = json.load(f) + name = e['component'] + recipe = e.pop('recipe', None) + prev = components.get(name) + if prev is None: + e['recipes'] = [recipe] if recipe else [] + components[name] = e + return + if prev.get('srcrev') != e.get('srcrev') or prev.get('files') != e.get('files'): + bb.fatal("forgefirm-image-manifest: component '%s' is claimed by %s and %s with different sources" + % (name, prev['recipes'], recipe)) + if recipe and recipe not in prev['recipes']: + prev['recipes'].append(recipe) + prev['recipes'].sort() + + for p in sorted(glob.glob(rootfs + d.getVar('FORGEFIRM_MANIFEST_DIR') + '/*.json')): + add(p) + for p in sorted(glob.glob(os.path.join(d.getVar('DEPLOY_DIR_IMAGE'), 'forgefirm-manifest.d', '*.json'))): + add(p) + if not components: + bb.fatal("forgefirm-image-manifest: no component entries found") + + layer_identity, layer_build = forgefirm_manifest_layers(d) + platform = {'machine': d.getVar('MACHINE'), 'layers': layer_identity} + platform['kernel_modules'] = sorted(os.listdir(os.path.join(rootfs, 'lib', 'modules'))) \ + if os.path.isdir(os.path.join(rootfs, 'lib', 'modules')) else [] + dtbs = {} + for p in sorted(glob.glob(os.path.join(rootfs, 'boot', '**', '*.dtb'), recursive=True)): + dtbs[os.path.relpath(p, os.path.join(rootfs, 'boot'))] = forgefirm_manifest_sha256_file(p) + platform['dtb'] = dtbs + + canonical = json.dumps({'components': components, 'platform': platform}, + sort_keys=True, separators=(',', ':')) + manifest = { + 'format': 1, + 'image': {'name': d.getVar('PN'), 'version': d.getVar('FORGEFIRM_VERSION_STRING')}, + 'build': {'layers': layer_build}, + 'content_sha256': hashlib.sha256(canonical.encode('utf-8')).hexdigest(), + 'components': components, + 'platform': platform, + } + text = json.dumps(manifest, sort_keys=True, indent=1) + '\n' + + target = os.path.join(rootfs, 'etc', 'forgefirm-manifest.json') + with open(target, 'w') as f: + f.write(text) + os.chmod(target, 0o644) + + deploy = d.getVar('IMGDEPLOYDIR') + if deploy: + name = d.getVar('IMAGE_NAME') + '.forgefirm-manifest.json' + with open(os.path.join(deploy, name), 'w') as f: + f.write(text) + link = os.path.join(deploy, d.getVar('IMAGE_LINK_NAME') + '.forgefirm-manifest.json') + if os.path.lexists(link): + os.remove(link) + os.symlink(name, link) + bb.note("forgefirm-image-manifest: %d components, content_sha256 %s" + % (len(components), manifest['content_sha256'])) +} diff --git a/meta-forgefirm/classes/forgefirm-manifest.bbclass b/meta-forgefirm/classes/forgefirm-manifest.bbclass new file mode 100644 index 0000000..3698d2d --- /dev/null +++ b/meta-forgefirm/classes/forgefirm-manifest.bbclass @@ -0,0 +1,132 @@ +# forgefirm-manifest.bbclass - source fingerprint of one ForgeFIRM component +# +# A component recipe that inherits this class records what source it was +# built from: the component name, the pinned revision, and one +# [path, blob-id] pair per source file (git blob ids, so equal content +# means equal ids regardless of the commit that carried it). The image +# collects the entries into /etc/forgefirm-manifest.json +# (forgefirm-image-manifest.bbclass); the acceptance tool (forgetest) and +# the release gate (scripts/acceptance-gate.py) read that file to decide +# which acceptance results still apply to a build. +# +# Modes (FORGEFIRM_MANIFEST_MODE): +# rootfs (default) the entry is installed as +# /etc/forgefirm-manifest.d/.json in the recipe's main package +# deploy the recipe writes the entry into ${DEPLOYDIR} itself (see the +# kernel-module bbappend: its .ko is packaged by kernel-module-split +# into a versioned package, so a rootfs file from the recipe would +# not reach the image); this class then only provides the helpers. +# +# Sources: a git checkout in ${S} is fingerprinted with `git ls-tree` +# (submodules recursed, the gitlink kept). Non-git sources (file:// recipes) +# set FORGEFIRM_MANIFEST_SRC to the directory to fingerprint; the ids are +# computed with `git hash-object`, so they compare with tree ids. + +FORGEFIRM_MANIFEST_NAME ?= "${PN}" +FORGEFIRM_MANIFEST_MODE ?= "rootfs" +FORGEFIRM_MANIFEST_SRC ?= "" +FORGEFIRM_MANIFEST_DIR = "${sysconfdir}/forgefirm-manifest.d" + +def forgefirm_manifest_git(args, cwd): + import subprocess + return subprocess.check_output(['git'] + args, cwd=cwd, + stderr=subprocess.STDOUT).decode('utf-8', 'replace') + +def forgefirm_manifest_tree(files, repo, prefix): + import os + out = forgefirm_manifest_git(['ls-tree', '-r', '--full-tree', 'HEAD'], repo) + for line in out.splitlines(): + if not line.strip(): + continue + meta, path = line.split('\t', 1) + fields = meta.split() + typ, obj = fields[1], fields[2] + files.append([prefix + path, obj]) + if typ == 'commit': + sub = os.path.join(repo, path) + if os.path.exists(os.path.join(sub, '.git')): + forgefirm_manifest_tree(files, sub, prefix + path + '/') + +def forgefirm_manifest_git_prefix(cwd): + """`git hash-object --stdin-paths` resolves paths against the repository + top level, not the cwd; this is the cwd's prefix inside the enclosing + repository ('' when there is none).""" + import subprocess + try: + return subprocess.check_output(['git', 'rev-parse', '--show-prefix'], cwd=cwd, + stderr=subprocess.DEVNULL).decode().strip() + except (subprocess.CalledProcessError, OSError): + return '' + +def forgefirm_manifest_hash_dir(src): + import os, subprocess + paths = [] + for root, dirs, fns in os.walk(src): + dirs[:] = sorted(x for x in dirs if x not in ('.git', '__pycache__')) + for fn in fns: + if fn.endswith(('.pyc', '.pyo')): + continue + p = os.path.join(root, fn) + if os.path.isfile(p) and not os.path.islink(p): + paths.append(os.path.relpath(p, src)) + paths.sort() + if not paths: + return [] + prefix = forgefirm_manifest_git_prefix(src) + out = subprocess.run(['git', 'hash-object', '--stdin-paths'], cwd=src, + input=('\n'.join(prefix + p for p in paths) + '\n').encode(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + ids = out.stdout.decode().split() + if len(ids) != len(paths): + bb.fatal("forgefirm-manifest: hash-object returned %d ids for %d paths" + % (len(ids), len(paths))) + return [[p, i] for p, i in zip(paths, ids)] + +def forgefirm_manifest_entry(d): + import os + pn = d.getVar('PN') + entry = {'component': d.getVar('FORGEFIRM_MANIFEST_NAME'), + 'recipe': pn, 'pv': d.getVar('PV'), 'files': []} + src = (d.getVar('FORGEFIRM_MANIFEST_SRC') or '').strip() + s = d.getVar('S') + if src: + if not os.path.isdir(src): + bb.fatal("forgefirm-manifest: %s: FORGEFIRM_MANIFEST_SRC '%s' is not a directory" % (pn, src)) + entry['srcrev'] = None + entry['source'] = 'files' + entry['files'] = forgefirm_manifest_hash_dir(src) + elif os.path.exists(os.path.join(s, '.git')): + entry['srcrev'] = forgefirm_manifest_git(['rev-parse', 'HEAD'], s).strip() + uri = (d.getVar('SRC_URI') or '').split() + entry['source'] = uri[0].split(';')[0] if uri else None + forgefirm_manifest_tree(entry['files'], s, '') + else: + bb.fatal("forgefirm-manifest: %s: ${S} is not a git checkout and FORGEFIRM_MANIFEST_SRC is unset" % pn) + entry['files'].sort() + return entry + +def forgefirm_manifest_write(entry, path): + import json, os + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + json.dump(entry, f, sort_keys=True, separators=(',', ':')) + f.write('\n') + os.chmod(path, 0o644) + +fakeroot python do_forgefirm_manifest() { + import os + entry = forgefirm_manifest_entry(d) + forgefirm_manifest_write(entry, os.path.join(d.getVar('D') + d.getVar('FORGEFIRM_MANIFEST_DIR'), + d.getVar('PN') + '.json')) +} +do_forgefirm_manifest[depends] += "virtual/fakeroot-native:do_populate_sysroot" +do_forgefirm_manifest[vardeps] += "FORGEFIRM_MANIFEST_NAME FORGEFIRM_MANIFEST_SRC SRCREV SRC_URI \ + forgefirm_manifest_entry forgefirm_manifest_tree forgefirm_manifest_git \ + forgefirm_manifest_hash_dir forgefirm_manifest_git_prefix forgefirm_manifest_write" + +python __anonymous() { + if d.getVar('FORGEFIRM_MANIFEST_MODE') == 'rootfs': + bb.build.addtask('do_forgefirm_manifest', 'do_package do_populate_sysroot', 'do_install', d) +} + +FILES:${PN}:append = " ${FORGEFIRM_MANIFEST_DIR}" diff --git a/meta-forgefirm/recipes-devtools/python/python3-gfhardware.bbappend b/meta-forgefirm/recipes-devtools/python/python3-gfhardware.bbappend new file mode 100644 index 0000000..f718cca --- /dev/null +++ b/meta-forgefirm/recipes-devtools/python/python3-gfhardware.bbappend @@ -0,0 +1,3 @@ +# ForgeFIRM image manifest: source fingerprint of the hardware library +# (forgefirm-manifest.bbclass in this layer). +inherit forgefirm-manifest diff --git a/meta-forgefirm/recipes-devtools/python/python3-gfutilities_%.bbappend b/meta-forgefirm/recipes-devtools/python/python3-gfutilities_%.bbappend new file mode 100644 index 0000000..dc2632a --- /dev/null +++ b/meta-forgefirm/recipes-devtools/python/python3-gfutilities_%.bbappend @@ -0,0 +1,3 @@ +# ForgeFIRM image manifest: source fingerprint of the cloud-mode service +# layer (forgefirm-manifest.bbclass in this layer). +inherit forgefirm-manifest diff --git a/meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl.bb b/meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl.bb index 0eed75c..2ccfaff 100644 --- a/meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl.bb +++ b/meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl.bb @@ -13,7 +13,7 @@ SRCREV = "b4e01c5b24c75d09d6784c690d1c10654bc46da3" S = "${WORKDIR}/git" -inherit cmake update-rc.d +inherit cmake update-rc.d forgefirm-manifest DEPENDS += "ulfius jpeg" # media-ctl / v4l2-ctl configure the imx-media pipeline at runtime; diff --git a/meta-forgefirm/recipes-forgefirm/forgefirm-app/forgefirm-app.inc b/meta-forgefirm/recipes-forgefirm/forgefirm-app/forgefirm-app.inc index 2fa1ba2..e19eb59 100644 --- a/meta-forgefirm/recipes-forgefirm/forgefirm-app/forgefirm-app.inc +++ b/meta-forgefirm/recipes-forgefirm/forgefirm-app/forgefirm-app.inc @@ -19,3 +19,8 @@ SRCREV = "a0a174d8e9a6b4b6e3abe63acc51018199df4202" PV = "0.1.6+git" S = "${WORKDIR}/git" + +# One image-manifest component for the three app recipes (same repository, +# same pinned revision): the entries merge in the image manifest. +inherit forgefirm-manifest +FORGEFIRM_MANIFEST_NAME = "forgefirm-app" diff --git a/meta-forgefirm/recipes-forgefirm/grblhal-glowforge/grblhal-glowforge.bb b/meta-forgefirm/recipes-forgefirm/grblhal-glowforge/grblhal-glowforge.bb index dbaf36d..f5b59d3 100644 --- a/meta-forgefirm/recipes-forgefirm/grblhal-glowforge/grblhal-glowforge.bb +++ b/meta-forgefirm/recipes-forgefirm/grblhal-glowforge/grblhal-glowforge.bb @@ -17,7 +17,7 @@ SRC_URI += "file://grblhal.init" S = "${WORKDIR}/git" -inherit cmake update-rc.d +inherit cmake update-rc.d forgefirm-manifest INITSCRIPT_NAME = "grblhal" # stop 80 < forgectrl's 90: at runlevel 0/6 the controller goes down diff --git a/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb b/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb index c6afd21..2154c5e 100644 --- a/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb +++ b/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb @@ -1,5 +1,9 @@ require recipes-glowforge/images/glowforge-image.bb +# /etc/forgefirm-manifest.json: the build-input identity the acceptance tool +# and the release gate compare (forgefirm-image-manifest.bbclass). +inherit forgefirm-image-manifest + DESCRIPTION = "OpenGlow/ForgeFIRM image for Glowforge" # ForgeFIRM cuts the cloud dependency: drop the Glowforge cloud client diff --git a/meta-forgefirm/recipes-kernel/kernel-modules/kernel-module-glowforge.bbappend b/meta-forgefirm/recipes-kernel/kernel-modules/kernel-module-glowforge.bbappend new file mode 100644 index 0000000..ad340db --- /dev/null +++ b/meta-forgefirm/recipes-kernel/kernel-modules/kernel-module-glowforge.bbappend @@ -0,0 +1,18 @@ +# ForgeFIRM image manifest: source fingerprint of glowforge.ko. +# +# kernel-module-split packages the .ko into a versioned package that the +# recipe's main package only RPROVIDES, so a rootfs file installed here +# would never reach the image. The entry is deployed instead; the image +# collects it from DEPLOY_DIR_IMAGE (forgefirm-image-manifest.bbclass). +inherit forgefirm-manifest deploy +FORGEFIRM_MANIFEST_MODE = "deploy" + +python do_deploy() { + import os + entry = forgefirm_manifest_entry(d) + forgefirm_manifest_write(entry, os.path.join(d.getVar('DEPLOYDIR'), 'forgefirm-manifest.d', + d.getVar('PN') + '.json')) +} +do_deploy[vardeps] += "FORGEFIRM_MANIFEST_NAME SRCREV SRC_URI forgefirm_manifest_entry \ + forgefirm_manifest_tree forgefirm_manifest_git forgefirm_manifest_write" +addtask deploy after do_install before do_build diff --git a/meta-forgefirm/recipes-kernel/linux/linux-fslc_%.bbappend b/meta-forgefirm/recipes-kernel/linux/linux-fslc_%.bbappend new file mode 100644 index 0000000..0e004ae --- /dev/null +++ b/meta-forgefirm/recipes-kernel/linux/linux-fslc_%.bbappend @@ -0,0 +1,15 @@ +# ForgeFIRM image manifest: kernel identity (forgefirm-image-manifest.bbclass +# collects it from DEPLOY_DIR_IMAGE). The kernel tree is too large to list +# per file and every acceptance test depends on it anyway, so the entry +# carries the pinned revision and the hash of the configuration actually +# built as pseudo-files ("@srcrev", "@config") that the fingerprint globs +# match like any other path. Device-tree sources and config fragments come +# from the BSP layer, which the image manifest hashes by content. +do_deploy:append:glowforge() { + install -d ${DEPLOYDIR}/forgefirm-manifest.d + cfg=$(sha256sum ${B}/.config | cut -d' ' -f1) + printf '{"component":"%s","config_sha256":"%s","files":[["@config","%s"],["@srcrev","%s"]],"linux_version":"%s","pv":"%s","recipe":"%s","source":"%s","srcrev":"%s"}\n' \ + "${PN}" "$cfg" "$cfg" "${SRCREV}" "${LINUX_VERSION}" "${PV}" "${PN}" \ + "git://github.com/Freescale/linux-fslc.git" "${SRCREV}" \ + > ${DEPLOYDIR}/forgefirm-manifest.d/${PN}.json +}