meta-forgefirm: the image manifest (/etc/forgefirm-manifest.json)

Every image records the identity of its build inputs. forgefirm-manifest.bbclass
gives a component recipe a per-file source fingerprint (git ls-tree blob ids,
submodules recursed; file:// sources hashed with git hash-object); the kernel
and glowforge.ko publish theirs through do_deploy because kernel-module-split
leaves the module recipe's main package empty. forgefirm-image-manifest.bbclass
assembles the entries with the platform identity - machine, modules directory,
device tree hashes, layer content hashes (meta-forgefirm and the meta-openglow
layers by content, the kas-managed layers by revision) - into
/etc/forgefirm-manifest.json and deploys a copy next to the image. Layer
revisions and dirty flags stay outside the identity, so a rebuild from an
unchanged tree keeps its content hash and the release and dev images of one
build share it. The acceptance tool and the release gate read this file.

meta-forgefirm/classes/forgefirm-image-manifest.bbclass

@@ -0,0 +1,166 @@
+# forgefirm-image-manifest.bbclass - assemble /etc/forgefirm-manifest.json
+#
+# Inherited by the ForgeFIRM images. At rootfs postprocess it collects the
+# per-component entries (forgefirm-manifest.bbclass) from the rootfs
+# (/etc/forgefirm-manifest.d/*.json) and from DEPLOY_DIR_IMAGE
+# (forgefirm-manifest.d/*.json: the kernel and the kernel module, which
+# cannot ship a rootfs file), adds the platform identity, and writes
+#
+#   /etc/forgefirm-manifest.json          (in the image)
+#   ${IMAGE_NAME}.forgefirm-manifest.json (deployed next to the image,
+#                                          with the usual link name)
+#
+# The platform section records what the components' fingerprints do not:
+# the machine, the kernel modules directory (carries the kernel's local
+# version), the device tree blob hashes, and the layers. Layers named in
+# FORGEFIRM_MANIFEST_CONTENT_LAYERS are identified by content (tracked and
+# untracked-unignored files under the layer directory, git blob ids), so
+# a build from a dirty working tree and a later build from the identical
+# committed tree agree; the other layers are identified by revision.
+# Revisions and dirty flags of the content-hashed layers are informational
+# and live in the "build" section, outside the identity. Nothing
+# host-specific (paths, user, time) is recorded: the manifest travels in
+# a public release artifact.
+#
+# content_sha256 is the identity of the build's inputs: sha256 over the
+# canonical JSON (sorted keys, no whitespace) of {"components", "platform"}.
+# The image name, version string and the build section are metadata
+# outside that hash, so the release image and the dev image of one build
+# share the same identity, and so does a rebuild from an unchanged tree.
+
+FORGEFIRM_MANIFEST_DIR ?= "${sysconfdir}/forgefirm-manifest.d"
+FORGEFIRM_MANIFEST_CONTENT_LAYERS ?= "meta-forgefirm meta-glowforge-bsp meta-openglow-core"
+
+do_rootfs[depends] += "virtual/kernel:do_deploy kernel-module-glowforge:do_deploy"
+
+ROOTFS_POSTPROCESS_COMMAND += "forgefirm_manifest_assemble;"
+forgefirm_manifest_assemble[vardepsexclude] += "DATETIME"
+
+def forgefirm_manifest_layer_content(path):
+    import hashlib, os, subprocess
+    out = subprocess.run(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', '.'],
+                         cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout
+    paths = sorted(set(p.decode('utf-8', 'replace') for p in out.split(b'\0') if p))
+    paths = [p for p in paths
+             if os.path.isfile(os.path.join(path, p)) and not p.endswith('.md')]
+    if not paths:
+        return None
+    # hash-object --stdin-paths resolves against the repository top level
+    prefix = subprocess.check_output(['git', 'rev-parse', '--show-prefix'], cwd=path,
+                                     stderr=subprocess.DEVNULL).decode().strip()
+    ids = subprocess.run(['git', 'hash-object', '--stdin-paths'], cwd=path,
+                         input=('\n'.join(prefix + p for p in paths) + '\n').encode(),
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().split()
+    if len(ids) != len(paths):
+        bb.fatal("forgefirm-image-manifest: hash-object returned %d ids for %d paths under %s"
+                 % (len(ids), len(paths), path))
+    h = hashlib.sha256()
+    for p, i in zip(paths, ids):
+        h.update(p.encode('utf-8'))
+        h.update(b'\0')
+        h.update(i.encode('ascii'))
+        h.update(b'\n')
+    return h.hexdigest()
+
+def forgefirm_manifest_layers(d):
+    """Returns (identity, build): identity[name] = {"content_sha256"} for
+    content-hashed layers or {"rev"} for the rest; build[name] = revision
+    and dirty flag of every layer checkout (informational)."""
+    import os, subprocess
+    content_layers = (d.getVar('FORGEFIRM_MANIFEST_CONTENT_LAYERS') or '').split()
+    identity, build = {}, {}
+    for layer in (d.getVar('BBLAYERS') or '').split():
+        name = os.path.basename(layer.rstrip('/'))
+        try:
+            rev = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=layer,
+                                          stderr=subprocess.DEVNULL).decode().strip()
+            status = subprocess.check_output(['git', 'status', '--porcelain', '--', '.'], cwd=layer,
+                                             stderr=subprocess.DEVNULL).decode()
+            build[name] = {'rev': rev, 'dirty': bool(status.strip())}
+        except (subprocess.CalledProcessError, OSError):
+            rev = None
+            build[name] = {'rev': None, 'dirty': None}
+        if name in content_layers:
+            if rev is None:
+                bb.fatal("forgefirm-image-manifest: layer %s must be a git checkout to be content-hashed" % name)
+            identity[name] = {'content_sha256': forgefirm_manifest_layer_content(layer)}
+        else:
+            identity[name] = {'rev': rev}
+    return identity, build
+
+def forgefirm_manifest_sha256_file(path):
+    import hashlib
+    h = hashlib.sha256()
+    with open(path, 'rb') as f:
+        for chunk in iter(lambda: f.read(1 << 20), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+python forgefirm_manifest_assemble() {
+    import glob, hashlib, json, os
+
+    rootfs = d.getVar('IMAGE_ROOTFS')
+    components = {}
+
+    def add(path):
+        with open(path) as f:
+            e = json.load(f)
+        name = e['component']
+        recipe = e.pop('recipe', None)
+        prev = components.get(name)
+        if prev is None:
+            e['recipes'] = [recipe] if recipe else []
+            components[name] = e
+            return
+        if prev.get('srcrev') != e.get('srcrev') or prev.get('files') != e.get('files'):
+            bb.fatal("forgefirm-image-manifest: component '%s' is claimed by %s and %s with different sources"
+                     % (name, prev['recipes'], recipe))
+        if recipe and recipe not in prev['recipes']:
+            prev['recipes'].append(recipe)
+            prev['recipes'].sort()
+
+    for p in sorted(glob.glob(rootfs + d.getVar('FORGEFIRM_MANIFEST_DIR') + '/*.json')):
+        add(p)
+    for p in sorted(glob.glob(os.path.join(d.getVar('DEPLOY_DIR_IMAGE'), 'forgefirm-manifest.d', '*.json'))):
+        add(p)
+    if not components:
+        bb.fatal("forgefirm-image-manifest: no component entries found")
+
+    layer_identity, layer_build = forgefirm_manifest_layers(d)
+    platform = {'machine': d.getVar('MACHINE'), 'layers': layer_identity}
+    platform['kernel_modules'] = sorted(os.listdir(os.path.join(rootfs, 'lib', 'modules'))) \
+        if os.path.isdir(os.path.join(rootfs, 'lib', 'modules')) else []
+    dtbs = {}
+    for p in sorted(glob.glob(os.path.join(rootfs, 'boot', '**', '*.dtb'), recursive=True)):
+        dtbs[os.path.relpath(p, os.path.join(rootfs, 'boot'))] = forgefirm_manifest_sha256_file(p)
+    platform['dtb'] = dtbs
+
+    canonical = json.dumps({'components': components, 'platform': platform},
+                           sort_keys=True, separators=(',', ':'))
+    manifest = {
+        'format': 1,
+        'image': {'name': d.getVar('PN'), 'version': d.getVar('FORGEFIRM_VERSION_STRING')},
+        'build': {'layers': layer_build},
+        'content_sha256': hashlib.sha256(canonical.encode('utf-8')).hexdigest(),
+        'components': components,
+        'platform': platform,
+    }
+    text = json.dumps(manifest, sort_keys=True, indent=1) + '\n'
+
+    target = os.path.join(rootfs, 'etc', 'forgefirm-manifest.json')
+    with open(target, 'w') as f:
+        f.write(text)
+    os.chmod(target, 0o644)
+
+    deploy = d.getVar('IMGDEPLOYDIR')
+    if deploy:
+        name = d.getVar('IMAGE_NAME') + '.forgefirm-manifest.json'
+        with open(os.path.join(deploy, name), 'w') as f:
+            f.write(text)
+        link = os.path.join(deploy, d.getVar('IMAGE_LINK_NAME') + '.forgefirm-manifest.json')
+        if os.path.lexists(link):
+            os.remove(link)
+        os.symlink(name, link)
+    bb.note("forgefirm-image-manifest: %d components, content_sha256 %s"
+            % (len(components), manifest['content_sha256']))
+}

meta-forgefirm/classes/forgefirm-manifest.bbclass

@@ -0,0 +1,132 @@
+# forgefirm-manifest.bbclass - source fingerprint of one ForgeFIRM component
+#
+# A component recipe that inherits this class records what source it was
+# built from: the component name, the pinned revision, and one
+# [path, blob-id] pair per source file (git blob ids, so equal content
+# means equal ids regardless of the commit that carried it). The image
+# collects the entries into /etc/forgefirm-manifest.json
+# (forgefirm-image-manifest.bbclass); the acceptance tool (forgetest) and
+# the release gate (scripts/acceptance-gate.py) read that file to decide
+# which acceptance results still apply to a build.
+#
+# Modes (FORGEFIRM_MANIFEST_MODE):
+#   rootfs  (default) the entry is installed as
+#           /etc/forgefirm-manifest.d/<PN>.json in the recipe's main package
+#   deploy  the recipe writes the entry into ${DEPLOYDIR} itself (see the
+#           kernel-module bbappend: its .ko is packaged by kernel-module-split
+#           into a versioned package, so a rootfs file from the recipe would
+#           not reach the image); this class then only provides the helpers.
+#
+# Sources: a git checkout in ${S} is fingerprinted with `git ls-tree`
+# (submodules recursed, the gitlink kept). Non-git sources (file:// recipes)
+# set FORGEFIRM_MANIFEST_SRC to the directory to fingerprint; the ids are
+# computed with `git hash-object`, so they compare with tree ids.
+
+FORGEFIRM_MANIFEST_NAME ?= "${PN}"
+FORGEFIRM_MANIFEST_MODE ?= "rootfs"
+FORGEFIRM_MANIFEST_SRC ?= ""
+FORGEFIRM_MANIFEST_DIR = "${sysconfdir}/forgefirm-manifest.d"
+
+def forgefirm_manifest_git(args, cwd):
+    import subprocess
+    return subprocess.check_output(['git'] + args, cwd=cwd,
+                                   stderr=subprocess.STDOUT).decode('utf-8', 'replace')
+
+def forgefirm_manifest_tree(files, repo, prefix):
+    import os
+    out = forgefirm_manifest_git(['ls-tree', '-r', '--full-tree', 'HEAD'], repo)
+    for line in out.splitlines():
+        if not line.strip():
+            continue
+        meta, path = line.split('\t', 1)
+        fields = meta.split()
+        typ, obj = fields[1], fields[2]
+        files.append([prefix + path, obj])
+        if typ == 'commit':
+            sub = os.path.join(repo, path)
+            if os.path.exists(os.path.join(sub, '.git')):
+                forgefirm_manifest_tree(files, sub, prefix + path + '/')
+
+def forgefirm_manifest_git_prefix(cwd):
+    """`git hash-object --stdin-paths` resolves paths against the repository
+    top level, not the cwd; this is the cwd's prefix inside the enclosing
+    repository ('' when there is none)."""
+    import subprocess
+    try:
+        return subprocess.check_output(['git', 'rev-parse', '--show-prefix'], cwd=cwd,
+                                       stderr=subprocess.DEVNULL).decode().strip()
+    except (subprocess.CalledProcessError, OSError):
+        return ''
+
+def forgefirm_manifest_hash_dir(src):
+    import os, subprocess
+    paths = []
+    for root, dirs, fns in os.walk(src):
+        dirs[:] = sorted(x for x in dirs if x not in ('.git', '__pycache__'))
+        for fn in fns:
+            if fn.endswith(('.pyc', '.pyo')):
+                continue
+            p = os.path.join(root, fn)
+            if os.path.isfile(p) and not os.path.islink(p):
+                paths.append(os.path.relpath(p, src))
+    paths.sort()
+    if not paths:
+        return []
+    prefix = forgefirm_manifest_git_prefix(src)
+    out = subprocess.run(['git', 'hash-object', '--stdin-paths'], cwd=src,
+                         input=('\n'.join(prefix + p for p in paths) + '\n').encode(),
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
+    ids = out.stdout.decode().split()
+    if len(ids) != len(paths):
+        bb.fatal("forgefirm-manifest: hash-object returned %d ids for %d paths"
+                 % (len(ids), len(paths)))
+    return [[p, i] for p, i in zip(paths, ids)]
+
+def forgefirm_manifest_entry(d):
+    import os
+    pn = d.getVar('PN')
+    entry = {'component': d.getVar('FORGEFIRM_MANIFEST_NAME'),
+             'recipe': pn, 'pv': d.getVar('PV'), 'files': []}
+    src = (d.getVar('FORGEFIRM_MANIFEST_SRC') or '').strip()
+    s = d.getVar('S')
+    if src:
+        if not os.path.isdir(src):
+            bb.fatal("forgefirm-manifest: %s: FORGEFIRM_MANIFEST_SRC '%s' is not a directory" % (pn, src))
+        entry['srcrev'] = None
+        entry['source'] = 'files'
+        entry['files'] = forgefirm_manifest_hash_dir(src)
+    elif os.path.exists(os.path.join(s, '.git')):
+        entry['srcrev'] = forgefirm_manifest_git(['rev-parse', 'HEAD'], s).strip()
+        uri = (d.getVar('SRC_URI') or '').split()
+        entry['source'] = uri[0].split(';')[0] if uri else None
+        forgefirm_manifest_tree(entry['files'], s, '')
+    else:
+        bb.fatal("forgefirm-manifest: %s: ${S} is not a git checkout and FORGEFIRM_MANIFEST_SRC is unset" % pn)
+    entry['files'].sort()
+    return entry
+
+def forgefirm_manifest_write(entry, path):
+    import json, os
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, 'w') as f:
+        json.dump(entry, f, sort_keys=True, separators=(',', ':'))
+        f.write('\n')
+    os.chmod(path, 0o644)
+
+fakeroot python do_forgefirm_manifest() {
+    import os
+    entry = forgefirm_manifest_entry(d)
+    forgefirm_manifest_write(entry, os.path.join(d.getVar('D') + d.getVar('FORGEFIRM_MANIFEST_DIR'),
+                                                 d.getVar('PN') + '.json'))
+}
+do_forgefirm_manifest[depends] += "virtual/fakeroot-native:do_populate_sysroot"
+do_forgefirm_manifest[vardeps] += "FORGEFIRM_MANIFEST_NAME FORGEFIRM_MANIFEST_SRC SRCREV SRC_URI \
+    forgefirm_manifest_entry forgefirm_manifest_tree forgefirm_manifest_git \
+    forgefirm_manifest_hash_dir forgefirm_manifest_git_prefix forgefirm_manifest_write"
+
+python __anonymous() {
+    if d.getVar('FORGEFIRM_MANIFEST_MODE') == 'rootfs':
+        bb.build.addtask('do_forgefirm_manifest', 'do_package do_populate_sysroot', 'do_install', d)
+}
+
+FILES:${PN}:append = " ${FORGEFIRM_MANIFEST_DIR}"

meta-forgefirm/recipes-devtools/python/python3-gfhardware.bbappend

@@ -0,0 +1,3 @@
+# ForgeFIRM image manifest: source fingerprint of the hardware library
+# (forgefirm-manifest.bbclass in this layer).
+inherit forgefirm-manifest

meta-forgefirm/recipes-devtools/python/python3-gfutilities_%.bbappend

@@ -0,0 +1,3 @@
+# ForgeFIRM image manifest: source fingerprint of the cloud-mode service
+# layer (forgefirm-manifest.bbclass in this layer).
+inherit forgefirm-manifest

meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl.bb

@@ -13,7 +13,7 @@ SRCREV = "b4e01c5b24c75d09d6784c690d1c10654bc46da3"
 
 S = "${WORKDIR}/git"
 
-inherit cmake update-rc.d
+inherit cmake update-rc.d forgefirm-manifest
 
 DEPENDS += "ulfius jpeg"
 # media-ctl / v4l2-ctl configure the imx-media pipeline at runtime;

meta-forgefirm/recipes-forgefirm/forgefirm-app/forgefirm-app.inc

@@ -19,3 +19,8 @@ SRCREV = "a0a174d8e9a6b4b6e3abe63acc51018199df4202"
 PV = "0.1.6+git"
 
 S = "${WORKDIR}/git"
+
+# One image-manifest component for the three app recipes (same repository,
+# same pinned revision): the entries merge in the image manifest.
+inherit forgefirm-manifest
+FORGEFIRM_MANIFEST_NAME = "forgefirm-app"

meta-forgefirm/recipes-forgefirm/grblhal-glowforge/grblhal-glowforge.bb

@@ -17,7 +17,7 @@ SRC_URI += "file://grblhal.init"
 
 S = "${WORKDIR}/git"
 
-inherit cmake update-rc.d
+inherit cmake update-rc.d forgefirm-manifest
 
 INITSCRIPT_NAME = "grblhal"
 # stop 80 < forgectrl's 90: at runlevel 0/6 the controller goes down

meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb

@@ -1,5 +1,9 @@
 require recipes-glowforge/images/glowforge-image.bb
 
+# /etc/forgefirm-manifest.json: the build-input identity the acceptance tool
+# and the release gate compare (forgefirm-image-manifest.bbclass).
+inherit forgefirm-image-manifest
+
 DESCRIPTION = "OpenGlow/ForgeFIRM image for Glowforge"
 
 # ForgeFIRM cuts the cloud dependency: drop the Glowforge cloud client

meta-forgefirm/recipes-kernel/kernel-modules/kernel-module-glowforge.bbappend

@@ -0,0 +1,18 @@
+# ForgeFIRM image manifest: source fingerprint of glowforge.ko.
+#
+# kernel-module-split packages the .ko into a versioned package that the
+# recipe's main package only RPROVIDES, so a rootfs file installed here
+# would never reach the image. The entry is deployed instead; the image
+# collects it from DEPLOY_DIR_IMAGE (forgefirm-image-manifest.bbclass).
+inherit forgefirm-manifest deploy
+FORGEFIRM_MANIFEST_MODE = "deploy"
+
+python do_deploy() {
+    import os
+    entry = forgefirm_manifest_entry(d)
+    forgefirm_manifest_write(entry, os.path.join(d.getVar('DEPLOYDIR'), 'forgefirm-manifest.d',
+                                                 d.getVar('PN') + '.json'))
+}
+do_deploy[vardeps] += "FORGEFIRM_MANIFEST_NAME SRCREV SRC_URI forgefirm_manifest_entry \
+    forgefirm_manifest_tree forgefirm_manifest_git forgefirm_manifest_write"
+addtask deploy after do_install before do_build

meta-forgefirm/recipes-kernel/linux/linux-fslc_%.bbappend

@@ -0,0 +1,15 @@
+# ForgeFIRM image manifest: kernel identity (forgefirm-image-manifest.bbclass
+# collects it from DEPLOY_DIR_IMAGE). The kernel tree is too large to list
+# per file and every acceptance test depends on it anyway, so the entry
+# carries the pinned revision and the hash of the configuration actually
+# built as pseudo-files ("@srcrev", "@config") that the fingerprint globs
+# match like any other path. Device-tree sources and config fragments come
+# from the BSP layer, which the image manifest hashes by content.
+do_deploy:append:glowforge() {
+    install -d ${DEPLOYDIR}/forgefirm-manifest.d
+    cfg=$(sha256sum ${B}/.config | cut -d' ' -f1)
+    printf '{"component":"%s","config_sha256":"%s","files":[["@config","%s"],["@srcrev","%s"]],"linux_version":"%s","pv":"%s","recipe":"%s","source":"%s","srcrev":"%s"}\n' \
+        "${PN}" "$cfg" "$cfg" "${SRCREV}" "${LINUX_VERSION}" "${PV}" "${PN}" \
+        "git://github.com/Freescale/linux-fslc.git" "${SRCREV}" \
+        > ${DEPLOYDIR}/forgefirm-manifest.d/${PN}.json
+}