vh/forgefirm
1
0
Fork 0
mirror of https://github.com/openglow-org/forgefirm.git synced 2026-09-27 08:41:13 -07:00
Code Issues Packages Projects Releases Wiki Activity

A release publishes the source of the software it installs

Browse Source 
The release build merges kas/source-bundle.yml, which turns on the Yocto
archiver: the upstream source of each recipe as upstream publishes it, the
patches with their series file, and the recipe with its includes. The
overlay adds tasks only, so the image manifest is unchanged and an
acceptance result still applies; proven on the build host, where the
archiver build and a plain rebuild of the same tree give the same
content_sha256.

scripts/source-bundle.py packs forgefirm-source-v<version>.tar.gz: the
archives, both license manifests, the license texts, the ForgeFIRM layers,
the kas configuration, the layer revisions and the build identity of the
image. What the bundle must hold comes from the image, not from a list in
the script: every recipe of license.manifest and image_license.manifest
whose license is in the include list must have an archive, or the release
stops with the recipe named. release.sh attaches the bundle and covers it
with sha256sums.txt; FORGEFIRM_SOURCE_SKIP=1 bypasses deliberately.

On the build host: 68 of the image's 111 recipes carry source, 313.7 MiB,
under the 2 GiB limit of a release asset.

No acceptance catalog consequence: the change is release tooling on the
build host and puts no file and no behavior on the machine. The host-side
proof is forgetest/tests/test_source_bundle.py, which holds the license
decision, the choice of archive and the refusal.
This commit is contained in:
ScottW514
2026-09-07 09:49:18 -04:00
parent 9f715f08de
commit b0fa4ccaf5
5 changed files with 932 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/forgetest-ci.yml
+5
View File
@@ -6,6 +6,9 @@
# suites replayed on the machine's own log lines, and the check that
# every log phrase the cloud suite greps for is one the pinned cloud
# app can log (it reads the app sources from the manifest cache)
# - the source bundle of a release: which recipe of the image needs
# source, which archive answers for it, and the refusal that stops a
# release with source missing
# - shared UI files: theme.css and the vendored Bootstrap the page
# carries are byte-identical to forgectrl's at its pinned revision
# - coverage lint: every source path of every component pinned by the
@@ -21,6 +24,8 @@ on:
- 'forgetest/**'
- 'scripts/acceptance-gate.py'
- 'scripts/manifest-from-tree.py'
- 'scripts/source-bundle.py'
- 'kas/source-bundle.yml'
- 'meta-forgefirm/recipes-forgefirm/**'
- '.github/workflows/forgetest-ci.yml'
pull_request:
forgetest/tests/test_source_bundle.py
+225
View File
@@ -0,0 +1,225 @@
"""scripts/source-bundle.py - the source that a release publishes.
The bundle must hold the source of every recipe of the image whose license
makes source necessary. The list comes from the license manifests that the
build writes, so a package cannot reach a machine with its source left
behind. These tests hold the decision (which recipe needs source), the
choice of archive (the version of the image, not an older build), and the
refusal that stops a release with source missing."""
import importlib.util
import json
import os
import tarfile
import unittest
import helpers # noqa: F401 (sys.path)
REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
SCRIPT = os.path.join(REPO, "scripts", "source-bundle.py")
META_OPENGLOW = os.path.join(REPO, "..", "meta-openglow")
IMAGE = "forgefirm-image-glowforge.rootfs-20260101000000"
def load_script():
spec = importlib.util.spec_from_file_location("source_bundle", SCRIPT)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def package_block(recipe, version, license_, package=None):
return ("PACKAGE NAME: %s\nPACKAGE VERSION: %s\nRECIPE NAME: %s\nLICENSE: %s\n\n"
% (package or recipe, version, recipe, license_))
def write(path, text):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
def make_deploy(root, packages, archives, image_packages=()):
"""A deploy directory with the license manifests of one image and the
archive directories of the archiver pass."""
licdir = os.path.join(root, "licenses", "glowforge", IMAGE)
# The bundle answers for one image: its license manifests and its build
# identity must be in the deploy directory.
write(os.path.join(root, "images", "glowforge",
IMAGE + ".forgefirm-manifest.json"),
'{"content_sha256": "0"}\n')
write(os.path.join(licdir, "license.manifest"),
"".join(package_block(*p) for p in packages))
write(os.path.join(licdir, "image_license.manifest"),
"".join("RECIPE NAME: %s\nVERSION: %s\nLICENSE: %s\nFILES: x\n\n" % p
for p in image_packages))
for name in archives:
write(os.path.join(root, "sources", "arm-fslc-linux-gnueabi", name,
"%s.tar.gz" % name), "source of %s\n" % name)
return licdir
class Policy(unittest.TestCase):
"""The archiver pass and the check read one license list."""
def setUp(self):
self.mod = load_script()
self.policy = self.mod.read_policy(self.mod.KAS_OVERLAY)
def test_the_overlay_carries_the_filter(self):
self.assertIn("GPL*", self.policy["COPYLEFT_LICENSE_INCLUDE"])
self.assertIn("LGPL*", self.policy["COPYLEFT_LICENSE_INCLUDE"])
self.assertIn("Proprietary", self.policy["COPYLEFT_LICENSE_EXCLUDE"])
# The ForgeFIRM components are MIT and travel with the release too.
self.assertIn("forgectrl", self.policy["COPYLEFT_PN_INCLUDE"])
self.assertIn("gfcloud", self.policy["COPYLEFT_PN_INCLUDE"])
def test_a_copyleft_license_needs_source(self):
for expr in ("GPL-2.0-only",
"LGPL-2.1-or-later",
"GPL-2.0-only & bzip2-1.0.4",
"AFL-2.1 | GPL-2.0-or-later",
"(GPL-2.0-or-later | LGPL-3.0-only) & Unicode-DFS-2016",
"MIT ; LGPL-2.1-or-later",
"GPL-3.0-with-GCC-exception"):
self.assertTrue(self.mod.source_needed(expr, self.policy), expr)
def test_a_permissive_license_does_not(self):
for expr in ("MIT", "BSD-3-Clause", "Apache-2.0", "ISC", "PSF-2.0",
"Proprietary", "Firmware-imx-sdma_firmware", "PD"):
self.assertFalse(self.mod.source_needed(expr, self.policy), expr)
class Manifests(unittest.TestCase):
def setUp(self):
self.mod = load_script()
def test_a_recipe_collects_the_licenses_of_its_packages(self):
path = os.path.join(self.tmp(), "license.manifest")
write(path, package_block("avahi", "0.8", "LGPL-2.1-or-later", "avahi-daemon")
+ package_block("avahi", "0.8", "GPL-2.0-or-later & LGPL-2.1-or-later",
"avahi-locale-en-gb"))
recipes = self.mod.parse_manifest(path, "PACKAGE VERSION")
self.assertEqual(list(recipes), ["avahi"])
self.assertEqual(recipes["avahi"]["version"], "0.8")
self.assertEqual(len(recipes["avahi"]["licenses"]), 2)
def test_the_archive_of_the_image_version_wins(self):
index = {"linux-fslc": [("6.11.0", "/old"), ("6.12.20+git0+844aa34", "/new")],
"busybox": [("1.36.1", "/b1"), ("1.35.0", "/b0")],
"forgectrl": [("1_0.1.4", "/fc")]}
# An exact version first.
self.assertEqual(self.mod.select_archives(index, "busybox", "1.36.1"), ["/b1"])
# Then a version that starts with it: the kernel carries the
# revision of its git source in PV, and the manifest does not.
self.assertEqual(self.mod.select_archives(index, "linux-fslc", "6.12.20+git"),
["/new"])
# The name of an archive carries the epoch of the recipe, and a
# license manifest does not.
self.assertEqual(self.mod.select_archives(index, "forgectrl", "0.1.4"), ["/fc"])
self.assertEqual(self.mod.select_archives(index, "curl", "8.7.1"), [])
def test_a_directory_belongs_to_the_longest_recipe_name(self):
root = self.tmp()
make_deploy(root, [], ["python3-3.12.13-r0", "python3-certifi-2024.2.2-r0"])
index = self.mod.index_archives(os.path.join(root, "sources"),
{"python3", "python3-certifi"})
self.assertEqual([v for v, _ in index["python3"]], ["3.12.13"])
self.assertEqual([v for v, _ in index["python3-certifi"]], ["2024.2.2"])
def tmp(self):
import tempfile
d = tempfile.mkdtemp()
self.addCleanup(__import__("shutil").rmtree, d, True)
return d
@unittest.skipUnless(os.path.isdir(META_OPENGLOW),
"the meta-openglow sibling checkout is not here")
class Pack(unittest.TestCase):
"""The bundle end to end, on a deploy directory of made-up recipes."""
def setUp(self):
import tempfile
self.mod = load_script()
self.root = tempfile.mkdtemp()
self.addCleanup(__import__("shutil").rmtree, self.root, True)
self.deploy = os.path.join(self.root, "deploy")
self.out = os.path.join(self.root, "out")
def run_main(self, packages, archives, image_packages=()):
make_deploy(self.deploy, packages, archives, image_packages)
import sys
argv = sys.argv
sys.argv = ["source-bundle.py", "0.0.1", "--deploy", self.deploy,
"--image-name", IMAGE, "--out", self.out]
try:
return self.mod.main()
finally:
sys.argv = argv
def test_the_bundle_holds_the_source_and_the_accounting(self):
self.run_main(
packages=[("busybox", "1.36.1", "GPL-2.0-only"),
("jansson", "2.14", "MIT"),
("forgectrl", "0.1.4", "MIT")],
archives=["busybox-1.36.1-r0", "forgectrl-0.1.4-r0", "jansson-2.14-r0",
"u-boot-2020.01-r0"],
image_packages=[("u-boot", "2020.01", "GPL-2.0-or-later")])
path = os.path.join(self.out, "forgefirm-source-v0.0.1.tar.gz")
self.assertTrue(os.path.isfile(path))
with tarfile.open(path) as tar:
names = tar.getnames()
manifest = json.loads(tar.extractfile(
"forgefirm-source-v0.0.1/MANIFEST.json").read().decode())
sums = tar.extractfile(
"forgefirm-source-v0.0.1/sha256sums.txt").read().decode()
# The copyleft recipe and the ForgeFIRM component are in, the
# permissive third party is not.
self.assertIn("forgefirm-source-v0.0.1/sources/busybox-1.36.1-r0/busybox-1.36.1-r0.tar.gz",
names)
self.assertIn("forgefirm-source-v0.0.1/sources/forgectrl-0.1.4-r0/forgectrl-0.1.4-r0.tar.gz",
names)
self.assertNotIn("forgefirm-source-v0.0.1/sources/jansson-2.14-r0/jansson-2.14-r0.tar.gz",
names)
# The accounting travels with the source.
for name in ("README.md", "SOURCES.txt", "MANIFEST.json",
"licenses/license.manifest", "licenses/image_license.manifest",
"metadata/LAYERS.txt", "metadata/meta-forgefirm.tar.gz",
"metadata/kas/source-bundle.yml"):
self.assertIn("forgefirm-source-v0.0.1/" + name, names)
self.assertEqual(manifest["release"], "v0.0.1")
self.assertEqual(sorted(r["recipe"] for r in manifest["recipes"]),
["busybox", "forgectrl", "u-boot"])
self.assertEqual(manifest["recipes_without_source"],
[{"recipe": "jansson", "version": "2.14", "license": "MIT"}])
# Every file of the bundle is in the checksums, and only files that
# the bundle holds.
listed = sorted(line.split(" ", 1)[1] for line in sums.splitlines())
held = sorted(n.split("/", 1)[1] for n in names
if n != "forgefirm-source-v0.0.1/sha256sums.txt")
self.assertEqual(listed, held)
def test_a_recipe_with_no_source_stops_the_release(self):
with self.assertRaises(SystemExit) as caught:
self.run_main(packages=[("busybox", "1.36.1", "GPL-2.0-only"),
("gnutls", "3.8.4", "LGPL-2.1-or-later")],
archives=["busybox-1.36.1-r0"])
self.assertEqual(caught.exception.code, 1)
def test_the_source_of_gcc_covers_libgcc(self):
self.run_main(
packages=[("libgcc", "13.4.0", "GPL-3.0-with-GCC-exception"),
("glibc", "2.39+git", "GPL-2.0-only & LGPL-2.1-or-later"),
("glibc-locale", "2.39+git", "GPL-2.0-only & LGPL-2.1-or-later")],
archives=["gcc-source-13.4.0-13.4.0-r0", "glibc-2.39+git-r0"])
with tarfile.open(os.path.join(self.out, "forgefirm-source-v0.0.1.tar.gz")) as tar:
manifest = json.loads(tar.extractfile(
"forgefirm-source-v0.0.1/MANIFEST.json").read().decode())
holders = {r["recipe"]: r["holder"] for r in manifest["recipes"]}
self.assertEqual(holders["libgcc"], "gcc-source-13.4.0")
self.assertEqual(holders["glibc-locale"], "glibc")
if __name__ == "__main__":
unittest.main()
kas/source-bundle.yml
+53
View File
@@ -0,0 +1,53 @@
# ============================================================================
# ForgeFIRM - source-bundle build variant
# ============================================================================
# This overlay turns on the Yocto archiver, so the build writes the source of
# every recipe whose license makes source necessary. The archives go to
# build/tmp/deploy/sources/<arch>/<recipe>-<version>-<revision>/.
#
# The release pipeline (scripts/release.sh) merges this overlay into the
# release build, then packs the archives with scripts/source-bundle.py into
# the release asset forgefirm-source-v<version>.tar.gz.
#
# cd forgefirm
# kas build kas/source-bundle.yml # image + source archives
#
# The overlay adds tasks. It adds nothing to the root filesystem, and it
# changes no component, layer or device tree. The image manifest
# (/etc/forgefirm-manifest.json) is thus the same as without the overlay, and
# an acceptance result stays applicable
# (https://docs.forgefirm.org/developers/acceptance/).
#
# ARCHIVER_MODE[src] = "original" keeps the upstream archive as upstream
# publishes it. The patches of the recipe are archived beside it, with the
# `series` file that gives the order and the strip level, and with the recipe
# and its includes (ARCHIVER_MODE[recipe]). A recipe that gets its source from
# git is archived as a tar of the checkout at the pinned revision.
# ============================================================================
header:
version: 14
includes:
- forgefirm-glowforge.yml
local_conf_header:
source-bundle: |
INHERIT += "archiver"
ARCHIVER_MODE[src] = "original"
ARCHIVER_MODE[recipe] = "1"
ARCHIVER_MODE[diff] = "0"
ARCHIVER_MODE[dumpdata] = "0"
ARCHIVER_MODE[srpm] = "0"
ARCHIVER_MODE[compression] = "xz"
# Only the software that the machine gets. A native, cross or SDK recipe
# builds the image but is not part of it.
COPYLEFT_RECIPE_TYPES = "target"
# The license families that make source necessary. scripts/source-bundle.py
# reads these two lines from this file, so the archive and the check of the
# bundle use one list. Keep the syntax on one line each.
COPYLEFT_LICENSE_INCLUDE = "GPL* LGPL* AGPL* MPL* EPL* CDDL* CPL* OSL* EUPL* Artistic*"
COPYLEFT_LICENSE_EXCLUDE = "CLOSED Proprietary"
# The ForgeFIRM components are MIT, and the bundle carries their source
# too. forgetest is on the dev image only, and the bundle of a release
# does not carry it.
COPYLEFT_PN_INCLUDE = "forgectrl gfcloud gfhome python3-ffmachine python3-gfhardware python3-gfutilities ffboot slotmigrate forgefirm-users forgefirm-banner forgefirm-logging forgefirm-keys"
scripts/release.sh
+51 -4
View File
@@ -26,6 +26,17 @@
# FORGEFIRM_ACCEPTANCE_SKIP set to 1 to bypass the acceptance gate
# deliberately (never the default; see the site,
# Developers, "Acceptance")
# FORGEFIRM_SOURCE_SKIP set to 1 to build a release without the source
# bundle. The licenses of the software in the image
# make source necessary, so this is never the
# default.
#
# The source bundle: a release build merges kas/source-bundle.yml, so the
# build writes the source of every recipe of the image beside the image.
# scripts/source-bundle.py packs that source, the license manifests, the
# license texts and the ForgeFIRM layers into
# forgefirm-source-v<version>.tar.gz, and refuses to pack a bundle in which
# a recipe that needs source has none.
#
# Version contract: <version> == FORGEFIRM_RELEASE in forgefirm-image.bb
# == /etc/forgefirm-version ("v<version>") in the built rootfs == .fw
@@ -34,7 +45,8 @@
set -euo pipefail
REPO="$(cd "$(dirname "$0")/.." && pwd)"
DEPLOY="$REPO/build/tmp/deploy/images/glowforge"
DEPLOY_ROOT="$REPO/build/tmp/deploy"
DEPLOY="$DEPLOY_ROOT/images/glowforge"
IMAGE_BB="$REPO/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb"
INSTALLER="$REPO/scripts/install-forgefirm.sh"
WARN_BYTES=$((170 * 1024 * 1024))
@@ -63,10 +75,30 @@ FWUP="${FWUP:-fwup}"
command -v "$FWUP" >/dev/null || die "fwup not found (set FWUP=)"
command -v kas >/dev/null || die "kas not found on PATH"
# The source bundle belongs to a release. A --dev archive goes to one bench
# and publishes nothing, so it builds without the archiver.
SOURCE_BUNDLE=0
if [ "$MODE" = release ] && [ -z "${FORGEFIRM_SOURCE_SKIP:-}" ]; then
SOURCE_BUNDLE=1
fi
build_images () {
echo "== building images =="
( cd "$REPO" && kas shell kas/forgefirm-glowforge.yml \
-c 'bitbake forgefirm-image forgefirm-image-dev' ) \
CFG="kas/forgefirm-glowforge.yml"
TARGETS="forgefirm-image forgefirm-image-dev"
if [ "$SOURCE_BUNDLE" = 1 ]; then
# The archiver rides the release build, so the source that the bundle
# publishes is the source that this image is built from. The overlay
# adds tasks and nothing else: no file of the root filesystem and no
# component changes, so the image manifest and the acceptance result
# are the same with it and without it. The boot loader, the kernel and
# the kernel module reach the machine outside the root filesystem, so
# they are named as targets as well (scripts/source-bundle.py,
# BUILD_TARGETS).
CFG="$CFG:kas/source-bundle.yml"
TARGETS="$TARGETS u-boot virtual/kernel kernel-module-glowforge"
fi
echo "== building images ($CFG) =="
( cd "$REPO" && kas shell "$CFG" -c "bitbake $TARGETS" ) \
|| die "bitbake failed"
}
@@ -257,6 +289,21 @@ NOTE
ASSETS="$ASSETS NO-ACCEPTANCE.txt"
PRERELEASE="--prerelease"
fi
# The source bundle. It is packed from the license manifests of THIS
# rootfs, and source-bundle.py stops the release when a recipe of the image
# has no source (see the site, Developers, "Release flow").
rm -f "$STAGE"/forgefirm-source-v*.tar.gz
if [ "$SOURCE_BUNDLE" = 1 ]; then
echo "== source bundle =="
python3 "$REPO/scripts/source-bundle.py" "$VERSION" \
--deploy "$DEPLOY_ROOT" --image-name "$(basename "$EXT4" .ext4)" \
--out "$STAGE" \
|| die "the source bundle failed"
ASSETS="$ASSETS forgefirm-source-v$VERSION.tar.gz"
else
warn "source bundle SKIPPED by FORGEFIRM_SOURCE_SKIP - this release publishes no source"
fi
# Every attached file is bound to the release by the sums, the artifact
# included.
( cd "$STAGE" && sha256sum $(echo "$ASSETS" | tr ' ' '\n' | grep -v '^sha256sums.txt$') > sha256sums.txt )
scripts/source-bundle.py
+598
View File
@@ -0,0 +1,598 @@
#!/usr/bin/env python3
# (C) Copyright 2020-2026
# Scott Wiederhold, s.e.wiederhold@gmail.com
# https://community.openglow.org
# SPDX-License-Identifier: MIT
#
# Pack the source bundle of a ForgeFIRM release.
#
# source-bundle.py <version> [--build] [--out DIR] [--deploy DIR]
# [--image-name NAME]
#
# The bundle holds the source of the software that the release image
# carries: the archives that the Yocto archiver writes when the build runs
# with kas/source-bundle.yml, the license manifests of the image, the
# license texts, the ForgeFIRM layers, and the kas configuration.
#
# What the bundle must hold comes from the image itself, not from a list in
# this file:
#
# deploy/licenses/<arch>/<image>/license.manifest root filesystem
# deploy/licenses/<arch>/<image>/image_license.manifest kernel, DTB, U-Boot
#
# Every recipe in those two files whose license is in
# COPYLEFT_LICENSE_INCLUDE (kas/source-bundle.yml, the list the archiver
# filters with) must have an archive. A recipe with no archive stops the
# script, because the release would go out with source missing.
#
# --build runs the archiver pass first. Without it the script packs what
# build/tmp/deploy/sources already holds. scripts/release.sh builds the
# images with the overlay merged, so the release path needs no second build.
import argparse
import gzip
import hashlib
import io
import json
import os
import re
import subprocess
import sys
import tarfile
import time
from fnmatch import fnmatchcase
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
KAS_OVERLAY = os.path.join(REPO, "kas", "source-bundle.yml")
KAS_CONFIG = "kas/forgefirm-glowforge.yml:kas/source-bundle.yml"
# The targets of the archiver pass. The image pulls in the source of
# everything in the root filesystem; the boot loader, the kernel and the
# kernel module reach the machine outside the root filesystem, so they are
# named too. scripts/release.sh builds the same list.
BUILD_TARGETS = ["forgefirm-image", "u-boot", "virtual/kernel",
"kernel-module-glowforge"]
# The layers that ForgeFIRM controls. The bundle carries them whole, because
# they are the recipes that build the image. The upstream layers are in
# metadata/LAYERS.txt with their revisions.
FORGEFIRM_LAYERS = [
("meta-forgefirm", os.path.join(REPO, "meta-forgefirm")),
("meta-glowforge-bsp", os.path.join(REPO, "..", "meta-openglow", "meta-glowforge-bsp")),
("meta-openglow-core", os.path.join(REPO, "..", "meta-openglow", "meta-openglow-core")),
]
# A recipe whose source is in the archive of another recipe. The archiver
# skips these by design (archiver.bbclass, include_package).
COVERED_BY = {
"glibc-locale": "glibc",
"gcc": "gcc-source-{pv}",
"gcc-runtime": "gcc-source-{pv}",
"gcc-sanitizers": "gcc-source-{pv}",
"libgcc": "gcc-source-{pv}",
}
# GitHub refuses a release asset above 2 GiB.
ASSET_WARN = 1536 * 1024 * 1024
ASSET_FAIL = 2048 * 1024 * 1024
EXCLUDE_NAMES = {".git", "__pycache__", ".pytest_cache"}
def die(msg):
print("SOURCE BUNDLE FAILED: %s" % msg, file=sys.stderr)
sys.exit(1)
def warn(msg):
print("WARNING: %s" % msg, file=sys.stderr)
# --- the policy of the archiver pass -----------------------------------------
def read_policy(path):
"""The license filter of kas/source-bundle.yml. The archiver pass and
this check read one list, so a recipe cannot be necessary here and
filtered out there."""
try:
text = open(path, encoding="utf-8").read()
except OSError as exc:
die("cannot read the archiver policy: %s" % exc)
policy = {}
for var in ("COPYLEFT_LICENSE_INCLUDE", "COPYLEFT_LICENSE_EXCLUDE",
"COPYLEFT_PN_INCLUDE"):
m = re.search(r'^\s*%s\s*=\s*"([^"]*)"\s*$' % var, text, re.M)
if not m:
die("%s is not in %s" % (var, path))
policy[var] = m.group(1).split()
return policy
def license_tokens(expr):
"""The license identifiers of a LICENSE expression, without the
operators and the brackets."""
return [t for t in re.split(r"[()&|;\s]+", expr or "") if t]
def source_needed(expr, policy):
"""True when a license of the expression is in the include list.
This follows the decision that oe.license.is_included makes for the
archiver: an OR selects the branch with the most included licenses, so
one included license anywhere in the expression archives the recipe.
A recipe that names an included license and an excluded one in the same
AND expression is the one case where the two decisions can differ: the
archiver drops it, and this check keeps it. That direction is the safe
one. The check then fails and asks for a decision, instead of letting
the release go out with source missing."""
return any(fnmatchcase(t, p) for t in license_tokens(expr)
for p in policy["COPYLEFT_LICENSE_INCLUDE"])
# --- the manifests of the image ----------------------------------------------
def parse_manifest(path, version_key):
"""A license manifest is blocks of `KEY: value` lines, one block for
each package. Returns {recipe: {"version", "licenses"}}; a recipe with
several packages collects every license that its packages name."""
recipes = {}
block = {}
def flush():
name = block.get("RECIPE NAME")
if not name:
return
entry = recipes.setdefault(name, {"version": block.get(version_key, ""),
"licenses": []})
lic = block.get("LICENSE")
if lic and lic not in entry["licenses"]:
entry["licenses"].append(lic)
for line in open(path, encoding="utf-8"):
line = line.strip()
if not line:
flush()
block = {}
continue
if ":" in line:
key, val = line.split(":", 1)
block[key.strip()] = val.strip()
flush()
return recipes
def image_recipes(licdir):
"""Every recipe that the image carries, from both manifests. Both are
necessary: license.manifest alone leaves out the kernel, the device
tree and the boot loader, which reach the machine outside the root
filesystem."""
recipes = {}
for fn, key in (("license.manifest", "PACKAGE VERSION"),
("image_license.manifest", "VERSION")):
path = os.path.join(licdir, fn)
if not os.path.isfile(path):
die("no %s in %s" % (fn, licdir))
for name, entry in parse_manifest(path, key).items():
recipes.setdefault(name, entry)
return recipes
# --- the archives ------------------------------------------------------------
def index_archives(srcdir, names):
"""{recipe: [(version, directory)]} from
deploy/sources/<arch>/<recipe>-<version>-<revision>. A directory belongs
to the longest recipe name that is a prefix of it, so python3-certifi
does not become python3."""
index = {}
if not os.path.isdir(srcdir):
return index
ordered = sorted(names, key=len, reverse=True)
for arch in sorted(os.listdir(srcdir)):
archdir = os.path.join(srcdir, arch)
if arch == "mirror" or not os.path.isdir(archdir):
continue
for pf in sorted(os.listdir(archdir)):
path = os.path.join(archdir, pf)
if not os.path.isdir(path):
continue
stem = re.sub(r"-r\d+$", "", pf)
for name in ordered:
if stem.startswith(name + "-"):
index.setdefault(name, []).append((stem[len(name) + 1:], path))
break
return index
def select_archives(index, recipe, version):
"""The archive directories of one recipe. The deploy directory keeps
the archives of earlier builds, so the version of the image decides:
the exact version first, then a version that starts with it (the kernel
carries the revision of its git source in PV), and every archive of the
recipe last. The name of an archive carries the epoch of the recipe
(`1_0.1.4`) and a license manifest does not, so the epoch comes off
before the comparison."""
found = [(re.sub(r"^\d+_", "", v), p) for v, p in index.get(recipe, [])]
for pick in ([p for v, p in found if v == version],
[p for v, p in found if v.startswith(version)],
[p for _, p in found]):
if pick:
return sorted(pick)
return []
def archive_files(path):
"""The files of one archive directory."""
out = []
for root, dirs, files in os.walk(path):
dirs[:] = sorted(d for d in dirs if d not in EXCLUDE_NAMES)
for fn in sorted(files):
full = os.path.join(root, fn)
if os.path.isfile(full) and not os.path.islink(full):
out.append(full)
return out
# --- the layers --------------------------------------------------------------
def git_out(args, cwd):
try:
return subprocess.check_output(["git"] + args, cwd=cwd,
stderr=subprocess.DEVNULL).decode().strip()
except (subprocess.CalledProcessError, OSError):
return ""
def layer_records(repo):
"""Every layer checkout of the build, with its remote and its
revision."""
roots = [("forgefirm", repo),
("meta-openglow", os.path.join(repo, "..", "meta-openglow"))]
layers_dir = os.path.join(repo, "layers")
if os.path.isdir(layers_dir):
for name in sorted(os.listdir(layers_dir)):
path = os.path.join(layers_dir, name)
if os.path.isdir(os.path.join(path, ".git")):
roots.append((name, path))
out = []
for name, path in roots:
if not os.path.isdir(path):
continue
out.append({
"checkout": name,
"url": git_out(["remote", "get-url", "origin"], path),
"revision": git_out(["rev-parse", "HEAD"], path),
"branch": git_out(["rev-parse", "--abbrev-ref", "HEAD"], path),
"modified": bool(git_out(["status", "--porcelain"], path)),
})
return out
def tar_tree(name, path):
"""One layer as a reproducible tar.gz in memory."""
files = []
for root, dirs, fns in os.walk(path):
dirs[:] = sorted(d for d in dirs if d not in EXCLUDE_NAMES)
files += [os.path.join(root, f) for f in sorted(fns)
if not f.endswith((".pyc", ".pyo"))]
buf = io.BytesIO()
with gzip.GzipFile(fileobj=buf, mode="wb", compresslevel=6, mtime=0) as gz:
with tarfile.open(fileobj=gz, mode="w", format=tarfile.GNU_FORMAT) as tar:
for f in sorted(files):
if os.path.islink(f):
continue
arc = "%s/%s" % (name, os.path.relpath(f, path).replace(os.sep, "/"))
info = tar.gettarinfo(f, arcname=arc)
info.mtime, info.uid, info.gid = 0, 0, 0
info.uname, info.gname = "", ""
with open(f, "rb") as fh:
tar.addfile(info, fh)
return buf.getvalue()
# --- the bundle --------------------------------------------------------------
class Bundle:
"""The files of the bundle: a name inside the archive with either a path
on disk or the bytes to write."""
def __init__(self, root):
self.root = root
self.members = []
def add_file(self, name, path):
self.members.append((name, path, None))
def add_bytes(self, name, data):
self.members.append((name, None, data.encode("utf-8")
if isinstance(data, str) else data))
def add_tree(self, name, path):
for root, dirs, files in os.walk(path):
dirs[:] = sorted(d for d in dirs if d not in EXCLUDE_NAMES)
for fn in sorted(files):
full = os.path.join(root, fn)
if os.path.islink(full) or fn.endswith((".pyc", ".pyo")):
continue
self.add_file("%s/%s" % (name, os.path.relpath(full, path).replace(os.sep, "/")),
full)
def checksums(self):
lines = []
for name, path, data in sorted(self.members):
h = hashlib.sha256()
if path is not None:
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
else:
h.update(data)
lines.append("%s %s\n" % (h.hexdigest(), name))
return "".join(lines)
def write(self, out):
"""One reproducible tar.gz: sorted names, no timestamps, no
owners."""
self.add_bytes("sha256sums.txt", self.checksums())
with open(out, "wb") as raw:
with gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=6, mtime=0) as gz:
with tarfile.open(fileobj=gz, mode="w", format=tarfile.GNU_FORMAT) as tar:
for name, path, data in sorted(self.members):
arc = "%s/%s" % (self.root, name)
if path is not None:
info = tar.gettarinfo(path, arcname=arc)
info.mtime, info.uid, info.gid, info.mode = 0, 0, 0, 0o644
info.uname, info.gname = "", ""
with open(path, "rb") as f:
tar.addfile(info, f)
else:
info = tarfile.TarInfo(arc)
info.size, info.mtime, info.mode = len(data), 0, 0o644
tar.addfile(info, io.BytesIO(data))
README = """\
# ForgeFIRM {version} - source
This archive holds the source of the software in ForgeFIRM {version}.
ForgeFIRM is built with the Yocto Project. The archive holds one directory
for each recipe of the release image:
sources/<recipe>-<version>-<revision>/
A recipe directory holds the upstream source as upstream publishes it, the
patches that the recipe applies, the `series` file that gives their order
and their strip level, and the recipe with its includes. A recipe that gets
its source from git holds a tar of the checkout at the pinned revision.
`SOURCES.txt` maps each recipe of the image to its directory.
`MANIFEST.json` holds the same information for a program to read.
## What the archive holds
| Path | Content |
|---|---|
| `sources/` | The source of each recipe of the image. |
| `licenses/license.manifest` | Every package of the root filesystem with its license. |
| `licenses/image_license.manifest` | The kernel, the device tree and the boot loader. |
| `licenses/texts/` | The license text of each recipe. |
| `metadata/LAYERS.txt` | Each layer of the build with its remote and its revision. |
| `metadata/kas/` | The kas configuration that builds the image. |
| `metadata/meta-*.tar.gz` | The ForgeFIRM layers, whole. |
| `metadata/forgefirm-manifest.json` | The build identity of the release image. |
| `sha256sums.txt` | The checksum of every file above. |
The archive does not hold the upstream layers (poky, meta-openembedded,
meta-freescale, meta-freescale-distro). They are public git repositories.
`metadata/LAYERS.txt` and the kas lock file in `metadata/kas/` give the
revision of each one.
## How to build the image again
1. Get the ForgeFIRM repository. The kas configuration in `metadata/kas/`
names every layer and revision.
2. Install kas.
3. Build:
cd forgefirm
kas build kas/forgefirm-glowforge.yml
The build documentation is at
https://docs.forgefirm.org/developers/building/.
## The ForgeFIRM components
The components that OpenGlow writes are in this archive too. Their
repositories are at https://github.com/openglow-org/.
"""
def main():
ap = argparse.ArgumentParser(
description="Pack the source bundle of a ForgeFIRM release")
ap.add_argument("version", help="release version, without the leading v")
ap.add_argument("--build", action="store_true",
help="run the archiver build pass before packing")
ap.add_argument("--deploy", default=os.path.join(REPO, "build", "tmp", "deploy"),
help="the deploy directory of the build")
ap.add_argument("--image", default="forgefirm-image-glowforge.rootfs",
help="the link name of the release image")
ap.add_argument("--image-name",
help="the exact image name (IMAGE_NAME) that the release is "
"cut from; its license manifests decide what the bundle holds")
ap.add_argument("--out",
help="output directory (default: release-staging/v<version>)")
args = ap.parse_args()
version = args.version.lstrip("v")
out_dir = args.out or os.path.join(REPO, "release-staging", "v" + version)
policy = read_policy(KAS_OVERLAY)
if args.build:
cmd = ["kas", "shell", KAS_CONFIG, "-c", "bitbake " + " ".join(BUILD_TARGETS)]
print("== archiver pass: %s ==" % " ".join(cmd))
if subprocess.call(cmd, cwd=REPO) != 0:
die("the archiver build pass failed")
# The license manifests must belong to the image that the release
# carries. The link name follows the last build of any image, so the
# exact image name decides.
licroot = os.path.join(args.deploy, "licenses")
archs = sorted(os.listdir(licroot)) if os.path.isdir(licroot) else []
name = args.image_name
if not name:
for arch in archs:
cand = os.path.join(licroot, arch, args.image)
if os.path.islink(cand):
name = os.path.basename(os.readlink(cand).rstrip("/"))
break
if not name:
die("cannot find %s under %s (name the image with --image-name)"
% (args.image, licroot))
licdir = None
for arch in archs:
cand = os.path.join(licroot, arch, name)
if os.path.isdir(cand) and not os.path.islink(cand):
licdir = cand
break
if not licdir:
die("no license manifest directory for image '%s' under %s" % (name, licroot))
print("image: %s" % name)
print("licenses: %s" % licdir)
recipes = image_recipes(licdir)
needed = {}
for recipe, entry in recipes.items():
# A recipe with several packages can carry several license
# expressions. They are kept side by side, because joining them with
# an operator would change what they say.
expr = " ; ".join(entry["licenses"])
if source_needed(expr, policy) or \
any(fnmatchcase(recipe, p) for p in policy["COPYLEFT_PN_INCLUDE"]):
needed[recipe] = dict(entry, license=expr)
print("recipes: %d in the image, %d with source in the bundle"
% (len(recipes), len(needed)))
srcdir = os.path.join(args.deploy, "sources")
lookup = set(needed) | {COVERED_BY[r].format(pv=e["version"])
for r, e in needed.items() if r in COVERED_BY}
index = index_archives(srcdir, lookup)
# Every recipe that needs source must have an archive.
missing, records = [], []
for recipe in sorted(needed):
entry = needed[recipe]
holder, note = recipe, None
dirs = select_archives(index, recipe, entry["version"])
if not dirs and recipe in COVERED_BY:
alt = COVERED_BY[recipe].format(pv=entry["version"])
dirs = select_archives(index, alt, entry["version"])
if dirs:
holder, note = alt, "the source is in the archive of %s" % alt
if not dirs:
missing.append((recipe, entry["version"], entry["license"]))
continue
records.append({"recipe": recipe, "version": entry["version"],
"license": entry["license"], "holder": holder,
"archives": [os.path.basename(d) for d in dirs],
"note": note, "dirs": dirs})
if missing:
print("\nrecipes of the image with no source archive:", file=sys.stderr)
for recipe, ver, lic in missing:
print(" %-28s %-18s %s" % (recipe, ver, lic), file=sys.stderr)
die("%d recipe(s) have no source. Run the build with "
"kas/source-bundle.yml, or name the recipe in COPYLEFT_PN_INCLUDE "
"in that file." % len(missing))
# --- collect ---
bundle = Bundle("forgefirm-source-v" + version)
packed, total = set(), 0
for rec in records:
for d in rec["dirs"]:
if d in packed:
continue
packed.add(d)
for f in archive_files(d):
total += os.path.getsize(f)
bundle.add_file("sources/%s/%s"
% (os.path.basename(d),
os.path.relpath(f, d).replace(os.sep, "/")), f)
bundle.add_file("licenses/license.manifest",
os.path.join(licdir, "license.manifest"))
if os.path.isfile(os.path.join(licdir, "image_license.manifest")):
bundle.add_file("licenses/image_license.manifest",
os.path.join(licdir, "image_license.manifest"))
for recipe in sorted(recipes):
for arch in archs:
texts = os.path.join(licroot, arch, recipe)
if os.path.isdir(texts) and not os.path.islink(texts):
bundle.add_tree("licenses/texts/%s" % recipe, texts)
break
for fn in sorted(os.listdir(os.path.join(REPO, "kas"))):
if fn.endswith(".yml"):
bundle.add_file("metadata/kas/%s" % fn, os.path.join(REPO, "kas", fn))
# The build identity of the image the bundle answers for. Its absence
# means the deploy directory no longer holds that image, so the bundle
# would speak for a build that is not there.
image_manifest = os.path.join(args.deploy, "images", "glowforge",
name + ".forgefirm-manifest.json")
if not os.path.isfile(image_manifest):
die("no manifest for image '%s' at %s" % (name, image_manifest))
bundle.add_file("metadata/forgefirm-manifest.json", image_manifest)
layers = layer_records(REPO)
table = ["%-22s %-56s %-42s %s" % ("CHECKOUT", "URL", "REVISION", "BRANCH")]
for lay in layers:
table.append("%-22s %-56s %-42s %s%s"
% (lay["checkout"], lay["url"] or "-", lay["revision"] or "-",
lay["branch"] or "-", " (modified)" if lay["modified"] else ""))
bundle.add_bytes("metadata/LAYERS.txt", "\n".join(table) + "\n")
for layer, path in FORGEFIRM_LAYERS:
path = os.path.normpath(path)
if not os.path.isdir(path):
die("layer %s is not at %s" % (layer, path))
bundle.add_bytes("metadata/%s.tar.gz" % layer, tar_tree(layer, path))
table = ["ForgeFIRM v%s - the source of each recipe of the release image" % version,
"",
"%-28s %-20s %-46s %s" % ("RECIPE", "VERSION", "ARCHIVE", "LICENSE")]
for rec in records:
table.append("%-28s %-20s %-46s %s"
% (rec["recipe"], rec["version"], ",".join(rec["archives"]),
rec["license"]))
bundle.add_bytes("SOURCES.txt", "\n".join(table) + "\n")
bundle.add_bytes("README.md", README.format(version="v" + version))
bundle.add_bytes("MANIFEST.json", json.dumps({
"format": 1,
"release": "v" + version,
"image": name,
"policy": {"license_include": policy["COPYLEFT_LICENSE_INCLUDE"],
"license_exclude": policy["COPYLEFT_LICENSE_EXCLUDE"],
"recipe_include": policy["COPYLEFT_PN_INCLUDE"],
"recipe_types": ["target"]},
"recipes": [{k: v for k, v in rec.items() if k != "dirs"} for rec in records],
# The other recipes of the image, with the license that keeps them
# out. The record says what was decided, not only what was packed.
"recipes_without_source": [
{"recipe": r, "version": recipes[r]["version"],
"license": " ; ".join(recipes[r]["licenses"])}
for r in sorted(set(recipes) - set(needed))],
"layers": layers,
}, indent=1, sort_keys=True) + "\n")
os.makedirs(out_dir, exist_ok=True)
out = os.path.join(out_dir, "forgefirm-source-v%s.tar.gz" % version)
print("packing: %d archives, %d files, %.1f MiB of source"
% (len(packed), len(bundle.members), total / 1048576.0))
start = time.time()
bundle.write(out)
size = os.path.getsize(out)
print("== %s (%.1f MiB, %.0f s) ==" % (out, size / 1048576.0, time.time() - start))
if size >= ASSET_FAIL:
die("the bundle is %.2f GiB. A release asset of GitHub must stay below 2 GiB."
% (size / 1073741824.0))
if size >= ASSET_WARN:
warn("the bundle is %.2f GiB, close to the 2 GiB limit of a release asset"
% (size / 1073741824.0))
return 0
if __name__ == "__main__":
sys.exit(main())