forgefirm/.github/workflows/forgetest-ci.yml

# forgetest: the release acceptance tool's host-side proof.
#
#  - unit tests: campaign rules, fingerprints, artifact build/verify (the
#    release gate's decision, including the negative fixtures), runner +
#    HTTP API end to end with a fake catalog and a fake bench tool, the
#    suites replayed on the machine's own log lines, and the check that
#    every log phrase the cloud suite greps for is one the pinned cloud
#    app can log (it reads the app sources from the manifest cache)
#  - the source bundle of a release: which recipe of the image needs
#    source, which archive answers for it, and the refusal that stops a
#    release with source missing
#  - shared UI files: theme.css and the vendored Bootstrap the page
#    carries are byte-identical to forgectrl's at its pinned revision
#  - the extension key: the public key the image installs for the official
#    tier is byte-identical to the one the extension catalog publishes
#    (keys/forgefirm-ext.pub on its main branch)
#  - the catalog's address: EXTPKG_INDEX_URL in the pinned forgectrl's
#    src/extpkg.h is the catalog's latest-release download of the file its
#    tools/publish.py passes to gh release create; read, never requested
#    (every request of it is counted as a download)
#  - coverage lint: every source path of every component pinned by the
#    recipes must be selected by some catalog test's coverage globs or by a
#    host-tested row naming the host test that proves it (the tree manifest
#    is generated from the pins with git - no Yocto build); enforced: an
#    uncovered path fails the job (https://docs.forgefirm.org/developers/testing/).

name: forgetest-ci

on:
  push:
    branches: [master]
    paths:
      - 'forgetest/**'
      - 'scripts/acceptance-gate.py'
      - 'scripts/manifest-from-tree.py'
      - 'scripts/source-bundle.py'
      - 'kas/source-bundle.yml'
      - 'meta-forgefirm/recipes-forgefirm/**'
      - '.github/workflows/forgetest-ci.yml'
  pull_request:
  workflow_dispatch:

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout forgefirm
        uses: actions/checkout@v4
        with:
          path: forgefirm

      - name: Checkout meta-openglow (sibling)
        uses: actions/checkout@v4
        with:
          repository: openglow-org/meta-openglow
          ref: scarthgap
          path: meta-openglow

      - name: Checkout forgefirm-extensions-catalog (sibling)
        uses: actions/checkout@v4
        with:
          repository: openglow-org/forgefirm-extensions-catalog
          ref: main
          path: forgefirm-extensions-catalog

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Tree manifest from the recipe pins (also fetches the pinned app sources the needle check reads)
        working-directory: forgefirm
        run: python scripts/manifest-from-tree.py --out tree-manifest.json

      - name: The image's extension key is the catalog's published key
        run: |
          image=forgefirm/meta-forgefirm/recipes-forgefirm/forgefirm-keys/files/ext/forgefirm-ext.pub
          published=forgefirm-extensions-catalog/keys/forgefirm-ext.pub
          for f in "$image" "$published"; do
            test -f "$f" || { echo "missing: $f"; exit 1; }
          done
          cmp -s "$image" "$published" || { echo "$image is not $published"; exit 1; }

      - name: The catalog's address in the pinned forgectrl is the file the catalog publishes (read, never requested)
        run: |
          python3 - <<'EOF'
          import ast, glob, os, re, subprocess, sys

          PREFIX = "https://github.com/openglow-org/forgefirm-extensions-catalog/releases/latest/download/"

          def fail(why):
              print(why)
              sys.exit(1)

          # forgectrl's pin, and its src/extpkg.h from the tree manifest's cache
          pin = open("forgefirm/meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl-pin.inc").read()
          m = re.search(r'^SRCREV = "([0-9a-f]{40})"$', pin, re.M)
          if not m:
              fail("forgectrl-pin.inc names no SRCREV")
          rev = m.group(1)
          header = None
          for repo in sorted(glob.glob("forgefirm/.manifest-cache/*")):
              p = subprocess.run(["git", "-C", repo, "show", rev + ":src/extpkg.h"], capture_output=True, text=True)
              if p.returncode == 0:
                  header = p.stdout
                  break
          if header is None:
              fail("forgectrl %s is not in forgefirm/.manifest-cache" % rev)
          m = re.search(r'#define\s+EXTPKG_INDEX_URL\s*(?:\\\n\s*)?"([^"]*)"', header)
          if not m:
              fail("forgectrl %s's src/extpkg.h defines no EXTPKG_INDEX_URL" % rev[:7])
          url = m.group(1)

          # the file the catalog's publish tool passes to gh release create
          tree = ast.parse(open("forgefirm-extensions-catalog/tools/publish.py").read())
          assigned = {n.targets[0].id: n.value for n in ast.walk(tree)
                      if isinstance(n, ast.Assign) and len(n.targets) == 1 and isinstance(n.targets[0], ast.Name)}

          def file_name(v):
              if isinstance(v, ast.Name):
                  v = assigned.get(v.id)
              if isinstance(v, ast.Call) and v.args:
                  v = v.args[-1]
              return os.path.basename(v.value) if isinstance(v, ast.Constant) and isinstance(v.value, str) else None

          names = [file_name(n.args[3]) if len(n.args) > 3 else None for n in ast.walk(tree)
                   if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id == "gh"
                   and [a.value for a in n.args[:2] if isinstance(a, ast.Constant)] == ["release", "create"]]
          if len(names) != 1 or not names[0]:
              fail("tools/publish.py passes no one file to gh release create: %s" % names)

          print("forgectrl %s: %s; the catalog publishes %s" % (rev[:7], url, names[0]))
          if url != PREFIX + names[0]:
              fail("the address is not %s" % (PREFIX + names[0]))
          EOF

      - name: Unit tests
        working-directory: forgefirm/forgetest
        # -B: no bytecode caches under the package directory (the recipe's fetch
        # checksum would otherwise move without a source change)
        run: python -B -m unittest discover -s tests -v

      - name: No undefined names (the live drills never execute on the host)
        working-directory: forgefirm/forgetest
        # pyflakes exits 1 for any message, unused imports included, so the
        # grep is the gate. A run stopped by SIGINT exits 1 too, and its
        # "... stopped" line fails the grep. Any other status (a usage
        # error, a signal pyflakes does not handle) fails the step and
        # prints the capture, and a capture that cannot be written fails it
        # too. pyflakes ends a SIGPIPE with status 1 and no line, and a
        # write to the capture file raises none.
        run: |
          python -m pip install pyflakes
          python -B -m pyflakes --version
          { python -B -m pyflakes forgetest tests || test $? -eq 1; } > pyflakes.txt 2>&1 || { cat pyflakes.txt; exit 1; }
          if grep -E "undefined name|Traceback|^\.\.\. stopped$" pyflakes.txt; then exit 1; fi

      - name: Shared UI files match forgectrl at its pinned revision
        working-directory: forgefirm
        run: python scripts/check-ui-vendor.py

      - name: Coverage lint (enforced)
        working-directory: forgefirm/forgetest
        # The step runs under bash -e with no pipefail of its own: set here,
        # the status of the step is the lint's, not tee's.
        run: |
          set -o pipefail
          python -B -m forgetest.coverage --manifest ../tree-manifest.json --enforce | tee "$GITHUB_STEP_SUMMARY"

      - name: Gate self-check on the tree manifest
        working-directory: forgefirm
        run: |
          # The gate must load the catalog and refuse a missing/unauthorized
          # artifact cleanly (exit 1/2, never a traceback).
          set +e
          echo '{}' > empty.json
          python scripts/acceptance-gate.py empty.json tree-manifest.json --quiet
          rc=$?
          set -e
          test "$rc" -eq 1