zmk/.github/workflows/build.yml

name: Build

on:
  push:
    paths:
      - ".github/workflows/build.yml"
      - "app/**"
  pull_request:
    paths:
      - ".github/workflows/build.yml"
      - "app/**"
  schedule:
    - cron: "22 4 * * *"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'schedule' }}
  cancel-in-progress: true

permissions: {}

jobs:
  build:
    if: ${{ always() }}
    runs-on: ubuntu-latest
    container:
      image: docker.io/zmkfirmware/zmk-build-arm:4.1
    needs: compile-matrix
    strategy:
      matrix:
        include: ${{ fromJSON(needs.compile-matrix.outputs.include-list) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Cache west modules
        uses: actions/cache@v4
        env:
          cache-name: cache-zephyr-modules
        with:
          path: |
            modules/
            tools/
            zephyr/
            bootloader/
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('app/west.yml') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
        timeout-minutes: 2
        continue-on-error: true
      - name: Initialize workspace (west init)
        run: west init -l app
      - name: Update modules (west update)
        run: west update --fetch-opt=--filter=tree:0
      - name: Export Zephyr CMake package (west zephyr-export)
        run: west zephyr-export
      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "14.x"
      - name: Install @actions/artifact
        run: npm install @actions/artifact
      - name: Build
        uses: actions/github-script@v7
        id: boards-list
        with:
          script: |
            const fs = require('fs');

            const execSync = require('child_process').execSync;

            const buildShieldArgs = JSON.parse(`${{ matrix.shieldArgs }}`);

            let error = false;

            for (const shieldArgs of buildShieldArgs) {
              try {
                console.log(`::group::${{ matrix.board}} ${shieldArgs.shield} Build`)

                const output = execSync(`west build -s app -p -b ${{ matrix.board }} ${shieldArgs.snippet ? '-S ' + shieldArgs.snippet : ''} -- ${shieldArgs.shield ? '-DSHIELD="' + shieldArgs.shield + '"' : ''} ${shieldArgs['cmake-args'] || ''}`);

                console.log(output.toString());
              } catch (e) {
                console.error(`::error::Failed to build ${{ matrix.board }} ${shieldArgs.shield} ${shieldArgs['cmake-args']}`);
                console.error(e);
                error = true;
              } finally {
                console.log('::endgroup::');
              }
            }

            if (error) {
              throw new Error('Failed to build one or more configurations');
            }
      - name: Upload artifacts
        uses: actions/github-script@v7
        continue-on-error: ${{ github.event_name == 'pull_request' }}
        id: boards-upload
        with:
          script: |
            const fs = require('fs');
            const {default: artifact} = require('@actions/artifact');

            const buildShieldArgs = JSON.parse(`${{ matrix.shieldArgs }}`);

            let error = false;

            for (const shieldArgs of buildShieldArgs) {
              try {
                console.log(`::group::${{ matrix.board}} ${shieldArgs.shield} Upload`)

                const fileExtensions = ["hex", "uf2"];

                const files = fileExtensions
                  .map(extension => "build/zephyr/zmk." + extension)
                  .filter(path => fs.existsSync(path));

                const rootDirectory = 'build/zephyr';
                const options = {
                    continueOnError: true
                }

                const cmakeName = shieldArgs['cmake-args'] ? '-' + (shieldArgs.nickname || shieldArgs['cmake-args'].split(' ').join('')) : '';
                const artifactName = `${{ matrix.board }}${shieldArgs.shield ? '-' + shieldArgs.shield : ''}${cmakeName}-zmk`.replaceAll('/', '_');

                await artifact.uploadArtifact(artifactName, files, rootDirectory, options);
              } catch (e) {
                console.error(`::error::Failed to upload ${{ matrix.board }} ${shieldArgs.shield} ${shieldArgs['cmake-args']}`);
                console.error(e);
                error = true;
              } finally {
                console.log('::endgroup::');
              }
            }

            if (error) {
              throw new Error('Failed to build one or more configurations');
            }
  compile-matrix:
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    needs: [core-coverage, board-changes, nightly]
    outputs:
      include-list: ${{ steps.compile-list.outputs.result }}
    steps:
      - name: Join build lists
        uses: actions/github-script@v7
        id: compile-list
        with:
          script: |
            const coreCoverage = `${{ needs.core-coverage.outputs.core-include }}` || "[]";
            const boardChanges = `${{ needs.board-changes.outputs.boards-include }}` || "[]";
            const nightly = `${{ needs.nightly.outputs.nightly-include }}` || "[]";

            const combined = [
              ...JSON.parse(coreCoverage),
              ...JSON.parse(boardChanges),
              ...JSON.parse(nightly)
            ];
            const combinedUnique = [...new Map(combined.map(el => [JSON.stringify(el), el])).values()];

            const perBoard = {};

            for (const configuration of combinedUnique) {
              if (!perBoard[configuration.board])
                perBoard[configuration.board] = [];

              perBoard[configuration.board].push({
                shield: configuration.shield,
                'cmake-args': configuration['cmake-args'],
                snippet: configuration.snippet,
                nickname: configuration.nickname
              })
            }

            return Object.entries(perBoard).map(([board, shieldArgs]) => ({
              board,
              shieldArgs: JSON.stringify(shieldArgs),
            }));
  core-coverage:
    if: ${{ needs.get-changed-files.outputs.core-changes == 'true' }}
    runs-on: ubuntu-latest
    needs: get-changed-files
    outputs:
      core-include: ${{ steps.core-list.outputs.result }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "14.x"
      - name: Install js-yaml
        run: npm install js-yaml
      - uses: actions/github-script@v7
        id: core-list
        with:
          script: |
            const fs = require('fs');
            const yaml = require('js-yaml');

            const coreCoverage = yaml.load(fs.readFileSync('app/core-coverage.yml', 'utf8'));

            let include = coreCoverage.board.flatMap(board =>
              coreCoverage.shield.map(shield => ({ board, shield }))
            );

            return [...include, ...coreCoverage.include];
  board-changes:
    if: ${{ needs.get-changed-files.outputs.board-changes == 'true' }}
    runs-on: ubuntu-latest
    needs: [get-grouped-hardware, get-changed-files]
    outputs:
      boards-include: ${{ steps.boards-list.outputs.result }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "14.x"
      - name: Install js-yaml
        run: npm install js-yaml
      - uses: actions/github-script@v7
        id: boards-list
        with:
          script: |
            const fs = require('fs');
            const yaml = require('js-yaml');

            const changedFiles = JSON.parse(`${{ needs.get-changed-files.outputs.changed-files }}`);
            const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`);
            const boardChanges = new Set(changedFiles.filter(f => f.startsWith('app/boards')).map(f => f.split('/').slice(0, 4).join('/')));

            return (await Promise.all([...boardChanges].flatMap(async bc => {
              const globber = await glob.create(bc + "/*.zmk.yml");
              const files = await globber.glob();

              const aggregated = files.flatMap((f) =>
                yaml.loadAll(fs.readFileSync(f, "utf8"))
              );

              const boardAndShield = (b, s) => {
                if (s.siblings) {
                  return s.siblings.map(shield => ({
                    board: b.id,
                    shield,
                  }));
                } else {
                  return {
                    board: b.id,
                    shield: s.id
                  };
                }
              }

              return aggregated.flatMap(hm => {
                switch (hm.type) {
                  case "board":
                    if (hm.features && hm.features.includes("keys")) {
                      if (hm.siblings) {
                        return hm.siblings.map(board => ({
                          board,
                        }));
                      } else {
                        return {
                          board: hm.id
                        };
                      }
                    } else if (hm.exposes) {
                      return hm.exposes.flatMap(i =>
                        metadata.interconnects[i].shields.flatMap(s => boardAndShield(hm, s))
                      );
                    } else {
                      console.error("Board without keys or interconnect");
                      return [];
                    }
                    break;
                  case "shield":
                    if (hm.features && hm.features.includes("keys")) {
                      return hm.requires.flatMap(i =>
                        metadata.interconnects[i].boards.flatMap(b => boardAndShield(b, hm))
                      );
                    } else {
                      console.warn("Unhandled shield without keys");
                      return [];
                    }
                    break;
                  case "interconnect":
                    return [];
                }
              });
            }))).flat();
  nightly:
    if: ${{ github.event_name == 'schedule' && github.repository_owner == 'zmkfirmware' }}
    runs-on: ubuntu-latest
    needs: get-grouped-hardware
    outputs:
      nightly-include: ${{ steps.nightly-list.outputs.result }}
    steps:
      - name: Create nightly list
        uses: actions/github-script@v7
        id: nightly-list
        with:
          script: |
            const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`);

            let includeOnboard = metadata.onboard.flatMap(b => {
              if (b.siblings) {
                return b.siblings.map(board => ({
                  board,
                }));
              } else {
                return {
                  board: b.id,
                };
              }
            });

            let includeInterconnect = Object.values(metadata.interconnects).flatMap(i =>
              i.boards.flatMap(b =>
                i.shields.flatMap(s => {
                  if (s.siblings) {
                    return s.siblings.map(shield => ({
                      board: b.id,
                      shield,
                    }));
                  } else {
                    return {
                      board: b.id,
                      shield: s.id,
                    };
                  }
                })
              )
            );

            return [...includeOnboard, ...includeInterconnect];
  get-grouped-hardware:
    runs-on: ubuntu-latest
    outputs:
      organized-metadata: ${{ steps.organize-metadata.outputs.result }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "14.x"
      - name: Install js-yaml
        run: npm install js-yaml
      - name: Aggregate Metadata
        uses: actions/github-script@v7
        id: aggregate-metadata
        with:
          script: |
            const fs = require('fs');
            const yaml = require('js-yaml');

            const globber = await glob.create("app/boards/**/*.zmk.yml");
            const files = await globber.glob();

            const aggregated = files.flatMap((f) =>
              yaml.loadAll(fs.readFileSync(f, "utf8"))
            );

            return JSON.stringify(aggregated).replace(/\\/g,"\\\\").replace(/`/g,"\\`");
          result-encoding: string

      - name: Organize Metadata
        uses: actions/github-script@v7
        id: organize-metadata
        with:
          script: |
            const hardware = JSON.parse(`${{ steps.aggregate-metadata.outputs.result }}`);

            const grouped = hardware.reduce((agg, hm) => {
              switch (hm.type) {
                case "board":
                  if (hm.features && hm.features.includes("keys")) {
                    agg.onboard.push(hm);
                  } else if (hm.exposes) {
                    hm.exposes.forEach((element) => {
                      let ic = agg.interconnects[element] || {
                        boards: [],
                        shields: [],
                      };
                      ic.boards.push(hm);
                      agg.interconnects[element] = ic;
                    });
                  } else {
                    console.error("Board without keys or interconnect");
                  }
                  break;
                case "shield":
                  if (hm.features && hm.features.includes("keys")) {
                    hm.requires.forEach((id) => {
                      let ic = agg.interconnects[id] || { boards: [], shields: [] };
                      ic.shields.push(hm);
                      agg.interconnects[id] = ic;
                    });
                  }
                  break;
                case "interconnect":
                  let ic = agg.interconnects[hm.id] || { boards: [], shields: [] };
                  ic.interconnect = hm;
                  agg.interconnects[hm.id] = ic;
                  break;
              }
              return agg;
            },
            { onboard: [], interconnects: {} });

            return JSON.stringify(grouped).replace(/\\/g,"\\\\").replace(/`/g,"\\`");
          result-encoding: string
  get-changed-files:
    if: ${{ github.event_name != 'schedule' }}
    runs-on: ubuntu-latest
    outputs:
      changed-files: ${{ steps.changed-files.outputs.all_changed_files }}
      board-changes: ${{ steps.board-changes.outputs.result }}
      core-changes: ${{ steps.core-changes.outputs.result }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: tj-actions/changed-files@9200e69727eb73eb060652b19946b8a2fdfb654b # pin to v45.0.8 due to https://github.com/tj-actions/changed-files/issues/2463 https://www.stepsecurity.io/blog/harden-runner-detection-tj-actions-changed-files-action-is-compromised
        id: changed-files
        with:
          json: true
          escape_json: false
      - uses: actions/github-script@v7
        id: board-changes
        with:
          script: |
            const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all_changed_files }}`);
            const boardChanges = changedFiles.filter(f => f.startsWith('app/boards'));
            return boardChanges.length ? 'true' : 'false';
          result-encoding: string
      - uses: actions/github-script@v7
        id: core-changes
        with:
          script: |
            const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all_changed_files }}`);
            const boardChanges = changedFiles.filter(f => f.startsWith('app/boards'));
            const appChanges = changedFiles.filter(f => f.startsWith('app'));
            const ymlChanges = changedFiles.includes('.github/workflows/build.yml');
            return boardChanges.length < appChanges.length || ymlChanges ? 'true' : 'false';
          result-encoding: string