Image.yaml

name: Image
run-name: 'Image - ${{ inputs.oci-image-name || github.triggering_actor }} - ${{ github.ref }}'

on:
  push:
    paths:
      - "oci/*/image.y*ml"
      - "!oci/mock*"
  pull_request:
    paths:
      - "oci/*/image.y*ml"
      - "!oci/mock*"
  workflow_dispatch:
    inputs:
      oci-image-name:
        description: "OCI image to build and test"
        required: true
      b64-image-trigger:
        description: "(Base64 encoded) Pass the image.yaml as an argument"
        required: false
        type: string
      upload:
        description: "Upload image to GHCR"
        required: true
        type: boolean
        default: false
      external_ref_id:  # (1)
        description: 'Optional ID for unique run detection'
        required: false
        type: string
        default: "default-id"
  workflow_call:
    inputs:
      oci-image-name:
        description: "OCI image to build and test"
        type: string
        required: true
      upload:
        description: "Upload image to GHCR"
        required: true
        type: boolean
        default: false

env:
  VULNERABILITY_REPORT_SUFFIX: ".vulnerability-report.json"

jobs:
  prepare-build:
    runs-on: ubuntu-22.04
    name: Prepare build
    outputs:
      build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
      release-to: ${{ steps.prepare-matrix.outputs.release-to }}
      oci-img-path: ${{ steps.validate-image.outputs.img-path }}
      oci-img-name: ${{ steps.validate-image.outputs.img-name }}
    steps:
      - name: ${{ inputs.external_ref_id }}  # (2)
        if: ${{ github.event_name == 'workflow_dispatch' }}
        run: echo 'Started by ${{ inputs.external_ref_id }}' >> "$GITHUB_STEP_SUMMARY"

      - uses: actions/checkout@v4

      - name: Infer number of image triggers
        uses: tj-actions/changed-files@v35
        id: changed-files
        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
        with:
          separator: ","
          dir_names: "true"
          files: |
            oci/*/image.y*ml

      - name: Validate image from dispatch
        id: validate-image
        run: |
          set -ex

          # check if this is coming from a workflow dispatch/call
          # as checking github.event_name isn't reliable here
          if [ "${{ inputs.oci-image-name }}" != "" ]
          then
            img_path="oci/${{ inputs.oci-image-name }}"
          else
            img_path="${{ steps.changed-files.outputs.all_changed_files }}"
            occurrences="${img_path//[^,]}"
            if [ ${#occurrences} -ne 0 ]
            then
              echo "ERR: can only build 1 image at a time, but trying to trigger ${img_path}"
              exit 1
            fi
          fi
          test -d "${img_path}"

          echo "img-name=$(basename ${img_path})" >> "$GITHUB_OUTPUT"
          echo "img-path=${img_path}" >> "$GITHUB_OUTPUT"

      - name: Validate access to triggered image
        uses: ./.github/actions/validate-actor
        if: ${{ github.repository == 'canonical/oci-factory' }}
        with:
          admin-only: true
          image-path: ${{ steps.validate-image.outputs.img-path }}
          github-token: ${{ secrets.ROCKSBOT_TOKEN }}

      - name: Use custom image trigger
        if: ${{ inputs.b64-image-trigger != '' }}
        run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ steps.validate-image.outputs.img-path }}/image.yaml

      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"

      - run: pip install -r src/image/requirements.txt

      - name: Validate and prepare builds matrix
        id: prepare-matrix
        env:
          DATA_DIR: "revision-data"
        run: |
          mkdir ${{ env.DATA_DIR }}

          ./src/image/prepare_single_image_build_matrix.py \
            --oci-path ${{ steps.validate-image.outputs.img-path }} \
            --revision-data-dir ${{ env.DATA_DIR }}

  run-build:
    needs: [prepare-build]
    strategy:
      fail-fast: true
      matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
    uses: ./.github/workflows/Build-Rock.yaml
    with:
      oci-archive-name: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
      oci-factory-path: ${{ matrix.path }}
      rock-name: ${{ matrix.name }}
      rock-repo: ${{ matrix.source }}
      rock-repo-commit: ${{ matrix.commit }}
      rockfile-directory: ${{ matrix.directory }}
    secrets: inherit

  test:
    needs: [prepare-build, run-build]
    name: Test
    strategy:
      fail-fast: true
      matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
    uses: ./.github/workflows/Tests.yaml
    with:
      oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}"
      oci-image-path: "oci/${{ matrix.name }}"
      test-from: "cache"
      cache-key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}        
    secrets: inherit

  
  prepare-upload:
    runs-on: ubuntu-22.04
    needs: [prepare-build, run-build, test]
    name: Prepare upload
    if: ${{ inputs.upload || (github.ref_name == 'main' && github.event_name == 'push') }}
    env:
      OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
      OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
      OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
      OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
      OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
      IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
      SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
      DATA_DIR: "revision-data"
    outputs:
      build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
      revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
    steps:
      - uses: actions/checkout@v4

      - name: Use custom image trigger
        if: ${{ inputs.b64-image-trigger != '' }}
        run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml

      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"

      - run: |
          ./src/uploads/requirements.sh
          pip install -r src/image/requirements.txt -r src/uploads/requirements.txt

      - name: Upload the lockfile for the image
        id: swift-lock
        run: |
          ./src/uploads/swift_lockfile_lock.sh \
            ${{ needs.prepare-build.outputs.oci-img-name }}

      # Here starts the critical section, have to be executed in sequence outside of matrix.
      - name: Get next revision number
        id: get-next-revision
        run: ./src/image/define_image_revision.sh

      - name: Prepare builds matrix for upload
        id: prepare-matrix
        run: |
          set -ex

          mkdir ${{ env.DATA_DIR }}

          ./src/image/prepare_single_image_build_matrix.py \
            --oci-path ${{ needs.prepare-build.outputs.oci-img-path }} \
            --revision-data-dir ${{ env.DATA_DIR }} \
            --next-revision ${{ steps.get-next-revision.outputs.revision }} \
            --infer-image-track

          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}-$(date +%s)" >> "$GITHUB_OUTPUT"

      - name: Preempt Swift slot
        run: |
          ./src/uploads/preempt_swift_slots.sh ${{ env.DATA_DIR }}

      # Here leaves the critical section.
      # The lock will be removed even if the steps above fail,
      # or the workflow is cancelled.
      - name: Remove the lockfile for the image
        # Failing to lock the swift container can mean there are multiple
        # workflows trying to upload the same image at the same time.
        # Therefore we should not remove the lockfile if the swift lock failed.
        if: ${{ always() && steps.swift-lock.outcome != 'failure' }}
        run: |
          ./src/uploads/swift_lockfile_unlock.sh \
            ${{ needs.prepare-build.outputs.oci-img-name }}

      # The revision files have to be sanitised before merging,
      # since the `track` field should not be present.
      - name: Sanitise revision files
        run: |
          set -ex
          for revision_file in `ls ${{ env.DATA_DIR }}`
          do
            jq 'del(.track, .base)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
            mv ${{ env.DATA_DIR }}/$revision_file.tmp ${{ env.DATA_DIR }}/$revision_file
          done

      - uses: actions/cache/save@v4
        with:
          path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
          key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}

  upload:
    runs-on: ubuntu-22.04
    needs: [prepare-build, prepare-upload]
    name: Upload
    strategy:
      fail-fast: true
      matrix: ${{ fromJSON(needs.prepare-upload.outputs.build-matrix) }}
    env:
      OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
    outputs:
      artefacts-hashes: ${{ steps.artefacts-hashes.outputs.hashes }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Setup environment
        env:
          ROCKS_DEV_LP_SSH_PRIVATE: ${{ secrets.ROCKS_DEV_LP_SSH_PRIVATE }}
          ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
          CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
          # CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
          SKOPEO_BRANCH: "v1.9.1"
          SKOPEO_URL: "https://github.com/containers/skopeo"
        run: |
          ./src/image/requirements.sh
          ./src/uploads/requirements.sh
          pip install -r src/uploads/requirements.txt -r src/image/requirements.txt

      - uses: actions/cache/restore@v4
        with:
          path: ${{ env.OCI_ARCHIVE_NAME }}
          key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
          fail-on-cache-miss: true

      - name: Name output artefact
        id: rename-oci-archive
        run: |
          # Rename the OCI archive tarball
          canonical_tag="${{ matrix.track }}_${{ matrix.revision }}"
          name="${{ matrix.name }}_${canonical_tag}"
          mv ${{ env.OCI_ARCHIVE_NAME }} $name

          echo "name=${name}" >> "$GITHUB_OUTPUT"
          echo "canonical-tag=${canonical_tag}" >> "$GITHUB_OUTPUT"

      - uses: actions/cache/save@v4
        with:
          path: ${{ steps.rename-oci-archive.outputs.name }}
          key: ${{ github.run_id }}-${{ steps.rename-oci-archive.outputs.name }}

      - name: Install Syft
        uses: anchore/sbom-action/download-syft@v0
        with:
          syft-version: "v0.75.0"

      - name: Infer architectures
        id: get-arches
        run: |
          set -ex
          arches=$(skopeo inspect --raw \
            oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
            | jq -r 'if has("manifests") then .manifests[].platform.architecture else "${{ runner.arch }}" end' \
            | jq -nRcr '[inputs] | join(",")')

          echo "Detected architectures: ${arches}"
          echo "arches='${arches}'" >> "$GITHUB_OUTPUT"

      - name: Generate SBOMs
        id: generate-sboms
        run: |
          set -ex
          echo "Generating SBOMs for ${{ steps.get-arches.outputs.arches }}"

          IFS=',' read -ra arch_list <<< ${{ steps.get-arches.outputs.arches }}
          for arch in "${arch_list[@]}"; do
            if [[ "${arch}" == "unknown" ]]; then
              continue
            fi

            echo "Generate SBOM for ${arch}"

            skopeo --override-arch $arch copy \
              oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
              oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch}

            syft oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch} \
              -o spdx-json \
              --file ${{ steps.rename-oci-archive.outputs.name }}.${arch}.sbom.spdx.json
          done

          all_sboms_zip="${{ steps.rename-oci-archive.outputs.name }}.sbom.spdx.zip"
          zip "${all_sboms_zip}" ${{ steps.rename-oci-archive.outputs.name }}.*.sbom.spdx.json

          echo "sboms=${all_sboms_zip}" >> "$GITHUB_OUTPUT"

      - name: Fetch vulnerability artifacts for hashing
        uses: actions/cache/restore@v4
        with:
          path: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
          key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
          fail-on-cache-miss: true

      # https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md
      - name: Calculate artefacts hashes
        id: artefacts-hashes
        env:
          VULN_REPORT: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
          SBOMS: ${{ steps.generate-sboms.outputs.sboms }}
          OCI_IMAGE_ARCHIVE: ${{ steps.rename-oci-archive.outputs.name }}
        run: |
          set -ex
          echo "hashes=$(sha256sum ${{ env.VULN_REPORT }} ${{ env.OCI_IMAGE_ARCHIVE }} ${{ env.SBOMS }} | base64 -w0)"

      - name: Login to GHCR
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Upload SBOM
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.generate-sboms.outputs.sboms }}
          path: ${{ steps.generate-sboms.outputs.sboms }}
          if-no-files-found: error

      - name: Upload image
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.rename-oci-archive.outputs.name }}
          path: ${{ steps.rename-oci-archive.outputs.name }}
          if-no-files-found: error

      - name: Upload to GHCR
        id: upload-image
        env:
          GHCR_REPO: ghcr.io/${{ github.repository_owner }}/oci-factory
          GHCR_USERNAME: ${{ github.actor }}
          GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
        run: |
          oci_images="${PWD}/images-oci"
          rm -fr $oci_images
          mkdir -p $oci_images  

          tar -xf ${{ steps.rename-oci-archive.outputs.name }} -C $oci_images

          source="oci:${oci_images}"

          digest="$(skopeo inspect $source | jq -r .Digest)"
          echo "digest=$digest" >> $GITHUB_OUTPUT

          ./src/image/tag_and_publish.sh "${source}" \
            ${{ matrix.name }} \
            ${{ steps.rename-oci-archive.outputs.canonical-tag }}

      - name: Upload build metadata to Swift
        env:
          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
          IMAGE_NAME: ${{ matrix.name }}
          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
        run: |
          jq --arg base "${{ matrix.base }}" \
            --arg digest "${{ steps.upload-image.outputs.digest }}" \
            '. + {base: $base, digest: $digest}' <<< '${{ toJSON(matrix) }}' > build_metadata.json
          ./src/uploads/upload_to_swift.sh \
            ${{ matrix.name }} \
            ${{ matrix.track }} \
            ${{ matrix.revision }} \
            build_metadata.json \
            ${{ steps.generate-sboms.outputs.sboms }} \
            ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}

      - name: Create Git tag
        uses: rickstaa/action-create-tag@v1
        with:
          tag: "${{ steps.rename-oci-archive.outputs.name }}"
          message: "upload(${{ matrix.name }}): Build and upload new image revision ${{ matrix.revision }}"
          force_push_tag: true
          github_token: ${{ secrets.ROCKSBOT_TOKEN }}

  # We need this job because we want to make the releases all in one go,
  # so that we know which revisions to release, and so that we can update
  # and commit the _releases.json file in a single commit, outside a matrix job
  prepare-releases:
    name: Prepare releases
    needs: [prepare-build, prepare-upload, upload]
    runs-on: ubuntu-22.04
    if: ${{ needs.prepare-build.outputs.release-to != '' }}
    concurrency:
      group: ${{ needs.prepare-build.outputs.oci-img-path }}
      cancel-in-progress: true
    env:
      REVISION_DATA_DIR: revision-data
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"

      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Use custom image trigger
        if: ${{ inputs.b64-image-trigger != '' }}
        run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml

      - uses: actions/cache/restore@v4
        with:
          path: ${{ env.REVISION_DATA_DIR }}
          key: ${{ needs.prepare-upload.outputs.revision-data-cache-key }}
          fail-on-cache-miss: true

      - run: pip install -r src/image/requirements.txt

      - name: Merge release requests
        id: merge-release-requests
        env:
          PYTHONUNBUFFERED: 1
        run: |
          set -ex
          for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
          do
            ret=0
            release=$(jq -er .release < ${{ env.REVISION_DATA_DIR }}/$revision_file) || ret=1
            
            if [ $ret -eq 1 ]
            then
              echo "Revision $revision_file not marked for release"
              continue
            fi

            echo "Merge revision $revision_file with requested releases"

            ./src/image/merge_release_info.py \
              --image-trigger "${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml" \
              --revision-data-file "${{ env.REVISION_DATA_DIR }}/${revision_file}"
          done

      - uses: actions/cache/save@v4
        with:
          path: ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
          key: ${{ github.run_id }}-image-trigger

  release:
    name: Release
    needs: [prepare-build, prepare-releases]
    uses: ./.github/workflows/Release.yaml
    with:
      oci-image-name: "${{ needs.prepare-build.outputs.oci-img-name }}"
      image-trigger-cache-key: "${{ github.run_id }}-image-trigger"
    secrets: inherit

  generate-provenance:
    name: Generate SLSA provenance
    needs: [upload]
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/download-artifact@v4
        id: download
        with:
          path: artifacts

      - name: Generate provenance
        uses: philips-labs/slsa-provenance-action@v0.8.0
        with:
          command: generate
          subcommand: files
          arguments: --artifact-path ${{ steps.download.outputs.download-path }} --output-path provenance.json

      # - uses: sigstore/cosign-installer@v3.0.1
      #   with:
      #     cosign-release: 'v2.0.0'

      # - name: Sign provenance file
      #   env:
      #     PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
      #     COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
      #     PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
      #   run: |
      #     echo "${PRIVATE_KEY}" > cosign.key
      #     echo "${PUBLIC_KEY}" > cosign.pub
      #     cosign sign-blob --key cosign.key --output-signature provenance.json.sig provenance.json

      # - name: Upload verification keys
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: verify
      #     path: |
      #       cosign.pub
      #       provenance.json.sig
      #     if-no-files-found: error

      # - name: Generate checksums
      #   working-directory: ${{ steps.download.outputs.download-path }}
      #   run: sha256sum * > SHA256SUMS || true

      # - name: Upload SHA256SUMS file
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: SHA256SUMS
      #     path: SHA256SUMS
      #     if-no-files-found: error

      - name: Upload provenance file
        uses: actions/upload-artifact@v4
        with:
          name: provenance
          path: provenance.json
          if-no-files-found: error

  # generate-provenance:
  #   name: Generate SLSA provenance
  #   needs: [upload]
  #   permissions:
  #     actions: read # Needed for detection of GitHub Actions environment.
  #     id-token: write # Needed for provenance signing and ID
  #     contents: write # Needed for release uploads
  #   uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.5.0
  #   with:
  #     base64-subjects: "${{ needs.upload.outputs.artefacts-hashes }}"

  notify:
    runs-on: ubuntu-22.04
    name: Notify
    needs: [prepare-build, run-build, upload, prepare-releases, generate-provenance]
    if: ${{ !cancelled() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.x'
          
      - name: Summarize workflow failure message
        id: get-summary
        run: |
          echo '${{ toJson(needs) }}' > jobs.json
          ./src/notifications/summarize_workflow_results.py --jobs-file jobs.json

      - name: Get contacts for ${{ needs.prepare-build.outputs.oci-img-name }}
        id: get-contacts
        working-directory: ${{ needs.prepare-build.outputs.oci-img-path }}
        run: |
          mm_channels=$(yq -r '.notify | ."mattermost-channels" | join(",")' < contacts.y*ml)
          echo "mattermost-channels=${mm_channels}" >> "$GITHUB_OUTPUT"

      - name: Notify via Mattermost
        env:
          MM_BOT_TOKEN: ${{ secrets.MM_BOT_TOKEN }}
          FINAL_STATUS: failure
          MM_SERVER: ${{ secrets.MM_SERVER }}
          URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          SUMMARY: ${{ steps.get-summary.outputs.summary }}
          FOOTER: "Triggered by ${{ github.triggering_actor }}. Ref: ${{ github.ref }}. Attempts: ${{ github.run_attempt }}"
          TITLE: '${{ needs.prepare-build.outputs.oci-img-name }}: failed to build->upload->release'
        run: |
          for channel in $(echo ${{ steps.get-contacts.outputs.mattermost-channels }} | tr ',' ' ')
          do
            MM_CHANNEL_ID="${channel}" ./src/notifications/send_to_mattermost.sh
          done