Image - zhijie-yang - refs/pull/249/merge #1271

Workflow file for this run

	name: Image
	run-name: 'Image - ${{ inputs.oci-image-name \|\| github.triggering_actor }} - ${{ github.ref }}'

	on:
	push:
	paths:
	- "oci//image.yml"
	- "!oci/mock*"
	pull_request:
	paths:
	- "oci//image.yml"
	- "!oci/mock*"
	workflow_dispatch:
	inputs:
	oci-image-name:
	description: "OCI image to build and test"
	required: true
	b64-image-trigger:
	description: "(Base64 encoded) Pass the image.yaml as an argument"
	required: false
	type: string
	upload:
	description: "Upload image to GHCR"
	required: true
	type: boolean
	default: false
	external_ref_id: # (1)
	description: 'Optional ID for unique run detection'
	required: false
	type: string
	default: "default-id"
	workflow_call:
	inputs:
	oci-image-name:
	description: "OCI image to build and test"
	type: string
	required: true
	upload:
	description: "Upload image to GHCR"
	required: true
	type: boolean
	default: false

	env:
	VULNERABILITY_REPORT_SUFFIX: ".vulnerability-report.json"

	jobs:
	prepare-build:
	runs-on: ubuntu-22.04
	name: Prepare build
	outputs:
	build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
	release-to: ${{ steps.prepare-matrix.outputs.release-to }}
	oci-img-path: ${{ steps.validate-image.outputs.img-path }}
	oci-img-name: ${{ steps.validate-image.outputs.img-name }}
	steps:
	- name: ${{ inputs.external_ref_id }} # (2)
	if: ${{ github.event_name == 'workflow_dispatch' }}
	run: echo 'Started by ${{ inputs.external_ref_id }}' >> "$GITHUB_STEP_SUMMARY"

	- uses: actions/checkout@v4

	- name: Infer number of image triggers
	uses: tj-actions/changed-files@v35
	id: changed-files
	if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
	with:
	separator: ","
	dir_names: "true"
	files: \|
	oci//image.yml

	- name: Validate image from dispatch
	id: validate-image
	run: \|
	set -ex

	# check if this is coming from a workflow dispatch/call
	# as checking github.event_name isn't reliable here
	if [ "${{ inputs.oci-image-name }}" != "" ]
	then
	img_path="oci/${{ inputs.oci-image-name }}"
	else
	img_path="${{ steps.changed-files.outputs.all_changed_files }}"
	occurrences="${img_path//[^,]}"
	if [ ${#occurrences} -ne 0 ]
	then
	echo "ERR: can only build 1 image at a time, but trying to trigger ${img_path}"
	exit 1
	fi
	fi
	test -d "${img_path}"

	echo "img-name=$(basename ${img_path})" >> "$GITHUB_OUTPUT"
	echo "img-path=${img_path}" >> "$GITHUB_OUTPUT"

	- name: Validate access to triggered image
	uses: ./.github/actions/validate-actor
	if: ${{ github.repository == 'canonical/oci-factory' }}
	with:
	admin-only: true
	image-path: ${{ steps.validate-image.outputs.img-path }}
	github-token: ${{ secrets.ROCKSBOT_TOKEN }}

	- name: Use custom image trigger
	if: ${{ inputs.b64-image-trigger != '' }}
	run: echo ${{ inputs.b64-image-trigger }} \| base64 -d > ${{ steps.validate-image.outputs.img-path }}/image.yaml

	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"

	- run: pip install -r src/image/requirements.txt

	- name: Validate and prepare builds matrix
	id: prepare-matrix
	env:
	DATA_DIR: "revision-data"
	run: \|
	mkdir ${{ env.DATA_DIR }}

	./src/image/prepare_single_image_build_matrix.py \
	--oci-path ${{ steps.validate-image.outputs.img-path }} \
	--revision-data-dir ${{ env.DATA_DIR }}

	run-build:
	needs: [prepare-build]
	strategy:
	fail-fast: true
	matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
	uses: ./.github/workflows/Build-Rock.yaml
	with:
	oci-archive-name: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
	oci-factory-path: ${{ matrix.path }}
	rock-name: ${{ matrix.name }}
	rock-repo: ${{ matrix.source }}
	rock-repo-commit: ${{ matrix.commit }}
	rockfile-directory: ${{ matrix.directory }}
	secrets: inherit

	test:
	needs: [prepare-build, run-build]
	name: Test
	strategy:
	fail-fast: true
	matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
	uses: ./.github/workflows/Tests.yaml
	with:
	oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}"
	oci-image-path: "oci/${{ matrix.name }}"
	test-from: "cache"
	cache-key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
	secrets: inherit


	prepare-upload:
	runs-on: ubuntu-22.04
	needs: [prepare-build, run-build, test]
	name: Prepare upload
	if: ${{ inputs.upload \|\| (github.ref_name == 'main' && github.event_name == 'push') }}
	env:
	OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
	OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
	OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
	OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
	OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
	IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
	SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
	DATA_DIR: "revision-data"
	outputs:
	build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
	revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
	steps:
	- uses: actions/checkout@v4

	- name: Use custom image trigger
	if: ${{ inputs.b64-image-trigger != '' }}
	run: echo ${{ inputs.b64-image-trigger }} \| base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml

	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"

	- run: \|
	./src/uploads/requirements.sh
	pip install -r src/image/requirements.txt -r src/uploads/requirements.txt

	- name: Upload the lockfile for the image
	id: swift-lock
	run: \|
	./src/uploads/swift_lockfile_lock.sh \
	${{ needs.prepare-build.outputs.oci-img-name }}

	# Here starts the critical section, have to be executed in sequence outside of matrix.
	- name: Get next revision number
	id: get-next-revision
	run: ./src/image/define_image_revision.sh

	- name: Prepare builds matrix for upload
	id: prepare-matrix
	run: \|
	set -ex

	mkdir ${{ env.DATA_DIR }}

	./src/image/prepare_single_image_build_matrix.py \
	--oci-path ${{ needs.prepare-build.outputs.oci-img-path }} \
	--revision-data-dir ${{ env.DATA_DIR }} \
	--next-revision ${{ steps.get-next-revision.outputs.revision }} \
	--infer-image-track

	echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}-$(date +%s)" >> "$GITHUB_OUTPUT"

	- name: Preempt Swift slot
	run: \|
	./src/uploads/preempt_swift_slots.sh ${{ env.DATA_DIR }}

	# Here leaves the critical section.
	# The lock will be removed even if the steps above fail,
	# or the workflow is cancelled.
	- name: Remove the lockfile for the image
	# Failing to lock the swift container can mean there are multiple
	# workflows trying to upload the same image at the same time.
	# Therefore we should not remove the lockfile if the swift lock failed.
	if: ${{ always() && steps.swift-lock.outcome != 'failure' }}
	run: \|
	./src/uploads/swift_lockfile_unlock.sh \
	${{ needs.prepare-build.outputs.oci-img-name }}

	# The revision files have to be sanitised before merging,
	# since the `track` field should not be present.
	- name: Sanitise revision files
	run: \|
	set -ex
	for revision_file in `ls ${{ env.DATA_DIR }}`
	do
	jq 'del(.track, .base)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
	mv ${{ env.DATA_DIR }}/$revision_file.tmp ${{ env.DATA_DIR }}/$revision_file
	done

	- uses: actions/cache/save@v4
	with:
	path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
	key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}

	upload:
	runs-on: ubuntu-22.04
	needs: [prepare-build, prepare-upload]
	name: Upload
	strategy:
	fail-fast: true
	matrix: ${{ fromJSON(needs.prepare-upload.outputs.build-matrix) }}
	env:
	OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
	outputs:
	artefacts-hashes: ${{ steps.artefacts-hashes.outputs.hashes }}
	steps:
	- uses: actions/checkout@v4

	- uses: actions/setup-python@v5
	with:
	python-version: "3.10"

	- name: Setup environment
	env:
	ROCKS_DEV_LP_SSH_PRIVATE: ${{ secrets.ROCKS_DEV_LP_SSH_PRIVATE }}
	ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
	CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
	# CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
	SKOPEO_BRANCH: "v1.9.1"
	SKOPEO_URL: "https://github.com/containers/skopeo"
	run: \|
	./src/image/requirements.sh
	./src/uploads/requirements.sh
	pip install -r src/uploads/requirements.txt -r src/image/requirements.txt

	- uses: actions/cache/restore@v4
	with:
	path: ${{ env.OCI_ARCHIVE_NAME }}
	key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
	fail-on-cache-miss: true

	- name: Name output artefact
	id: rename-oci-archive
	run: \|
	# Rename the OCI archive tarball
	canonical_tag="${{ matrix.track }}_${{ matrix.revision }}"
	name="${{ matrix.name }}_${canonical_tag}"
	mv ${{ env.OCI_ARCHIVE_NAME }} $name

	echo "name=${name}" >> "$GITHUB_OUTPUT"
	echo "canonical-tag=${canonical_tag}" >> "$GITHUB_OUTPUT"

	- uses: actions/cache/save@v4
	with:
	path: ${{ steps.rename-oci-archive.outputs.name }}
	key: ${{ github.run_id }}-${{ steps.rename-oci-archive.outputs.name }}

	- name: Install Syft
	uses: anchore/sbom-action/download-syft@v0
	with:
	syft-version: "v0.75.0"

	- name: Infer architectures
	id: get-arches
	run: \|
	set -ex
	arches=$(skopeo inspect --raw \
	oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
	\| jq -r 'if has("manifests") then .manifests[].platform.architecture else "${{ runner.arch }}" end' \
	\| jq -nRcr '[inputs] \| join(",")')

	echo "Detected architectures: ${arches}"
	echo "arches='${arches}'" >> "$GITHUB_OUTPUT"

	- name: Generate SBOMs
	id: generate-sboms
	run: \|
	set -ex
	echo "Generating SBOMs for ${{ steps.get-arches.outputs.arches }}"

	IFS=',' read -ra arch_list <<< ${{ steps.get-arches.outputs.arches }}
	for arch in "${arch_list[@]}"; do
	if [[ "${arch}" == "unknown" ]]; then
	continue
	fi

	echo "Generate SBOM for ${arch}"

	skopeo --override-arch $arch copy \
	oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
	oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch}

	syft oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch} \
	-o spdx-json \
	--file ${{ steps.rename-oci-archive.outputs.name }}.${arch}.sbom.spdx.json
	done

	all_sboms_zip="${{ steps.rename-oci-archive.outputs.name }}.sbom.spdx.zip"
	zip "${all_sboms_zip}" ${{ steps.rename-oci-archive.outputs.name }}.*.sbom.spdx.json

	echo "sboms=${all_sboms_zip}" >> "$GITHUB_OUTPUT"

	- name: Fetch vulnerability artifacts for hashing
	uses: actions/cache/restore@v4
	with:
	path: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
	key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
	fail-on-cache-miss: true

	# https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md
	- name: Calculate artefacts hashes
	id: artefacts-hashes
	env:
	VULN_REPORT: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
	SBOMS: ${{ steps.generate-sboms.outputs.sboms }}
	OCI_IMAGE_ARCHIVE: ${{ steps.rename-oci-archive.outputs.name }}
	run: \|
	set -ex
	echo "hashes=$(sha256sum ${{ env.VULN_REPORT }} ${{ env.OCI_IMAGE_ARCHIVE }} ${{ env.SBOMS }} \| base64 -w0)"

	- name: Login to GHCR
	uses: docker/login-action@v2
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Upload SBOM
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.generate-sboms.outputs.sboms }}
	path: ${{ steps.generate-sboms.outputs.sboms }}
	if-no-files-found: error

	- name: Upload image
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.rename-oci-archive.outputs.name }}
	path: ${{ steps.rename-oci-archive.outputs.name }}
	if-no-files-found: error

	- name: Upload to GHCR
	id: upload-image
	env:
	GHCR_REPO: ghcr.io/${{ github.repository_owner }}/oci-factory
	GHCR_USERNAME: ${{ github.actor }}
	GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	oci_images="${PWD}/images-oci"
	rm -fr $oci_images
	mkdir -p $oci_images

	tar -xf ${{ steps.rename-oci-archive.outputs.name }} -C $oci_images

	source="oci:${oci_images}"

	digest="$(skopeo inspect $source \| jq -r .Digest)"
	echo "digest=$digest" >> $GITHUB_OUTPUT

	./src/image/tag_and_publish.sh "${source}" \
	${{ matrix.name }} \
	${{ steps.rename-oci-archive.outputs.canonical-tag }}

	- name: Upload build metadata to Swift
	env:
	OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
	OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
	OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
	OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
	OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
	IMAGE_NAME: ${{ matrix.name }}
	SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
	run: \|
	jq --arg base "${{ matrix.base }}" \
	--arg digest "${{ steps.upload-image.outputs.digest }}" \
	'. + {base: $base, digest: $digest}' <<< '${{ toJSON(matrix) }}' > build_metadata.json
	./src/uploads/upload_to_swift.sh \
	${{ matrix.name }} \
	${{ matrix.track }} \
	${{ matrix.revision }} \
	build_metadata.json \
	${{ steps.generate-sboms.outputs.sboms }} \
	${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}

	- name: Create Git tag
	uses: rickstaa/action-create-tag@v1
	with:
	tag: "${{ steps.rename-oci-archive.outputs.name }}"
	message: "upload(${{ matrix.name }}): Build and upload new image revision ${{ matrix.revision }}"
	force_push_tag: true
	github_token: ${{ secrets.ROCKSBOT_TOKEN }}

	# We need this job because we want to make the releases all in one go,
	# so that we know which revisions to release, and so that we can update
	# and commit the _releases.json file in a single commit, outside a matrix job
	prepare-releases:
	name: Prepare releases
	needs: [prepare-build, prepare-upload, upload]
	runs-on: ubuntu-22.04
	if: ${{ needs.prepare-build.outputs.release-to != '' }}
	concurrency:
	group: ${{ needs.prepare-build.outputs.oci-img-path }}
	cancel-in-progress: true
	env:
	REVISION_DATA_DIR: revision-data
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"

	- uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Use custom image trigger
	if: ${{ inputs.b64-image-trigger != '' }}
	run: echo ${{ inputs.b64-image-trigger }} \| base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml

	- uses: actions/cache/restore@v4
	with:
	path: ${{ env.REVISION_DATA_DIR }}
	key: ${{ needs.prepare-upload.outputs.revision-data-cache-key }}
	fail-on-cache-miss: true

	- run: pip install -r src/image/requirements.txt

	- name: Merge release requests
	id: merge-release-requests
	env:
	PYTHONUNBUFFERED: 1
	run: \|
	set -ex
	for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
	do
	ret=0
	release=$(jq -er .release < ${{ env.REVISION_DATA_DIR }}/$revision_file) \|\| ret=1

	if [ $ret -eq 1 ]
	then
	echo "Revision $revision_file not marked for release"
	continue
	fi

	echo "Merge revision $revision_file with requested releases"

	./src/image/merge_release_info.py \
	--image-trigger "${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml" \
	--revision-data-file "${{ env.REVISION_DATA_DIR }}/${revision_file}"
	done

	- uses: actions/cache/save@v4
	with:
	path: ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
	key: ${{ github.run_id }}-image-trigger

	release:
	name: Release
	needs: [prepare-build, prepare-releases]
	uses: ./.github/workflows/Release.yaml
	with:
	oci-image-name: "${{ needs.prepare-build.outputs.oci-img-name }}"
	image-trigger-cache-key: "${{ github.run_id }}-image-trigger"
	secrets: inherit

	generate-provenance:
	name: Generate SLSA provenance
	needs: [upload]
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/download-artifact@v4
	id: download
	with:
	path: artifacts

	- name: Generate provenance
	uses: philips-labs/slsa-provenance-action@v0.8.0
	with:
	command: generate
	subcommand: files
	arguments: --artifact-path ${{ steps.download.outputs.download-path }} --output-path provenance.json

	# - uses: sigstore/cosign-installer@v3.0.1
	# with:
	# cosign-release: 'v2.0.0'

	# - name: Sign provenance file
	# env:
	# PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
	# COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
	# PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
	# run: \|
	# echo "${PRIVATE_KEY}" > cosign.key
	# echo "${PUBLIC_KEY}" > cosign.pub
	# cosign sign-blob --key cosign.key --output-signature provenance.json.sig provenance.json

	# - name: Upload verification keys
	# uses: actions/upload-artifact@v4
	# with:
	# name: verify
	# path: \|
	# cosign.pub
	# provenance.json.sig
	# if-no-files-found: error

	# - name: Generate checksums
	# working-directory: ${{ steps.download.outputs.download-path }}
	# run: sha256sum * > SHA256SUMS \|\| true

	# - name: Upload SHA256SUMS file
	# uses: actions/upload-artifact@v4
	# with:
	# name: SHA256SUMS
	# path: SHA256SUMS
	# if-no-files-found: error

	- name: Upload provenance file
	uses: actions/upload-artifact@v4
	with:
	name: provenance
	path: provenance.json
	if-no-files-found: error

	# generate-provenance:
	# name: Generate SLSA provenance
	# needs: [upload]
	# permissions:
	# actions: read # Needed for detection of GitHub Actions environment.
	# id-token: write # Needed for provenance signing and ID
	# contents: write # Needed for release uploads
	# uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.5.0
	# with:
	# base64-subjects: "${{ needs.upload.outputs.artefacts-hashes }}"

	notify:
	runs-on: ubuntu-22.04
	name: Notify
	needs: [prepare-build, run-build, upload, prepare-releases, generate-provenance]
	if: ${{ !cancelled() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
	steps:
	- uses: actions/checkout@v4

	- uses: actions/setup-python@v5
	with:
	python-version: '3.x'

	- name: Summarize workflow failure message
	id: get-summary
	run: \|
	echo '${{ toJson(needs) }}' > jobs.json
	./src/notifications/summarize_workflow_results.py --jobs-file jobs.json

	- name: Get contacts for ${{ needs.prepare-build.outputs.oci-img-name }}
	id: get-contacts
	working-directory: ${{ needs.prepare-build.outputs.oci-img-path }}
	run: \|
	mm_channels=$(yq -r '.notify \| ."mattermost-channels" \| join(",")' < contacts.y*ml)
	echo "mattermost-channels=${mm_channels}" >> "$GITHUB_OUTPUT"

	- name: Notify via Mattermost
	env:
	MM_BOT_TOKEN: ${{ secrets.MM_BOT_TOKEN }}
	FINAL_STATUS: failure
	MM_SERVER: ${{ secrets.MM_SERVER }}
	URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
	SUMMARY: ${{ steps.get-summary.outputs.summary }}
	FOOTER: "Triggered by ${{ github.triggering_actor }}. Ref: ${{ github.ref }}. Attempts: ${{ github.run_attempt }}"
	TITLE: '${{ needs.prepare-build.outputs.oci-img-name }}: failed to build->upload->release'
	run: \|
	for channel in $(echo ${{ steps.get-contacts.outputs.mattermost-channels }} \| tr ',' ' ')
	do
	MM_CHANNEL_ID="${channel}" ./src/notifications/send_to_mattermost.sh
	done

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Image - zhijie-yang - refs/pull/249/merge #1271

Workflow file

Image - zhijie-yang - refs/pull/249/merge #1271

Jobs

Run details

Workflow file for this run