From 38eb5bf1934ba175149c6e5a8257639f58436d89 Mon Sep 17 00:00:00 2001 From: Alexander Bachmann Date: Tue, 12 Nov 2024 00:08:55 +0100 Subject: [PATCH 1/4] hack: add separate function to create common container engine args Add a separate function to create common container engine arguments to reduce complexity in container_build function. Signed-off-by: Alexander Bachmann --- hack/build-image | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/hack/build-image b/hack/build-image index feec9df..92d6aee 100755 --- a/hack/build-image +++ b/hack/build-image @@ -219,17 +219,25 @@ def container_build(cli, target): # --arch is not provided. So if the target arch and the host_arch # are the same, skip passing the extra argument. args.append(f"--arch={target.arch}") + run(cli, args + create_common_container_engine_args(cli, target), check=True) + +def create_common_container_engine_args(cli, target): + args = [] + pkgs_from = PACKAGES_FROM[target.pkg_source] + if pkgs_from: + args.append(f"--build-arg=INSTALL_PACKAGES_FROM={pkgs_from}") + if cli.extra_build_arg: args.extend(cli.extra_build_arg) + for tname in target.all_names(baseless=cli.without_repo_bases): args.append("-t") args.append(tname) + args.append("-f") args.append(target_containerfile(target)) args.append(kind_source_dir(target.name)) - args = [str(a) for a in args] - run(cli, args, check=True) - + return [str(a) for a in args] def container_push(cli, push_name): """Construct and execute a command to push a container image.""" From c03dca9470b4bee390ef92db2f4dd80111219d0f Mon Sep 17 00:00:00 2001 From: Alexander Bachmann Date: Tue, 12 Nov 2024 00:11:41 +0100 Subject: [PATCH 2/4] hack: use buildx to cross-build images with docker When Docker is used and the target architecture differs from the host architecture, buildx is used for cross-building. Depending on the container engine and the target architecture, multiple commands need to be executed to prepare the environment, build the image, and clean up artifacts afterward. To streamline this process, a tasks array (an array of lambdas) has been introduced, its elements are executed sequencally at the end of the container_build function. Signed-off-by: Alexander Bachmann --- hack/build-image | 53 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/hack/build-image b/hack/build-image index 92d6aee..34e8530 100755 --- a/hack/build-image +++ b/hack/build-image @@ -203,23 +203,42 @@ def container_engine(cli): def container_build(cli, target): """Construct and execute a command to build the target container image.""" - args = [container_engine(cli), "build"] - pkgs_from = PACKAGES_FROM[target.pkg_source] - if pkgs_from: - args.append(f"--build-arg=INSTALL_PACKAGES_FROM={pkgs_from}") - # docker doesn't currently support alt. architectures - if "docker" in args[0]: - if target.arch != host_arch(): - raise RuntimeError("Docker does not support --arch") - elif target.arch != host_arch() or FORCE_ARCH_FLAG: - # We've noticed a few small quirks when using podman with the --arch - # option. The main issue is that building the client image works - # but then the toolbox image fails because it somehow doesn't see - # the image we just built as usable. This doesn't happen when - # --arch is not provided. So if the target arch and the host_arch - # are the same, skip passing the extra argument. - args.append(f"--arch={target.arch}") - run(cli, args + create_common_container_engine_args(cli, target), check=True) + eng = container_engine(cli) + tasks = [] + + # For docker cross-builds we need to use buildx + if "docker" in eng and target.arch != host_arch(): + args = [eng, "buildx"] + + # Docker's default builder only supports the host architecture. + # Therefore, we need to create a new builder to support other + # architectures, and we must ensure we start with a fresh builder + # that does not contain any images from previous builds. + tasks.append(lambda : run(cli, args + ["rm", target.flat_name()], check=False)) + tasks.append(lambda : run(cli, args + ["create", f"--name={target.flat_name()}"], check=True)) + + tasks.append(lambda : run(cli, args + [ + "build", + f"--builder={target.flat_name()}", + f"--platform=linux/{target.arch}", + "--load"] + create_common_container_engine_args(cli, target), check=True)) + + tasks.append(lambda : run(cli, args + ["rm", target.flat_name()], check=True)) + else: + args = [eng, "build"] + if target.arch != host_arch() or FORCE_ARCH_FLAG: + # We've noticed a few small quirks when using podman with the --arch + # option. The main issue is that building the client image works + # but then the toolbox image fails because it somehow doesn't see + # the image we just built as usable. This doesn't happen when + # --arch is not provided. So if the target arch and the host_arch + # are the same, skip passing the extra argument. + args += [f"--arch={target.arch}"] + + tasks.append(lambda : run(cli, args + create_common_container_engine_args(cli, target), check=True)) + + for task in tasks: + task() def create_common_container_engine_args(cli, target): args = [] From 4ec2571086f9c2ffbb4c6f2e3e87ec7fe1375966 Mon Sep 17 00:00:00 2001 From: Alexander Bachmann Date: Tue, 22 Oct 2024 23:28:43 +0200 Subject: [PATCH 3/4] workflows: build and push fedora arm64 images Extend the build matrix to include fedora arm64 images for samba-server, samba-ad-server and samba-client. Signed-off-by: Alexander Bachmann --- .github/workflows/container-image.yml | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 1affbee..3e75c4b 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -54,6 +54,8 @@ jobs: - package_source: devbuilds os: centos arch: amd64 + - os: fedora + arch: arm64 runs-on: ubuntu-latest env: BUILDAH_FORMAT: oci @@ -82,6 +84,9 @@ jobs: # the distro packages for centos do not include an ad-dc - package_source: default os: centos + include: + - os: fedora + arch: arm64 runs-on: ubuntu-latest env: BUILDAH_FORMAT: oci @@ -102,6 +107,9 @@ jobs: matrix: os: [centos, fedora, opensuse] arch: [amd64] + include: + - os: fedora + arch: arm64 runs-on: ubuntu-latest env: BUILDAH_FORMAT: oci @@ -250,11 +258,21 @@ jobs: with: image: "samba-server:default-fedora-amd64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch server default-fedora-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-server:default-fedora-arm64" + container_engine: ${{ env.CONTAINER_CMD }} - name: Fetch server nightly-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "samba-server:nightly-fedora-amd64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch server nightly-fedora-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-server:nightly-fedora-arm64" + container_engine: ${{ env.CONTAINER_CMD }} - name: Fetch server nightly-centos-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 with: @@ -271,17 +289,32 @@ jobs: with: image: "samba-ad-server:default-fedora-amd64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch ad-server default-fedora-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-ad-server:default-fedora-arm64" + container_engine: ${{ env.CONTAINER_CMD }} - name: Fetch ad-server nightly-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "samba-ad-server:nightly-fedora-amd64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch ad-server nightly-fedora-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-ad-server:nightly-fedora-arm64" + container_engine: ${{ env.CONTAINER_CMD }} # (client images) - name: Fetch client default-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "samba-client:default-fedora-amd64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch client default-fedora-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-client:default-fedora-arm64" + container_engine: ${{ env.CONTAINER_CMD }} # (toolbox images) - name: Fetch toolbox default-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 @@ -297,12 +330,17 @@ jobs: --repo-base=${REPO_BASE} --no-distro-qualified -i samba-server:default-fedora-amd64 + -i samba-server:default-fedora-arm64 -i samba-server:nightly-fedora-amd64 + -i samba-server:nightly-fedora-arm64 -i samba-server:nightly-centos-amd64 -i samba-server:devbuilds-centos-amd64 -i samba-ad-server:default-fedora-amd64 + -i samba-ad-server:default-fedora-arm64 -i samba-ad-server:nightly-fedora-amd64 + -i samba-ad-server:nightly-fedora-arm64 -i samba-client:default-fedora-amd64 + -i samba-client:default-fedora-arm64 -i samba-toolbox:default-fedora-amd64 - name: Push images run: > @@ -313,10 +351,13 @@ jobs: --push-state=exists --push-selected-tags=mixed -i ${REPO_BASE}/samba-server:default-fedora-amd64 + -i ${REPO_BASE}/samba-server:default-fedora-arm64 -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 + -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-amd64 + -i ${REPO_BASE}/samba-client:default-fedora-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64 From e49ca064d7f5cdb5db303e77a4d0ae06c7c8d75d Mon Sep 17 00:00:00 2001 From: Alexander Bachmann Date: Tue, 12 Nov 2024 01:00:06 +0100 Subject: [PATCH 4/4] workflows: build and push opensuse arm64 images Extend the build matrix to include opensuse arm64 images for samba-server, samba-ad-server and samba-client. Signed-off-by: Alexander Bachmann --- .github/workflows/container-image.yml | 38 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 3e75c4b..64cb25d 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -45,17 +45,17 @@ jobs: matrix: package_source: [default, nightly] os: [centos, fedora, opensuse] - arch: [amd64] + arch: [amd64, arm64] exclude: # there are no nightly packages for opensuse - package_source: nightly os: opensuse + - os: centos + arch: arm64 include: - package_source: devbuilds os: centos arch: amd64 - - os: fedora - arch: arm64 runs-on: ubuntu-latest env: BUILDAH_FORMAT: oci @@ -76,7 +76,7 @@ jobs: matrix: package_source: [default, nightly] os: [centos, fedora, opensuse] - arch: [amd64] + arch: [amd64, arm64] exclude: # there are no nightly packages for opensuse - package_source: nightly @@ -84,8 +84,7 @@ jobs: # the distro packages for centos do not include an ad-dc - package_source: default os: centos - include: - - os: fedora + - os: centos arch: arm64 runs-on: ubuntu-latest env: @@ -106,9 +105,9 @@ jobs: strategy: matrix: os: [centos, fedora, opensuse] - arch: [amd64] - include: - - os: fedora + arch: [amd64, arm64] + exclude: + - os: centos arch: arm64 runs-on: ubuntu-latest env: @@ -263,6 +262,11 @@ jobs: with: image: "samba-server:default-fedora-arm64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch server default-opensuse-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-server:default-opensuse-arm64" + container_engine: ${{ env.CONTAINER_CMD }} - name: Fetch server nightly-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 with: @@ -294,6 +298,11 @@ jobs: with: image: "samba-ad-server:default-fedora-arm64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch ad-server default-opensuse-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-ad-server:default-opensuse-arm64" + container_engine: ${{ env.CONTAINER_CMD }} - name: Fetch ad-server nightly-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 with: @@ -315,6 +324,11 @@ jobs: with: image: "samba-client:default-fedora-arm64" container_engine: ${{ env.CONTAINER_CMD }} + - name: Fetch client default-opensuse-arm64 + uses: ishworkh/container-image-artifact-download@v2.0.0 + with: + image: "samba-client:default-opensuse-arm64" + container_engine: ${{ env.CONTAINER_CMD }} # (toolbox images) - name: Fetch toolbox default-fedora-amd64 uses: ishworkh/container-image-artifact-download@v2.0.0 @@ -331,16 +345,19 @@ jobs: --no-distro-qualified -i samba-server:default-fedora-amd64 -i samba-server:default-fedora-arm64 + -i samba-server:default-opensuse-arm64 -i samba-server:nightly-fedora-amd64 -i samba-server:nightly-fedora-arm64 -i samba-server:nightly-centos-amd64 -i samba-server:devbuilds-centos-amd64 -i samba-ad-server:default-fedora-amd64 -i samba-ad-server:default-fedora-arm64 + -i samba-ad-server:default-opensuse-arm64 -i samba-ad-server:nightly-fedora-amd64 -i samba-ad-server:nightly-fedora-arm64 -i samba-client:default-fedora-amd64 -i samba-client:default-fedora-arm64 + -i samba-client:default-opensuse-arm64 -i samba-toolbox:default-fedora-amd64 - name: Push images run: > @@ -352,12 +369,15 @@ jobs: --push-selected-tags=mixed -i ${REPO_BASE}/samba-server:default-fedora-amd64 -i ${REPO_BASE}/samba-server:default-fedora-arm64 + -i ${REPO_BASE}/samba-server:default-opensuse-arm64 -i ${REPO_BASE}/samba-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-server:nightly-centos-amd64 -i ${REPO_BASE}/samba-server:devbuilds-centos-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-amd64 -i ${REPO_BASE}/samba-ad-server:default-fedora-arm64 + -i ${REPO_BASE}/samba-ad-server:default-opensuse-arm64 -i ${REPO_BASE}/samba-ad-server:nightly-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-amd64 -i ${REPO_BASE}/samba-client:default-fedora-arm64 + -i ${REPO_BASE}/samba-client:default-opensuse-arm64 -i ${REPO_BASE}/samba-toolbox:default-fedora-amd64