From 6ce747fae7709eedfccca768e306eed4d863d06b Mon Sep 17 00:00:00 2001 From: Elie CHARRA Date: Mon, 29 Jul 2024 11:35:08 +0200 Subject: [PATCH] feat: add new ansible build matrix --- .github/workflows/docker.yml | 381 +++++++++++++++++++++++++++++++++++ Dockerfile | 12 ++ README.md | 40 +++- build-matrix/go.mod | 11 + build-matrix/go.sum | 11 + build-matrix/main.go | 116 +++++++++++ build-matrix/main_test.go | 49 +++++ 7 files changed, 610 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 Dockerfile create mode 100644 build-matrix/go.mod create mode 100644 build-matrix/go.sum create mode 100755 build-matrix/main.go create mode 100644 build-matrix/main_test.go diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..a7d9834 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,381 @@ +name: Build docker images +on: + push: +# TODO Enable schedule + #schedule: + # At 00:00 every Sunday + #- cron: 0 0 * * 0 +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: true +jobs: + matrix: + name: Compute build matrix from pypi API + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: './build-matrix/go.mod' + cache-dependency-path: './build-matrix/go.mod' + + - name: Run matrix generator tests + working-directory: build-matrix + run: go test ./ + + # TODO Remove the fake matrix + - id: matrix + working-directory: build-matrix + run: | + MATRIX=$(go run ./) + echo ${MATRIX} | jq + echo 'matrix=[{"ansible":"10.2","additional_tags":["10"]},{"ansible":"10.1","additional_tags":[]}]' >> $GITHUB_OUTPUT + #echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + build: + needs: [ matrix ] + runs-on: ubuntu-latest + name: Build ansible ${{ matrix.versions.ansible }}-${{ matrix.target }}/${{ matrix.platform }} + permissions: + packages: write + contents: read + strategy: + fail-fast: false + matrix: + target: + - base + - aws + - gcp + platform: + - linux/amd64 + - linux/arm64 + versions: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Set up QEMU + if: matrix.platform == 'linux/arm64' + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Github Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Docker images + uses: docker/build-push-action@v6 + id: build + with: + pull: true + target: ${{ matrix.target }} + build-args: | + ANSIBLE_VERSION=${{ matrix.versions.ansible }} + platforms: ${{ matrix.platform }} + outputs: type=docker,dest=/tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar + tags: ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}-${{ github.sha }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }} + path: /tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar + retention-days: 1 + if-no-files-found: error + + test: + name: Test image ${{ matrix.versions.ansible }}-${{ matrix.target }}/${{ matrix.platform }} + needs: [ matrix, build ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: + - base + - aws + - gcp + platform: + - linux/amd64 + versions: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + export PLATFORM_PAIR=${platform//\//-} + echo "PLATFORM_PAIR=${PLATFORM_PAIR}" >> $GITHUB_ENV + echo "IMAGE=ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ matrix.target }}-${PLATFORM_PAIR}-${{ github.sha }}" >> $GITHUB_ENV + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }} + path: /tmp + + - name: Load image + run: | + docker load --input /tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar + docker image ls -a + + - name: Test ansible version + run: docker run --rm ${{ env.IMAGE }} ansible-community --version | grep 'Ansible community version ${{ matrix.versions.ansible }}' + + - name: Test aws flavor + if: matrix.target == 'aws' + run: | + docker run --rm ${{ env.IMAGE }} sh -c "python3 -c \"import boto3; print(boto3.__version__)\"" + + - name: Test gcp flavor + if: matrix.target == 'gcp' + run: | + docker run --rm ${{ env.IMAGE_NAME }} sh -c "python3 -c \"import google.auth; print(google.auth.__version__)\"" + + deploy: + name: Push image ${{ matrix.versions.ansible }} + needs: [ matrix, security, test ] + runs-on: ubuntu-latest + env: + AWS_REGION: "us-east-1" + strategy: + fail-fast: false + matrix: + versions: ${{ fromJson(needs.matrix.outputs.matrix) }} + if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ansible_build_matrix' }} # TODO(eliecharra): Remove condition + permissions: + id-token: write + packages: write + contents: read + steps: + - name: Download base/amd64 artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-base-linux-amd64 + path: /tmp + + - name: Download gcp/amd64 artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-gcp-linux-amd64 + path: /tmp + + - name: Download aws/amd64 artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-aws-linux-amd64 + path: /tmp + + - name: Download base/arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-base-linux-arm64 + path: /tmp + + - name: Download gcp/arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-gcp-linux-arm64 + path: /tmp + + - name: Download aws/arm64 artifact + uses: actions/download-artifact@v4 + with: + name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-aws-linux-arm64 + path: /tmp + + - name: Load image + run: | + docker load --input /tmp/${{ matrix.versions.ansible }}-base-linux-amd64.tar + docker load --input /tmp/${{ matrix.versions.ansible }}-gcp-linux-amd64.tar + docker load --input /tmp/${{ matrix.versions.ansible }}-aws-linux-amd64.tar + docker load --input /tmp/${{ matrix.versions.ansible }}-base-linux-arm64.tar + docker load --input /tmp/${{ matrix.versions.ansible }}-gcp-linux-arm64.tar + docker load --input /tmp/${{ matrix.versions.ansible }}-aws-linux-arm64.tar + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-duration-seconds: 900 + + - name: Login to Amazon ECR + run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${REPOSITORY_PATH} + env: + REPOSITORY_PATH: ${{ secrets.PUBLIC_RUNNER_ANSIBLE_ECR_REPOSITORY_URL }} + + - name: Log in to Github Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push images + env: + ECR_IMAGE: ${{ secrets.PUBLIC_RUNNER_ANSIBLE_ECR_REPOSITORY_URL }}/${{ github.repository }} + run: | + echo "Pushing ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-amd64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-base-linux-amd64-${{ github.sha }}\ + ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-amd64 + docker push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-amd64 + + echo "Pushing ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-amd64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64-${{ github.sha }}\ + ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-amd64 + docker push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-amd64 + + echo "Pushing ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-amd64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64-${{ github.sha }}\ + ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-amd64 + docker push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-amd64 + + echo "Pushing ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-arm64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-base-linux-arm64-${{ github.sha }}\ + ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-arm64 + docker push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-arm64 + + echo "Pushing ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-arm64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64-${{ github.sha }}\ + ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 + docker push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 + + echo "Pushing ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-arm64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64-${{ github.sha }}\ + ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-arm64 + docker push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-arm64 + + echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-base-linux-amd64-${{ github.sha }}\ + ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64 + docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64 + + echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64-${{ github.sha }}\ + ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64 + docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64 + + echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64-${{ github.sha }}\ + ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64 + docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64 + + echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-base-linux-arm64-${{ github.sha }}\ + ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64 + docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64 + + echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64-${{ github.sha }}\ + ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 + docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 + + echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64" + docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64-${{ github.sha }}\ + ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64 + docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64 + + echo "Create manifest ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}" + docker manifest create ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }} \ + --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64 \ + --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64 + docker manifest push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }} + + echo "Create manifest ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp" + docker manifest create ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp \ + --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 \ + --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64 + docker manifest push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp + + echo "Create manifest ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws" + docker manifest create ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws \ + --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64 \ + --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64 + docker manifest push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws + + echo "Create manifest ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}" + docker manifest create ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }} \ + --amend ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-arm64 \ + --amend ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-linux-amd64 + docker manifest push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }} + + echo "Create manifest ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp" + docker manifest create ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp \ + --amend ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 \ + --amend ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp-linux-amd64 + docker manifest push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-gcp + + echo "Create manifest ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws" + docker manifest create ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws \ + --amend ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-arm64 \ + --amend ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws-linux-amd64 + docker manifest push ${{ env.ECR_IMAGE }}:${{ matrix.versions.ansible }}-aws + + security: + name: Security scan + needs: [ matrix, deploy ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + versions: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Run Trivy vulnerability scanner for base image + uses: aquasecurity/trivy-action@0.24.0 + with: + image-ref: "ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ env.PLATFORM_PAIR }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "base.sarif" + severity: "CRITICAL,HIGH" + + - name: Run Trivy vulnerability scanner for gcp image + uses: aquasecurity/trivy-action@0.24.0 + with: + image-ref: "ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-${{ env.PLATFORM_PAIR }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "gcp.sarif" + severity: "CRITICAL,HIGH" + + - name: Run Trivy vulnerability scanner for aws image + uses: aquasecurity/trivy-action@0.24.0 + with: + image-ref: "ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-${{ env.PLATFORM_PAIR }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "aws.sarif" + severity: "CRITICAL,HIGH" + + - name: Upload base image scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "base.sarif" + + - name: Upload gcp image scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "gcp.sarif" + + - name: Upload aws image scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "aws.sarif" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4042cb7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-alpine as base +ARG ANSIBLE_VERSION=10.0 +RUN apt update && DEBIAN_FRONTEND=noninteractive apt upgrade -y &&\ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* &&\ + pip install --no-cache-dir ansible==${ANSIBLE_VERSION}.* ansible-runner~=2.4 + +FROM base as gcp +RUN pip install --no-cache-dir requests==2.32.3 google-auth==2.32.0 + +FROM base as aws +RUN pip install --no-cache-dir boto3==1.34.151 diff --git a/README.md b/README.md index ba6a361..5d1e655 100644 --- a/README.md +++ b/README.md @@ -14,24 +14,44 @@ The image is pushed to the `public.ecr.aws/spacelift/runner-ansible` public repo Altogether we have 3 flavors of the image: -- `public.ecr.aws/spacelift/runner-ansible` - built on top of the [Spacelift Terraform runner image](https://github.com/spacelift-io/runner-terraform), with Ansible installed. -- `public.ecr.aws/spacelift/runner-ansible-aws` - built on top of `runner-ansible`, with `boto3` installed. -- `public.ecr.aws/spacelift/runner-ansible-gcp` - built on top of `runner-ansible`, with `google-auth` installed. +- `runner-ansible:${ANSIBLE_VERSION}` - built on top of `python` base image, with `ansible` and `ansible-runner` installed. +- `runner-ansible:${ANSIBLE_VERSION}-aws` - built on top of `runner-ansible`, with `boto3` installed. +- `runner-ansible:${ANSIBLE_VERSION}-gcp` - built on top of `runner-ansible`, with `google-auth` installed. -## Branch Model +Every image is available for the following architectures: -This repository uses two main branches: +- linux/amd64 +- linux/arm64 -- `main` - contains the production version of the runner image. -- `future` - used to test development changes. +## Tag Model -Pushes to main deploy to the `latest` tag, whereas pushes to future deploy to the `future` tag. This -means that to use the development version you can use the `public.ecr.aws/spacelift/runner-ansible:future` image. +This repository create a tag for each minor version of ansible. -## Development +In case you don't care about locking the minor version, we also create a tag for the major version that is automatically +bumped when a new minor is released. + +You can find below is a non-exhaustive list of tags. This may get outdated with time. + +- `10`, `10.2` +- `10.1` +- `9`, `9.8` +- `9.7` +- `...` + +All tags are rebuild every sunday at midnight to be able to get latest security fixes. + +## Contributing The only requirement for working on this repo is a [Docker](https://www.docker.com/) installation. +**ℹī¸ Please do not open PR to add a new package to those base images because your workflow need it.** + +We want to keep the size of those base image as small as possible 🙏 Only package that are a **strong requirement** to run ansible will be accepted in base images. + +If you need a specific package, please maintain your own version using those image as base image with `FROM public.ecr.aws/spacelift/runner-ansible:10`. + +We are open to add new image flavors if needed to support common ansible roles and use cases. The rule of thumb is to keep base image small. + ### Testing a new Image The easiest way to test a new image before opening a pull request is to push it to your own diff --git a/build-matrix/go.mod b/build-matrix/go.mod new file mode 100644 index 0000000..5285690 --- /dev/null +++ b/build-matrix/go.mod @@ -0,0 +1,11 @@ +module github.com/spacelift-io/build-matrix + +go 1.22.1 + +require ( + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/build-matrix/go.sum b/build-matrix/go.sum new file mode 100644 index 0000000..79eafcc --- /dev/null +++ b/build-matrix/go.sum @@ -0,0 +1,11 @@ +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/build-matrix/main.go b/build-matrix/main.go new file mode 100755 index 0000000..679ce9e --- /dev/null +++ b/build-matrix/main.go @@ -0,0 +1,116 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "sort" + + "github.com/Masterminds/semver/v3" +) + +const ( + // Define the oldest major version we care about, we do not want to build image starting ansible 1.0 + minSupportedMajor = 7 +) + +type ReleaseResponse struct { + Releases map[string]any `json:"releases"` +} + +type matrixVersion struct { + Ansible string `json:"ansible"` + AdditionalTags []string `json:"additional_tags"` +} +type Matrix []matrixVersion + +// This small script reads ansible versions from pypi and returns an aggregated list of deduplicated minor version. +// This is used to compute the build matrix in github to build all minor versions in parallel. +// This script will also find the latest minor version for every major to be able to tag the docker image dynamically. +// For example if we have the following version returned: +// - 10.1.1 +// - 10.3.1 +// - 10.4.3 +// - 10.4.3. +// The script will return the following versions: +// - 10.1 +// - 10.3 +// - 10.4, additional_tags: 10 +// Check main_test.go for a quick overview of the expected behavior. +func main() { + resp, err := http.Get("https://pypi.org/pypi/ansible/json") + if err != nil { + log.Fatal(err) + } + + matrixOutput := GenerateBuildMatrix(resp.Body, minSupportedMajor) + + output, err := json.Marshal(matrixOutput) + if err != nil { + log.Fatal(err) + } + fmt.Print(string(output)) +} + +func GenerateBuildMatrix(reader io.Reader, minSupportedMajor uint64) Matrix { + releases := ReleaseResponse{} + if err := json.NewDecoder(reader).Decode(&releases); err != nil { + log.Fatal(err) + } + + var versions []*semver.Version + + for v := range releases.Releases { + version, err := semver.NewVersion(v) + if err != nil { + log.Printf("Unable to parse version %s\n", v) + continue + } + versions = append(versions, version) + } + + sort.Slice(versions, func(i, j int) bool { + return versions[j].LessThan(versions[i]) + }) + + versionGroupedByMajor := make(map[int][]*semver.Version) + // Just used for stable ordering + var majorVersions []int + + for _, version := range versions { + if version.Major() < minSupportedMajor { + break + } + major := int(version.Major()) + if _, exists := versionGroupedByMajor[major]; !exists { + majorVersions = append(majorVersions, major) + } + versionGroupedByMajor[major] = append(versionGroupedByMajor[major], version) + } + + sort.Sort(sort.Reverse(sort.IntSlice(majorVersions))) + + minorVersionDeduplication := map[string]any{} + matrix := Matrix{} + for _, majorVersion := range majorVersions { + for i, version := range versionGroupedByMajor[majorVersion] { + key := fmt.Sprintf("%d.%d", version.Major(), version.Minor()) + if _, exists := minorVersionDeduplication[key]; exists { + continue + } + additionalTags := make([]string, 0) + if i == 0 { + additionalTags = append(additionalTags, fmt.Sprintf("%d", version.Major())) + } + minorVersionDeduplication[key] = struct{}{} + matrix = append(matrix, matrixVersion{ + Ansible: key, + AdditionalTags: additionalTags, + }) + } + } + + return matrix +} diff --git a/build-matrix/main_test.go b/build-matrix/main_test.go new file mode 100644 index 0000000..0398763 --- /dev/null +++ b/build-matrix/main_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateBuildMatrix(t *testing.T) { + fakePythonVersions := ReleaseResponse{ + Releases: map[string]any{ + "1.1.0": struct{}{}, + "1.1.1": struct{}{}, + "2.10.0": struct{}{}, + "2.11.0": struct{}{}, + "2.11.2": struct{}{}, + "3.1.0": struct{}{}, + "3.1.1": struct{}{}, + "3.2.0": struct{}{}, + }, + } + + fakeJsonResponse, err := json.Marshal(fakePythonVersions) + require.NoError(t, err) + + matrix := GenerateBuildMatrix(bytes.NewReader(fakeJsonResponse), 2) + expectedMatrix := Matrix{ + { + Ansible: "3.2", + AdditionalTags: []string{"3"}, + }, + { + Ansible: "3.1", + AdditionalTags: []string{}, + }, + { + Ansible: "2.11", + AdditionalTags: []string{"2"}, + }, + { + Ansible: "2.10", + AdditionalTags: []string{}, + }, + } + assert.Equal(t, expectedMatrix, matrix) +}