diff --git a/build/build-image/cache/Dockerfile-base b/build/build-image/cache/Dockerfile-base new file mode 100644 index 0000000000..c352db1e55 --- /dev/null +++ b/build/build-image/cache/Dockerfile-base @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-builders/gcloud-slim + +RUN apt-get update \ + && apt-get install -y gcc python3-dev python3-setuptools python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && pip3 install --no-cache-dir -U crcmod + +COPY checksum /bin +COPY save_cache /bin +COPY restore_cache /bin + +RUN chmod +x /bin/checksum /bin/save_cache /bin/restore_cache \ No newline at end of file diff --git a/build/build-image/cache/Dockerfile-restore b/build/build-image/cache/Dockerfile-restore new file mode 100644 index 0000000000..bd4d33a068 --- /dev/null +++ b/build/build-image/cache/Dockerfile-restore @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG project_id +FROM us-docker.pkg.dev/${project_id}/ci/cache + +ENTRYPOINT ["restore_cache"] \ No newline at end of file diff --git a/build/build-image/cache/Dockerfile-save b/build/build-image/cache/Dockerfile-save new file mode 100644 index 0000000000..1941874a13 --- /dev/null +++ b/build/build-image/cache/Dockerfile-save @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG project_id +FROM us-docker.pkg.dev/${project_id}/ci/cache + +ENTRYPOINT ["save_cache"] \ No newline at end of file diff --git a/build/build-image/cache/README.md b/build/build-image/cache/README.md new file mode 100644 index 0000000000..d2590a76f8 --- /dev/null +++ b/build/build-image/cache/README.md @@ -0,0 +1,142 @@ +# Origin + +This `README.md` was originally based on [cloud-builders-community](https://github.com/GoogleCloudPlatform/cloud-builders-community), which contains source code for community-contributed Docker images. You can use these images as build steps for [Google Cloud Build](https://cloud.google.com/build/docs). + +The key change made upon importing the [cache builder's](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache) code into Agones was the update of image tags to use Google Artifact Registry instead of Google Container Registry. + +# Cache builders + +This includes a pair of builders, `save_cache` and `restore_cache`, that work together to cache files between builds to a GCS bucket (or local file). + +## Using the `save_cache` builder + +All options that require a value use the form `--option=value` or `-o=value` so that they look nice in Yaml files. + +| Option | Description | +| ---------------- | ----------------------------------------------------------- | +| -b, --bucket | The cloud storage bucket to upload the cache to. [optional] | +| -o, --out | The output directory to write the cache to. [optional] | +| -k, --key | The cache key used for this cache file. [optional] | +| -p, --path | The files to store in the cache. Can be repeated. | +| -t, --threshold | The parallel composite upload threshold [default: 50M] | +| -n, --no-clobber | Skips the save if the cache file already exists in GCS. | + +One of `--bucket` or `--out` parameters are required. If `--bucket` then the cache file will be uploaded to the provided GCS bucket path. If `--out` then the cache file will be stored in the directory specified on disk. + +The key provided by `--key` is used to identify the cache file. Any other cache files for the same key will be overwritten by this one. + +The `--path` parameters can be repeated for as many folders as you'd like to cache. When restored, they will retain folder structure on disk. + +The `--no-clobber` flag is used to skip creating and uploading the cache to GCS if the cache file already exists. This will shorten the time for builds when a cache was restored and is not changed by your build process. For example, this flag can be used if you are caching your dependencies and all of your dependencies are pinned to a specific version. This flag is valid only when `--bucket` is used. + +## Using the `restore_cache` builder + +All options use the form `--option=value` or `-o=value` so that they look nice in Yaml files. + +| Option | Description | +| ---------------------- | -------------------------------------------------------------------------------------- | +| -b, --bucket | The cloud storage bucket to download the cache from. [optional] | +| -s, --src | The local directory in which the cache is stored. [optional] | +| -k, --key | The cache key used for this cache file. [optional] | +| -kf, --key_fallback | The cache key fallback pattern to be used if exact cache key is not found. [optional] | + +One of `--bucket` or `--src` parameters are required. If `--bucket` then the cache file will be downloaded from the provided GCS bucket path. If `--src` then the cache file will be read from the directory specified on disk. + +The key provided by `--key` is used to identify the cache file. + +The fallback key pattern provide by `--key_fallback`, will be used to fetch the most recent cache file matching that pattern in case there is a cache miss from the specified `--key`. + +### `checksum` Helper + +As apps develop, cache needs change. For instance when dependencies are removed from a project, or versions are updated, there is no need to cache the older versions of dependencies. Therefore it's recommended that you update the cache key when these changes occur. + +This builder includes a `checksum` helper script, which you can use to create a simple checksum of files in your project to use as a cache key. + +To use it in the `--key` arguemnt, simply surround the command with `$()`: + +```bash +--key=build-cache-$(checksum build.gradle)-$(checksum dependencies.gradle) +``` + +To ensure you aren't paying for storage of obsolete cache files you can add an Object Lifecycle Rule to the cache bucket to delete object older than 30 days. + +## Examples + +The following examples demonstrate build requests that use this builder. + +### Saving a cache with checksum to GCS bucket + +This `cloudbuild.yaml` saves the files and folders in the `path` arguments to a cache file in the GCS bucket `gs://$CACHE_BUCKET/`. In this example the key will be updated, resulting in a new cache, every time the `cloudbuild.yaml` build file is changed. + +```yaml +- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache' + args: + - '--bucket=gs://$CACHE_BUCKET/' + - '--key=resources-$( checksum cloudbuild.yaml )' + - '--path=.cache/folder1' + - '--path=.cache/folder2/subfolder3' +``` + +If your build process only changes the cache contents whenever `cloudbuild.yaml` changes, then you can skip saving the cache again if it already exists in GCS: +```yaml +- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache' + args: + - '--bucket=gs://$CACHE_BUCKET/' + - '--key=resources-$( checksum cloudbuild.yaml )' + - '--path=.cache/folder1' + - '--path=.cache/folder2/subfolder3' + - '--no-clobber' +``` + +### Saving a cache with checksum to a local file + +This `cloudbuild.yaml` saves the files and folders in the `path` arguments to a cache file in the directory passed to the `out` parameter. In this example the key will be updated, resulting in a new cache, every time the `cloudbuild.yaml` build file is changed. + +```yaml +- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache' + args: + - '--out=/cache/' + - '--key=resources-$( checksum cloudbuild.yaml )' + - '--path=.cache/folder1' + - '--path=.cache/folder2/subfolder3' + volumes: + - name: 'cache' + path: '/cache' +``` + +### Restore a cache from a GCS bucket + +This `cloudbuild.yaml` restores the files from the compressed cache file identified by `key` on the cache bucket provided, if it exists. + +```yaml +- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache' + args: + - '--bucket=gs://$CACHE_BUCKET/' + - '--key=resources-$( checksum cloudbuild.yaml )' +``` + +### Restore a cache from a local file + +This `cloudbuild.yaml` restores the files from the compressed cache file identified by `key` on the local filesystem, if it exists. + +```yaml +- name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache' + args: + - '--src=/cache/' + - '--key=resources-$( checksum cloudbuild.yaml )' + volumes: + - name: 'cache' + path: '/cache' +``` + +### Restore a cache with a fallback key + +```yaml +- name: us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache + id: restore_cache + args: [ + '--bucket=gs://${_CACHE_BUCKET}', + '--key=gradle-$( checksum checksum.txt )', + '--key_fallback=gradle-', + ] +``` diff --git a/build/build-image/cache/checksum b/build/build-image/cache/checksum new file mode 100755 index 0000000000..5dbe2e9438 --- /dev/null +++ b/build/build-image/cache/checksum @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo $(cksum $@ | cut -d' ' -f1) | tr ' ' - diff --git a/build/build-image/cache/cloudbuild.yaml b/build/build-image/cache/cloudbuild.yaml new file mode 100644 index 0000000000..d78ccefa86 --- /dev/null +++ b/build/build-image/cache/cloudbuild.yaml @@ -0,0 +1,146 @@ +--- +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + + - name: 'gcr.io/cloud-builders/docker' + id: build_base_image + args: + - 'build' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest' + - '--file=Dockerfile-base' + - '.' + + - name: 'gcr.io/cloud-builders/docker' + id: build_save_cache + args: + - 'build' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest' + - '--file=Dockerfile-save' + - '--build-arg=project_id=$PROJECT_ID' + - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest' + - '.' + + - name: 'gcr.io/cloud-builders/docker' + id: build_restore_cache + args: + - 'build' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}' + - '--tag=us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest' + - '--file=Dockerfile-restore' + - '--build-arg=project_id=$PROJECT_ID' + - '--cache-from=us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest' + - '.' + + # Test the script + - name: 'gcr.io/cloud-builders/gsutil' + id: setup_test + entrypoint: 'bash' + args: + - '-c' + - | + echo "Creating test cache file structure." + mkdir -p /original/folder1 /original/folder2/subfolder3 rel_folder + touch /original/folder1/file1.txt + touch /original/folder1/file2.txt + touch /original/folder2/ignore.txt + touch /original/folder2/subfolder3/file1.txt + touch rel_folder/file3.txt + volumes: + - name: original + path: /original + + - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache' + id: save_cache + args: + - '--out=/cached' + - '--key=simple-key-$( checksum cloudbuild.yaml )' + - '--path=/original/folder1' + - '--path=/original/folder2/subfolder3' + - '--path=rel_folder' + volumes: + - name: original + path: /original + - name: cached + path: /cached + + - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache' + id: verify_cache + entrypoint: 'bash' + args: + - '-c' + - | + echo "Verifying cache file exists." + cache_file="/cached/simple-key-$( checksum cloudbuild.yaml ).tgz" + if [[ ! -f "${cache_file}" ]];then + echo "Missing cache file at ${cache_file}" + echo "Contents:" + echo "$(ls -al /cached)" + exit 1 + fi + echo "Cache tests passed." + volumes: + - name: cached + path: /cached + + - name: 'gcr.io/cloud-builders/gsutil' + id: clean_cache + entrypoint: bash + args: + - -c + - | + echo "Clearing original files..." + rm -rf /original/* + rm -rf rel_folder/ + volumes: + - name: original + path: /original + + - name: 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache' + id: restore_cache + args: + - '--src=/cached' + - '--key=simple-key-$( checksum cloudbuild.yaml )' + volumes: + - name: original + path: /original + - name: cached + path: /cached + + - name: 'gcr.io/cloud-builders/gsutil' + id: verify_restore + entrypoint: bash + args: + - '-c' + - | + test -f /original/folder1/file1.txt + test -f /original/folder1/file2.txt + test -f /original/folder2/ignore.txt + test -f /original/folder2/subfolder3/file1.txt + test -f rel_folder/file3.txt + test -f /workspace/rel_folder/file3.txt + volumes: + - name: original + path: /original + +images: + - 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:${_VERSION}' + - 'us-docker.pkg.dev/$PROJECT_ID/ci/cache:latest' + - 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:${_VERSION}' + - 'us-docker.pkg.dev/$PROJECT_ID/ci/save_cache:latest' + - 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:${_VERSION}' + - 'us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache:latest' diff --git a/build/build-image/cache/restore_cache b/build/build-image/cache/restore_cache new file mode 100755 index 0000000000..69b339f69d --- /dev/null +++ b/build/build-image/cache/restore_cache @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BUCKET="" +SRC_DIR="." +KEY="cache" +KEY_FALLBACK_PATTERN="" +PATHS=() + +function print_usage { + echo "Usage: $0 --bucket=gs://my-cache-bucket --key=cache-key" + echo + echo "Saves the specified paths to a cache file located in the out directory." + echo + echo " -b, --bucket The cloud storage bucket to download the cache from. [optional]" + echo " -s, --src The local directory in which the cache is stored. [optional]" + echo " -k, --key The cache key used for this cache file. [optional]" + echo " -kf, --key_fallback The cache key fallback pattern to be used if exact cache key is not found. It will try to find a file based on the pattern specified and will retrieve the most recent one. [optional]" + echo +} + +for i in "$@"; do + case $i in + -b=*|--bucket=*) + BUCKET="${i#*=}" + shift + ;; + -s=*|--src=* ) + SRC_DIR="${i#*=}" + shift + ;; + -k=*|--key=* ) + KEY="${i#*=}" + shift + ;; + -kf=*|--key_fallback=* ) + KEY_FALLBACK_PATTERN="${i#*=}" + shift + ;; + -h|--help ) + print_usage + exit 0 + ;; + *) + echo "Unknown option: ${i}" + print_usage + exit 1 + ;; + esac +done + +if [ -z "${BUCKET}" ] && [ -z "${SRC_DIR}" ];then + echo "Require one of [--bucket, --src]" + print_usage + exit 1 +fi + +# Evaluate the key string so that bash variables work +eval "KEY=\"$KEY\"" +eval "KEY_FALLBACK_PATTERN=\"$KEY_FALLBACK_PATTERN\"" + +if [ -n "${BUCKET}" ];then + REMOTE_CACHE_FILE="${BUCKET}/${KEY}.tgz" + + echo "Checking cache file existence for: ${REMOTE_CACHE_FILE}" + REMOTE_CACHE_FILE_EXISTENCE=$(gsutil -q stat ${REMOTE_CACHE_FILE}; echo $?) + + # If cache file exists download and decompress it + if [ $REMOTE_CACHE_FILE_EXISTENCE -eq 0 ];then + echo "Downloading cache file: ${REMOTE_CACHE_FILE}..." + gsutil -q cp "${REMOTE_CACHE_FILE}" "${SRC_DIR}" + + CACHE_FILE="${SRC_DIR}/${KEY}.tgz" + + echo "Restoring cache from file ${CACHE_FILE}..." + tar xpzf "$CACHE_FILE" -P + rm "$CACHE_FILE" + else + # Check if fallback key pattern has been specified + if [ -z "${KEY_FALLBACK_PATTERN}" ]; then + echo "No fallback key pattern specified. Can not restore cache!" + exit 0 + fi + + # Try to find a fallback file + FALLBACK_CACHE_FILE=$(gsutil ls -l $BUCKET/$KEY_FALLBACK_PATTERN** | sort -k 2,2r | grep -m 1 -i "$KEY_FALLBACK_PATTERN" | awk '{print $3}') + + # Check if fallback cache file has been found + if [ -z "${FALLBACK_CACHE_FILE}" ]; then + echo "No fallback cache file found! Can not restore cache!" + exit 0 + fi + + # Download found fallback cache file + echo "Downloading fallback cache file: ${FALLBACK_CACHE_FILE}..." + gsutil -q cp "${FALLBACK_CACHE_FILE}" "${SRC_DIR}" + + FALLBACK_CACHE_FILE_NAME=$(echo ${FALLBACK_CACHE_FILE} | awk '{n=split($0,parts,"/"); print parts[n]}') + + echo "Restoring cache from fallback file ${FALLBACK_CACHE_FILE}..." + tar xpzf "$FALLBACK_CACHE_FILE_NAME" -P + rm "$FALLBACK_CACHE_FILE_NAME" + fi +# If SRC_DIR is specified, restore cache from local file +elif [ -n "${SRC_DIR}" ]; then + LOCAL_CACHE_FILE="${SRC_DIR}/${KEY}.tgz" + echo "Restoring from local cache file ${LOCAL_CACHE_FILE}..." + tar xpzf "$LOCAL_CACHE_FILE" -P +fi diff --git a/build/build-image/cache/save_cache b/build/build-image/cache/save_cache new file mode 100755 index 0000000000..8b8b217e3e --- /dev/null +++ b/build/build-image/cache/save_cache @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BUCKET="" +OUT_DIR="." +KEY="cache" +THRESHOLD="50M" +PATHS=() + +function print_usage { + echo "Usage: $0 --out=/cache_dir --key=cache-key --path=/path/to/files/1 --path=/path/to/files/2" + echo + echo "Saves the specified paths to a cache file located in the out directory." + echo + echo " -b, --bucket The cloud storage bucket to upload the cache to. [optional]" + echo " -o, --out The output directory to write the cache to. [optional]" + echo " -k, --key The cache key used for this cache file." + echo " -p, --path The files to store in the cache. Can be repeated." + echo " -t, --threshold The parallel composite upload threshold [default: 50M]" + echo " -n, --no-clobber Skips the save if the cache file already exists in GCS." + echo +} + +for i in "$@"; do + case $i in + -b=*|--bucket=*) + BUCKET="${i#*=}" + shift + ;; + -o=*|--out=*) + OUT_DIR="${i#*=}" + shift + ;; + -k=*|--key=*) + KEY="${i#*=}" + shift + ;; + -p=*|--path=*) + PATHS+="${i#*=} " + shift + ;; + -t=*|--threshold=*) + THRESHOLD="${i#*=} " + shift + ;; + -n|--no-clobber) + NO_CLOBBER=true + shift + ;; + -h|--help ) + print_usage + exit 0 + ;; + *) + echo "Unknown option: ${i}" + print_usage + exit 1 + ;; + esac +done + +if [ -z "$KEY" ] || [ -z "$PATHS" ];then + print_usage + exit 1 +fi + +if [ "$NO_CLOBBER" = true ] && [ -z "$BUCKET" ];then + echo "--bucket must be specified if --no-clobber is used" + echo + print_usage + exit 1 +fi + +eval "KEY=$KEY" + +CACHE_FILE="${OUT_DIR}/${KEY}.tgz" + +if [ "$NO_CLOBBER" = true ];then + BUCKET_FILE="$BUCKET/$KEY.tgz" + FILE_LS=$(gsutil ls "$BUCKET_FILE") + if [ "$FILE_LS" == "$BUCKET_FILE" ];then + echo "Cache file exists, exiting save_cache without over-writing cache file." + exit 0 + fi +fi + +echo "Compressing cache to ${CACHE_FILE}..." +tar cpzf "$CACHE_FILE" ${PATHS[@]} -P + +if [ -n "$BUCKET" ];then + echo "Uploading cache to Google Cloud Storage..." + gsutil -o GSUtil:parallel_composite_upload_threshold=${THRESHOLD} cp -R "$CACHE_FILE" "$BUCKET" +fi