From 1863d80cea920a69d8222c58c6c0f991d8354c3a Mon Sep 17 00:00:00 2001 From: Yihong Wang Date: Mon, 6 May 2024 22:12:51 -0700 Subject: [PATCH] Download node binary in GH action Add a script to download node/yarn binary in GH action with retry feature to prevent the unstable connection when downloading the node/yarn binary in the Maven build. Signed-off-by: Yihong Wang --- .github/bin/download_nodejs | 93 +++++++++++++++++++ .github/workflows/hive-tests.yml | 4 +- .github/workflows/kudu.yml | 2 +- .github/workflows/maven-checks.yml | 2 +- .../product-tests-basic-environment.yml | 2 +- .../product-tests-specific-environment.yml | 4 +- .github/workflows/singlestore-tests.yml | 2 +- .github/workflows/spark-integration.yml | 2 +- .github/workflows/test-other-modules.yml | 2 +- .github/workflows/tests.yml | 2 +- 10 files changed, 104 insertions(+), 11 deletions(-) create mode 100755 .github/bin/download_nodejs diff --git a/.github/bin/download_nodejs b/.github/bin/download_nodejs new file mode 100755 index 000000000000..7b5f9eb30ca3 --- /dev/null +++ b/.github/bin/download_nodejs @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -euo pipefail + +NODE_VERSION="18.20.2" +YARN_VERSION="1.22.22" + +wget_retry() { + local max=$1; shift; + local interval=$1; shift; + local dest=$1; shift; + local src=$1; shift; + local check=$1; shift; + + fileList=$(mktemp) + until wget -q -O "${dest}" "${src}" && tar -tf "${dest}" > "$fileList" && grep -q "${check}" "${fileList}" ; do + max=$((max-1)) + if [[ "$max" -eq 0 ]]; then + return 1 + fi + sleep "$interval" + done + + return 0 +} + +# GH action variables: https://docs.github.com/en/actions/learn-github-actions/variables +# Use the RUNNER_OS variable to get the os string for the download file +get_os() { + case "${RUNNER_OS}" in + Linux) echo "linux" ;; + macOS) echo "darwin" ;; + Windows) echo "win" ;; + *) echo "unknown" ;; + esac +} + +# Use the RUNNER_ARCH variable to get the arch string for the download file +get_arch() { + case "${RUNNER_ARCH}" in + X86) echo "x86" ;; + X64) echo "x64" ;; + ARM) echo "armv7l" ;; + ARM64) echo "arm64" ;; + *) echo "unknown" ;; + esac +} + +download_node() { + if [[ -a "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}/node-${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" ]]; then + echo "Node binary exists. Skipped download" + return 0 + fi + + if ! wget_retry 3 10 "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}/node-${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" \ + "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" "node"; then + rm "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}/node-${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" + return 1 + fi +} + +download_yarn() { + if [[ -a "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}/yarn-${YARN_VERSION}.tar.gz" ]]; then + echo "Yarn binary exists. Skipped download" + return 0 + fi + + if ! wget_retry 3 10 "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}/yarn-${YARN_VERSION}.tar.gz" \ + "https://github.com/yarnpkg/yarn/releases/download/v${YARN_VERSION}/yarn-v${YARN_VERSION}.tar.gz" "yarn"; then + rm "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}/yarn-${YARN_VERSION}.tar.gz" + return 1 + fi +} + +NODE_OS=$(get_os) +NODE_ARCH=$(get_arch) + +mkdir -p "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}" +mkdir -p "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}" + +if download_node; then + echo "node-v${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz is ready for use" +else + echo "failed to download node binary" + exit 1 +fi + +if download_yarn; then + echo "yarn-v${YARN_VERSION}.tar.gz is ready for use" +else + echo "failed to download yarn binary" + exit 1 +fi diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index 7b3b568d5576..a5362c7332ea 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -56,7 +56,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Install Hive Module run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" @@ -108,7 +108,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies && .github/bin/download_nodejs - name: Install Hive Module run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml index c33cc2cfc533..713ec18d9c2a 100644 --- a/.github/workflows/kudu.yml +++ b/.github/workflows/kudu.yml @@ -54,7 +54,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index e01a6e3fb9da..9799de4d797e 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -40,7 +40,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress -P ci + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress -P ci && .github/bin/download_nodejs - name: Maven Checks run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml index 159860fbd8f7..0495b289c8a4 100644 --- a/.github/workflows/product-tests-basic-environment.yml +++ b/.github/workflows/product-tests-basic-environment.yml @@ -60,7 +60,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Maven install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index ed3b6499b581..3691bc73fb5c 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -60,7 +60,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Maven install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" @@ -111,7 +111,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies && .github/bin/download_nodejs - name: Maven install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml index 9e6e669513d2..9080223120bb 100644 --- a/.github/workflows/singlestore-tests.yml +++ b/.github/workflows/singlestore-tests.yml @@ -67,7 +67,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Install SingleStore Module run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml index b786fe93ddaa..36d20c1f4d9b 100644 --- a/.github/workflows/spark-integration.yml +++ b/.github/workflows/spark-integration.yml @@ -55,7 +55,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index 24cab9ad2239..238c486eb0b2 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -55,7 +55,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 657b32eeb5a9..8c09378b978e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -91,7 +91,7 @@ jobs: ${{ runner.os }}-maven-2- - name: Populate maven cache if: steps.cache-maven.outputs.cache-hit != 'true' && needs.changes.outputs.codechange == 'true' - run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs - name: Maven Install if: needs.changes.outputs.codechange == 'true' run: |