Download node binary in GH action

Add a script to download node/yarn binary in GH action with retry feature to prevent the unstable connection when downloading the node/yarn binary in the Maven build. Signed-off-by: Yihong Wang <yh.wang@ibm.com>
prestodb · May 14, 2024 · 1863d80 · 1863d80
1 parent b15f0c5
commit 1863d80
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 11 deletions.
diff --git a/.github/bin/download_nodejs b/.github/bin/download_nodejs
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+NODE_VERSION="18.20.2"
+YARN_VERSION="1.22.22"
+
+wget_retry() {
+  local max=$1; shift;
+  local interval=$1; shift;
+  local dest=$1; shift;
+  local src=$1; shift;
+  local check=$1; shift;
+
+  fileList=$(mktemp)
+  until wget -q -O "${dest}" "${src}" && tar -tf "${dest}" > "$fileList"  && grep -q "${check}" "${fileList}" ; do
+    max=$((max-1))
+    if [[ "$max" -eq 0 ]]; then
+      return 1
+    fi
+    sleep "$interval"
+  done
+
+  return 0
+}
+
+# GH action variables: https://docs.github.com/en/actions/learn-github-actions/variables
+# Use the RUNNER_OS variable to get the os string for the download file
+get_os() {
+  case "${RUNNER_OS}" in
+    Linux) echo "linux" ;;
+    macOS) echo "darwin" ;;
+    Windows) echo "win" ;;
+    *) echo "unknown" ;;
+  esac
+}
+
+# Use the RUNNER_ARCH variable to get the arch string for the download file
+get_arch() {
+  case "${RUNNER_ARCH}" in
+    X86) echo "x86" ;;
+    X64) echo "x64" ;;
+    ARM) echo "armv7l" ;;
+    ARM64) echo "arm64" ;;
+    *) echo "unknown" ;;
+  esac
+}
+
+download_node() {
+  if [[ -a "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}/node-${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" ]]; then
+    echo "Node binary exists. Skipped download"
+    return 0
+  fi
+
+  if ! wget_retry 3 10 "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}/node-${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" \
+      "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz" "node"; then
+    rm "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}/node-${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz"
+    return 1
+  fi
+}
+
+download_yarn() {
+  if [[ -a "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}/yarn-${YARN_VERSION}.tar.gz" ]]; then
+    echo "Yarn binary exists. Skipped download"
+    return 0
+  fi
+
+  if ! wget_retry 3 10 "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}/yarn-${YARN_VERSION}.tar.gz" \
+      "https://github.com/yarnpkg/yarn/releases/download/v${YARN_VERSION}/yarn-v${YARN_VERSION}.tar.gz" "yarn"; then
+      rm "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}/yarn-${YARN_VERSION}.tar.gz"
+      return 1
+  fi
+}
+
+NODE_OS=$(get_os)
+NODE_ARCH=$(get_arch)
+
+mkdir -p "${HOME}/.m2/repository/com/github/eirslett/node/${NODE_VERSION}"
+mkdir -p "${HOME}/.m2/repository/com/github/eirslett/yarn/${YARN_VERSION}"
+
+if download_node; then
+  echo "node-v${NODE_VERSION}-${NODE_OS}-${NODE_ARCH}.tar.gz is ready for use"
+else
+  echo "failed to download node binary"
+  exit 1
+fi
+
+if download_yarn; then
+  echo "yarn-v${YARN_VERSION}.tar.gz is ready for use"
+else
+  echo "failed to download yarn binary"
+  exit 1
+fi
diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml
@@ -56,7 +56,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Install Hive Module
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
@@ -108,7 +108,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies && .github/bin/download_nodejs
       - name: Install Hive Module
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml
@@ -54,7 +54,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress 
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Maven Install
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml
@@ -40,7 +40,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies  --no-transfer-progress -P ci
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies  --no-transfer-progress -P ci && .github/bin/download_nodejs
       - name: Maven Checks
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml
@@ -60,7 +60,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Maven install
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml
@@ -60,7 +60,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Maven install
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
@@ -111,7 +111,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies && .github/bin/download_nodejs
       - name: Maven install
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml
@@ -67,7 +67,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Install SingleStore Module
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml
@@ -55,7 +55,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Maven Install
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml
@@ -55,7 +55,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Maven Install
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -91,7 +91,7 @@ jobs:
             ${{ runner.os }}-maven-2-
       - name: Populate maven cache
         if: steps.cache-maven.outputs.cache-hit != 'true' && needs.changes.outputs.codechange == 'true'
-        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress 
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs
       - name: Maven Install
         if: needs.changes.outputs.codechange == 'true'
         run: |