From 29f1050deaebfbaca31cf461acf79434c704091e Mon Sep 17 00:00:00 2001 From: Joe deKoning Date: Thu, 14 Jun 2018 16:29:12 -0400 Subject: [PATCH] Add artifactory support * adds the capability of reducing footprint on jenkins jobs concerning the storage of artifacts * [skip ci] Signed-off-by: Joe deKoning --- buildenv/jenkins/common/build | 26 +++++++++++--- buildenv/jenkins/common/pipeline-functions | 37 +++++++++++++++++--- buildenv/jenkins/common/variables-functions | 38 +++++++++++++++------ 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/buildenv/jenkins/common/build b/buildenv/jenkins/common/build index e4cc42d004c..ff56c89a7fe 100644 --- a/buildenv/jenkins/common/build +++ b/buildenv/jenkins/common/build @@ -261,10 +261,28 @@ def build() { def archive() { stage('Archive') { timestamps { - sh "tar -zcvf ${TEST_PREFIX}`git -C openj9 rev-parse --short HEAD`${TEST_SUFFIX} openj9/test/" - archiveArtifacts artifacts: "**/${TEST_PREFIX}*${TEST_SUFFIX}", fingerprint: true, onlyIfSuccessful: true - sh "tar -C build/$RELEASE/images/ -zcvf ${SDK_PREFIX}`date +%Y%d%m%H%M`${SDK_SUFFIX} ${JDK_FOLDER}" - archiveArtifacts artifacts: "**/${SDK_PREFIX}*${SDK_SUFFIX}", fingerprint: true, onlyIfSuccessful: true + sh "tar -C build/${RELEASE}/images/ -zcvf ${SDK_FILENAME} ${JDK_FOLDER}" + if (ARTIFACTORY_SERVER) { + def server = Artifactory.server ARTIFACTORY_SERVER + // if the archive location is modified the definition of CUSTOMIZED_SDK_URL will also need to be updated + def uploadSpec = """{ + "files":[ + { + "pattern": "${env.WORKSPACE}/${SDK_FILENAME}", + "target": "${ARTIFACTORY_REPO}/${env.JOB_NAME}/${env.BUILD_ID}/" + } + ] + }""" + + def buildInfo = Artifactory.newBuildInfo() + // hardcoded to 30 for now, this need follow up for usage on per platform level or across all jobs + buildInfo.retention maxBuilds: 30, deleteBuildArtifacts: true + server.upload spec: uploadSpec, buildInfo: buildInfo + server.publishBuildInfo buildInfo + } else { + echo "ARTIFACTORY server is not set saving artifacts on jenkins." + archiveArtifacts artifacts: "**/${SDK_FILENAME}", fingerprint: true, onlyIfSuccessful: true + } } } } diff --git a/buildenv/jenkins/common/pipeline-functions b/buildenv/jenkins/common/pipeline-functions index 1a23c052f5d..2afab088d8f 100644 --- a/buildenv/jenkins/common/pipeline-functions +++ b/buildenv/jenkins/common/pipeline-functions @@ -135,7 +135,7 @@ def get_date() { ).trim() } -def build(BUILD_JOB_NAME, OPENJDK_REPO, OPENJDK_BRANCH, OPENJDK_SHA, OPENJ9_REPO, OPENJ9_BRANCH, OPENJ9_SHA, OMR_REPO, OMR_BRANCH, OMR_SHA, VARIABLE_FILE, VENDOR_REPO, VENDOR_BRANCH, VENDOR_CREDENTIALS_ID, NODE) { +def build(BUILD_JOB_NAME, OPENJDK_REPO, OPENJDK_BRANCH, OPENJDK_SHA, OPENJ9_REPO, OPENJ9_BRANCH, OPENJ9_SHA, OMR_REPO, OMR_BRANCH, OMR_SHA, VARIABLE_FILE, VENDOR_REPO, VENDOR_BRANCH, VENDOR_CREDENTIALS_ID, NODE, SDK_FILENAME) { stage ("${BUILD_JOB_NAME}") { return build_with_slack(BUILD_JOB_NAME, [string(name: 'OPENJDK_REPO', value: OPENJDK_REPO), @@ -151,7 +151,8 @@ def build(BUILD_JOB_NAME, OPENJDK_REPO, OPENJDK_BRANCH, OPENJDK_SHA, OPENJ9_REPO string(name: 'VENDOR_REPO', value: VENDOR_REPO), string(name: 'VENDOR_BRANCH', value: VENDOR_BRANCH), string(name: 'VENDOR_CREDENTIALS_ID', value: VENDOR_CREDENTIALS_ID), - string(name: 'NODE', value: NODE)]) + string(name: 'NODE', value: NODE), + string(name: 'SDK_FILENAME', value: SDK_FILENAME)]) } } @@ -177,6 +178,28 @@ def build_with_one_upstream(JOB_NAME, UPSTREAM_JOB_NAME, UPSTREAM_JOB_NUMBER, VA } } +def build_with_artifactory(JOB_NAME, VARIABLE_FILE, VENDOR_REPO, VENDOR_BRANCH, VENDOR_CREDENTIALS_ID, NODE, OPENJ9_REPO, OPENJ9_BRANCH, OPENJ9_SHA, VENDOR_TEST_REPOS, VENDOR_TEST_BRANCHES, VENDOR_TEST_SHAS, VENDOR_TEST_DIRS, USER_CREDENTIALS_ID, BUILD_LIST, CUSTOMIZED_SDK_URL, ARTIFACTORY_CREDS) { + stage ("${JOB_NAME}") { + return build_with_slack(JOB_NAME, + [string(name: 'VARIABLE_FILE', value: VARIABLE_FILE), + string(name: 'VENDOR_REPO', value: VENDOR_REPO), + string(name: 'VENDOR_BRANCH', value: VENDOR_BRANCH), + string(name: 'VENDOR_CREDENTIALS_ID', value: VENDOR_CREDENTIALS_ID), + string(name: 'NODE', value: NODE), + string(name: 'OPENJ9_REPO', value: OPENJ9_REPO), + string(name: 'OPENJ9_BRANCH', value: OPENJ9_BRANCH), + string(name: 'OPENJ9_SHA', value: OPENJ9_SHA), + string(name: 'VENDOR_TEST_REPOS', value: VENDOR_TEST_REPOS), + string(name: 'VENDOR_TEST_BRANCHES', value: VENDOR_TEST_BRANCHES), + string(name: 'VENDOR_TEST_SHAS', value: VENDOR_TEST_SHAS), + string(name: 'VENDOR_TEST_DIRS', value: VENDOR_TEST_DIRS), + string(name: 'USER_CREDENTIALS_ID', value: USER_CREDENTIALS_ID), + string(name: 'BUILD_LIST', value: BUILD_LIST), + string(name: 'CUSTOMIZED_SDK_URL', value: CUSTOMIZED_SDK_URL), + string(name: 'CUSTOMIZED_SDK_URL_CREDENTIAL_ID', value: ARTIFACTORY_CREDS)]) + } +} + def build_with_slack(JOB_NAME, PARAMETERS) { def DOWNSTREAM_JOB_NUMBER = '' def DOWNSTREAM_JOB_URL = '' @@ -215,7 +238,7 @@ def workflow(SDK_VERSION, SPEC, SHAS, OPENJDK_REPO, OPENJDK_BRANCH, OPENJ9_REPO, // compile the source and build the SDK def BUILD_JOB_NAME = "Build-JDK${SDK_VERSION}-${SPEC}" - jobs["build"] = build(BUILD_JOB_NAME, OPENJDK_REPO, OPENJDK_BRANCH, SHAS['OPENJDK'], OPENJ9_REPO, OPENJ9_BRANCH, SHAS['OPENJ9'], OMR_REPO, OMR_BRANCH, SHAS['OMR'], params.VARIABLE_FILE, params.VENDOR_REPO, params.VENDOR_BRANCH, params.VENDOR_CREDENTIALS_ID, params.BUILD_NODE) + jobs["build"] = build(BUILD_JOB_NAME, OPENJDK_REPO, OPENJDK_BRANCH, SHAS['OPENJDK'], OPENJ9_REPO, OPENJ9_BRANCH, SHAS['OPENJ9'], OMR_REPO, OMR_BRANCH, SHAS['OMR'], params.VARIABLE_FILE, params.VENDOR_REPO, params.VENDOR_BRANCH, params.VENDOR_CREDENTIALS_ID, params.BUILD_NODE, SDK_FILENAME) echo "JOB: ${BUILD_JOB_NAME} PASSED in: ${jobs['build'].getDurationString()}" if (TESTS_TARGETS.trim() != "none") { @@ -252,7 +275,13 @@ def workflow(SDK_VERSION, SPEC, SHAS, OPENJDK_REPO, OPENJDK_BRANCH, OPENJ9_REPO, def TEST_JOB_NAME = "Test-${name}-JDK${SDK_VERSION}-${SPEC}" testjobs["${TEST_JOB_NAME}"] = { // run tests against the SDK build in the upstream job: Build-JDK${SDK_VERSION}-${SPEC} - jobs["${TEST_JOB_NAME}"] = build_with_one_upstream(TEST_JOB_NAME, BUILD_JOB_NAME, jobs["build"].getNumber(), params.VARIABLE_FILE, params.VENDOR_REPO, params.VENDOR_BRANCH, params.VENDOR_CREDENTIALS_ID, params.TEST_NODE, OPENJ9_REPO, OPENJ9_BRANCH, SHAS['OPENJ9'], VENDOR_TEST_REPOS, VENDOR_TEST_BRANCHES, VENDOR_TEST_SHAS, VENDOR_TEST_DIRS, USER_CREDENTIALS_ID, BUILD_LIST) + if (ARTIFACTORY_SERVER) { + def CUSTOMIZED_SDK_URL = "${ARTIFACTORY_BASE_URL}/${BUILD_JOB_NAME}/${jobs["build"].getNumber()}/${SDK_FILENAME}" + echo "Using CUSTOMIZED_SDK_URL = ${CUSTOMIZED_SDK_URL}, ARTIFACTORY_CREDS = ${ARTIFACTORY_CREDS}" + jobs["${TEST_JOB_NAME}"] = build_with_artifactory(TEST_JOB_NAME, params.VARIABLE_FILE, params.VENDOR_REPO, params.VENDOR_BRANCH, params.VENDOR_CREDENTIALS_ID, params.TEST_NODE, OPENJ9_REPO, OPENJ9_BRANCH, SHAS['OPENJ9'], VENDOR_TEST_REPOS, VENDOR_TEST_BRANCHES, VENDOR_TEST_SHAS, VENDOR_TEST_DIRS, USER_CREDENTIALS_ID, BUILD_LIST, CUSTOMIZED_SDK_URL, ARTIFACTORY_CREDS) + } else { + jobs["${TEST_JOB_NAME}"] = build_with_one_upstream(TEST_JOB_NAME, BUILD_JOB_NAME, jobs["build"].getNumber(), params.VARIABLE_FILE, params.VENDOR_REPO, params.VENDOR_BRANCH, params.VENDOR_CREDENTIALS_ID, params.TEST_NODE, OPENJ9_REPO, OPENJ9_BRANCH, SHAS['OPENJ9'], VENDOR_TEST_REPOS, VENDOR_TEST_BRANCHES, VENDOR_TEST_SHAS, VENDOR_TEST_DIRS, USER_CREDENTIALS_ID, BUILD_LIST) + } echo "JOB: ${TEST_JOB_NAME} PASSED in: ${jobs[TEST_JOB_NAME].getDurationString()}" } } diff --git a/buildenv/jenkins/common/variables-functions b/buildenv/jenkins/common/variables-functions index 66953d2cbd1..a13426a2d1c 100644 --- a/buildenv/jenkins/common/variables-functions +++ b/buildenv/jenkins/common/variables-functions @@ -333,16 +333,6 @@ def set_build_variables() { set_release() set_jdk_folder() - try{ - SDK_PREFIX = "OpenJ9-JDK${SDK_VERSION}-PR${ghprbPullId}-${SPEC}-" - } catch (e) { - SDK_PREFIX = "OpenJ9-JDK${SDK_VERSION}-${SPEC}-" - } - - SDK_SUFFIX = ".tar.gz" - TEST_PREFIX = "test-" - TEST_SUFFIX = ".tar.gz" - // set variables for the build environment configuration // check job parameters, if not provided default to variables file BUILD_ENV_VARS = params.BUILD_ENV_VARS @@ -378,6 +368,18 @@ def set_build_variables() { echo "Using BUILD_ENV_CMD = ${BUILD_ENV_CMD}, BUILD_ENV_VARS_LIST = ${BUILD_ENV_VARS_LIST.toString()}" } +def set_sdk_variables() { + SDK_FILENAME = params.SDK_FILENAME + if (!SDK_FILENAME) { + DATESTAMP = sh ( + script: 'date +%Y%d%m%H%M', + returnStdout: true + ).trim() + SDK_FILENAME = "OpenJ9-JDK${SDK_VERSION}-${SPEC}-${DATESTAMP}.tar.gz" + echo "Using SDK_FILENAME = ${SDK_FILENAME}" + } +} + /* * Sets variables for a test job on a given platform. * Note that JAVA_BIN is set separately within buildenv/jenkins/common/test @@ -412,6 +414,18 @@ def set_slack_channel() { } } +def set_artifactory_config() { + ARTIFACTORY_SERVER = VARIABLES.artifactory_server + ARTIFACTORY_REPO = VARIABLES.artifactory_repo + ARTIFACTORY_CREDS = VARIABLES.artifactory_creds + if (ARTIFACTORY_SERVER) { + ARTIFACTORY_BASE_URL = "https://" + "${ARTIFACTORY_SERVER}" + ".com/artifactory/" + "${ARTIFACTORY_REPO}" + echo "Using artifactory server/repo: ${ARTIFACTORY_SERVER} / ${ARTIFACTORY_REPO}" + echo "Using artifactory base url: ${ARTIFACTORY_BASE_URL}" + echo "Using artifactory credentials: ${ARTIFACTORY_CREDS}" + } +} + /* * Sets the names of the downstream build & test jobs */ @@ -437,6 +451,7 @@ def set_job_variables(job_type) { // fetch credentials required to connect to GitHub using ssh protocol set_user_credentials() + set_artifactory_config() switch (job_type) { case "build": @@ -444,6 +459,7 @@ def set_job_variables(job_type) { set_node('build') // set variables for a build job set_build_variables() + set_sdk_variables() break case "test": // set the node the tests would run on @@ -456,12 +472,14 @@ def set_job_variables(job_type) { set_node('build') // set variables for a pull request job that builds an SDK set_build_variables() + set_sdk_variables() set_test_variables() break case "pipeline": // set variables for a pipeline job set_repos_variables() set_vendor_variables() + set_sdk_variables() set_test_targets() set_slack_channel() set_job_names()