diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index c488fa3b75..eda34d7a2e 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -19,7 +19,7 @@ pipeline { } } steps { - script { + script { dockerAgent = detectDockerAgent() currentBuild.description = "$INPUT_MANIFEST" } diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy index c3ba01cbb1..b533a79f0b 100644 --- a/src/jenkins/BuildManifest.groovy +++ b/src/jenkins/BuildManifest.groovy @@ -25,30 +25,72 @@ class BuildManifest implements Serializable { String getFilename() { return this.name.toLowerCase().replaceAll(' ', '-') } + + String getPackageName() { + return [ + this.getFilename(), + this.version, + this.platform, + this.architecture, + ].join('-') + '.tar.gz' + } + } + + class Components extends HashMap { + + Components(ArrayList data) { + data.each { item -> + Component component = new Component(item) + this[component.name] = component + } + } + } + + class Component implements Serializable { + String name + String version + String ref + String commit_id + String repository + Map artifacts + + Component(Map data) { + this.name = data.name + this.version = data.version + this.ref = data.ref + this.commit_id = data.commit_id + this.repository = data.repository + this.artifacts = new HashMap<>(data.artifacts) + } + } Build build + Components components BuildManifest(Map data) { this.build = new BuildManifest.Build(data.build) + this.components = new BuildManifest.Components(data.components) } public String getArtifactRoot(String jobName, String buildNumber) { return [ - jobName, - this.build.version, - buildNumber, - this.build.platform, - this.build.architecture + jobName, + this.build.version, + buildNumber, + this.build.platform, + this.build.architecture ].join("/") } public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) { return [ - publicArtifactUrl, - this.getArtifactRoot(jobName, buildNumber) + publicArtifactUrl, + this.getArtifactRoot(jobName, buildNumber) ].join('/') } -} - + public String getMinArtifact() { + components.get(build.name.replace(' ','-'))?.artifacts?.get("dist")?.first() + } +} diff --git a/tests/jenkins/TestArchiveAssembleUpload.groovy b/tests/jenkins/TestArchiveAssembleUpload.groovy index f95679c431..f8e269ce3a 100644 --- a/tests/jenkins/TestArchiveAssembleUpload.groovy +++ b/tests/jenkins/TestArchiveAssembleUpload.groovy @@ -24,6 +24,9 @@ class TestArchiveAssembleUpload extends BuildPipelineTest { binding.setVariable('STAGE_NAME', 'stage') binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42') binding.setVariable('BUILD_NUMBER', '33') + binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role') + binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy') + binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket') binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role') helper.registerAllowedMethod("s3Upload", [Map]) diff --git a/tests/jenkins/TestAssembleUpload.groovy b/tests/jenkins/TestAssembleUpload.groovy index bdec0ca56c..d25f203230 100644 --- a/tests/jenkins/TestAssembleUpload.groovy +++ b/tests/jenkins/TestAssembleUpload.groovy @@ -22,6 +22,9 @@ class TestAssembleUpload extends BuildPipelineTest { binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account') binding.setVariable('STAGE_NAME', 'stage') binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42') + binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role') + binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy') + binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket') binding.setVariable('BUILD_NUMBER', '33') binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role') diff --git a/tests/jenkins/TestBuildAssembleUpload.groovy b/tests/jenkins/TestBuildAssembleUpload.groovy index 1fc065b53f..1af44ea13c 100644 --- a/tests/jenkins/TestBuildAssembleUpload.groovy +++ b/tests/jenkins/TestBuildAssembleUpload.groovy @@ -24,6 +24,9 @@ class TestBuildAssembleUpload extends BuildPipelineTest { binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account') binding.setVariable('STAGE_NAME', 'stage') + binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role') + binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy') + binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket') binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role') helper.registerAllowedMethod("withCredentials", [List, Closure], { list, closure -> diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt index 0bdd77fd36..44e6261014 100644 --- a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt @@ -27,6 +27,7 @@ uploadArtifacts.library({identifier=jenkins@20211123, retriever=null}) uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml}) BuildManifest.asBoolean() + BuildManifest.getMinArtifact() BuildManifest.getArtifactRoot(vars-build, 33) uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64) uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds}) @@ -35,6 +36,10 @@ uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist}) uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist}) + uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64) + uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/}) + uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/}) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) Messages.asBoolean() Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt index 196f431d89..ae57cbd088 100644 --- a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt @@ -18,6 +18,7 @@ uploadArtifacts.library({identifier=jenkins@20211123, retriever=null}) uploadArtifacts.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) BuildManifest.asBoolean() + BuildManifest.getMinArtifact() BuildManifest.getArtifactRoot(vars-build, 33) uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64) uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds}) @@ -26,6 +27,10 @@ uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist}) uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist}) + uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64) + uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/}) + uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/}) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) Messages.asBoolean() Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt index b3077f7941..c5099d83e6 100644 --- a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt @@ -25,6 +25,7 @@ uploadArtifacts.library({identifier=jenkins@20211123, retriever=null}) uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml}) BuildManifest.asBoolean() + BuildManifest.getMinArtifact() BuildManifest.getArtifactRoot(vars-build, 33) uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64) uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds}) @@ -33,6 +34,10 @@ uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist}) uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist}) + uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64) + uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/}) + uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/}) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) Messages.asBoolean() Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile b/tests/jenkins/jobs/BuildManifest_Jenkinsfile index 594e04a12c..c83931dddb 100644 --- a/tests/jenkins/jobs/BuildManifest_Jenkinsfile +++ b/tests/jenkins/jobs/BuildManifest_Jenkinsfile @@ -5,7 +5,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ - + def lib = library("jenkins") pipeline { @@ -22,8 +22,10 @@ pipeline { echo buildManifest.build.getFilename() echo buildManifest.getArtifactRoot('bundle-build', '1') echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1') + echo buildManifest.build.getPackageName() + echo buildManifest.components.getMinArtifact() } } } } -} \ No newline at end of file +} diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt index 61e95d2690..fb9893a0e1 100644 --- a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt @@ -15,3 +15,6 @@ BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1) BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64) + BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-linux-x64.tar.gz) + BuildManifest.getMinArtifact() + BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz) diff --git a/vars/uploadArtifacts.groovy b/vars/uploadArtifacts.groovy index 528ec884f2..b2870c129a 100644 --- a/vars/uploadArtifacts.groovy +++ b/vars/uploadArtifacts.groovy @@ -2,6 +2,9 @@ void call(Map args = [:]) { def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest)) + def minArtifactPath = buildManifest.getMinArtifact() + def productFilename = buildManifest.build.getFilename() + def packageName = buildManifest.build.getPackageName() def artifactPath = buildManifest.getArtifactRoot("${JOB_NAME}", "${BUILD_NUMBER}") echo "Uploading to s3://${ARTIFACT_BUCKET_NAME}/${artifactPath}" @@ -18,10 +21,17 @@ void call(Map args = [:]) { path: "${artifactPath}/dist" ) + echo "Uploading to s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${artifactPath}" + + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + s3Upload(file: "builds/${productFilename}/${minArtifactPath}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/core/${productFilename}/${buildManifest.build.version}/") + s3Upload(file: "dist/${productFilename}/${packageName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/bundle/${productFilename}/${buildManifest.build.version}/") + } + def baseUrl = buildManifest.getArtifactRootUrl("${PUBLIC_ARTIFACT_URL}", "${JOB_NAME}", "${BUILD_NUMBER}") lib.jenkins.Messages.new(this).add("${STAGE_NAME}", [ - "${baseUrl}/builds/${buildManifest.build.getFilename()}/manifest.yml", - "${baseUrl}/dist/${buildManifest.build.getFilename()}/manifest.yml" + "${baseUrl}/builds/${productFilename}/manifest.yml", + "${baseUrl}/dist/${productFilename}/manifest.yml" ].join('\n') ) }