sovrin-foundation · andkononykhin · Jun 27, 2019 · Jun 24, 2019 · Jun 24, 2019 · Jun 24, 2019
diff --git a/devops/aws-codebuild/Jenkinsfile.cd b/devops/aws-codebuild/Jenkinsfile.cd
@@ -1,5 +1,7 @@
 #!groovy
 
+String projectName = 'token-plugin'
+
 def sovLibrary = library(identifier: 'sovrin-aws-codebuild@v1.0.1', retriever: modernSCM(
     github(credentialsId: 'sovbot-github', repoOwner: 'sovrin-foundation', repository: 'aws-codebuild-pipeline-plugin')
 )).com.sovrin.pipeline
@@ -18,12 +20,11 @@ def downloadPackagingUtils() {
 }
 
 def nodeLabels = [
-        codeBuild: env.LIBSOVTOKEN_CODEBUILD_NODE_LABEL ?: 'codebuild',
-        macos: env.LIBSOVTOKEN_MACOS_NODE_LABEL ?: 'macos',
+    codeBuild: env.LIBSOVTOKEN_CODEBUILD_NODE_LABEL ?: 'codebuild',
+    macos: env.LIBSOVTOKEN_MACOS_NODE_LABEL ?: 'macos',
 ]
 
 pipelineWrapper({
-
     //put code build containers inside a vpc under our dev account
     env.USE_VPC_CONFIG = true
     //this IP is outdated, should use the one from env
@@ -32,6 +33,8 @@ pipelineWrapper({
     nodeWrapper(nodeLabels.codeBuild) {
         def osname = 'xenial'
         List projects = ['sovtoken', 'sovtokenfees']
+        List pkgsList = []
+        String indyNodeVersion
 
         stage('Checkout sources from SCM') {
             checkout scm
@@ -43,6 +46,16 @@ pipelineWrapper({
             ['devops', 'sovtoken/sovtoken/__metadata__.py', 'sovtokenfees/sovtokenfees/__metadata__.py'])
         def awsCBHelper = sovLibrary.AwsCodeBuildHelper.new(this, buildCtx)
 
+        stage('Get indy-node version') {
+            indyNodeVersion = utils.shStdout("""
+                sed -n "s/.*indy-node==\\([^\\"\\']\\+\\).*/\\1/p" sovtoken/setup.py
+            """)
+            if (!indyNodeVersion) {
+                throw new Exception('indy-node version is not found')
+            }
+            logger.info("Detected indy-node version: '$indyNodeVersion'")
+        }
+
         Map builds = [
             "s3-upload": {  // performs on the current jenkins node
                 stage('Upload source to S3') {
@@ -65,126 +78,143 @@ pipelineWrapper({
         }
 
         builds = projects.collectEntries { proj ->
-            ["$osname:$proj": [
-                 nodeLabel: "${nodeLabels.codeBuild}",
-                 build: {
-
-                     def packageName = proj
-                     def prTag = "ci-$osname-$proj" // use ci project
-                     def srcVersion
-                     def lastRevision
-                     def debPVersion
-                     def goals = ['package']
-
-                     def ciImageTag
-
-                     stage('Get source version') {
-                         srcVersion = utils.shStdout("SRC_DIR_NAME=$proj make -C devops src_version -s")
-                         logger.info("Current source version: $srcVersion")
-                     }
-
-//                stage('Get last revision') {
-//                    lastRevision = evernymRepo.getLastRevision {
-//                        delegate.packageName = packageName
-//                        packageSrcVersion = srcVersion
-//                        repoDistr = 'agency-dev'
-//                    }
+            String projLabel = "$osname-$proj"
+
+            ["$projLabel": [
+                nodeLabel: "${nodeLabels.codeBuild}",
+                build: {
+                    def packageName = proj
+                    def prTag = "ci-$osname-$proj" // use ci project
+                    def srcVersion
+                    def lastRevision
+                    def debPVersion
+                    def goals = ['package']
+
+                    def ciImageTag
+
+                    stage('Get source version') {
+                        srcVersion = utils.shStdout("SRC_DIR_NAME=$proj make -C devops src_version -s")
+                        logger.info("Current source version: $srcVersion")
+                    }
+
+//               stage('Get last revision') {
+//                   lastRevision = evernymRepo.getLastRevision {
+//                       delegate.packageName = packageName
+//                       packageSrcVersion = srcVersion
+//                       repoDistr = 'agency-dev'
+//                   }
 //
-//                    if (lastRevision) {
-//                        logger.info("Found last revision number: $lastRevision")
-//                    } else {
-//                        logger.info("No previous revision was found")
-//                    }
-//                }
-
-
-                     stage('Set release parameters') {
-                         def releaseVersion =  env.BRANCH_NAME == 'stable' ? '' : "$BUILD_NUMBER"
-
-                         debPVersion = "$srcVersion${releaseVersion ? '~' + releaseVersion : ''}"
-                         logger.info("Package version for sovrin repo: $debPVersion")
-                     }
-
-                     stage("$osname:$proj: Resolve image tag") {
-                         def _imgVersion = utils.shStdout("OSNAME=$osname make -C devops image_ci_version -s")
-                         ciImageTag = "$_imgVersion-$osname-$proj-ci"
-                         logger.info("CI docker image tag: $ciImageTag")
-                     }
-
-                     awsCBHelper.build() {
-                         projectTag = prTag
-
-                         // build spec for env image
-                         envBuildSrc = ['devops', "${proj}/${proj}/__metadata__.py"] // TODO make more accurate
-                         envBuildCmds = [
-                                 'export PROJECT_DIR=$PWD',
-                                 'make -C devops image_ci'
-                         ]
-                         envBuildEnvv = [
-                                 [name: 'OSNAME', value: osname],
-                                 [name: 'SRC_DIR_NAME', value: proj],
-                                 [name: 'DOCKER_NAME', value: awsCBHelper.buildCtx.projectName],
-                                 [name: 'DOCKER_TAG', value: ciImageTag],
-                         ]
-
-                         // env and build spec
-                         imageTag = ciImageTag
-                         buildspec = 'devops/aws-codebuild/buildspec.yml'
-                         envv = [
-                                 [name: 'OSNAME', value: osname],
-                                 [name: 'MAKE_GOALS', value: 'package'],
-                                 [name: 'SRC_DIR_NAME', value: proj],
-                                 [name: 'ARTIFACTS', value: "devops/_build/$proj/*$packageName*.*"], // TODO more accurate here
-                                 [name: 'FPM_P_VERSION', value: debPVersion],
-                         ]
-
-                         onArtifacts = {
-                             this.stage("$osname:$proj: Archive logs") {
-                                 utils.archiveArtifacts("logs/*.log*") {
-                                     truncate = true
-                                     allowEmptyArchive = true
-                                 }
-                             }
-                         }
-                     }
-
-
-                     stage('Upload debs to repos') {
-
-                         String debName
-
-                         dir("${awsCBHelper.buildCtx.projects[prTag].artifactsDir}") {
-                             dir("sovrin-packaging") {
-                                 downloadPackagingUtils()
-                             }
-
-                             debName = utils.shStdout("ls $packageName*$debPVersion*.deb")
-
-                             logger.info("Uploading debian package '$debName' to sovrin repo")
-                             sh "mkdir debs &&  mv $debName ./debs/"
-                             withCredentials([file(credentialsId: 'SovrinRepoSSHKey', variable: 'sovrin_key')]) {
-                                 sh "./sovrin-packaging/upload_debs.py ./debs $env.SOVRIN_CORE_REPO_NAME $env.BRANCH_NAME --host $env.SOVRIN_REPO_HOST --ssh-key $sovrin_key"
-                             }
-
-                         }
-
-                         notifier.email {
-                             to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
-                             subject = '$PROJECT_NAME - Build # $BUILD_NUMBER: ' + "new deb '$debName' was published"
-                             body = ("New debian package '$debName' was built and published" +
-                                     '\n\nCheck console output at $BUILD_URL to view the results.')
-                         }
-                     }
-
-                 }
-                ]
-            ]
+//                   if (lastRevision) {
+//                       logger.info("Found last revision number: $lastRevision")
+//                   } else {
+//                       logger.info("No previous revision was found")
+//                   }
+//               }
+
+
+                    stage('Set release parameters') {
+                        def releaseVersion =  env.BRANCH_NAME == 'stable' ? '' : "$BUILD_NUMBER"
+
+                        debPVersion = "$srcVersion${releaseVersion ? '~' + releaseVersion : ''}"
+                        logger.info("Package version for sovrin repo: $debPVersion")
+                    }
+
+                    stage("$projLabel: Resolve image tag") {
+                        def _imgVersion = utils.shStdout("OSNAME=$osname make -C devops image_ci_version -s")
+                        ciImageTag = "$_imgVersion-$osname-$proj-ci"
+                        logger.info("CI docker image tag: $ciImageTag")
+                    }
+
+                    awsCBHelper.build() {
+                        projectTag = prTag
+
+                        // build spec for env image
+                        envBuildSrc = ['devops', "${proj}/${proj}/__metadata__.py"] // TODO make more accurate
+                        envBuildCmds = [
+                                'export PROJECT_DIR=$PWD',
+                                'make -C devops image_ci'
+                        ]
+                        envBuildEnvv = [
+                                [name: 'OSNAME', value: osname],
+                                [name: 'SRC_DIR_NAME', value: proj],
+                                [name: 'DOCKER_NAME', value: awsCBHelper.buildCtx.projectName],
+                                [name: 'DOCKER_TAG', value: ciImageTag],
+                        ]
+
+                        // env and build spec
+                        imageTag = ciImageTag
+                        buildspec = 'devops/aws-codebuild/buildspec.yml'
+                        envv = [
+                                [name: 'OSNAME', value: osname],
+                                [name: 'MAKE_GOALS', value: 'package'],
+                                [name: 'SRC_DIR_NAME', value: proj],
+                                [name: 'ARTIFACTS', value: "devops/_build/${proj}/*${packageName}*.*"], // TODO more accurate here
+                                [name: 'FPM_P_VERSION', value: debPVersion],
+                        ]
+
+                        onArtifacts = {
+                            this.stage("$projLabel: Archive logs") {
+                                utils.archiveArtifacts("logs/*.log*") {
+                                    truncate = true
+                                    allowEmptyArchive = true
+                                }
+
+                                String pkgBaseName = "${packageName}_${debPVersion}"
+
+                                this.stash includes: utils.shStdout("ls ${pkgBaseName}_*.deb"), name: pkgBaseName
+                                pkgsList += pkgBaseName
+                            }
+                        }
+                    }
+                }
+            ]]
         }
 
-        stage("Package and publish") {
+        stage("Build artifacts") {
             builds.failFast = false
             utils.parallel builds
         }
+
+        stage('Release artifacts') {
+            String repoName = env.SOVRIN_CORE_REPO_NAME
+            String repoComponent = env.BRANCH_NAME
+            String debsDir = 'debs'
+
+            if ((env.BRANCH_NAME == 'stable') && indyNodeVersion.find(/[^0-9\.]/)) {
+                repoComponent = 'rc'
+            }
+            logger.info("Debian repo component is set to: '$repoComponent'")
+
+            dir(debsDir) {
+                pkgsList.each {
+                    unstash name: it
+                }
+            }
+            logger.info("Uploading debian packages to '${repoName}/${repoComponent}' sovrin repo: $pkgsList")
+
+            dir("sovrin-packaging") {
+                downloadPackagingUtils()
+            }
+
+            withCredentials([file(credentialsId: 'SovrinRepoSSHKey', variable: 'sovrin_key')]) {
+                sh "./sovrin-packaging/upload_debs.py $debsDir $repoName $repoComponent --host $env.SOVRIN_REPO_HOST --ssh-key $sovrin_key --force-backup"
+            }
+
+            notifier.email {
+                to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
+                subject = "[$projectName] new packages ${pkgsList.join(' ')} published to '${repoName}/${repoComponent}' repo"
+                body = """
+                    |New debian packages ${pkgsList.join(' ')} were built and published to '${repoName}/${repoComponent}' repo
+                    |
+                    |Build:
+                    |   Project: \$PROJECT_NAME
+                    |   Url: ${this.env.BUILD_URL}
+                    |   Number: ${this.env.BUILD_NUMBER}
+                    |
+                    |Check console output at ${this.env.BUILD_URL}console to view the details.
+                """.stripMargin()
+            }
+        }
     }
 }, { err ->
     if (err) {
@@ -193,6 +223,7 @@ pipelineWrapper({
 
     stage("Build result notification") {
         notifier.email {
+            subject = "[$projectName] \$DEFAULT_SUBJECT"
             to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
         }
     }

diff --git a/docs/release.md b/docs/release.md
@@ -0,0 +1,29 @@
+# Release Workflow
+
+## Overview
+
+Release workflow is engined by Jenkins and described by the [pipeline](../devops/aws-codebuild/Jenkinsfile.cd).
+
+For now release artifacts are created only for debian package [repository](https://repo.sovrin.org/deb/)
+maintained by [The Sovrin Foundation](https://sovrin.org)
+
+There are several types of releases:
+- dev releases are built for each commit to `master` branch;
+- rc (release candidate) releases are built for each commit to `stable` when `indy-node` dependency is set to *non stable* version;
+- stable releases are built for each commit to `stable` branch when `indy-node` dependency is set to *stable* version.
+
+## Release Pipeline
+
+1. Preparation
+    1. `indy-node` version is checked.
+    2. Necessary AWS resources are created.
+2. Build
+    1. AWS CodeBuild project builds debian packages for each plugin
+3. Publishing and Notification
+    1. Built packages are published to the repository component depending on the `indy-node` dependency version and GitHub branch as it was mentioned above:
+        - `master` commits are published to `master`, debian packages include unique pipeline build number as part of their versions.
+        - `stable` commits are published to `rc` if `indy-node` version is *non stable* (e.g. `1.8.0~rc1`) and to `stable` otherwise. These artifacts' versions match ones from the source code.
+
+### Important Note
+
+Commits to `stable` with plugins versions and `indy-node` version that were already published can't be published again since release logic detects duplicates in the debian repository and skips publishing for such cases.