From d9a143828c3f08dd329ba54b8d5f19db3b79e604 Mon Sep 17 00:00:00 2001 From: Wei Chu Date: Tue, 20 Oct 2020 21:55:43 -0700 Subject: [PATCH] add website artifacts pipeline --- ci/docker/runtime_functions.sh | 21 ++++++ ci/jenkins/Jenkins_steps.groovy | 49 ++++++++++++++ .../Jenkinsfile_website_version_artifacts | 67 +++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 ci/jenkins/Jenkinsfile_website_version_artifacts diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 2e90b5e8d25e..68ecf30d96ea 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1920,6 +1920,27 @@ build_docs_beta() { popd } +push_docs() { + folder_name=$1 + set -ex + pip3 install --user awscli + export PATH=~/.local/bin:$PATH + pushd docs/_build + wget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip && unzip versions.zip && rm versions.zip + mkdir $folder_name && tar -xzf full_website.tgz -C $folder_name --strip-components 1 + # check if folder_name already exists in versions + pushd versions + if [ -d "$folder_name" ]; then + echo "Folder $folder_name already exists in versions. Please double check the FOLDER_NAME variable in Jenkens pipeline" + exit 1 + fi + popd + mv $folder_name versions + zip -r9 versions.zip versions/. + aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read + popd +} + create_repo() { repo_folder=$1 mxnet_url=$2 diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 73a47d164871..a40e66a407d5 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1587,6 +1587,55 @@ def docs_jekyll() { }] } +// This is for building the full website +// Assumes you have run all of the docs generation functions +def docs_full_website() { + return ['Build artifacts full_website.tgz': { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + + unstash 'jekyll-artifacts' + unstash 'c-artifacts' + unstash 'python-artifacts' + unstash 'r-artifacts' + unstash 'julia-artifacts' + unstash 'scala-artifacts' + unstash 'java-artifacts' + unstash 'clojure-artifacts' + + utils.docker_run('ubuntu_cpu_jekyll', 'build_docs', false) + utils.pack_lib('full_website', 'docs/_build/full_website.tgz', false) + } + } + } + }] +} + +// This is for uploading website artifacts to S3 bucket +// Assumes you have run docs_full_website function +def docs_upload_s3() { + return ['Upload artifacts to s3 bucket': { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + timeout(time: max_time, unit: 'MINUTES') { + if(env.FOLDER_NAME) { + utils.unpack_and_init('full_website', 'docs/_build/full_website.tgz') + + utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false) + + archiveArtifacts 'docs/_build/versions.zip' + } else { + sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline' + sh 'exit 1' + } + + } + } + } + }] +} // This is for publishing the full website // Assumes you have run all of the docs generation functions diff --git a/ci/jenkins/Jenkinsfile_website_version_artifacts b/ci/jenkins/Jenkinsfile_website_version_artifacts new file mode 100644 index 000000000000..df11a4cbaa71 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_version_artifacts @@ -0,0 +1,67 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline to build website artifacit and upload to private s3 bucket +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('restricted-utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} + +utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Build Api Docs', [ + custom_steps.docs_jekyll(), + custom_steps.docs_c(), + custom_steps.docs_python(), + custom_steps.docs_julia(), + custom_steps.docs_r(), + custom_steps.docs_scala(), + custom_steps.docs_java(), + custom_steps.docs_clojure() + ]) + + utils.parallel_stage('Build Full Website', [ + custom_steps.docs_full_website() + ]) + + utils.parallel_stage('Upload Docs', [ + custom_steps.docs_upload_s3() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +)