Skip to content

Commit

Permalink
Build: Split up task in the CI pipeline to ease running outside circl…
Browse files Browse the repository at this point in the history
…eci (grafana#18861)

* build: make sign rpm packages not depend on checking out private key

* build: move commands from circleci config into verify signed packages script

* build: split update and publish of deb and rpm into two scripts

* use files argument for sign and verify packages

* validate files argument for sign and verify packages

* update test publish of deb/rpm readme
  • Loading branch information
marefr authored Sep 23, 2019
1 parent 8f9c487 commit 4386604
Show file tree
Hide file tree
Showing 14 changed files with 164 additions and 48 deletions.
57 changes: 42 additions & 15 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -214,15 +214,15 @@ jobs:
- run:
name: build and package grafana
command: './scripts/build/build-all.sh'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: verify signed packages
command: |
mkdir -p ~/.rpmdb/pubkeys
curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
./scripts/build/verify_signed_packages.sh dist/*.rpm
command: './scripts/build/verify_signed_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
Expand All @@ -249,9 +249,12 @@ jobs:
- run:
name: build and package grafana
command: './scripts/build/build.sh'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
Expand Down Expand Up @@ -360,9 +363,12 @@ jobs:
- run:
name: package grafana
command: './scripts/build/build.sh --fast --package-only'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
Expand Down Expand Up @@ -435,9 +441,12 @@ jobs:
- run:
name: build and package enterprise
command: './scripts/build/build.sh -enterprise'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
Expand Down Expand Up @@ -476,15 +485,15 @@ jobs:
- run:
name: build and package grafana
command: './scripts/build/build-all.sh -enterprise'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
command: './scripts/build/sign_packages.sh dist/*.rpm'
- run:
name: verify signed packages
command: |
mkdir -p ~/.rpmdb/pubkeys
curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
./scripts/build/verify_signed_packages.sh dist/*.rpm
command: './scripts/build/verify_signed_packages.sh dist/*.rpm'
- run:
name: sha-sum packages
command: 'go run build.go sha-dist'
Expand Down Expand Up @@ -537,15 +546,24 @@ jobs:
- run:
name: Deploy to Grafana.com
command: './scripts/build/publish.sh --enterprise'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: Load GPG private key
command: './scripts/build/load-signing-key.sh'
command: './scripts/build/update_repo/load-signing-key.sh'
- run:
name: Update Debian repository
command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
- run:
name: Publish Debian repository
command: './scripts/build/update_repo/publish-deb.sh "enterprise"'
- run:
name: Update RPM repository
command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
- run:
name: Publish RPM repository
command: './scripts/build/update_repo/publish-rpm.sh "enterprise" "$CIRCLE_TAG"'


deploy-master:
Expand Down Expand Up @@ -591,15 +609,24 @@ jobs:
- run:
name: Deploy to Grafana.com
command: './scripts/build/publish.sh'
- run:
name: Prepare GPG private key
command: './scripts/build/prepare_signing_key.sh'
- run:
name: Load GPG private key
command: './scripts/build/load-signing-key.sh'
command: './scripts/build/update_repo/load-signing-key.sh'
- run:
name: Update Debian repository
command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
- run:
name: Publish Debian repository
command: './scripts/build/update_repo/publish-deb.sh "oss"'
- run:
name: Update RPM repository
command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
- run:
name: Publish RPM repository
command: './scripts/build/update_repo/publish-rpm.sh "oss" "$CIRCLE_TAG"'

build-oss-msi:
docker:
Expand Down
7 changes: 0 additions & 7 deletions scripts/build/load-signing-key.sh

This file was deleted.

6 changes: 6 additions & 0 deletions scripts/build/prepare_signing_key.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash

set -e

git clone git@github.com:torkelo/private.git ~/private-repo
cp ~/private-repo/signing/private.key /private.key
18 changes: 15 additions & 3 deletions scripts/build/sign_packages.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
#!/bin/bash

git clone git@github.com:torkelo/private.git ~/private-repo
set -e

gpg --allow-secret-key-import --import ~/private-repo/signing/private.key
_files=$*

if [ -z "$_files" ]; then
echo "_files (arg 1) has to be set"
exit 1
fi

if [ -z "$GPG_KEY_PASSWORD" ]; then
echo "GPG_KEY_PASSWORD has to be set"
exit 1
fi

gpg --allow-secret-key-import --import /private.key

cp ./scripts/build/rpmmacros ~/.rpmmacros

for package in dist/*.rpm; do
for package in $_files; do
[ -e "$package" ] || continue
./scripts/build/sign_expect "$GPG_KEY_PASSWORD" "$package"
done
17 changes: 9 additions & 8 deletions scripts/build/update_repo/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,23 @@
It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.

```bash
docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing
docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.2 bash # 1.2.2 is the newest image at the time of writing
# in the container:
mkdir -p /go/src/github.com/grafana/dist
cd /go/src/github.com/grafana
mkdir -p /dist

#outside of container:
cd <grafana project dir>/..
docker cp grafana <container_name>:/go/src/github.com/grafana/.
docker cp grafana <container_name>:/
docker cp <gpg.key used for signing> <container_name>:/private.key

#in container:
gpg --batch --allow-secret-key-import --import /private.key
./scripts/build/update_repo/load-signing-key.sh
cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..

#run these scripts:
./script/build/update_repo/test-update-deb-repo.sh <gpg key password>
./script/build/update_repo/test-update-rpm-repo.sh <gpg key password>
#run these scripts to update local deb and rpm repos and publish them:
./scripts/build/update_repo/test-update-deb-repo.sh <gpg key password>
./scripts/build/update_repo/test-publish-deb-repo.sh
./scripts/build/update_repo/test-update-rpm-repo.sh <gpg key password>
./scripts/build/update_repo/test-publish-rpm-repo.sh

```
6 changes: 6 additions & 0 deletions scripts/build/update_repo/load-signing-key.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash

set -e

gpg --batch --allow-secret-key-import --import /private.key
pkill gpg-agent
30 changes: 30 additions & 0 deletions scripts/build/update_repo/publish-deb.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash

RELEASE_TYPE="${1:-}"
GCP_DB_BUCKET="${2:-grafana-aptly-db}"
GCP_REPO_BUCKET="${3:-grafana-repo}"

if [ -z "$RELEASE_TYPE" ]; then
echo "RELEASE_TYPE (arg 1) has to be set"
exit 1
fi

if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
exit 1
fi

set -e

# Update the repo and db on gcp

gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"

# Uploads the binaries before the metadata (to prevent 404's for debs)
gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"

gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"

# usage:
#
# deb https://packages.grafana.com/oss/deb stable main
41 changes: 41 additions & 0 deletions scripts/build/update_repo/publish-rpm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/usr/bin/env bash

RELEASE_TYPE="${1:-}"
RELEASE_TAG="${2:-}"
GCP_REPO_BUCKET="${3:-grafana-repo}"

REPO="rpm"

if [ -z "$RELEASE_TYPE" ]; then
echo "RELEASE_TYPE (arg 1) has to be set"
exit 1
fi

if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
exit 1
fi

if echo "$RELEASE_TAG" | grep -q "beta"; then
REPO="rpm-beta"
fi

set -e

# Setup environment
BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO"

# Update the repo and db on gcp
gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
gsutil -m rsync -r -d /rpm-repo "$BUCKET"

# usage:
# [grafana]
# name=grafana
# baseurl=https://packages.grafana.com/oss/rpm
# repo_gpgcheck=1
# enabled=1
# gpgcheck=1
# gpgkey=https://packages.grafana.com/gpg.key
# sslverify=1
# sslcacert=/etc/pki/tls/certs/ca-bundle.crt
3 changes: 3 additions & 0 deletions scripts/build/update_repo/test-publish-deb-repo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

./scripts/build/update_repo/publish-deb.sh "oss" "grafana-testing-aptly-db" "grafana-testing-repo"
3 changes: 3 additions & 0 deletions scripts/build/update_repo/test-publish-rpm-repo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

./scripts/build/update_repo/publish-rpm.sh "oss" "v5.4.3" "grafana-testing-repo"
2 changes: 1 addition & 1 deletion scripts/build/update_repo/test-update-deb-repo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

GPG_PASS=${1:-}

./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"
./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db"
10 changes: 0 additions & 10 deletions scripts/build/update_repo/update-deb.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ GPG_PASS="${2:-}"
RELEASE_TAG="${3:-}"
DIST_PATH="${4:-}"
GCP_DB_BUCKET="${5:-grafana-aptly-db}"
GCP_REPO_BUCKET="${6:-grafana-repo}"

REPO="grafana"

Expand Down Expand Up @@ -54,15 +53,6 @@ rm /tmp/sign-this /tmp/sign-this.asc
aptly publish update stable filesystem:repo:grafana
aptly publish update beta filesystem:repo:grafana

# Update the repo and db on gcp

gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"

# Uploads the binaries before the metadata (to prevent 404's for debs)
gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"

gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"

# usage:
#
# deb https://packages.grafana.com/oss/deb stable main
4 changes: 0 additions & 4 deletions scripts/build/update_repo/update-rpm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,6 @@ rm /rpm-repo/repodata/repomd.xml.asc || true
pkill gpg-agent || true
./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"

# Update the repo and db on gcp
gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
gsutil -m rsync -r -d /rpm-repo "$BUCKET"

# usage:
# [grafana]
# name=grafana
Expand Down
8 changes: 8 additions & 0 deletions scripts/build/verify_signed_packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@

_files=$*

if [ -z "$_files" ]; then
echo "_files (arg 1) has to be set"
exit 1
fi

mkdir -p ~/.rpmdb/pubkeys
curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key

ALL_SIGNED=0

for file in $_files; do
Expand Down

0 comments on commit 4386604

Please sign in to comment.