[LDC] Set up CI / package generation with Azure Pipelines and Shippable

Azure: Windows x86 & x64, macOS x64, Linux x64, Android ARMv7-A & AArch64
Shippable: Linux AArch64

2 packages per platform (enabled/disabled assertions), except for cross-
compiled Android.

Tagged builds are uploaded to the corresponding GitHub release, untagged
builds to the GitHub 'CI' release.

Author: kinke

.azure-pipelines/posix.yml

@@ -0,0 +1,59 @@
+# Each step starts in the checked-out source directory,
+# environment variables aren't persisted across steps.
+
+steps:
+- script: |
+    set -ex
+    cd ..
+    if [ "$CI_OS" = "osx" ]; then export PATH="$PWD/ninja:$PATH"; fi
+    cmake --version
+    ninja --version
+    mkdir build
+    cd build
+    cmake -G Ninja $BUILD_SOURCESDIRECTORY/llvm \
+      -DLLVM_ENABLE_PROJECTS='compiler-rt;lld' \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=$PWD/../install \
+      -DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
+      -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
+      -DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
+      $BASE_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS
+    ninja install
+  displayName: Build & install LLVM incl. LLD, compiler-rt and the Khronos SPIRV-LLVM-Translator
+- script: |
+    set -ex
+    cd ..
+    if [ "${BUILD_SOURCEBRANCH:0:10}" = "refs/tags/" ]; then
+      ARTIFACT_ID=${BUILD_SOURCEBRANCH:15}
+    else
+      ARTIFACT_ID=${BUILD_SOURCEVERSION:0:8}
+    fi
+    ASSERTS_SUFFIX=""
+    if [ "$LLVM_ENABLE_ASSERTIONS" = "ON" ]; then
+      ASSERTS_SUFFIX="-withAsserts"
+    fi
+    artifactName=llvm-$ARTIFACT_ID-$CI_OS-$ARCH$ASSERTS_SUFFIX
+    mv install $artifactName
+    mkdir artifacts
+    chmod -R go=rX $artifactName
+    if [ "$CI_OS" = "osx" ]; then
+      sudo chown -R root:wheel $artifactName
+      tar -cJf artifacts/$artifactName.tar.xz --options='compression-level=9' $artifactName
+    else
+      XZ_OPT=-9 tar -cJf artifacts/$artifactName.tar.xz --owner=0 --group=0 $artifactName
+    fi
+    # Persist the ARTIFACT_ID and ASSERTS_SUFFIX variables for future steps
+    set +x
+    echo "##vso[task.setvariable variable=ARTIFACT_ID]$ARTIFACT_ID"
+    echo "##vso[task.setvariable variable=ASSERTS_SUFFIX]$ASSERTS_SUFFIX"
+  displayName: Pack installation dir
+- script: |
+    set -ex
+    cd ..
+    artifactName=llvm-$ARTIFACT_ID.src
+    chmod -R go=rX $BUILD_SOURCESDIRECTORY
+    XZ_OPT=-9 tar -cJf artifacts/$artifactName.tar.xz --exclude-vcs --owner=0 --group=0 --transform=s,${BUILD_SOURCESDIRECTORY:1},$artifactName, $BUILD_SOURCESDIRECTORY
+  displayName: Pack source dir
+  condition: and(succeeded(), and(eq(variables['CI_OS'], 'linux'), eq(variables['LLVM_ENABLE_ASSERTIONS'], 'OFF')))
+- publish: ../artifacts
+  artifact: $(CI_OS)-$(ARCH)$(ASSERTS_SUFFIX)

.azure-pipelines/windows.yml

@@ -0,0 +1,65 @@
+# Each step starts in the checked-out source directory,
+# environment variables aren't persisted across steps.
+
+steps:
+- script: |
+    set
+  displayName: Print environment variables
+- checkout: self
+  submodules: true
+  fetchDepth: 50
+- script: |
+    echo on
+    cd ..
+    :: Download & extract Ninja
+    curl -L -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip 2>&1
+    mkdir ninja
+    cd ninja
+    7z x ../ninja.zip > nul
+    cd ..
+    :: Download & install clang
+    curl -L -o clang.exe http://releases.llvm.org/%CLANG_VERSION%/LLVM-%CLANG_VERSION%-win64.exe 2>&1
+    clang.exe /S
+  displayName: Install prerequisites
+- script: |
+    echo on
+    cd ..
+    set PATH=C:\Program Files\LLVM\bin;%CD%\ninja;%PATH%
+    call "%VSINSTALLDIR%Common7\Tools\VsDevCmd.bat" -arch=%ARCH%
+    echo on
+    cmake --version
+    ninja --version
+    clang-cl --version
+    mkdir build
+    cd build
+    cmake -G Ninja %BUILD_SOURCESDIRECTORY%\llvm -DCMAKE_C_COMPILER:PATH=clang-cl.exe -DCMAKE_CXX_COMPILER:PATH=clang-cl.exe -DLLVM_ENABLE_PROJECTS=compiler-rt;lld -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%CD%/../install -DLLVM_TARGETS_TO_BUILD=%LLVM_TARGETS_TO_BUILD% -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=%LLVM_EXPERIMENTAL_TARGETS_TO_BUILD% -DLLVM_ENABLE_ASSERTIONS=%LLVM_ENABLE_ASSERTIONS% -DLLVM_DEFAULT_TARGET_TRIPLE=%LLVM_DEFAULT_TARGET_TRIPLE% %BASE_CMAKE_FLAGS% %EXTRA_CMAKE_FLAGS%
+    ninja install
+  displayName: Build & install LLVM incl. LLD, compiler-rt and the Khronos SPIRV-LLVM-Translator
+- script: |
+    echo on
+    cd ..
+    :: On Windows, the lld symlinks are copies
+    rm install/bin/ld.lld.exe
+    rm install/bin/ld64.lld.exe
+    rm install/bin/lld.exe
+    rm install/bin/wasm-ld.exe
+    if "%BUILD_SOURCEBRANCH:~0,10%" == "refs/tags/" (
+      set ARTIFACT_ID=%BUILD_SOURCEBRANCH:~15%
+    ) else (
+      set ARTIFACT_ID=%BUILD_SOURCEVERSION:~0,8%
+    )
+    set ASSERTS_SUFFIX=
+    if "%LLVM_ENABLE_ASSERTIONS%" == "ON" (
+      set ASSERTS_SUFFIX=-withAsserts
+    )
+    set ARTIFACT_NAME=llvm-%ARTIFACT_ID%-windows-%ARCH%%ASSERTS_SUFFIX%
+    mv install %ARTIFACT_NAME%
+    mkdir artifacts
+    cd %ARTIFACT_NAME%
+    7z a -mx=9 ../artifacts/%ARTIFACT_NAME%.7z * > nul
+    :: Persist the ASSERTS_SUFFIX env variable for future steps
+    echo off
+    echo ##vso[task.setvariable variable=ASSERTS_SUFFIX]%ASSERTS_SUFFIX%
+  displayName: Pack installation dir
+- publish: ../artifacts
+  artifact: windows-$(ARCH)$(ASSERTS_SUFFIX)

azure-pipelines.yml

@@ -0,0 +1,219 @@
+variables:
+  LLVM_TARGETS_TO_BUILD: AArch64;ARM;Mips;MSP430;NVPTX;PowerPC;RISCV;WebAssembly;X86
+  LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: AVR
+  # Prevent dependencies on some purely optional libraries, and a dependency on clang
+  # for compiler-rt and SPIRV-LLVM-Translator tests.
+  BASE_CMAKE_FLAGS: -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_LIBEDIT=OFF -DCOMPILER_RT_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_TESTS=OFF -DCOMPILER_RT_USE_LIBCXX=OFF
+
+# Although tags are supposed to trigger builds by default, they don't.
+trigger:
+- '*'
+- refs/tags/ldc-v*
+
+jobs:
+- job: Windows
+  timeoutInMinutes: 180
+  pool:
+    vmImage: 'windows-2019'
+  variables:
+    CLANG_VERSION: 9.0.0
+    VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\
+    EXTRA_CMAKE_FLAGS: -DLLVM_USE_CRT_RELEASE=MT
+  strategy:
+    matrix:
+      x64:
+        ARCH: x64
+        LLVM_DEFAULT_TARGET_TRIPLE: x86_64-pc-windows-msvc
+        LLVM_ENABLE_ASSERTIONS: OFF
+      x64_withAsserts:
+        ARCH: x64
+        LLVM_DEFAULT_TARGET_TRIPLE: x86_64-pc-windows-msvc
+        LLVM_ENABLE_ASSERTIONS: ON
+      x86:
+        ARCH: x86
+        LLVM_DEFAULT_TARGET_TRIPLE: i686-pc-windows-msvc
+        LLVM_ENABLE_ASSERTIONS: OFF
+        # Let CMake configure 64-bit clang-cl for 32-bit code emission.
+        CFLAGS: -m32
+        CXXFLAGS: -m32
+        ASMFLAGS: -m32
+      x86_withAsserts:
+        ARCH: x86
+        LLVM_DEFAULT_TARGET_TRIPLE: i686-pc-windows-msvc
+        LLVM_ENABLE_ASSERTIONS: ON
+        CFLAGS: -m32
+        CXXFLAGS: -m32
+        ASMFLAGS: -m32
+  steps:
+    - template: .azure-pipelines/windows.yml
+
+- job: Linux
+  timeoutInMinutes: 180
+  pool:
+    vmImage: 'ubuntu-16.04'
+  variables:
+    CI_OS: linux
+    ARCH: x86_64
+    EXTRA_CMAKE_FLAGS: -DLLVM_BINUTILS_INCDIR=/usr/include -DCMAKE_CXX_FLAGS=-static-libstdc++
+  strategy:
+    matrix:
+      x64:
+        LLVM_ENABLE_ASSERTIONS: OFF
+      x64_withAsserts:
+        LLVM_ENABLE_ASSERTIONS: ON
+  steps:
+    - checkout: self
+      submodules: true
+      fetchDepth: 50
+    - script: |
+        set -ex
+        cd ..
+        export DEBIAN_FRONTEND=noninteractive
+        sudo apt-get -q update
+        sudo apt-get -yq install \
+          git-core cmake ninja-build g++ binutils-dev python
+      displayName: Install prerequisites
+    - template: .azure-pipelines/posix.yml
+
+- job: macOS
+  timeoutInMinutes: 180
+  pool:
+    vmImage: 'macOS-10.14'
+  variables:
+    CI_OS: osx
+    ARCH: x86_64
+    EXTRA_CMAKE_FLAGS: -DHAVE_FUTIMENS=0
+    MACOSX_DEPLOYMENT_TARGET: 10.9
+  strategy:
+    matrix:
+      x64:
+        LLVM_ENABLE_ASSERTIONS: OFF
+      x64_withAsserts:
+        LLVM_ENABLE_ASSERTIONS: ON
+  steps:
+    - checkout: self
+      submodules: true
+      fetchDepth: 50
+    - script: |
+        set -ex
+        cd ..
+        # Download & extract Ninja
+        curl -L -o ninja-mac.zip https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-mac.zip
+        mkdir ninja
+        tar -xf ninja-mac.zip -C ninja
+      displayName: Install prerequisites
+    - template: .azure-pipelines/posix.yml
+
+- job: Android
+  timeoutInMinutes: 180
+  pool:
+    vmImage: 'ubuntu-16.04'
+  variables:
+    CI_OS: android
+    LLVM_TARGETS_TO_BUILD: AArch64;ARM;WebAssembly;X86
+    LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: ''
+    LLVM_ENABLE_ASSERTIONS: OFF
+  strategy:
+    matrix:
+      armv7a:
+        ARCH: armv7a
+        EXTRA_CMAKE_FLAGS: -DANDROID_ABI=armeabi-v7a -DANDROID_NATIVE_API_LEVEL=21 -DANDROID_STL=c++_static -DCMAKE_CROSSCOMPILING=True -DLLVM_TARGET_ARCH=ARM -DLLVM_DEFAULT_TARGET_TRIPLE=armv7a--linux-androideabi -DLLVM_ENABLE_PLUGINS=OFF
+      aarch64:
+        ARCH: aarch64
+        EXTRA_CMAKE_FLAGS: -DANDROID_ABI=arm64-v8a -DANDROID_NATIVE_API_LEVEL=21 -DANDROID_STL=c++_static -DCMAKE_CROSSCOMPILING=True -DLLVM_TARGET_ARCH=AArch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64--linux-android -DLLVM_ENABLE_PLUGINS=OFF -DSANITIZER_CXX_ABI=libc++
+  steps:
+    - checkout: self
+      submodules: true
+      fetchDepth: 50
+    - script: |
+        set -ex
+        cd ..
+        export DEBIAN_FRONTEND=noninteractive
+        sudo apt-get -q update
+        sudo apt-get -yq install \
+          git-core cmake ninja-build g++ curl python unzip
+      displayName: Install prerequisites
+    - script: |
+        set -ex
+        cd ..
+        cmake --version
+        ninja --version
+        mkdir build-native
+        cd build-native
+        cmake -G Ninja $BUILD_SOURCESDIRECTORY/llvm \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
+          -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
+          -DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
+          $BASE_CMAKE_FLAGS
+        ninja llvm-tblgen
+      displayName: Build native llvm-tblgen
+    - script: |
+        set -ex
+        cd ..
+        # Download & extract Android NDK
+        curl -OL https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip
+        unzip android-ndk-r20b-linux-x86_64.zip > /dev/null
+        rm android-ndk-r20b-linux-x86_64.zip
+        # The NDK toolchain file enforces `-g` as base C[XX] flag - remove it to
+        # *significantly* reduce executable sizes
+        TOOLCHAIN_FILE=$PWD/android-ndk-r20b/build/cmake/android.toolchain.cmake
+        sed -i 's|^  -g$||' $TOOLCHAIN_FILE
+        # Extend EXTRA_CMAKE_FLAGS for future steps
+        set +x
+        echo "##vso[task.setvariable variable=EXTRA_CMAKE_FLAGS]$EXTRA_CMAKE_FLAGS -DLLVM_TABLEGEN=$PWD/build-native/bin/llvm-tblgen -DLLVM_CONFIG_PATH=$PWD/build-native/bin/llvm-config -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN_FILE"
+      displayName: Set up cross-compilation
+    - template: .azure-pipelines/posix.yml
+
+# Uploads the packages to a GitHub release.
+- job: Upload_to_GitHub
+  timeoutInMinutes: 30
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn:
+    - Windows
+    - Linux
+    - macOS
+    - Android
+  # Only run for branches and tags, no PRs.
+  condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')))
+  steps:
+    - checkout: none
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        targetPath: artifacts
+    - bash: |
+        set -ex
+        cd artifacts
+        mv */*.* .
+        find . -type d -empty -delete
+        ls -l
+        sha256sum * > sha256sums.txt
+      displayName: Move artifact files out of their subdirs and compute hashes
+    - bash: |
+        set -ex
+        if [ "${BUILD_SOURCEBRANCH:0:10}" != "refs/tags/" ]; then
+          releaseTag=CI
+          isPreRelease=true
+          artifactID=${BUILD_SOURCEVERSION:0:8}
+        else
+          releaseTag=${BUILD_SOURCEBRANCH:10}
+          isPreRelease=false
+          artifactID=${BUILD_SOURCEBRANCH:15}
+        fi
+        mv artifacts/sha256sums.txt artifacts/llvm-$artifactID.sha256sums.txt
+        set +x
+        echo "##vso[task.setvariable variable=GITHUB_RELEASE_TAG]$releaseTag"
+        echo "##vso[task.setvariable variable=GITHUB_RELEASE_ISPRERELEASE]$isPreRelease"
+      displayName: Set GITHUB_RELEASE_{TAG,ISPRERELEASE} env variables
+    - task: GitHubRelease@0
+      inputs:
+        gitHubConnection: kinke
+        repositoryName: ldc-developers/llvm-project
+        action: edit
+        tag: $(GITHUB_RELEASE_TAG)
+        assets: artifacts/*
+        assetUploadMode: replace
+        isPreRelease: $(GITHUB_RELEASE_ISPRERELEASE)
+        addChangeLog: false
+      displayName: Upload artifacts to GitHub release