all: switch to LLVM 16

This commit adds support for LLVM 16 and switches to it by default. That
means three LLVM versions are supported at the same time: LLVM 14, 15,
and 16.

Part of this work was based on a PR by QuLogic:
#3649
But I also had parts of this already implemented in an old branch I
already made for LLVM 16.

The difference is that this commit is more complete:
  * It switches to LLVM 16 by default.
  * It updates some things to also make it work with a self-built LLVM.
  * It fixes the CGo bug in a slightly different way, and also fixes
    another one not included in the original PR.
  * It does not keep compiler tests passing on older LLVM versions. I
    have found this to be quite burdensome and therefore don't generally
    do this - the smoke tests should hopefully catch most regressions.

.github/workflows/build-macos.yml

@@ -33,7 +33,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-macos-v3
+          key: llvm-source-16-macos-v1
           path: |
             llvm-project/clang/lib/Headers
             llvm-project/clang/include
@@ -58,7 +58,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-build
         with:
-          key: llvm-build-15-macos-v4
+          key: llvm-build-16-macos-v1
           path: llvm-build
       - name: Build LLVM
         if: steps.cache-llvm-build.outputs.cache-hit != 'true'
@@ -120,7 +120,7 @@ jobs:
       - name: Install LLVM
         shell: bash
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=1 brew install llvm@15
+          HOMEBREW_NO_AUTO_UPDATE=1 brew install llvm@16
       - name: Checkout
         uses: actions/checkout@v3
       - name: Install Go

.github/workflows/docker.yml

@@ -53,7 +53,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          build-contexts: tinygo-llvm-build=docker-image://tinygo/llvm-15
+          build-contexts: tinygo-llvm-build=docker-image://tinygo/llvm-16
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Trigger Drivers repo build on Github Actions

.github/workflows/linux.yml

@@ -43,7 +43,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-linux-alpine-v3
+          key: llvm-source-16-linux-alpine-v1
           path: |
             llvm-project/clang/lib/Headers
             llvm-project/clang/include
@@ -68,7 +68,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-build
         with:
-          key: llvm-build-15-linux-alpine-v4
+          key: llvm-build-16-linux-alpine-v1
           path: llvm-build
       - name: Build LLVM
         if: steps.cache-llvm-build.outputs.cache-hit != 'true'
@@ -194,7 +194,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-linux-asserts-v3
+          key: llvm-source-16-linux-asserts-v1
           path: |
             llvm-project/clang/lib/Headers
             llvm-project/clang/include
@@ -219,7 +219,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-build
         with:
-          key: llvm-build-15-linux-asserts-v4
+          key: llvm-build-16-linux-asserts-v1
           path: llvm-build
       - name: Build LLVM
         if: steps.cache-llvm-build.outputs.cache-hit != 'true'
@@ -297,7 +297,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-linux-v3
+          key: llvm-source-16-linux-v1
           path: |
             llvm-project/clang/lib/Headers
             llvm-project/clang/include
@@ -322,7 +322,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-build
         with:
-          key: llvm-build-15-linux-arm-v4
+          key: llvm-build-16-linux-arm-v1
           path: llvm-build
       - name: Build LLVM
         if: steps.cache-llvm-build.outputs.cache-hit != 'true'
@@ -414,7 +414,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-linux-v3
+          key: llvm-source-16-linux-v1
           path: |
             llvm-project/clang/lib/Headers
             llvm-project/clang/include
@@ -439,7 +439,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-build
         with:
-          key: llvm-build-15-linux-arm64-v4
+          key: llvm-build-16-linux-arm64-v1
           path: llvm-build
       - name: Build LLVM
         if: steps.cache-llvm-build.outputs.cache-hit != 'true'

.github/workflows/llvm.yml

@@ -35,8 +35,8 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           images: |
-            tinygo/llvm-15
-            ghcr.io/${{ github.repository_owner }}/llvm-15
+            tinygo/llvm-16
+            ghcr.io/${{ github.repository_owner }}/llvm-16
           tags: |
             type=sha,format=long
             type=raw,value=latest

.github/workflows/sizediff.yml

@@ -24,19 +24,19 @@ jobs:
           submodules: true
       - name: Install apt dependencies
         run: |
-          echo 'deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main' | sudo tee /etc/apt/sources.list.d/llvm.list
+          echo 'deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main' | sudo tee /etc/apt/sources.list.d/llvm.list
           wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
           sudo apt-get update
           sudo apt-get install --no-install-recommends -y \
-              llvm-15-dev \
-              clang-15 \
-              libclang-15-dev \
-              lld-15
+              llvm-16-dev \
+              clang-16 \
+              libclang-16-dev \
+              lld-16
       - name: Restore LLVM source cache
         uses: actions/cache@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-sizediff-v1
+          key: llvm-source-16-sizediff-v1
           path: |
             llvm-project/compiler-rt
       - name: Download LLVM source

.github/workflows/windows.yml

@@ -41,7 +41,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-source
         with:
-          key: llvm-source-15-windows-v4
+          key: llvm-source-16-windows-v1
           path: |
             llvm-project/clang/lib/Headers
             llvm-project/clang/include
@@ -66,7 +66,7 @@ jobs:
         uses: actions/cache/restore@v3
         id: cache-llvm-build
         with:
-          key: llvm-build-15-windows-v6
+          key: llvm-build-16-windows-v1
           path: llvm-build
       - name: Build LLVM
         if: steps.cache-llvm-build.outputs.cache-hit != 'true'

Makefile

@@ -113,7 +113,7 @@ endif
 
 .PHONY: all tinygo test $(LLVM_BUILDDIR) llvm-source clean fmt gen-device gen-device-nrf gen-device-nxp gen-device-avr gen-device-rp
 
-LLVM_COMPONENTS = all-targets analysis asmparser asmprinter bitreader bitwriter codegen core coroutines coverage debuginfodwarf debuginfopdb executionengine frontendopenmp instrumentation interpreter ipo irreader libdriver linker lto mc mcjit objcarcopts option profiledata scalaropts support target windowsdriver windowsmanifest
+LLVM_COMPONENTS = all-targets analysis asmparser asmprinter bitreader bitwriter codegen core coroutines coverage debuginfodwarf debuginfopdb executionengine frontendhlsl frontendopenmp instrumentation interpreter ipo irreader libdriver linker lto mc mcjit objcarcopts option profiledata scalaropts support target windowsdriver windowsmanifest
 
 ifeq ($(OS),Windows_NT)
     EXE = .exe
@@ -154,7 +154,7 @@ LLD_LIB_NAMES = lldCOFF lldCommon lldELF lldMachO lldMinGW lldWasm
 LLD_LIBS = $(START_GROUP) $(addprefix -l,$(LLD_LIB_NAMES)) $(END_GROUP)
 
 # Other libraries that are needed to link TinyGo.
-EXTRA_LIB_NAMES = LLVMInterpreter LLVMMCA LLVMX86TargetMCA
+EXTRA_LIB_NAMES = LLVMInterpreter LLVMMCA LLVMRISCVTargetMCA LLVMX86TargetMCA
 
 # All libraries to be built and linked with the tinygo binary (lib/lib*.a).
 LIB_NAMES = clang $(CLANG_LIB_NAMES) $(LLD_LIB_NAMES) $(EXTRA_LIB_NAMES)
@@ -170,7 +170,7 @@ NINJA_BUILD_TARGETS = clang llvm-config llvm-ar llvm-nm $(addprefix lib/lib,$(ad
 # For static linking.
 ifneq ("$(wildcard $(LLVM_BUILDDIR)/bin/llvm-config*)","")
     CGO_CPPFLAGS+=$(shell $(LLVM_CONFIG_PREFIX) $(LLVM_BUILDDIR)/bin/llvm-config --cppflags) -I$(abspath $(LLVM_BUILDDIR))/tools/clang/include -I$(abspath $(CLANG_SRC))/include -I$(abspath $(LLD_SRC))/include
-    CGO_CXXFLAGS=-std=c++14
+    CGO_CXXFLAGS=-std=c++17
     CGO_LDFLAGS+=-L$(abspath $(LLVM_BUILDDIR)/lib) -lclang $(CLANG_LIBS) $(LLD_LIBS) $(shell $(LLVM_CONFIG_PREFIX) $(LLVM_BUILDDIR)/bin/llvm-config --ldflags --libs --system-libs $(LLVM_COMPONENTS)) -lstdc++ $(CGO_LDFLAGS_EXTRA)
 endif
 
@@ -238,7 +238,7 @@ gen-device-renesas: build/gen-device-svd
 
 # Get LLVM sources.
 $(LLVM_PROJECTDIR)/llvm:
-	git clone -b xtensa_release_15.x --depth=1 https://github.com/espressif/llvm-project $(LLVM_PROJECTDIR)
+	git clone -b xtensa_release_16.x --depth=1 https://github.com/espressif/llvm-project $(LLVM_PROJECTDIR)
 llvm-source: $(LLVM_PROJECTDIR)/llvm
 
 # Configure LLVM.

builder/cc1as.cpp

@@ -56,6 +56,7 @@
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
+#include <optional>
 #include <system_error>
 using namespace clang;
 using namespace clang::driver;
@@ -103,6 +104,14 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
   Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
   if (Arg *A = Args.getLastArg(options::OPT_darwin_target_variant_triple))
     Opts.DarwinTargetVariantTriple = llvm::Triple(A->getValue());
+  if (Arg *A = Args.getLastArg(OPT_darwin_target_variant_sdk_version_EQ)) {
+    VersionTuple Version;
+    if (Version.tryParse(A->getValue()))
+      Diags.Report(diag::err_drv_invalid_value)
+          << A->getAsString(Args) << A->getValue();
+    else
+      Opts.DarwinTargetVariantSDKVersion = Version;
+  }
 
   Opts.CPU = std::string(Args.getLastArgValue(OPT_target_cpu));
   Opts.Features = Args.getAllArgValues(OPT_target_feature);
@@ -122,11 +131,12 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
     Opts.CompressDebugSections =
         llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
             .Case("none", llvm::DebugCompressionType::None)
-            .Case("zlib", llvm::DebugCompressionType::Z)
+            .Case("zlib", llvm::DebugCompressionType::Zlib)
+            .Case("zstd", llvm::DebugCompressionType::Zstd)
             .Default(llvm::DebugCompressionType::None);
   }
 
-  Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
+  Opts.RelaxELFRelocations = !Args.hasArg(OPT_mrelax_relocations_no);
   if (auto *DwarfFormatArg = Args.getLastArg(OPT_gdwarf64, OPT_gdwarf32))
     Opts.Dwarf64 = DwarfFormatArg->getOption().matches(OPT_gdwarf64);
   Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
@@ -189,6 +199,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
   Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
   Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
   Opts.NoWarn = Args.hasArg(OPT_massembler_no_warn);
+  Opts.NoTypeCheck = Args.hasArg(OPT_mno_type_check);
   Opts.RelocationModel =
       std::string(Args.getLastArgValue(OPT_mrelocation_model, "pic"));
   Opts.TargetABI = std::string(Args.getLastArgValue(OPT_target_abi));
@@ -214,6 +225,8 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
             .Case("default", EmitDwarfUnwindType::Default);
   }
 
+  Opts.AsSecureLogFile = Args.getLastArgValue(OPT_as_secure_log_file);
+
   return Success;
 }
 
@@ -265,6 +278,7 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
 
   MCTargetOptions MCOptions;
   MCOptions.EmitDwarfUnwind = Opts.EmitDwarfUnwind;
+  MCOptions.AsSecureLogFile = Opts.AsSecureLogFile;
 
   std::unique_ptr<MCAsmInfo> MAI(
       TheTarget->createMCAsmInfo(*MRI, Opts.Triple, MCOptions));
@@ -314,6 +328,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
       TheTarget->createMCObjectFileInfo(Ctx, PIC));
   if (Opts.DarwinTargetVariantTriple)
     MOFI->setDarwinTargetVariantTriple(*Opts.DarwinTargetVariantTriple);
+  if (!Opts.DarwinTargetVariantSDKVersion.empty())
+    MOFI->setDarwinTargetVariantSDKVersion(Opts.DarwinTargetVariantSDKVersion);
   Ctx.setObjectFileInfo(MOFI.get());
 
   if (Opts.SaveTemporaryLabels)
@@ -353,6 +369,7 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
 
   MCOptions.MCNoWarn = Opts.NoWarn;
   MCOptions.MCFatalWarnings = Opts.FatalWarnings;
+  MCOptions.MCNoTypeCheck = Opts.NoTypeCheck;
   MCOptions.ABIName = Opts.TargetABI;
 
   // FIXME: There is a bit of code duplication with addPassesToEmitFile.

builder/cc1as.h

@@ -82,6 +82,7 @@ struct AssemblerInvocation {
   unsigned NoExecStack : 1;
   unsigned FatalWarnings : 1;
   unsigned NoWarn : 1;
+  unsigned NoTypeCheck : 1;
   unsigned IncrementalLinkerCompatible : 1;
   unsigned EmbedBitcode : 1;
 
@@ -97,7 +98,14 @@ struct AssemblerInvocation {
 
   /// Darwin target variant triple, the variant of the deployment target
   /// for which the code is being compiled.
-  llvm::Optional<llvm::Triple> DarwinTargetVariantTriple;
+  std::optional<llvm::Triple> DarwinTargetVariantTriple;
+
+  /// The version of the darwin target variant SDK which was used during the
+  /// compilation
+  llvm::VersionTuple DarwinTargetVariantSDKVersion;
+
+  /// The name of a file to use with \c .secure_log_unique directives.
+  std::string AsSecureLogFile;
   /// @}
 
 public:
@@ -114,6 +122,7 @@ struct AssemblerInvocation {
     NoExecStack = 0;
     FatalWarnings = 0;
     NoWarn = 0;
+    NoTypeCheck = 0;
     IncrementalLinkerCompatible = 0;
     Dwarf64 = 0;
     DwarfVersion = 0;

builder/sizes_test.go

@@ -41,9 +41,9 @@ func TestBinarySize(t *testing.T) {
 	// This is a small number of very diverse targets that we want to test.
 	tests := []sizeTest{
 		// microcontrollers
-		{"hifive1b", "examples/echo", 4612, 280, 0, 2252},
-		{"microbit", "examples/serial", 2724, 388, 8, 2256},
-		{"wioterminal", "examples/pininterrupt", 6039, 1485, 116, 6816},
+		{"hifive1b", "examples/echo", 4568, 280, 0, 2252},
+		{"microbit", "examples/serial", 2728, 388, 8, 2256},
+		{"wioterminal", "examples/pininterrupt", 5972, 1484, 116, 6816},
 
 		// TODO: also check wasm. Right now this is difficult, because
 		// wasm binaries are run through wasm-opt and therefore the

cgo/libclang.go

@@ -653,8 +653,19 @@ func (p *cgoPackage) addErrorAt(position token.Position, msg string) {
 func (f *cgoFile) makeDecayingASTType(typ C.CXType, pos token.Pos) ast.Expr {
 	// Strip typedefs, if any.
 	underlyingType := typ
+	if underlyingType.kind == C.CXType_Elaborated {
+		// Starting with LLVM 16, the elaborated type is used for more types.
+		// According to the Clang documentation, the elaborated type has no
+		// semantic meaning so can be stripped (it is used to better convey type
+		// name information).
+		// Source:
+		// https://clang.llvm.org/doxygen/classclang_1_1ElaboratedType.html#details
+		// > The type itself is always "sugar", used to express what was written
+		// > in the source code but containing no additional semantic information.
+		underlyingType = C.clang_Type_getNamedType(underlyingType)
+	}
 	if underlyingType.kind == C.CXType_Typedef {
-		c := C.tinygo_clang_getTypeDeclaration(typ)
+		c := C.tinygo_clang_getTypeDeclaration(underlyingType)
 		underlyingType = C.tinygo_clang_getTypedefDeclUnderlyingType(c)
 		// TODO: support a chain of typedefs. At the moment, it seems to get
 		// stuck in an endless loop when trying to get to the most underlying
@@ -788,6 +799,8 @@ func (f *cgoFile) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 			return f.makeASTType(underlying, pos)
 		case C.CXType_Enum:
 			return f.makeASTType(underlying, pos)
+		case C.CXType_Typedef:
+			return f.makeASTType(underlying, pos)
 		default:
 			typeKindSpelling := getString(C.clang_getTypeKindSpelling(underlying.kind))
 			f.addError(pos, fmt.Sprintf("unknown elaborated type (libclang type kind %s)", typeKindSpelling))
@@ -808,6 +821,7 @@ func (f *cgoFile) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 		}
 		if name == "" {
 			// Anonymous record, probably inside a typedef.
+			// Not reachable anymore since LLVM 16.
 			location := f.getUniqueLocationID(pos, cursor)
 			name = f.getUnnamedDeclName("_Ctype_"+cgoRecordPrefix+"__", location)
 		} else {

cgo/libclang_config_llvm15.go

@@ -1,4 +1,4 @@
-//go:build !byollvm && !llvm14
+//go:build !byollvm && llvm15
 
 package cgo
 

cgo/libclang_config_llvm16.go

@@ -0,0 +1,21 @@
+//go:build !byollvm && !llvm14 && !llvm15
+
+package cgo
+
+// As of 2023-05-05, there is a packaging issue on Debian:
+// https://github.com/llvm/llvm-project/issues/62199
+// A workaround is to fix this locally, using something like this:
+//
+//   ln -sf ../../x86_64-linux-gnu/libclang-16.so.1 /usr/lib/llvm-16/lib/libclang.so
+
+/*
+#cgo linux        CFLAGS:  -I/usr/lib/llvm-16/include
+#cgo darwin,amd64 CFLAGS:  -I/usr/local/opt/llvm@16/include
+#cgo darwin,arm64 CFLAGS:  -I/opt/homebrew/opt/llvm@16/include
+#cgo freebsd      CFLAGS:  -I/usr/local/llvm16/include
+#cgo linux        LDFLAGS: -L/usr/lib/llvm-16/lib -lclang
+#cgo darwin,amd64 LDFLAGS: -L/usr/local/opt/llvm@16/lib -lclang -lffi
+#cgo darwin,arm64 LDFLAGS: -L/opt/homebrew/opt/llvm@16/lib -lclang -lffi
+#cgo freebsd      LDFLAGS: -L/usr/local/llvm16/lib -lclang
+*/
+import "C"