Merge to main

protocolbuffers · Jul 16, 2024 · 1951078 · 1951078
2 parents 1e23a36 + 507ba36
commit 1951078
Show file tree

Hide file tree

Showing 442 changed files with 20,706 additions and 11,177 deletions.
diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml
@@ -29,6 +29,7 @@ jobs:
           # If we are in a continuous run, test other supported branches
           - branch: ${{ inputs.continuous-run && '25.x' }}
           - branch: ${{ inputs.continuous-run && '27.x' }}
+          - branch: ${{ inputs.continuous-run && '28.x' }}
 
     name: Test staleness ${{ matrix.os.name }} ${{ matrix.branch }}
     runs-on: ${{ matrix.os.value }}

diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml
@@ -32,10 +32,7 @@ jobs:
         include:
           - runner: ubuntu
             bazelversion: '6.4.0'
-            bzlmod: true
-            continuous-only: true
-          - runner: ubuntu
-            bazelversion: '6.4.0'
+            # Not running Bazel 6 with bzlmod, because it doesn't support use_repo_rule in rules_jvm_external
             bzlmod: false
             continuous-only: true
     runs-on: ${{ matrix.runner }}-latest

diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml
@@ -29,9 +29,9 @@ jobs:
         config:
           - { name: Optimized, flags: --config=opt }
           - { name: Debug, flags: --config=dbg, continuous-only: true }
-          - { name: ASAN, flags: --config=asan, runner: ubuntu-20-large }
-          - { name: MSAN, flags: --config=docker-msan, runner: ubuntu-20-large, continuous-only: true }
-          - { name: TSAN, flags: --config=tsan, runner: ubuntu-20-large, continuous-only: true }
+          - { name: ASAN, flags: --config=asan, runner: ubuntu-22-4core }
+          - { name: MSAN, flags: --config=docker-msan, runner: ubuntu-22-4core, continuous-only: true }
+          - { name: TSAN, flags: --config=tsan, runner: ubuntu-22-4core, continuous-only: true }
           - { name: UBSAN, flags: --config=ubsan, continuous-only: true }
           - { name: No-RTTI, flags: --cxxopt=-fno-rtti, continuous-only: true }
         include:
@@ -40,13 +40,22 @@ jobs:
           - targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/...
 
           # Override cases with custom images
-          - config: { name: "Bazel7" }
+          - config: { name: "Bazel7", flags: --noenable_bzlmod }
+            version: Bazel7
             image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e"
             targets: "//src/... //third_party/utf8_range/..."
+          - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod }
+            version: Bazel7bzlmod
+            image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e"
+            targets: "//src/... //third_party/utf8_range/..."
+            # TODO: enable back command_line_interface_unittest after bug is fixed
+            exclude-targets: "-//src/google/protobuf/compiler:command_line_interface_unittest"
           - config: { name: "TCMalloc" }
+            version: TcMalloc
             image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:1c5133455481f4d1bb8afa477029604f41f1a3c46cebe4d9958cf1af95b5c87c"
             targets: "//src/... //third_party/utf8_range/..."
           - config: { name: "aarch64" }
+            version: TcMalloc
             targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test //third_party/utf8_range/..."
             image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:6.3.0-aarch64-68e662b3a56b881804dc4e9d45f949791cbc4b94"
     name: ${{ matrix.config.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.config.name }}
@@ -63,8 +72,9 @@ jobs:
         with:
           image: ${{ matrix.image }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: cpp_linux/${{ matrix.config.name }}
+          bazel-cache: cpp_linux/${{ matrix.version }}
           bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }}
+          exclude-targets: ${{ matrix.exclude-targets }}
 
   linux-gcc:
     strategy:
@@ -92,7 +102,7 @@ jobs:
       matrix:
         arch: [x86_64, aarch64]
     name: Linux Release ${{ matrix.arch}}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20-4core
     steps:
       - name: Checkout pending changes
         uses: protocolbuffers/protobuf-ci/checkout@v3
@@ -466,6 +476,12 @@ jobs:
           arch: ${{ matrix.windows-arch || 'x64' }}
           vsversion: ${{ matrix.vsversion }}
 
+      # Workaround for Abseil incompatibility with CMake 3.30 (b/352354235).
+      - name: Downgrade CMake
+        if: ${{ runner.os == 'Windows' }}
+        run: choco install cmake --version 3.29.6 --force
+        shell: bash
+
       # Workaround for incompatibility between gcloud and windows-2019 runners.
       - name: Install Python
         if: ${{ matrix.python-version && (!matrix.continuous-only || inputs.continuous-run) }}

diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml
@@ -44,6 +44,17 @@ jobs:
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
             targets: //java/... //java/internal:java_version //compatibility/...
             continuous-only: false
+          - name: Bazel7
+            version: 'bazel7nobzlmod'
+            image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e
+            targets: //java/... //java/internal:java_version //compatibility/...
+            flags: --noenable_bzlmod
+          - name: Bazel7 with Bzlmod
+            version: 'bazel7bzlmod'
+            image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e
+            targets: //java/... //java/internal:java_version //compatibility/...
+            flags: --enable_bzlmod
+>>>>>>> upstream/main
           - name: aarch64
             version: 'aarch64'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17
@@ -65,7 +76,7 @@ jobs:
           image: ${{ matrix.image }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: java_linux/${{ matrix.version }}
-          bazel: test ${{ matrix.targets }} --test_env=KOKORO_JAVA_VERSION
+          bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_JAVA_VERSION
 
 # TODO restore this test (or a better one) when gRPC has rebuilt with 26.x
   # linkage-monitor:

diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml
@@ -30,7 +30,7 @@ jobs:
           - { name: "Bazel 7", bazel_version: "7.1.1", continuous-only: true }
           - { name: "Fastbuild" }
           - { name: "Optimized", flags: "-c opt", continuous-only: true }
-          - { name: "ASAN", flags: "--config=asan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/...", runner: ubuntu-20-large }
+          - { name: "ASAN", flags: "--config=asan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/...", runner: ubuntu-22-4core }
           - { name: "UBSAN", flags: "--config=ubsan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/... -//lua/...", continuous-only: true }
           - { name: "32-bit", flags: "--copt=-m32 --linkopt=-m32", exclude-targets: "-//benchmarks:benchmark -//python/..." }
           # TODO: Add 32-bit ASAN test
@@ -52,7 +52,7 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '6.3.0' }}-75f2a85ece6526cc3d54087018c0f1097d78d42b
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: upb-bazel
-          bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //protos/... //hpb_generator/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
+          bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
           exclude-targets: ${{ matrix.config.exclude-targets }}
 
   linux-gcc:
@@ -71,7 +71,7 @@ jobs:
           image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17"
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-gcc"
-          bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt //bazel/... //benchmarks/... //lua/... //protos/... //hpb_generator/... //python/... //upb/... //upb_generator/...
+          bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
 
   windows:
     strategy:
@@ -92,7 +92,7 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-windows"
-          bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/... //protos/... //hpb_generator/...
+          bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/...
           version: 6.3.0
           exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test
 
@@ -119,7 +119,7 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-macos"
-          bazel: ${{ matrix.config.bazel-command }} --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //protos/... //hpb_generator/... //python/... //upb/... //upb_generator/...
+          bazel: ${{ matrix.config.bazel-command }} --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
           version: 6.3.0
 
   no-python:
@@ -182,13 +182,13 @@ jobs:
           # supported Python versions and assume this gives us sufficient test
           # coverage.
           - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'binary' }
-          - { os: macos-11, python-version: "3.8", architecture: x64, type: 'binary' }
+          - { os: macos-12, python-version: "3.8", architecture: x64, type: 'binary' }
           - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'binary' }
-          - { os: macos-12, python-version: "3.12", architecture: x64, type: 'binary' }
+          - { os: macos-13, python-version: "3.12", architecture: x64, type: 'binary' }
           - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'source', continuous-only: true }
-          - { os: macos-11, python-version: "3.8", architecture: x64, type: 'source', continuous-only: true }
+          - { os: macos-12, python-version: "3.8", architecture: x64, type: 'source', continuous-only: true }
           - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'source', continuous-only: true }
-          - { os: macos-12, python-version: "3.12", architecture: x64, type: 'source', continuous-only: true }
+          - { os: macos-13, python-version: "3.12", architecture: x64, type: 'source', continuous-only: true }
 
           # Windows uses the full API up until Python 3.10.
           - { os: windows-2019, python-version: "3.8", architecture: x86, type: 'binary', continuous-only: true }

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -12,7 +12,10 @@ load(":protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library
 
 licenses(["notice"])
 
-exports_files(["LICENSE"])
+exports_files([
+    "LICENSE",
+    "PrivacyInfo.xcprivacy",
+])
 
 ################################################################################
 # Well Known Types Proto Library Rules

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -35,6 +35,9 @@ option(protobuf_BUILD_LIBUPB "Build libupb" ON)
 option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF)
 option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "")
 option(protobuf_ALLOW_CCACHE "Adjust build flags to allow for ccache support." OFF)
+
+# We support Unity (Jumbo) builds best-effort.
+option(protobuf_USE_UNITY_BUILD "Enable Unity (Jumbo) build for" OFF)
 if (BUILD_SHARED_LIBS)
   set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON)
 else (BUILD_SHARED_LIBS)
@@ -81,7 +84,7 @@ if (protobuf_BUILD_SHARED_LIBS)
 endif ()
 
 # Version metadata
-set(protobuf_VERSION_STRING "5.28.0")
+set(protobuf_VERSION_STRING "5.29.0")
 set(protobuf_DESCRIPTION "Protocol Buffers")
 set(protobuf_CONTACT "protobuf@googlegroups.com")
 

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -3,7 +3,7 @@
 
 module(
     name = "protobuf",
-    version = "28.0-dev", # Automatically updated on release
+    version = "29.0-dev", # Automatically updated on release
     compatibility_level = 1,
     repo_name = "com_google_protobuf",
 )
@@ -18,12 +18,14 @@ bazel_dep(name = "jsoncpp", version = "1.9.5")
 bazel_dep(name = "rules_cc", version = "0.0.9")
 bazel_dep(name = "rules_fuzzing", version = "0.5.2")
 bazel_dep(name = "rules_java", version = "5.3.5")
-bazel_dep(name = "rules_jvm_external", version = "5.1")
+bazel_dep(name = "rules_jvm_external", version = "6.0")
+bazel_dep(name = "rules_kotlin", version = "1.9.0")
 bazel_dep(name = "rules_pkg", version = "0.7.0")
 bazel_dep(name = "rules_python", version = "0.28.0")
 bazel_dep(name = "rules_rust", version = "0.45.1")
 bazel_dep(name = "platforms", version = "0.0.8")
 bazel_dep(name = "zlib", version = "1.3.1")
+bazel_dep(name = "bazel_features", version = "1.13.0", repo_name = "proto_bazel_features")
 
 # TODO: remove after toolchain types are moved to protobuf
 bazel_dep(name = "rules_proto", version = "4.0.0")
@@ -70,3 +72,30 @@ crate.spec(
 )
 crate.from_specs()
 use_repo(crate, crate_index = "crates")
+
+maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
+maven.install(
+    artifacts = [
+        "com.google.caliper:caliper:1.0-beta-3",
+        "com.google.code.findbugs:jsr305:3.0.2",
+        "com.google.code.gson:gson:2.8.9",
+        "com.google.errorprone:error_prone_annotations:2.5.1",
+        "com.google.j2objc:j2objc-annotations:2.8",
+        "com.google.guava:guava:32.0.1-jre",
+        "com.google.guava:guava-testlib:32.0.1-jre",
+        "com.google.truth:truth:1.1.2",
+        "junit:junit:4.13.2",
+        "org.mockito:mockito-core:4.3.1",
+        "biz.aQute.bnd:biz.aQute.bndlib:6.4.0",
+        "info.picocli:picocli:4.6.3",
+    ],
+    repositories = [
+        "https://repo1.maven.org/maven2",
+        "https://repo.maven.apache.org/maven2",
+    ],
+)
+use_repo(maven, "maven")
+
+# Development dependencies
+bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest", dev_dependency = True)
+bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec
@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name     = 'Protobuf-C++'
-  s.version  = '5.28.0'
+  s.version  = '5.29.0'
   s.summary  = 'Protocol Buffers v3 runtime library for C++.'
   s.homepage = 'https://github.com/google/protobuf'
   s.license  = 'BSD-3-Clause'

diff --git a/Protobuf.podspec b/Protobuf.podspec
@@ -5,7 +5,7 @@
 # dependent projects use the :git notation to refer to the library.
 Pod::Spec.new do |s|
   s.name     = 'Protobuf'
-  s.version  = '3.28.0'
+  s.version  = '3.29.0'
   s.summary  = 'Protocol Buffers v.3 runtime library for Objective-C.'
   s.homepage = 'https://github.com/protocolbuffers/protobuf'
   s.license  = 'BSD-3-Clause'
@@ -18,7 +18,7 @@ Pod::Spec.new do |s|
   s.source = { :git => 'https://github.com/protocolbuffers/protobuf.git',
                :tag => "v#{s.version}" }
 
-  s.source_files = 'objectivec/*.{h,m}',
+  s.source_files = 'objectivec/*.{h,m,swift}',
                    'objectivec/google/protobuf/Any.pbobjc.h',
                    'objectivec/google/protobuf/Api.pbobjc.h',
                    'objectivec/google/protobuf/Duration.pbobjc.h',
@@ -33,6 +33,9 @@ Pod::Spec.new do |s|
   # left out, as it's an umbrella implementation file.
   s.exclude_files = 'objectivec/GPBProtocolBuffers.m'
 
+  # Now that there is a Swift source file, set a version.
+  s.swift_version = '5.0'
+
   s.resource_bundle = {
     "Protobuf_Privacy" => "PrivacyInfo.xcprivacy"
   }

diff --git a/WORKSPACE b/WORKSPACE
@@ -99,11 +99,11 @@ load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies")
 rules_cc_dependencies()
 
 # For `kt_jvm_library`
-load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories")
+load("@rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories")
 
 kotlin_repositories()
 
-load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains")
+load("@rules_kotlin//kotlin:core.bzl", "kt_register_toolchains")
 
 kt_register_toolchains()
 
@@ -245,3 +245,12 @@ http_archive(
     strip_prefix = "utf8_range-d863bc33e15cba6d873c878dcca9e6fe52b2f8cb",
     url = "https://github.com/protocolbuffers/utf8_range/archive/d863bc33e15cba6d873c878dcca9e6fe52b2f8cb.zip",
 )
+
+# Needed for testing only
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+    name = "rules_testing",
+    sha256 = "02c62574631876a4e3b02a1820cb51167bb9cdcdea2381b2fa9d9b8b11c407c4",
+    strip_prefix = "rules_testing-0.6.0",
+    url = "https://github.com/bazelbuild/rules_testing/releases/download/v0.6.0/rules_testing-v0.6.0.tar.gz",
+)
diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -0,0 +1,40 @@
+# This is a WORKSPACE file used by bzlmod in combination with MODULE.bazel.
+# It's used for a gradual migration and it should be empty.
+# Don't remove this file. If the file doesn't exist, bzlmod falls back to WORKSPACE file.
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# TODO: either replace rules_ruby with a maintained version on BCR
+# or use bzlmod extensions to depend on this specific repo
+http_archive(
+    name = "rules_ruby",
+    urls = [
+      "https://github.com/protocolbuffers/rules_ruby/archive/b7f3e9756f3c45527be27bc38840d5a1ba690436.zip"
+    ],
+    strip_prefix = "rules_ruby-b7f3e9756f3c45527be27bc38840d5a1ba690436",
+    sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8",
+)
+
+load("@rules_ruby//ruby:defs.bzl", "ruby_runtime")
+
+ruby_runtime("system_ruby")
+
+register_toolchains("@system_ruby//:toolchain")
+
+# Follwing are just needed to run conformance tests, not really needed to support them via MODULE.bazel
+
+# For testing runtime against old gencode from a previous major version.
+http_archive(
+    name = "com_google_protobuf_v25.0",
+    strip_prefix = "protobuf-25.0",
+    url = "https://github.com/protocolbuffers/protobuf/releases/download/v25.0/protobuf-25.0.tar.gz",
+)
+
+# Needed as a dependency of @com_google_protobuf_v25.x, which was before
+# utf8_range was merged in.
+http_archive(
+    name = "utf8_range",
+    strip_prefix = "utf8_range-d863bc33e15cba6d873c878dcca9e6fe52b2f8cb",
+    url = "https://github.com/protocolbuffers/utf8_range/archive/d863bc33e15cba6d873c878dcca9e6fe52b2f8cb.zip",
+)
+
diff --git a/bazel/common/BUILD.bazel b/bazel/common/BUILD.bazel
@@ -7,7 +7,9 @@ bzl_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        "//bazel/private:native_bzl",
+        ":proto_lang_toolchain_info_bzl",
+        "//bazel/private:toolchain_helpers_bzl",
+        "@proto_bazel_features//:features",
     ],
 )
 
@@ -29,6 +31,6 @@ bzl_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        ":proto_common.bzl",
+        "//bazel/private:native_bzl",
     ],
 )