Merge branch 'develop' into MTRNord/add-tracing-to-appservices

matrix-org · Sep 6, 2023 · 3d9eb8e · 3d9eb8e
2 parents 2ac68cc + 5130303
commit 3d9eb8e
Show file tree

Hide file tree

Showing 105 changed files with 2,068 additions and 338 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -93,7 +93,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
 
       - name: Setup Poetry
@@ -150,7 +150,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -167,7 +167,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
         with:
             components: clippy
       - uses: Swatinem/rust-cache@v2
@@ -268,7 +268,7 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -308,7 +308,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
 
       # There aren't wheels for some of the older deps, so we need to install
@@ -416,7 +416,7 @@ jobs:
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
 
       - name: Run SyTest
@@ -556,7 +556,7 @@ jobs:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
 
       - uses: actions/setup-go@v4
@@ -584,7 +584,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.61.0
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo test

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,65 @@
+# Synapse 1.92.0rc1 (2023-09-05)
+
+### Features
+
+- Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816))
+- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113))
+- Return a `Retry-After` with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136))
+- Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218))
+- Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223))
+
+### Bugfixes
+
+- Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155))
+- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185))
+- Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205))
+- Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210))
+- Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211))
+- Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221))
+
+### Improved Documentation
+
+- Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008))
+- Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168))
+- Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178))
+
+### Internal Changes
+
+- Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099))
+- Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213))
+- Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135))
+- Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170))
+- Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172))
+- Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175))
+- Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180))
+- Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183))
+- Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184))
+- Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201))
+- Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187))
+- Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212))
+- Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241))
+
+### Updates to locked dependencies
+
+* Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141))
+* Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238))
+* Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237))
+* Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196))
+* Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195))
+* Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198))
+* Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236))
+* Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194))
+* Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140))
+* Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200))
+* Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199))
+
+# Synapse 1.91.1 (2023-09-04)
+
+### Bugfixes
+
+- Fix a performance regression introduced in Synapse 1.91.0 where event persistence would cause an excessive linear growth in CPU usage. ([\#16220](https://github.com/matrix-org/synapse/issues/16220))
+
+
 # Synapse 1.91.0 (2023-08-30)
 
 No significant changes since 1.91.0rc1.

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/changelog.d/15816.feature b/changelog.d/15816.feature
diff --git a/changelog.d/15997.misc b/changelog.d/15997.misc
@@ -0,0 +1 @@
+Allow modules to delete rooms.
diff --git a/changelog.d/16008.doc b/changelog.d/16008.doc
diff --git a/changelog.d/16066.bugfix b/changelog.d/16066.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where multi-device accounts could cause high load due to presence.
diff --git a/changelog.d/16090.misc b/changelog.d/16090.misc
@@ -0,0 +1 @@
+Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled.
diff --git a/changelog.d/16099.misc b/changelog.d/16099.misc
diff --git a/changelog.d/16113.feature b/changelog.d/16113.feature
diff --git a/changelog.d/16121.misc b/changelog.d/16121.misc
diff --git a/changelog.d/16135.misc b/changelog.d/16135.misc
diff --git a/changelog.d/16136.feature b/changelog.d/16136.feature
diff --git a/changelog.d/16137.feature b/changelog.d/16137.feature
@@ -0,0 +1 @@
+Support resolving homeservers using `matrix-fed` DNS SRV records from [MSC4040](https://github.com/matrix-org/matrix-spec-proposals/pull/4040).
diff --git a/changelog.d/16155.bugfix b/changelog.d/16155.bugfix
diff --git a/changelog.d/16168.doc b/changelog.d/16168.doc
diff --git a/changelog.d/16170.bugfix b/changelog.d/16170.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where multi-device accounts could cause high load due to presence.
diff --git a/changelog.d/16170.misc b/changelog.d/16170.misc
diff --git a/changelog.d/16171.bugfix b/changelog.d/16171.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where multi-device accounts could cause high load due to presence.
diff --git a/changelog.d/16171.misc b/changelog.d/16171.misc
diff --git a/changelog.d/16172.bugfix b/changelog.d/16172.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where multi-device accounts could cause high load due to presence.
diff --git a/changelog.d/16172.misc b/changelog.d/16172.misc
diff --git a/changelog.d/16174.bugfix b/changelog.d/16174.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where multi-device accounts could cause high load due to presence.
diff --git a/changelog.d/16175.misc b/changelog.d/16175.misc
diff --git a/changelog.d/16178.doc b/changelog.d/16178.doc
diff --git a/changelog.d/16179.misc b/changelog.d/16179.misc
diff --git a/changelog.d/16180.misc b/changelog.d/16180.misc
diff --git a/changelog.d/16183.misc b/changelog.d/16183.misc
diff --git a/changelog.d/16184.misc b/changelog.d/16184.misc
diff --git a/changelog.d/16185.bugfix b/changelog.d/16185.bugfix
diff --git a/changelog.d/16186.misc b/changelog.d/16186.misc
diff --git a/changelog.d/16187.misc b/changelog.d/16187.misc
diff --git a/changelog.d/16188.misc b/changelog.d/16188.misc
diff --git a/changelog.d/16201.misc b/changelog.d/16201.misc
diff --git a/changelog.d/16205.bugfix b/changelog.d/16205.bugfix
diff --git a/changelog.d/16210.bugfix b/changelog.d/16210.bugfix
diff --git a/changelog.d/16211.bugfix b/changelog.d/16211.bugfix
diff --git a/changelog.d/16212.misc b/changelog.d/16212.misc
diff --git a/changelog.d/16213.misc b/changelog.d/16213.misc
diff --git a/changelog.d/16219.feature b/changelog.d/16219.feature
@@ -0,0 +1 @@
+Add the ability to use `G` (GiB) and `T` (TiB) suffixes in configuration options that refer to numbers of bytes.
diff --git a/changelog.d/16220.misc b/changelog.d/16220.misc
diff --git a/changelog.d/16235.misc b/changelog.d/16235.misc
@@ -0,0 +1 @@
+Fix type checking when using the new version of Twisted.
diff --git a/changelog.d/16240.misc b/changelog.d/16240.misc
@@ -0,0 +1 @@
+Delete device messages asynchronously and in staged batches using the task scheduler.
diff --git a/changelog.d/16248.misc b/changelog.d/16248.misc
@@ -0,0 +1 @@
+Bump minimum supported Rust version to 1.61.0.
diff --git a/changelog.d/16251.bugfix b/changelog.d/16251.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where appservices using MSC2409 to receive to_device messages, would only get messages for one user.
diff --git a/changelog.d/16257.bugfix b/changelog.d/16257.bugfix
@@ -0,0 +1 @@
+Fix long-standing bug where we kept re-requesting a remote server's key repeatedly, potentially causing delays in receiving events over federation.
diff --git a/changelog.d/16260.misc b/changelog.d/16260.misc
@@ -0,0 +1 @@
+Update rust to version 1.71.1 in the nix development environment.
diff --git a/changelog.d/16263.misc b/changelog.d/16263.misc
@@ -0,0 +1 @@
+Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled.
diff --git a/debian/changelog b/debian/changelog
@@ -1,3 +1,15 @@
+matrix-synapse-py3 (1.92.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.92.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Sep 2023 11:21:43 +0100
+
+matrix-synapse-py3 (1.91.1) stable; urgency=medium
+
+  * New Synapse release 1.91.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 04 Sep 2023 14:03:18 +0100
+
 matrix-synapse-py3 (1.91.0) stable; urgency=medium
 
   * New Synapse release 1.91.0.

diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
@@ -242,6 +242,7 @@ The following parameters should be set in the URL:
   - `displayname` - Users are ordered alphabetically by `displayname`.
   - `avatar_url` - Users are ordered alphabetically by avatar URL.
   - `creation_ts` - Users are ordered by when the users was created in ms.
+  - `last_seen_ts` - Users are ordered by when the user was lastly seen in ms.
 
 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
   Setting this value to `b` will reverse the above sort order. Defaults to `f`.
@@ -272,6 +273,7 @@ The following fields are returned in the JSON response body:
   - `displayname` - string - The user's display name if they have set one.
   - `avatar_url` - string -  The user's avatar URL if they have set one.
   - `creation_ts` - integer - The user's creation timestamp in ms.
+  - `last_seen_ts` - integer - The user's last activity timestamp in ms.
 
 - `next_token`: string representing a positive integer - Indication for pagination. See above.
 - `total` - integer - Total number of media.

diff --git a/docs/upgrade.md b/docs/upgrade.md
@@ -88,6 +88,14 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
+# Upgrading to v1.93.0
+
+## Minimum supported Rust version
+The minimum supported Rust version has been increased from v1.60.0 to v1.61.0.
+Users building from source will need to ensure their `rustc` version is up to
+date.
+
+
 # Upgrading to v1.90.0
 
 ## App service query parameter authorization is now a configuration option

diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md
@@ -25,8 +25,10 @@ messages from the database after 5 minutes, rather than 5 months.
 
 In addition, configuration options referring to size use the following suffixes:
 
-* `M` = MiB, or 1,048,576 bytes
 * `K` = KiB, or 1024 bytes
+* `M` = MiB, or 1,048,576 bytes
+* `G` = GiB, or 1,073,741,824 bytes
+* `T` = TiB, or 1,099,511,627,776 bytes
 
 For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes
 for a user avatar.

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -82,14 +82,19 @@
                   #
                   # NOTE: We currently need to set the Rust version unnecessarily high
                   # in order to work around https://github.com/matrix-org/synapse/issues/15939
-                  (rust-bin.stable."1.70.0".default.override {
+                  (rust-bin.stable."1.71.1".default.override {
                     # Additionally install the "rust-src" extension to allow diving into the
                     # Rust source code in an IDE (rust-analyzer will also make use of it).
                     extensions = [ "rust-src" ];
                   })
                   # The rust-analyzer language server implementation.
                   rust-analyzer
 
+                  # GCC includes a linker; needed for building `ruff`
+                  gcc
+                  # Needed for building `ruff`
+                  gnumake
+
                   # Native dependencies for running Synapse.
                   icu
                   libffi
@@ -236,6 +241,19 @@
                   URI
                   YAMLLibYAML
                 ]}";
+
+                # Clear the LD_LIBRARY_PATH environment variable on shell init.
+                #
+                # By default, devenv will set LD_LIBRARY_PATH to point to .devenv/profile/lib. This causes
+                # issues when we include `gcc` as a dependency to build C libraries, as the version of glibc
+                # that the development environment's cc compiler uses may differ from that of the system.
+                #
+                # When LD_LIBRARY_PATH is set, system tools will attempt to use the development environment's
+                # libraries. Which, when built against a different glibc version lead, to "version 'GLIBC_X.YY'
+                # not found" errors.
+                enterShell = ''
+                  unset LD_LIBRARY_PATH
+                '';
               }
             ];
           };