Merge remote-tracking branch 'origin/release-v1.36' into hs/hacked-to…

…gether-event-cache
matrix-org · Jun 16, 2021 · 7d33ba7 · 7d33ba7
2 parents dacc395 + 1c8045f
commit 7d33ba7
Show file tree

Hide file tree

Showing 233 changed files with 11,287 additions and 4,938 deletions.
diff --git a/.buildkite/postgres-config.yaml b/.buildkite/postgres-config.yaml
@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: "/src/.buildkite/test.signing.key"
+signing_key_path: ".buildkite/test.signing.key"
 
 report_stats: false
 
@@ -16,6 +16,4 @@ database:
     database: synapse
 
 # Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true
+trusted_key_servers: []
diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh
@@ -33,6 +33,10 @@ scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
 echo "+++ Run synapse_port_db against test database"
 coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
 
+# We should be able to run twice against the same database.
+echo "+++ Run synapse_port_db a second time"
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+
 #####
 
 # Now do the same again, on an empty database.

diff --git a/.buildkite/sqlite-config.yaml b/.buildkite/sqlite-config.yaml
@@ -3,7 +3,7 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: "/src/.buildkite/test.signing.key"
+signing_key_path: ".buildkite/test.signing.key"
 
 report_stats: false
 
@@ -13,6 +13,4 @@ database:
     database: ".buildkite/test_db.db"
 
 # Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true
+trusted_key_servers: []
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -41,7 +41,7 @@ workflows:
       - dockerhubuploadlatest:
           filters:
             branches:
-              only: master
+              only: [ master, main ]
 
 commands:
   docker_prepare:

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -0,0 +1,31 @@
+name: Deploy the documentation
+
+on:
+  push:
+    branches:
+      - develop
+
+  workflow_dispatch:
+
+jobs:
+  pages:
+    name: GitHub Pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
+        with:
+          mdbook-version: '0.4.9'
+
+      - name: Build the documentation
+        run: mdbook build
+
+      - name: Deploy latest documentation
+        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          keep_files: true
+          publish_dir: ./book
+          destination_dir: ./develop
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -34,7 +34,13 @@ jobs:
     if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
     runs-on: ubuntu-latest
     steps:
+      # Note: This and the script can be simplified once we drop Buildkite. See:
+      #   https://github.com/actions/checkout/issues/266#issuecomment-638346893
+      #   https://github.com/actions/checkout/issues/416
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
       - uses: actions/setup-python@v2
       - run: pip install tox
       - name: Patch Buildkite-specific test script
@@ -226,9 +232,9 @@ jobs:
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
-      - name: Dump results.tap
+      - name: Summarise results.tap
         if: ${{ always() }}
-        run: cat /logs/results.tap
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}

diff --git a/.gitignore b/.gitignore
@@ -46,3 +46,6 @@ __pycache__/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+
+# docs
+book/
diff --git a/CHANGES.md b/CHANGES.md
diff --git a/INSTALL.md b/INSTALL.md
@@ -399,11 +399,9 @@ Once you have installed synapse as above, you will need to configure it.
 
 ### Using PostgreSQL
 
-By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
-very light workloads.
-
-Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
+By default Synapse uses an [SQLite](https://sqlite.org/) database and in doing so trades
+performance for convenience. Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org)
+instead. Advantages include:
 
 - significant performance improvements due to the superior threading and
   caching model, smarter query optimiser
@@ -412,6 +410,10 @@ Almost all installations should opt to use [PostgreSQL](https://www.postgresql.o
 For information on how to install and use PostgreSQL in Synapse, please see
 [docs/postgres.md](docs/postgres.md)
 
+SQLite is only acceptable for testing purposes. SQLite should not be used in
+a production server. Synapse will perform poorly when using
+SQLite, especially when participating in large rooms.
+
 ### TLS certificates
 
 The default configuration exposes a single HTTP port on the local

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -40,6 +40,7 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh
 
+include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 

diff --git a/README.rst b/README.rst
@@ -149,21 +149,45 @@ For details on having Synapse manage your federation TLS certificates
 automatically, please see `<docs/ACME.md>`_.
 
 
-Security Note
+Security note
 =============
 
-Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+Matrix serves raw, user-supplied data in some APIs -- specifically the `content
+repository endpoints`_.
 
-Whilst we have tried to mitigate against possible XSS attacks (e.g.
-https://github.com/matrix-org/synapse/pull/1021) we recommend running
-matrix homeservers on a dedicated domain name, to limit any malicious user generated
-content served to web browsers a matrix API from being able to attack webapps hosted
-on the same domain.  This is particularly true of sharing a matrix webclient and
-server on the same domain.
+.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
 
-See https://github.com/vector-im/riot-web/issues/1977 and
-https://developer.github.com/changes/2014-04-25-user-content-security for more details.
+Whilst we make a reasonable effort to mitigate against XSS attacks (for
+instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
+domain hosting other web applications. This especially applies to sharing
+the domain with Matrix web clients and other sensitive applications like
+webmail. See
+https://developer.github.com/changes/2014-04-25-user-content-security for more
+information.
+
+.. _CSP: https://github.com/matrix-org/synapse/pull/1021
+
+Ideally, the homeserver should not simply be on a different subdomain, but on
+a completely different `registered domain`_ (also known as top-level site or
+eTLD+1). This is because `some attacks`_ are still possible as long as the two
+applications share the same registered domain.
+
+.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
+
+.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
+
+To illustrate this with an example, if your Element Web or other sensitive web
+application is hosted on ``A.example1.com``, you should ideally host Synapse on
+``example2.com``. Some amount of protection is offered by hosting on
+``B.example1.com`` instead, so this is also acceptable in some scenarios.
+However, you should *not* host your Synapse on ``A.example1.com``.
+
+Note that all of the above refers exclusively to the domain used in Synapse's
+``public_baseurl`` setting. In particular, it has no bearing on the domain
+mentioned in MXIDs hosted on that server.
+
+Following this advice ensures that even if an XSS is found in Synapse, the
+impact to other applications will be minimal.
 
 
 Upgrading an existing Synapse

diff --git a/UPGRADE.rst b/UPGRADE.rst
@@ -88,7 +88,7 @@ for example:
 Upgrading to v1.34.0
 ====================
 
-`room_invite_state_types` configuration setting
+``room_invite_state_types`` configuration setting
 -----------------------------------------------
 
 The ``room_invite_state_types`` configuration setting has been deprecated and
@@ -106,13 +106,10 @@ remove it from your configuration file. The default value used to be:
       - "m.room.encryption"
       - "m.room.name"
 
-If you have customised this value by adding addition state types, you should
-remove ``room_invite_state_types`` and configure ``additional_event_types`` with
-your customisations.
+If you have customised this value, you should remove ``room_invite_state_types`` and
+configure ``room_prejoin_state`` instead.
+
 
-If you have customised this value by removing state types, you should rename
-``room_invite_state_types`` to ``additional_event_types``, and set
-``disable_default_event_types`` to ``true``.
 
 Upgrading to v1.33.0
 ====================

diff --git a/book.toml b/book.toml
@@ -0,0 +1,39 @@
+# Documentation for possible options in this file is at
+# https://rust-lang.github.io/mdBook/format/config.html
+[book]
+title = "Synapse"
+authors = ["The Matrix.org Foundation C.I.C."]
+language = "en"
+multilingual = false
+
+# The directory that documentation files are stored in
+src = "docs"
+
+[build]
+# Prevent markdown pages from being automatically generated when they're
+# linked to in SUMMARY.md
+create-missing = false
+
+[output.html]
+# The URL visitors will be directed to when they try to edit a page
+edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
+
+# Remove the numbers that appear before each item in the sidebar, as they can
+# get quite messy as we nest deeper
+no-section-label = true
+
+# The source code URL of the repository
+git-repository-url = "https://github.com/matrix-org/synapse"
+
+# The path that the docs are hosted on
+site-url = "/synapse/"
+
+# Additional HTML, JS, CSS that's injected into each page of the book.
+# More information available in docs/website_files/README.md
+additional-css = [
+    "docs/website_files/table-of-contents.css",
+    "docs/website_files/remove-nav-buttons.css",
+    "docs/website_files/indent-section-headers.css",
+]
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"
diff --git a/changelog.d/9881.feature b/changelog.d/9881.feature
diff --git a/changelog.d/9882.misc b/changelog.d/9882.misc
diff --git a/changelog.d/9885.misc b/changelog.d/9885.misc
diff --git a/changelog.d/9886.misc b/changelog.d/9886.misc
diff --git a/changelog.d/9889.feature b/changelog.d/9889.feature
diff --git a/changelog.d/9889.removal b/changelog.d/9889.removal
diff --git a/changelog.d/9895.bugfix b/changelog.d/9895.bugfix
diff --git a/changelog.d/9896.bugfix b/changelog.d/9896.bugfix
diff --git a/changelog.d/9896.misc b/changelog.d/9896.misc
diff --git a/changelog.d/9902.feature b/changelog.d/9902.feature
diff --git a/changelog.d/9904.misc b/changelog.d/9904.misc
diff --git a/changelog.d/9905.feature b/changelog.d/9905.feature
diff --git a/changelog.d/9910.bugfix b/changelog.d/9910.bugfix
diff --git a/changelog.d/9910.feature b/changelog.d/9910.feature
diff --git a/changelog.d/9911.doc b/changelog.d/9911.doc
diff --git a/changelog.d/9913.docker b/changelog.d/9913.docker
diff --git a/changelog.d/9915.feature b/changelog.d/9915.feature
diff --git a/changelog.d/9916.feature b/changelog.d/9916.feature
diff --git a/changelog.d/9928.bugfix b/changelog.d/9928.bugfix
diff --git a/changelog.d/9930.bugfix b/changelog.d/9930.bugfix
diff --git a/changelog.d/9931.misc b/changelog.d/9931.misc
diff --git a/changelog.d/9932.misc b/changelog.d/9932.misc
diff --git a/changelog.d/9935.feature b/changelog.d/9935.feature
diff --git a/changelog.d/9945.feature b/changelog.d/9945.feature
diff --git a/changelog.d/9947.feature b/changelog.d/9947.feature
diff --git a/changelog.d/9950.feature b/changelog.d/9950.feature
diff --git a/changelog.d/9954.feature b/changelog.d/9954.feature
diff --git a/changelog.d/9959.misc b/changelog.d/9959.misc
diff --git a/changelog.d/9961.bugfix b/changelog.d/9961.bugfix
diff --git a/changelog.d/9965.bugfix b/changelog.d/9965.bugfix
diff --git a/changelog.d/9966.feature b/changelog.d/9966.feature