From d786f99d098ad42d13abcfe52fb950ca38157d4c Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 8 Apr 2022 18:55:59 +0100 Subject: [PATCH 1/5] Use poetry lockfile in twisted trial environment Co-authored-by: Sean Quah Co-authored-by: Dan Callahan --- .ci/patch_for_twisted_trunk.sh | 8 ----- .github/workflows/twisted_trunk.yml | 47 +++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 20 deletions(-) delete mode 100755 .ci/patch_for_twisted_trunk.sh diff --git a/.ci/patch_for_twisted_trunk.sh b/.ci/patch_for_twisted_trunk.sh deleted file mode 100755 index f524581986a9..000000000000 --- a/.ci/patch_for_twisted_trunk.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# replaces the dependency on Twisted in `python_dependencies` with trunk. - -set -e -cd "$(dirname "$0")"/.. - -sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index fb9d46b7bfdd..03aaab241fc5 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -6,16 +6,25 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e mypy + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + extras: "all" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run mypy trial: runs-on: ubuntu-latest @@ -23,14 +32,15 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: - python-version: 3.7 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + python-version: "3.x" + extras: "all test" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run trial --jobs 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. @@ -55,11 +65,24 @@ jobs: steps: - uses: actions/checkout@v2 - name: Patch dependencies - run: .ci/patch_for_twisted_trunk.sh + # Note: The script below runs poetry against a virtualenv in /src/.venv/ + # ...but the sytest-synapse container expects it to be in /venv/ + # So we temporarily symlink it, run poetry, then remove the link. + # Tidying up is important because `/src` is a mounted volume. + run: | + ln -s -T /venv /src/.venv + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + rm /src/.venv working-directory: /src - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src + env: + # Use offline mode to avoid reinstalling the pinned version of + # twisted. + OFFLINE: 1 - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap From 09af56d44012b64a1dcc11140c5c06ea69be53f2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 8 Apr 2022 19:02:17 +0100 Subject: [PATCH 2/5] Changelog --- changelog.d/12425.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12425.misc diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc new file mode 100644 index 000000000000..3b076be0bd52 --- /dev/null +++ b/changelog.d/12425.misc @@ -0,0 +1 @@ +Run twisted trunk CI job in the locked poetry environment. From 3e8e5efef5fb69fc23487ba770c04328619b3454 Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Thu, 7 Apr 2022 01:27:26 +0100 Subject: [PATCH 3/5] Bump locked version of prometheus-client to 0.14.0 The typechecking fix in #12389 only works if your local copy of prometheus-client is at least version 0.14. Otherwise, it fails with: synapse/metrics/_types.py:22: error: Module "prometheus_client.registry" has no attribute "Collector" [attr-defined] Signed-off-by: Dan Callahan --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 17430cbfa6c1..c7324f9d20da 100644 --- a/poetry.lock +++ b/poetry.lock @@ -717,7 +717,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock [[package]] name = "prometheus-client" -version = "0.13.1" +version = "0.14.0" description = "Python client for the Prometheus monitoring system." category = "main" optional = false @@ -2225,8 +2225,8 @@ platformdirs = [ {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, ] prometheus-client = [ - {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"}, - {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"}, + {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"}, + {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"}, ] psycopg2 = [ {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"}, From 32b281b50595746dfb960ee8feeb00ce9fa6d4ec Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Wed, 13 Apr 2022 10:38:29 +0100 Subject: [PATCH 4/5] Apply review feedback Signed-off-by: Dan Callahan --- .github/workflows/twisted_trunk.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 03aaab241fc5..5a765065a0ff 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -21,7 +21,6 @@ jobs: python-version: "3.x" extras: "all" - run: | - poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" - run: poetry run mypy @@ -37,7 +36,6 @@ jobs: python-version: "3.x" extras: "all test" - run: | - poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" - run: poetry run trial --jobs 2 tests @@ -65,16 +63,14 @@ jobs: steps: - uses: actions/checkout@v2 - name: Patch dependencies - # Note: The script below runs poetry against a virtualenv in /src/.venv/ - # ...but the sytest-synapse container expects it to be in /venv/ - # So we temporarily symlink it, run poetry, then remove the link. - # Tidying up is important because `/src` is a mounted volume. + # Note: The poetry commands want to create a virtualenv in /src/.venv/, + # but the sytest-synapse container expects it to be in /venv/. + # We symlink it before running poetry so that poetry actually + # ends up installing to `/venv`. run: | ln -s -T /venv /src/.venv - poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" - rm /src/.venv working-directory: /src - name: Run SyTest run: /bootstrap.sh synapse From 0199f958eeb8ee94d7c96fd76f54a3543838c67f Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Wed, 13 Apr 2022 10:51:32 +0100 Subject: [PATCH 5/5] Restore calls to `poetry remove twisted` Signed-off-by: Dan Callahan --- .github/workflows/twisted_trunk.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 5a765065a0ff..8fc1affb7746 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -21,6 +21,7 @@ jobs: python-version: "3.x" extras: "all" - run: | + poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" - run: poetry run mypy @@ -36,6 +37,7 @@ jobs: python-version: "3.x" extras: "all test" - run: | + poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" - run: poetry run trial --jobs 2 tests @@ -69,6 +71,7 @@ jobs: # ends up installing to `/venv`. run: | ln -s -T /venv /src/.venv + poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" working-directory: /src