merge dev into mls

Makefile

@@ -94,14 +94,24 @@ endif
 # TASTY_PATTERN=".."  make ci package=brig
 #
 # If you want to pass arguments to the test-suite call cabal-run-integration.sh directly.
-.PHONY: ci
-ci: c db-migrate
+.PHONY: ci-fast
+ci-fast: c db-migrate
 ifeq ("$(package)", "all")
 	./hack/bin/cabal-run-integration.sh all
 	./hack/bin/cabal-run-integration.sh integration
 endif
 	./hack/bin/cabal-run-integration.sh $(package)
 
+# variant of `make ci-fast` that compiles the entire project even if `package` is specified.
+.PHONY: ci-safe
+ci-safe:
+	make c package=all
+	make ci-fast
+
+.PHONY: ci
+ci:
+	@echo -en "\n\n\nplease choose between goals ci-fast and ci-safe.\n\n\n"
+
 # Compile and run services
 # Usage: make crun `OR` make crun package=galley
 .PHONY: cr

cassandra-schema.cql

@@ -863,6 +863,7 @@ CREATE TABLE brig_test.connection_remote (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX connection_remote_right_domain_idx ON brig_test.connection_remote (right_domain);
 
 CREATE TABLE brig_test.users_pending_activation (
     user uuid PRIMARY KEY,
@@ -1183,14 +1184,13 @@ CREATE TABLE galley_test.team_features (
     mls_e2eid_lock_status int,
     mls_e2eid_status int,
     mls_e2eid_ver_exp timestamp,
-    mls_migration_clients_threshold int,
     mls_migration_finalise_regardless_after timestamp,
     mls_migration_lock_status int,
     mls_migration_start_time timestamp,
     mls_migration_status int,
-    mls_migration_users_threshold int,
     mls_protocol_toggle_users set<uuid>,
     mls_status int,
+    mls_supported_protocols set<int>,
     outlook_cal_integration_lock_status int,
     outlook_cal_integration_status int,
     search_visibility_inbound_status int,
@@ -1293,6 +1293,7 @@ CREATE TABLE galley_test.user_remote_conv (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX user_remote_conv_conv_remote_domain_idx ON galley_test.user_remote_conv (conv_remote_domain);
 
 CREATE TABLE galley_test.legalhold_whitelisted (
     team uuid PRIMARY KEY
@@ -1332,6 +1333,7 @@ CREATE TABLE galley_test.member_remote_user (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+CREATE INDEX member_remote_user_user_remote_domain_idx ON galley_test.member_remote_user (user_remote_domain);
 
 CREATE TABLE galley_test.team_member (
     team uuid,
@@ -1419,6 +1421,26 @@ CREATE TABLE galley_test.group_id_conv_id (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE galley_test.team_admin (
+    team uuid,
+    user uuid,
+    PRIMARY KEY (team, user)
+) WITH CLUSTERING ORDER BY (user ASC)
+    AND bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 -- NOTE: this table is unused. It was replaced by mls_group_member_client
 CREATE TABLE galley_test.member_client (
     conv uuid,

changelog.d/1-api-changes/WPB-1085

@@ -0,0 +1,3 @@
+Adding users in Proteus will only succeed if all federated backends hosting the
+users are available. Otherwise, the endpoint will fail with a Federation error,
+enumerating all unavailable domains.

changelog.d/1-api-changes/WPB-240

@@ -0,0 +1,3 @@
+Added a new notification event type, "federation.delete".
+This event contains a single domain for a remote server that the local server is de-federating from.
+This notification is sent twice during de-federation. Once before and once after cleaning up and removing references to the remote server from the local database.

changelog.d/2-features/remove-indexed-billing-members-feature

@@ -0,0 +1 @@
+Enable indexed billing members by default and remove the feature flag

changelog.d/3-bug-fixes/pr-3436

@@ -0,0 +1 @@
+`/i/user/meta-info` endpoint in backoffice/stern fixed

changelog.d/4-docs/sso-faq

@@ -0,0 +1 @@
+SSO Faq entry on CSP

changelog.d/5-internal/ci-redis-ephemeral

@@ -0,0 +1 @@
+In CI integration tests, use redis-ephemeral in master mode (may be reverted in the future, see PR details)

changelog.d/5-internal/federator-status

@@ -0,0 +1 @@
+Add the status endpoint to both federator ports

changelog.d/5-internal/fs-1179

@@ -0,0 +1,2 @@
+Adding a new internal APIs to Brig and Galley to defederate domains.
+Background-Worker has been reworked to seperate AMQP channel handling from processing. This was done to allow a defederation worker to share the same connection management process with notification pusher.

changelog.d/5-internal/jsonLower

@@ -0,0 +1 @@
+[hscim] make `jsonLower` fail on duplicate fields

changelog.d/5-internal/pr-3346

@@ -0,0 +1 @@
+Servantify brig internal api: misc

changelog.d/5-internal/pr-3425

@@ -0,0 +1 @@
+`stern` is added to the new run-services implementation for the integration tests

changelog.d/5-internal/wpb-2986

@@ -0,0 +1 @@
+Adding graceful shutdown handling to background-worker to allow it to finish processing its current message before the service quits.

changelog.d/6-federation/WPB-240

@@ -0,0 +1 @@
+De-federating from a remote server sends a pair of notifications to clients, announcing which server will no longer be federated with.

changelog.d/6-federation/fs-1179

@@ -0,0 +1 @@
+Removing a federation domain will now remove all conversations and users for that domain from the local database.

charts/background-worker/templates/configmap.yaml

@@ -20,6 +20,15 @@ data:
     federatorInternal:
       host: federator
       port: 8080
+
+    galley:
+      host: galley
+      port: 8080
+
+    brig:
+      host: brig
+      port: 8080
+
     rabbitmq:
 {{toYaml .rabbitmq | indent 6 }}
     backendNotificationPusher:

charts/federator/templates/deployment.yaml

@@ -73,16 +73,15 @@ spec:
               containerPort: {{ .Values.service.internalFederatorPort }}
             - name: external
               containerPort: {{ .Values.service.externalFederatorPort }}
-          # TODO ensure to have a status endpoint!
-          # livenessProbe:
-          #   httpGet:
-          #     scheme: HTTP
-          #     path: /i/status
-          #     port: {{ .Values.service.internalFederatorPort }}
-          # readinessProbe:
-          #   httpGet:
-          #     scheme: HTTP
-          #     path: /i/status
-          #     port: {{ .Values.service.internalFederatorPort }}
+          livenessProbe:
+            httpGet:
+              scheme: HTTP
+              path: /i/status
+              port: {{ .Values.service.internalFederatorPort }}
+          readinessProbe:
+            httpGet:
+              scheme: HTTP
+              path: /i/status
+              port: {{ .Values.service.internalFederatorPort }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

charts/galley/templates/configmap.yaml

@@ -60,9 +60,6 @@ data:
       exposeInvitationURLsTeamAllowlist: {{ .settings.exposeInvitationURLsTeamAllowlist }}
       {{- end }}
       conversationCodeURI: {{ .settings.conversationCodeURI | quote }}
-      {{- if .settings.enableIndexedBillingTeamMembers }}
-      enableIndexedBillingTeamMembers: {{ .settings.enableIndexedBillingTeamMembers }}
-      {{- end }}
       federationDomain: {{ .settings.federationDomain }}
       {{- if $.Values.secrets.mlsPrivateKeys }}
       mlsPrivateKeyPaths:

charts/galley/values.yaml

@@ -34,9 +34,6 @@ config:
     exposeInvitationURLsTeamAllowlist: []
     maxConvSize: 500
     intraListing: true
-    # Before making indexedBillingTeamMember true while upgrading, please
-    # refer to notes here: https://github.com/wireapp/wire-server-deploy/releases/tag/v2020-05-15
-    indexedBillingTeamMember: false
     # Disable one ore more API versions. Please make sure the configuration value is the same in all these charts:
     # brig, cannon, cargohold, galley, gundeck, proxy, spar.
     # disabledAPIVersions: [ v3 ]

charts/integration/templates/configmap.yaml

@@ -51,6 +51,10 @@ data:
       host: backgroundWorker.{{ .Release.Namespace }}.svc.cluster.local
       port: 8080
 
+    stern:
+      host: stern.{{ .Release.Namespace }}.svc.cluster.local
+      port: 8080
+
     originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
 
     backendTwo:
@@ -99,4 +103,8 @@ data:
         host: backgroundWorker.{{ .Release.Namespace }}-fed2.svc.cluster.local
         port: 8080
 
+      stern:
+        host: stern.{{ .Release.Namespace }}-fed2.svc.cluster.local
+        port: 8080
+
       originDomain: federation-test-helper.{{ .Release.Namespace }}-fed2.svc.cluster.local

charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml

@@ -5,7 +5,7 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: check-cluster-job
-  namespace: databases
+  namespace: {{ .Release.Namespace }}
 spec:
   template:
     spec:

charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml

@@ -2,7 +2,7 @@ apiVersion: k8ssandra.io/v1alpha1
 kind: K8ssandraCluster
 metadata:
   name: k8ssandra-cluster
-  namespace: databases
+  namespace: {{ .Release.Namespace }}
 spec:
   auth: false
   cassandra:

docs/src/developer/reference/config-options.md

@@ -9,25 +9,6 @@ the Wire backend services.
 
 ## Settings in galley
 
-```
-# [galley.yaml]
-settings:
-  enableIndexedBillingTeamMembers: false
-```
-
-### Indexed Billing Team Members
-
-Use indexed billing team members for journaling. When `enabled`,
-galley would use the `billing_team_member` table to send billing
-events with user ids of team owners (who have the `SetBilling`
-permission). Before enabling this flag, the `billing_team_member`
-table must be backfilled.
-
-Even when the flag is `disabled`, galley will keep writing to the
-`biling_team_member` table, this flag only affects the reads and has
-been added in order to deploy new code and backfill data in
-production.
-
 ### MLS private key paths
 
 Note: This developer documentation. Documentation for site operators can be found here: {ref}`mls-message-layer-security`

docs/src/how-to/install/version-requirements.md

@@ -3,9 +3,9 @@
 *Updated: 26.04.2021*
 
 ```{warning}
-If you already installed Wire by using `poetry`, please refer to the
+If you already installed Wire by using `poetry`, which would be the case **only** for ancient (`pre-2020`, version `0.01`) setups, please refer to the
 [old version](https://docs.wire.com/versions/install-with-poetry/how-to/index.html) of
-the installation guide.
+the installation guide. If you have never used `poetry` to install Wire, please ignore this note.
 ```
 
 ## Persistence

docs/src/understand/federation/api.md

@@ -199,7 +199,6 @@ to synchronize the state of the conversations of their members.
     propagate a message to local users. This is used whenever there is a
     remote user in a conversation (see end-to-end flows).
 - `on-mls-message-sent`: Receive a MLS message that originates in the calling backend
-- `on-new-remote-conversation`: Inform the called backend about a conversation that exists on the calling backend. This request is made before the first time the backend might learn about this conversation, e.g. when its first user is added to the conversation.
 - `update-typing-indicator`: Used by the calling backend (that does not own the conversation ) to inform the backend about a change of the typing indicator status of one of its users
 - `on-typing-indicator-updated`: Used by the calling backend (that owns a conversation) to inform the called backend about a change of the typing indicator status of remote user
 - `on-user-deleted-conversations`: When a user on calling backend this request is made for all conversations on the called backend was part of

docs/src/understand/single-sign-on/generic-setup.md

@@ -35,3 +35,7 @@ eg.
 See
 <https://support.wire.com/hc/en-us/articles/360000954617-Pro-How-to-log-in-with-SSO>-
 on how to use this to login on wire.
+
+## Trouble shooting
+
+See {ref}`FAQ <trouble-shooting-faq>`.

docs/src/understand/single-sign-on/trouble-shooting.md

@@ -367,3 +367,26 @@ clash.
 
 Do not rely on case sensitivity of `IssuerID` or `NameID`, or on
 `NameID` qualifiers for distinguishing user identifiers.
+
+## After logging in via IdP page, the redirection to the wire app is not happening
+
+**Problem:** when logging in using SSO, the user gets redirected to
+the IdP page. After entering the credentials, IdP successfully
+authenticates but is stuck at the stage where redirection needs to
+happen to Wire app.
+
+The console log may mention CSP violations.
+
+**Possible cause and fix:** Some browsers prevent redirects if there
+is a risk of leaking sensitive form data to the redirect target. In
+your setup (in particular when you're using email domain-based
+redirect from the wire cloud to your on-prem instance), your browser
+may decide to not trust the wire app with the results of the IdP login
+procedure.  In order to circumvent this issue your IdP needs to be
+configured to add the URL of the IdP (or 'self') to the CSP
+form-action header.
+
+See also:
+[1](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/form-action),
+[2](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy),
+[3](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP).

flaky-tests.yaml

@@ -38,18 +38,6 @@
         assertBool, called at test/integration/API/OAuth.hs:473:16 in main:API.OAuth
       Use -p '(!/turn/&&!/user.auth.cookies.limit/)&&/max active tokens/' to rerun this test only.
 
--
-  test_name: "send billing events to some owners in large teams (indexedBillingTeamMembers disabled)"
-  comments: |
-    Error message: create team: Expected 1 TeamActivate, got nothing
-
-    CallStack (from HasCallStack):
-      assertFailure, called at test/integration/API/SQS.hs:74:32 in main:API.SQS
-      tActivate, called at test/integration/API/SQS.hs:78:47 in main:API.SQS
-      assertTeamActivate, called at test/integration/API/Util.hs:191:3 in main:API.Util
-      createBindingTeam', called at test/integration/API/Util.hs:183:20 in main:API.Util
-      createBindingTeam, called at test/integration/API/Teams.hs:1546:25 in main:API.Teams
-
 -
   test_name: "delete team conversation"
   comments: |

hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl

@@ -14,8 +14,8 @@ ingress-nginx:
         # choose a random free port
         https: null
         http: null
-      # in CI, do not use ValidatingWebhooks, as these, if not properly cleaned up
-      # (i.e. the ingress controller was deleted in another namespace but the webhook remains)
-      # prevent new kind:Ingress resources to be created in the cluster.
-      admissionWebhooks:
-        enabled: false
+    # in CI, do not use ValidatingWebhooks, as these, if not properly cleaned up
+    # (i.e. the ingress controller was deleted in another namespace but the webhook remains)
+    # prevent new kind:Ingress resources to be created in the cluster.
+    admissionWebhooks:
+      enabled: false

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -184,7 +184,6 @@ galley:
       maxFanoutSize: 18
       maxConvSize: 16
       conversationCodeURI: https://kube-staging-nginz-https.zinfra.io/conversation-join/
-      enableIndexedBillingTeamMembers: true
       # See helmfile for the real value
       federationDomain: integration.example.com
       featureFlags:
@@ -232,8 +231,8 @@ gundeck:
       host: cassandra-ephemeral
       replicaCount: 1
     redis:
-      host: redis-cluster
-      connectionMode: cluster
+      host: redis-ephemeral-master
+      connectionMode: master
     aws:
       account: "123456789012"
       region: eu-west-1

hack/helmfile.yaml

@@ -55,18 +55,6 @@ releases:
     namespace: '{{ .Values.namespace2 }}'
     chart: '../.local/charts/databases-ephemeral'
 
-  - name: 'redis-cluster'
-    namespace: '{{ .Values.namespace1 }}'
-    chart: '../.local/charts/redis-cluster'
-    values:
-      - './helm_vars/redis-cluster/values.yaml.gotmpl'
-
-  - name: 'redis-cluster'
-    namespace: '{{ .Values.namespace2 }}'
-    chart: '../.local/charts/redis-cluster'
-    values:
-      - './helm_vars/redis-cluster/values.yaml.gotmpl'
-
   - name: 'rabbitmq'
     namespace: '{{ .Values.namespace1 }}'
     chart: '../.local/charts/rabbitmq'