Use locally built schema binaries for db migrations (#2791)

* Makefile: Allow db-migrate and db-reset to migrate/reset all keyspaces

* Use locally built schema binaries for db migrations

Also make sure DB is up to date every time tests are run and not just when DBs
are spun up.

Makefile

@@ -86,7 +86,7 @@ endif
 # Usage: make ci package=brig test=1
 # If you want to pass arguments to the test-suite call the script directly.
 .PHONY: ci
-ci: c
+ci: c db-migrate
 	./hack/bin/cabal-run-integration.sh $(package)
 
 .PHONY: cabal-fmt
@@ -139,18 +139,6 @@ regen-local-nix-derivations:
 check-local-nix-derivations: regen-local-nix-derivations
 	git diff --exit-code
 
-# reset db using cabal
-.PHONY: db-reset-package
-db-reset-package: c
-	$(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset
-
-# migrate db using cabal
-# For using stack see the Makefile of the package, e.g. services/brig/Makefile
-# Usage: make db-migrate-package package=galley
-.PHONY: db-migrate-package
-db-migrate-package: c
-	$(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1
-
 # Build everything (Haskell services and nginz)
 .PHONY: services
 services: init install
@@ -217,13 +205,53 @@ cqlsh:
 	@echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
 	docker exec -it $(CASSANDRA_CONTAINER) /usr/bin/cqlsh
 
+.PHONY: db-reset-package
+db-reset-package:
+	@echo "Deprecated! Please use 'db-reset' instead"
+	$(MAKE) db-reset package=$(package)
+
+.PHONY: db-migrate-package
+db-migrate-package:
+	@echo "Deprecated! Please use 'db-migrate' instead"
+	$(MAKE) db-migrate package=$(package)
+
+# Usage:
+#
+# Reset all keyspaces
+# make db-reset
+#
+# Reset keyspace for only one service, say galley:
+# make db-reset package=galley
 .PHONY: db-reset
-db-reset:
-	@echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
-	make db-reset-package package=brig
-	make db-reset-package package=galley
-	make db-reset-package package=gundeck
-	make db-reset-package package=spar
+db-reset: c
+	@echo "Make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
+ifeq ($(package), all)
+	./dist/brig-schema --keyspace brig_test --replication-factor 1 --reset
+	./dist/galley-schema --keyspace galley_test --replication-factor 1 --reset
+	./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --reset
+	./dist/spar-schema --keyspace spar_test --replication-factor 1 --reset
+else
+	$(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset
+endif
+
+# Usage:
+#
+# Migrate all keyspaces
+# make db-migrate
+#
+# Migrate keyspace for only one service, say galley:
+# make db-migrate package=galley
+.PHONY: db-migrate
+db-migrate: c
+ifeq ($(package), all)
+	./dist/brig-schema --keyspace brig_test --replication-factor 1
+	./dist/galley-schema --keyspace galley_test --replication-factor 1
+	./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1
+	./dist/spar-schema --keyspace spar_test --replication-factor 1
+else
+	$(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1
+endif
+
 
 #################################
 ## dependencies

changelog.d/5-internal/integration-db-migrate

@@ -0,0 +1 @@
+Use locally build schema binaries for db migrations and execute them right before running integration tests.

changelog.d/5-internal/make-db-migrate

@@ -0,0 +1 @@
+Rename the make targets from `db-migrate-package` and `db-reset-package` to `db-migrate` and `db-reset` and allow migrating and resetting all keyspaces.

deploy/dockerephemeral/docker-compose.yaml

@@ -198,66 +198,6 @@ services:
     networks:
       - demo_wire
 
-  db_migrations_brig_schema:
-    image: quay.io/wire/brig-schema
-    depends_on:
-      - cassandra
-    entrypoint: /scripts/db-migrate/brig-schema.sh
-    volumes:
-      - ./:/scripts
-    links:
-      - cassandra
-    networks:
-      - demo_wire
-
-  db_migrations_brig_index:
-    image: quay.io/wire/brig-index
-    depends_on:
-      - elasticsearch
-    entrypoint: /scripts/db-migrate/brig-index.sh
-    volumes:
-      - ./:/scripts
-    links:
-      - elasticsearch
-    networks:
-      - demo_wire
-
-  db_migrations_galley:
-    image: quay.io/wire/galley-schema
-    depends_on:
-      - cassandra
-    entrypoint: /scripts/db-migrate/galley-schema.sh
-    volumes:
-      - ./:/scripts
-    links:
-      - cassandra
-    networks:
-      - demo_wire
-
-  db_migrations_gundeck:
-    image: quay.io/wire/gundeck-schema
-    depends_on:
-      - cassandra
-    entrypoint: /scripts/db-migrate/gundeck-schema.sh
-    volumes:
-      - ./:/scripts
-    links:
-      - cassandra
-    networks:
-      - demo_wire
-
-  db_migrations_spar:
-    image: quay.io/wire/spar-schema
-    depends_on:
-      - cassandra
-    entrypoint: /scripts/db-migrate/spar-schema.sh
-    volumes:
-      - ./:/scripts
-    links:
-      - cassandra
-    networks:
-      - demo_wire
-
   # FIXME: replace aws_cli with an image that we build.
   aws_cli:
     image: mesosphere/aws-cli:1.14.5

deploy/dockerephemeral/run.sh

@@ -4,17 +4,4 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 DOCKER_FILE="$SCRIPT_DIR/docker-compose.yaml"
 
-# If there are schema changes and you don't force pull the docker
-# migrations, you may run out of sync and you would get this error
-# message (or a similar one):
-#
-#     brig: Schema Version too old! Expecting at least: 49, but got: 48
-#
-# So we always pull these migration images first.
-docker pull quay.io/wire/brig-schema
-docker pull quay.io/wire/galley-schema
-docker pull quay.io/wire/gundeck-schema
-docker pull quay.io/wire/spar-schema
-docker pull quay.io/wire/brig-index
-
 docker-compose --file "$DOCKER_FILE" up