From 655b7ad102f311b8e13eca75e14537fd17772139 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 1 Mar 2022 20:23:33 +0100 Subject: [PATCH 01/16] add driver --- terracotta/drivers/__init__.py | 9 ++- terracotta/drivers/postgresql_meta_store.py | 80 +++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 terracotta/drivers/postgresql_meta_store.py diff --git a/terracotta/drivers/__init__.py b/terracotta/drivers/__init__.py index 26f38988..5799ade4 100644 --- a/terracotta/drivers/__init__.py +++ b/terracotta/drivers/__init__.py @@ -24,6 +24,10 @@ def load_driver(provider: str) -> Type[MetaStore]: from terracotta.drivers.mysql_meta_store import MySQLMetaStore return MySQLMetaStore + if provider == 'postgresql': + from terracotta.drivers.postgresql_meta_store import PostgreSQLMetaStore + return PostgreSQLMetaStore + if provider == 'sqlite': from terracotta.drivers.sqlite_meta_store import SQLiteMetaStore return SQLiteMetaStore @@ -41,6 +45,9 @@ def auto_detect_provider(url_or_path: str) -> str: if scheme == 'mysql': return 'mysql' + if scheme == 'postgresql': + return 'postgresql' + return 'sqlite' @@ -61,7 +68,7 @@ def get_driver(url_or_path: URLOrPathType, provider: str = None) -> TerracottaDr url_or_path: A path identifying the database to connect to. The expected format depends on the driver provider. - provider: Driver provider to use (one of sqlite, sqlite-remote, mysql; + provider: Driver provider to use (one of sqlite, sqlite-remote, mysql, postgresql; default: auto-detect). Example: diff --git a/terracotta/drivers/postgresql_meta_store.py b/terracotta/drivers/postgresql_meta_store.py new file mode 100644 index 00000000..c53db883 --- /dev/null +++ b/terracotta/drivers/postgresql_meta_store.py @@ -0,0 +1,80 @@ +"""drivers/postgresql_meta_store.py + +PostgreSQL-backed metadata driver. Metadata is stored in a PostgreSQL database. +""" + +from typing import Mapping, Sequence + +import sqlalchemy as sqla +from terracotta.drivers.relational_meta_store import RelationalMetaStore + + +class PostgreSQLMetaStore(RelationalMetaStore): + """A PostgreSQL-backed metadata driver. + + Stores metadata and paths to raster files in PostgreSQL. + + Requires a running PostgreSQL server. + + The PostgreSQL database consists of 4 different tables: + + - ``terracotta``: Metadata about the database itself. + - ``key_names``: Contains two columns holding all available keys and their description. + - ``datasets``: Maps key values to physical raster path. + - ``metadata``: Contains actual metadata as separate columns. Indexed via key values. + + This driver caches key names. + """ + SQL_DIALECT = 'postgresql' + SQL_DRIVER = 'psycopg2' + SQL_TIMEOUT_KEY = 'connect_timeout' + + MAX_PRIMARY_KEY_SIZE = 2730 // 4 # Max B-tree index size in bytes + DEFAULT_PORT = 5432 + + def __init__(self, postgresql_path: str) -> None: + """Initialize the PostgreSQLDriver. + + This should not be called directly, use :func:`~terracotta.get_driver` instead. + + Arguments: + + postgresql_path: URL to running PostgreSQL server, in the form + ``postgresql://username:password@hostname/database`` + + """ + super().__init__(postgresql_path) + + # raise an exception if database name is invalid + if not self.url.database: + raise ValueError('database must be specified in PostgreSQL path') + if '/' in self.url.database.strip('/'): + raise ValueError('invalid database path') + + @classmethod + def _normalize_path(cls, path: str) -> str: + url = cls._parse_path(path) + + path = f'{url.drivername}://{url.host}:{url.port or cls.DEFAULT_PORT}/{url.database}' + path = path.rstrip('/') + return path + + def _create_database(self) -> None: + engine = sqla.create_engine( + self.url.set(database='postgres'), # `.set()` returns a copy with changed parameters + echo=False, + future=True, + isolation_level='AUTOCOMMIT' + ) + with engine.connect() as connection: + connection.execute(sqla.text(f'CREATE DATABASE {self.url.database}')) + connection.commit() + + def _initialize_database( + self, + keys: Sequence[str], + key_descriptions: Mapping[str, str] = None + ) -> None: + # Enforce max primary key length equal to max B-tree index size + self.SQL_KEY_SIZE = self.MAX_PRIMARY_KEY_SIZE // len(keys) + super()._initialize_database(keys, key_descriptions) From 428ed8a637c3dabedff47d13ddfd2147bffa8570 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 1 Mar 2022 20:24:24 +0100 Subject: [PATCH 02/16] postgres config vars --- terracotta/config.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/terracotta/config.py b/terracotta/config.py index ee44f563..86f0c66f 100644 --- a/terracotta/config.py +++ b/terracotta/config.py @@ -16,7 +16,7 @@ class TerracottaSettings(NamedTuple): #: Path to database DRIVER_PATH: str = '' - #: Driver provider to use (sqlite, sqlite-remote, mysql; auto-detected by default) + #: Driver provider to use (sqlite, sqlite-remote, mysql, postgresql; auto-detected by default) DRIVER_PROVIDER: Optional[str] = None #: Activate debug mode in Flask app @@ -73,6 +73,12 @@ class TerracottaSettings(NamedTuple): #: MySQL database password (if not given in driver path) MYSQL_PASSWORD: Optional[str] = None + #: PostgreSQL database username (if not given in driver path) + POSTGRESQL_USER: Optional[str] = None + + #: PostgreSQL database password (if not given in driver path) + POSTGRESQL_PASSWORD: Optional[str] = None + #: Use a process pool for band retrieval in parallel USE_MULTIPROCESSING: bool = True @@ -125,6 +131,8 @@ class SettingSchema(Schema): MYSQL_USER = fields.String() MYSQL_PASSWORD = fields.String() + POSTGRESQL_USER = fields.String() + POSTGRESQL_PASSWORD = fields.String() USE_MULTIPROCESSING = fields.Boolean() From 5063623827ce3f141e70cfc74a43a31b4cac3a58 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 1 Mar 2022 20:24:36 +0100 Subject: [PATCH 03/16] psycopg2 dependency --- setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index bf34bf66..7afbb20c 100644 --- a/setup.py +++ b/setup.py @@ -90,18 +90,21 @@ 'matplotlib', 'moto', 'aws-xray-sdk', - 'pymysql>=1.0.0' + 'pymysql>=1.0.0', + 'psycopg2' ], 'docs': [ 'sphinx', 'sphinx_autodoc_typehints', 'sphinx-click', - 'pymysql>=1.0.0' + 'pymysql>=1.0.0', + 'psycopg2' ], 'recommended': [ 'colorlog', 'crick', - 'pymysql>=1.0.0' + 'pymysql>=1.0.0', + 'psycopg2' ] }, # CLI From bde6a96f145992bbbf2051f69aacb574913dbb2e Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 1 Mar 2022 20:25:02 +0100 Subject: [PATCH 04/16] remove mysql mention from relational driver --- terracotta/drivers/relational_meta_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terracotta/drivers/relational_meta_store.py b/terracotta/drivers/relational_meta_store.py index 2ccdf45f..8e3e5a39 100644 --- a/terracotta/drivers/relational_meta_store.py +++ b/terracotta/drivers/relational_meta_store.py @@ -185,7 +185,7 @@ def db_version(self) -> str: def create(self, keys: Sequence[str], key_descriptions: Mapping[str, str] = None) -> None: """Create and initialize database with empty tables. - This must be called before opening the first connection. The MySQL database must not + This must be called before opening the first connection. The database must not exist already. Arguments: From b0dc3dbe18842e75c7f0a892ac95804f48a3e973 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:23:51 +0100 Subject: [PATCH 05/16] add postgres provider fixture --- tests/conftest.py | 81 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 44a85ffe..2eaed706 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,10 @@ def pytest_addoption(parser): '--mysql-server', help='MySQL server to use for testing in the form of user:password@host:port' ) + parser.addoption( + '--postgresql-server', + help='PostgreSQL server to use for testing in the form of user:password@host:port' + ) @pytest.fixture() @@ -46,6 +50,11 @@ def mysql_server(request): return request.config.getoption('mysql_server') +@pytest.fixture() +def postgresql_server(request): + return request.config.getoption('postgresql_server') + + def cloud_optimize(raster_file, outfile, create_mask=False, remove_nodata=False): import math import contextlib @@ -372,58 +381,80 @@ def test_server(testdb): @pytest.fixture() -def driver_path(provider, tmpdir, mysql_server): +def driver_path(provider, tmpdir, mysql_server, postgresql_server): """Get a valid, uninitialized driver path for given provider""" import random import string from urllib.parse import urlparse - def validate_con_info(con_info): - return (con_info.scheme == 'mysql' + def validate_con_info(con_info, db_scheme): + return (con_info.scheme == db_scheme and con_info.hostname and con_info.username and not con_info.path) def random_string(length): - return ''.join(random.choices(string.ascii_uppercase, k=length)) + return ''.join(random.choices(string.ascii_lowercase, k=length)) if provider == 'sqlite': dbfile = tmpdir.join('test.sqlite') yield str(dbfile) - elif provider == 'mysql': - if not mysql_server: - return pytest.skip('mysql_server argument not given') + elif provider == 'mysql' or provider == 'postgresql': + if provider == 'mysql': + db_server = mysql_server + con_db = '' + else: + db_server = postgresql_server + con_db = 'postgres' + + if not db_server: + return pytest.skip(f'{provider}_server argument not given') - if not mysql_server.startswith('mysql://'): - mysql_server = f'mysql://{mysql_server}' + if not db_server.startswith(f'{provider}://'): + db_server = f'{provider}://{db_server}' - con_info = urlparse(mysql_server) - if not validate_con_info(con_info): - raise ValueError('invalid value for mysql_server') + con_info = urlparse(db_server) + if not validate_con_info(con_info, provider): + raise ValueError(f'invalid value for {provider}_server') dbpath = random_string(24) - import pymysql + if provider == 'mysql': + import pymysql as driver + elif provider == 'postgresql': + import psycopg2 as driver + try: - with pymysql.connect(host=con_info.hostname, user=con_info.username, - password=con_info.password): + with driver.connect(host=con_info.hostname, user=con_info.username, + password=con_info.password, database=con_db): pass - except pymysql.OperationalError as exc: - raise RuntimeError('error connecting to MySQL server') from exc + except driver.OperationalError as exc: + raise RuntimeError(f'error connecting to {provider} server') from exc try: - yield f'{mysql_server}/{dbpath}' + yield f'{db_server}/{dbpath}' finally: # cleanup - with pymysql.connect(host=con_info.hostname, user=con_info.username, - password=con_info.password) as connection: - with connection.cursor() as cursor: - try: - cursor.execute(f'DROP DATABASE IF EXISTS {dbpath}') - except pymysql.Warning: - pass + connection = driver.connect( + host=con_info.hostname, + user=con_info.username, + password=con_info.password, + database=con_db + ) + connection.autocommit = True + with connection.cursor() as cursor: + try: + if provider == 'postgresql': + # Postgres refuses to drop DB if any sessions are hanging around + cursor.execute("SELECT pg_terminate_backend(pg_stat_activity.pid) " + "FROM pg_stat_activity " + f"WHERE pg_stat_activity.datname = '{dbpath}'") + cursor.execute(f'DROP DATABASE IF EXISTS {dbpath}') + except driver.Warning: + pass + connection.close() else: return NotImplementedError(f'unknown provider {provider}') From 4034fc62b79d7fc08be56342083d1c43bce9a5e0 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:24:41 +0100 Subject: [PATCH 06/16] parametrize over postgres provider --- tests/drivers/test_drivers.py | 5 +++-- tests/drivers/test_mysql.py | 22 +++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/drivers/test_drivers.py b/tests/drivers/test_drivers.py index d3881839..a584833f 100644 --- a/tests/drivers/test_drivers.py +++ b/tests/drivers/test_drivers.py @@ -1,10 +1,11 @@ import pytest -TESTABLE_DRIVERS = ['sqlite', 'mysql'] +TESTABLE_DRIVERS = ['sqlite', 'mysql', 'postgresql'] DRIVER_CLASSES = { 'sqlite': 'SQLiteMetaStore', 'sqlite-remote': 'SQLiteRemoteMetaStore', - 'mysql': 'MySQLMetaStore' + 'mysql': 'MySQLMetaStore', + 'postgresql': 'PostgreSQLMetaStore' } diff --git a/tests/drivers/test_mysql.py b/tests/drivers/test_mysql.py index e51aec44..4379bce3 100644 --- a/tests/drivers/test_mysql.py +++ b/tests/drivers/test_mysql.py @@ -1,16 +1,16 @@ import pytest TEST_CASES = { - 'mysql://root@localhost:5000/test': dict( + '{provider}://root@localhost:5000/test': dict( username='root', password=None, host='localhost', port=5000, database='test' ), 'root@localhost:5000/test': dict( username='root', password=None, host='localhost', port=5000, database='test' ), - 'mysql://root:foo@localhost/test': dict( + '{provider}://root:foo@localhost/test': dict( username='root', password='foo', host='localhost', port=None, database='test' ), - 'mysql://localhost/test': dict( + '{provider}://localhost/test': dict( password=None, host='localhost', port=None, database='test' ), 'localhost/test': dict( @@ -20,25 +20,29 @@ INVALID_TEST_CASES = [ 'http://localhost/test', # wrong scheme - 'mysql://localhost', # no database - 'mysql://localhost/test/foo/bar' # path too deep + '{provider}://localhost', # no database + '{provider}://localhost/test/foo/bar' # path too deep ] @pytest.mark.parametrize('case', TEST_CASES.keys()) -def test_path_parsing(case): +@pytest.mark.parametrize('provider', ('mysql', 'postgresql')) +def test_path_parsing(case, provider): from terracotta import drivers # empty cache drivers._DRIVER_CACHE = {} - db = drivers.get_driver(case, provider='mysql') + db_path = case.format(provider=provider) + db = drivers.get_driver(db_path, provider=provider) for attr in ('username', 'password', 'host', 'port', 'database'): assert getattr(db.meta_store.url, attr) == TEST_CASES[case].get(attr, None) @pytest.mark.parametrize('case', INVALID_TEST_CASES) -def test_invalid_paths(case): +@pytest.mark.parametrize('provider', ('mysql', 'postgresql')) +def test_invalid_paths(case, provider): from terracotta import drivers + db_path = case.format(provider=provider) with pytest.raises(ValueError): - drivers.get_driver(case, provider='mysql') + drivers.get_driver(db_path, provider=provider) From c6659c4e84966a4176203914a3880e79bdbe4414 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:25:15 +0100 Subject: [PATCH 07/16] rename test file --- tests/drivers/{test_mysql.py => test_paths.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/drivers/{test_mysql.py => test_paths.py} (100%) diff --git a/tests/drivers/test_mysql.py b/tests/drivers/test_paths.py similarity index 100% rename from tests/drivers/test_mysql.py rename to tests/drivers/test_paths.py From b7c49ed747671288522504094e7a0f9011496870 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:25:31 +0100 Subject: [PATCH 08/16] add postgresql server to CI --- .github/workflows/test.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b77ca7de..8a26a3c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,8 @@ on: - main env: - DB_PORT: 3306 + MYSQL_PORT: 3306 + POSTGRESQL_PORT: 5432 DB_USER: root DB_PASSWORD: root @@ -30,6 +31,18 @@ jobs: defaults: run: shell: bash + + services: + postgres: + image: postgres + env: + POSTGRES_USER: ${{ env.DB_USER }} + POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }} + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v2 @@ -76,8 +89,9 @@ jobs: - name: Run tests run: | - MYSQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@127.0.0.1:${{ env.DB_PORT }}" - python -m pytest . --color=yes --cov=terracotta --mysql-server=$MYSQL_SRV + MYSQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@127.0.0.1:${{ env.MYSQL_PORT }}" + POSTGRESQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@127.0.0.1:${{ env.POSTGRESQL_PORT }}" + python -m pytest . --color=yes --cov=terracotta --mysql-server=$MYSQL_SRV --postgresql-server=$POSTGRESQL_SRV - name: Run benchmarks run: | From 6604017a667f0b8f715c1072f3cfc3bf05868ee0 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:40:58 +0100 Subject: [PATCH 09/16] cleaner switch --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2eaed706..8f9a29be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -405,7 +405,7 @@ def random_string(length): if provider == 'mysql': db_server = mysql_server con_db = '' - else: + elif provider == 'postgresql': db_server = postgresql_server con_db = 'postgres' From 95e012dd555f082289615492e48bf86a5ead6c5c Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:41:20 +0100 Subject: [PATCH 10/16] fix postgres hostname in CI workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a26a3c8..caf9d20c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -90,7 +90,7 @@ jobs: - name: Run tests run: | MYSQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@127.0.0.1:${{ env.MYSQL_PORT }}" - POSTGRESQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@127.0.0.1:${{ env.POSTGRESQL_PORT }}" + POSTGRESQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@postgres:${{ env.POSTGRESQL_PORT }}" python -m pytest . --color=yes --cov=terracotta --mysql-server=$MYSQL_SRV --postgresql-server=$POSTGRESQL_SRV - name: Run benchmarks From dce4117f9115303b574f81fe94b07e2a3c9fdc55 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Wed, 2 Mar 2022 14:47:55 +0100 Subject: [PATCH 11/16] map postgres service to localhost --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index caf9d20c..20baafc3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + - 5432:5432 steps: - uses: actions/checkout@v2 @@ -90,7 +92,7 @@ jobs: - name: Run tests run: | MYSQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@127.0.0.1:${{ env.MYSQL_PORT }}" - POSTGRESQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@postgres:${{ env.POSTGRESQL_PORT }}" + POSTGRESQL_SRV="${{ env.DB_USER }}:${{ env.DB_PASSWORD }}@localhost:${{ env.POSTGRESQL_PORT }}" python -m pytest . --color=yes --cov=terracotta --mysql-server=$MYSQL_SRV --postgresql-server=$POSTGRESQL_SRV - name: Run benchmarks From 0a3ed4134556aa66a0a4b51d56fed1bf737fafb4 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Mon, 21 Mar 2022 17:01:12 +0100 Subject: [PATCH 12/16] DEFAULT_CONNECT_DB class var --- terracotta/drivers/postgresql_meta_store.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/terracotta/drivers/postgresql_meta_store.py b/terracotta/drivers/postgresql_meta_store.py index c53db883..c0bd907f 100644 --- a/terracotta/drivers/postgresql_meta_store.py +++ b/terracotta/drivers/postgresql_meta_store.py @@ -31,6 +31,8 @@ class PostgreSQLMetaStore(RelationalMetaStore): MAX_PRIMARY_KEY_SIZE = 2730 // 4 # Max B-tree index size in bytes DEFAULT_PORT = 5432 + # Will connect to this db before creatting the 'terracotta' db + DEFAULT_CONNECT_DB = 'postgres' def __init__(self, postgresql_path: str) -> None: """Initialize the PostgreSQLDriver. @@ -61,7 +63,8 @@ def _normalize_path(cls, path: str) -> str: def _create_database(self) -> None: engine = sqla.create_engine( - self.url.set(database='postgres'), # `.set()` returns a copy with changed parameters + # `.set()` returns a copy with changed parameters + self.url.set(database=self.DEFAULT_CONNECT_DB), echo=False, future=True, isolation_level='AUTOCOMMIT' From 05305c54816f67e4a35eef758069e82ed74e6c1a Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 22 Mar 2022 11:45:01 +0100 Subject: [PATCH 13/16] simplify test db cleanup --- tests/conftest.py | 48 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8f9a29be..a221e4fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -385,6 +385,9 @@ def driver_path(provider, tmpdir, mysql_server, postgresql_server): """Get a valid, uninitialized driver path for given provider""" import random import string + from terracotta import drivers + from terracotta.exceptions import InvalidDatabaseError + import sqlalchemy as sqla from urllib.parse import urlparse @@ -402,9 +405,11 @@ def random_string(length): yield str(dbfile) elif provider == 'mysql' or provider == 'postgresql': + dbpath = random_string(24) + if provider == 'mysql': db_server = mysql_server - con_db = '' + con_db = 'terracotta' elif provider == 'postgresql': db_server = postgresql_server con_db = 'postgres' @@ -419,42 +424,27 @@ def random_string(length): if not validate_con_info(con_info, provider): raise ValueError(f'invalid value for {provider}_server') - dbpath = random_string(24) - - if provider == 'mysql': - import pymysql as driver - elif provider == 'postgresql': - import psycopg2 as driver - + driver = drivers.get_driver(f'{db_server}/{con_db}', provider=provider) try: - with driver.connect(host=con_info.hostname, user=con_info.username, - password=con_info.password, database=con_db): + with driver.connect(verify=False): pass - except driver.OperationalError as exc: + except InvalidDatabaseError as exc: raise RuntimeError(f'error connecting to {provider} server') from exc try: yield f'{db_server}/{dbpath}' finally: # cleanup - connection = driver.connect( - host=con_info.hostname, - user=con_info.username, - password=con_info.password, - database=con_db - ) - connection.autocommit = True - with connection.cursor() as cursor: - try: - if provider == 'postgresql': - # Postgres refuses to drop DB if any sessions are hanging around - cursor.execute("SELECT pg_terminate_backend(pg_stat_activity.pid) " - "FROM pg_stat_activity " - f"WHERE pg_stat_activity.datname = '{dbpath}'") - cursor.execute(f'DROP DATABASE IF EXISTS {dbpath}') - except driver.Warning: - pass - connection.close() + with driver.meta_store.sqla_engine.connect().execution_options( + isolation_level='AUTOCOMMIT' + ) as conn: + if provider == 'postgresql': + # Postgres refuses to drop DB if any sessions are hanging around + conn.execute(sqla.text( + "SELECT pg_terminate_backend(pg_stat_activity.pid) " + "FROM pg_stat_activity " + f"WHERE pg_stat_activity.datname = '{dbpath}'")) + conn.execute(sqla.text(f'DROP DATABASE IF EXISTS {dbpath}')) else: return NotImplementedError(f'unknown provider {provider}') From c0e1ab450144f51e60aab4e1ae9dc644c7ac147c Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 22 Mar 2022 12:14:42 +0100 Subject: [PATCH 14/16] dummy commit --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index a221e4fc..124eb978 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -380,6 +380,7 @@ def test_server(testdb): assert not server_proc.is_alive() + @pytest.fixture() def driver_path(provider, tmpdir, mysql_server, postgresql_server): """Get a valid, uninitialized driver path for given provider""" From 9738fe6b9dda64791e8380119e95f2df934f3ed8 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 22 Mar 2022 12:15:02 +0100 Subject: [PATCH 15/16] dummy commit - trigger tests --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 124eb978..a221e4fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -380,7 +380,6 @@ def test_server(testdb): assert not server_proc.is_alive() - @pytest.fixture() def driver_path(provider, tmpdir, mysql_server, postgresql_server): """Get a valid, uninitialized driver path for given provider""" From f0257fcd8a97066f868ff54e8779b5ef6363fd93 Mon Sep 17 00:00:00 2001 From: Philip Graae Date: Tue, 22 Mar 2022 12:34:53 +0100 Subject: [PATCH 16/16] use mysql con_db --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a221e4fc..a32b8349 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -405,11 +405,10 @@ def random_string(length): yield str(dbfile) elif provider == 'mysql' or provider == 'postgresql': - dbpath = random_string(24) if provider == 'mysql': db_server = mysql_server - con_db = 'terracotta' + con_db = 'mysql' elif provider == 'postgresql': db_server = postgresql_server con_db = 'postgres' @@ -431,6 +430,7 @@ def random_string(length): except InvalidDatabaseError as exc: raise RuntimeError(f'error connecting to {provider} server') from exc + dbpath = random_string(24) try: yield f'{db_server}/{dbpath}'