From 87b24372bd2e512258f97c201c23a79dbc10d80e Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 19 Dec 2023 20:00:44 +0100 Subject: [PATCH] test: Fix `FATAL: sorry, too many clients already` By moving two Pytest fixtures responsible for database connectivity from the "function" scope to the "session" scope, the number of redundant invocations to `sqlalchemy.create_engine()` can be significantly reduced. Apparently, this fixes to connection pool overflow. --- target_postgres/tests/test_target_postgres.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/target_postgres/tests/test_target_postgres.py b/target_postgres/tests/test_target_postgres.py index d675dc90..251895f8 100644 --- a/target_postgres/tests/test_target_postgres.py +++ b/target_postgres/tests/test_target_postgres.py @@ -47,7 +47,7 @@ def postgres_config_ssh_tunnel_fixture(): return postgres_config_ssh_tunnel() -@pytest.fixture +@pytest.fixture(scope="session") def postgres_target(postgres_config) -> TargetPostgres: return TargetPostgres(config=postgres_config) @@ -81,6 +81,7 @@ class AssertionHelper: def __init__(self, target: TargetPostgres, metadata_column_prefix: str): self.target = target self.metadata_column_prefix = metadata_column_prefix + self.engine = create_engine(self.target) def remove_metadata_columns(self, row: dict) -> dict: new_row = {} @@ -107,9 +108,8 @@ def verify_data( table, as determined by lowest primary_key value, or else a list of dictionaries representing every row in the table. """ - engine = create_engine(self.target) full_table_name = f"{self.target.config['default_target_schema']}.{table_name}" - with engine.connect() as connection: + with self.engine.connect() as connection: if primary_key is not None and check_data is not None: if isinstance(check_data, dict): result = connection.execute( @@ -154,9 +154,8 @@ def verify_schema( it is all about the `type` attribute which is compared. metadata_column_prefix: The prefix string for metadata columns. Usually `_sdc`. """ - engine = create_engine(self.target) schema = self.target.config["default_target_schema"] - with engine.connect() as connection: + with self.engine.connect() as connection: meta = sqlalchemy.MetaData() table = sqlalchemy.Table( table_name, meta, schema=schema, autoload_with=connection @@ -178,7 +177,7 @@ def verify_schema( ) -@pytest.fixture +@pytest.fixture(scope="session") def helper(postgres_target) -> AssertionHelper: return AssertionHelper( target=postgres_target, metadata_column_prefix=METADATA_COLUMN_PREFIX