Skip to content

Commit

Permalink
test: Fix FATAL: sorry, too many clients already
Browse files Browse the repository at this point in the history
By moving two Pytest fixtures responsible for database connectivity
from the "function" scope to the "session" scope, the number of
redundant invocations to `sqlalchemy.create_engine()` can be
significantly reduced. Apparently, this fixes to connection pool
overflow.
  • Loading branch information
amotl committed Dec 19, 2023
1 parent c446d94 commit 87b2437
Showing 1 changed file with 5 additions and 6 deletions.
11 changes: 5 additions & 6 deletions target_postgres/tests/test_target_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def postgres_config_ssh_tunnel_fixture():
return postgres_config_ssh_tunnel()


@pytest.fixture
@pytest.fixture(scope="session")
def postgres_target(postgres_config) -> TargetPostgres:
return TargetPostgres(config=postgres_config)

Expand Down Expand Up @@ -81,6 +81,7 @@ class AssertionHelper:
def __init__(self, target: TargetPostgres, metadata_column_prefix: str):
self.target = target
self.metadata_column_prefix = metadata_column_prefix
self.engine = create_engine(self.target)

def remove_metadata_columns(self, row: dict) -> dict:
new_row = {}
Expand All @@ -107,9 +108,8 @@ def verify_data(
table, as determined by lowest primary_key value, or else a list of
dictionaries representing every row in the table.
"""
engine = create_engine(self.target)
full_table_name = f"{self.target.config['default_target_schema']}.{table_name}"
with engine.connect() as connection:
with self.engine.connect() as connection:
if primary_key is not None and check_data is not None:
if isinstance(check_data, dict):
result = connection.execute(
Expand Down Expand Up @@ -154,9 +154,8 @@ def verify_schema(
it is all about the `type` attribute which is compared.
metadata_column_prefix: The prefix string for metadata columns. Usually `_sdc`.
"""
engine = create_engine(self.target)
schema = self.target.config["default_target_schema"]
with engine.connect() as connection:
with self.engine.connect() as connection:
meta = sqlalchemy.MetaData()
table = sqlalchemy.Table(
table_name, meta, schema=schema, autoload_with=connection
Expand All @@ -178,7 +177,7 @@ def verify_schema(
)


@pytest.fixture
@pytest.fixture(scope="session")
def helper(postgres_target) -> AssertionHelper:
return AssertionHelper(
target=postgres_target, metadata_column_prefix=METADATA_COLUMN_PREFIX
Expand Down

0 comments on commit 87b2437

Please sign in to comment.