Parallelize tests (#34)

* Change charm database user * Fix tests user * Add password rotation action * Rework secrets management * Add unit tests * Add constants * Update part of the code to correct actions * Implement password rotation * Improve messages * Improve get password test * Fix comment and remove unneeded else block * Add set password unit test * Improve unit test * Change helper function * Add copyright notice * Improve username retrieval * Add initial code for password rotation test * Fix comment * Add test checks * Small fix * Add retry to helper function * Fix lint problems * Improve integration test * Add pytest mark * Add separate jobs for different tests * Add separate encironments for different tests * Define asyncio mode * Add pytest marks * Fix test name * Remove duplicate job * Squashed commit of the following: commit db8dbd8 Author: Marcelo Henrique Neppel <[email protected]> Date: Fri Aug 26 16:15:15 2022 -0300 Add pytest mark commit b6f2d8a Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 14:57:20 2022 -0300 Improve integration test commit 16dce63 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 13:50:42 2022 -0300 Fix lint problems commit 14ec0f0 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 13:48:51 2022 -0300 Add retry to helper function commit 4bc38b8 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 11:56:41 2022 -0300 Small fix commit 34421e5 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 11:47:51 2022 -0300 Add test checks commit 7c75e45 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 09:08:04 2022 -0300 Fix comment commit 8988aa3 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 09:07:22 2022 -0300 Add initial code for password rotation test commit ef2bd31 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 09:06:50 2022 -0300 Improve username retrieval commit be497bc Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 08:31:03 2022 -0300 Add copyright notice commit a728a20 Author: Marcelo Henrique Neppel <[email protected]> Date: Thu Aug 25 08:24:32 2022 -0300 Change helper function commit 71f86ac Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 16:34:58 2022 -0300 Improve unit test commit 0d7ea40 Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 16:32:52 2022 -0300 Add set password unit test commit c92fdbc Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 15:42:47 2022 -0300 Fix comment and remove unneeded else block commit 4b7a534 Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 13:59:35 2022 -0300 Improve get password test commit 9c9b211 Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 13:54:54 2022 -0300 Improve messages commit f7024e4 Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 13:46:00 2022 -0300 Implement password rotation commit b8a3fd2 Merge: 4735754 a336356 Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 24 08:36:35 2022 -0300 Merge branch 'main' into password-rotation commit 4735754 Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 23 16:22:37 2022 -0300 Update part of the code to correct actions commit e345432 Merge: ca8f525 36640b2 Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 23 16:15:32 2022 -0300 Merge branch 'rework-secrets' into password-rotation commit 36640b2 Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 23 11:44:53 2022 -0300 Add constants commit 05b707e Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 23 11:38:44 2022 -0300 Add unit tests commit fc17a5e Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 23 10:48:48 2022 -0300 Rework secrets management commit ca8f525 Merge: 4cf20fd 3e1ece8 Author: Marcelo Henrique Neppel <[email protected]> Date: Mon Aug 22 14:54:17 2022 -0300 Merge branch 'main' into password-rotation commit 4cf20fd Merge: d325da6 62c636f Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 17 16:31:52 2022 -0300 Merge branch 'main' into password-rotation commit d325da6 Author: Marcelo Henrique Neppel <[email protected]> Date: Wed Aug 17 16:27:28 2022 -0300 Add password rotation action commit dd395f5 Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 16 10:57:19 2022 -0300 Fix tests user commit fe8bd8b Author: Marcelo Henrique Neppel <[email protected]> Date: Tue Aug 16 10:48:45 2022 -0300 Change charm database user * Improve pytest marks * Register pytest markers * Fix pytest mark
canonical · Sep 5, 2022 · 6a11b8a · 6a11b8a
1 parent 5a4aa0f
commit 6a11b8a
Show file tree

Hide file tree

Showing 9 changed files with 227 additions and 20 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -13,6 +13,7 @@ jobs:
         run: python3 -m pip install tox
       - name: Run linters
         run: tox -e lint
+
   unit-test:
     name: Unit tests
     runs-on: ubuntu-latest
@@ -23,8 +24,69 @@ jobs:
         run: python -m pip install tox
       - name: Run tests
         run: tox -e unit
-  integration-test-microk8s:
-    name: Integration tests (microk8s)
+
+  integration-test-microk8s-charm:
+    name: Integration tests for charm deployment (microk8s)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e charm-integration
+
+  integration-test-microk8s-database-relation:
+    name: Integration tests for database relation (microk8s)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e database-relation-integration
+
+  integration-test-microk8s-db-relation:
+    name: Integration tests for db relation (microk8s)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e db-relation-integration
+
+  integration-test-microk8s-db-admin-relation:
+    name: Integration tests for db-admin relation (microk8s)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e db-admin-relation-integration
+
+  integration-test-microk8s-password-rotation:
+    name: Integration tests for password rotation (microk8s)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -36,4 +98,4 @@ jobs:
           # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
           bootstrap-options: "--agent-version 2.9.29"
       - name: Run integration tests
-        run: tox -e integration
+        run: tox -e password-rotation-integration
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -42,8 +42,8 @@ jobs:
       - name: Run tests
         run: tox -e unit
 
-  integration-test:
-    name: Integration tests
+  integration-test-charm:
+    name: Integration tests for charm deployment
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -55,15 +55,79 @@ jobs:
           # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
           bootstrap-options: "--agent-version 2.9.29"
       - name: Run integration tests
-        run: tox -e integration
+        run: tox -e charm-integration
+
+  integration-test-database-relation:
+    name: Integration tests for database relation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e database-relation-integration
+
+  integration-test-db-relation:
+    name: Integration tests for db relation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e db-relation-integration
+
+  integration-test-db-admin-relation:
+    name: Integration tests for db-admin relation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e db-admin-relation-integration
+
+  integration-test-password-rotation:
+    name: Integration tests for password rotation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e password-rotation-integration
 
   release-to-charmhub:
     name: Release to CharmHub
     needs:
       - lib-check
       - lint
       - unit-test
-      - integration-test
+      - integration-test-charm
+      - integration-test-database-relation
+      - integration-test-db-relation
+      - integration-test-db-admin-relation
+      - integration-test-password-rotation
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

diff --git a/pyproject.toml b/pyproject.toml
@@ -11,6 +11,14 @@ show_missing = true
 [tool.pytest.ini_options]
 minversion = "6.0"
 log_cli_level = "INFO"
+asyncio_mode = "auto"
+markers = [
+    "charm_tests",
+    "database_relation_tests",
+    "db_relation_tests",
+    "db_admin_relation_tests",
+    "password_rotation_tests",
+]
 
 # Formatting tools configuration
 [tool.black]

diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations.py
@@ -31,6 +31,7 @@
 
 
 @pytest.mark.abort_on_fail
+@pytest.mark.database_relation_tests
 async def test_deploy_charms(ops_test: OpsTest, application_charm, database_charm):
     """Deploy both charms (application and database) to use in the tests."""
     # Deploy both charms (multiple units for each application to test that later they correctly
@@ -68,7 +69,7 @@ async def test_deploy_charms(ops_test: OpsTest, application_charm, database_char
         await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", wait_for_units=1)
 
 
-@pytest.mark.abort_on_fail
+@pytest.mark.database_relation_tests
 async def test_database_relation_with_charm_libraries(ops_test: OpsTest):
     """Test basic functionality of database relation interface."""
     # Relate the charms and wait for them exchanging some connection data.
@@ -122,6 +123,7 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest):
             cursor.execute("DROP TABLE test;")
 
 
+@pytest.mark.database_relation_tests
 async def test_user_with_extra_roles(ops_test: OpsTest):
     """Test superuser actions and the request for more permissions."""
     # Get the connection string to connect to the database.
@@ -142,6 +144,7 @@ async def test_user_with_extra_roles(ops_test: OpsTest):
     connection.close()
 
 
+@pytest.mark.database_relation_tests
 async def test_two_applications_doesnt_share_the_same_relation_data(
     ops_test: OpsTest, application_charm
 ):
@@ -176,6 +179,7 @@ async def test_two_applications_doesnt_share_the_same_relation_data(
     assert application_connection_string != another_application_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_an_application_can_connect_to_multiple_database_clusters(
     ops_test: OpsTest, database_charm
 ):
@@ -208,6 +212,7 @@ async def test_an_application_can_connect_to_multiple_database_clusters(
     assert application_connection_string != another_application_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_an_application_can_connect_to_multiple_aliased_database_clusters(
     ops_test: OpsTest, database_charm
 ):
@@ -243,6 +248,7 @@ async def test_an_application_can_connect_to_multiple_aliased_database_clusters(
     assert application_connection_string != another_application_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, application_charm):
     """Test that an application can request additional databases using the same interface."""
     # Relate the charms using another relation and wait for them exchanging some connection data.
@@ -263,6 +269,7 @@ async def test_an_application_can_request_multiple_databases(ops_test: OpsTest,
     assert first_database_connection_string != second_database_connection_string
 
 
+@pytest.mark.database_relation_tests
 async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest):
     """Test that there is no read-only endpoint in a standalone cluster."""
     async with ops_test.fast_forward():
@@ -280,6 +287,7 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest):
         )
 
 
+@pytest.mark.database_relation_tests
 async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest):
     """Test that there is read-only endpoint in a scaled up cluster."""
     async with ops_test.fast_forward():

diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py
@@ -3,7 +3,6 @@
 # See LICENSE file for licensing details.
 
 import logging
-import os
 
 import psycopg2
 import pytest
@@ -31,11 +30,9 @@
 UNIT_IDS = [0, 1, 2]
 
 
-@pytest.mark.skipif(
-    os.environ.get("PYTEST_SKIP_DEPLOY", False),
-    reason="skipping deploy, model expected to be provided.",
-)
 @pytest.mark.abort_on_fail
+@pytest.mark.charm_tests
+@pytest.mark.skip_if_deployed
 async def test_build_and_deploy(ops_test: OpsTest):
     """Build the charm-under-test and deploy it.
 
@@ -54,7 +51,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
         assert ops_test.model.applications[APP_NAME].units[unit_id].workload_status == "active"
 
 
-@pytest.mark.charm
+@pytest.mark.charm_tests
 async def test_application_created_required_resources(ops_test: OpsTest) -> None:
     # Compare the k8s resources that the charm and Patroni should create with
     # the currently created k8s resources.
@@ -64,6 +61,7 @@ async def test_application_created_required_resources(ops_test: OpsTest) -> None
     assert set(existing_resources) == set(expected_resources)
 
 
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("unit_id", UNIT_IDS)
 async def test_labels_consistency_across_pods(ops_test: OpsTest, unit_id: int) -> None:
     model = ops_test.model.info
@@ -75,6 +73,7 @@ async def test_labels_consistency_across_pods(ops_test: OpsTest, unit_id: int) -
     assert pod.metadata.labels["cluster-name"] == f"patroni-{APP_NAME}"
 
 
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("unit_id", UNIT_IDS)
 async def test_database_is_up(ops_test: OpsTest, unit_id: int):
     # Query Patroni REST API and check the status that indicates
@@ -84,6 +83,7 @@ async def test_database_is_up(ops_test: OpsTest, unit_id: int):
     assert result.status_code == 200
 
 
+@pytest.mark.charm_tests
 @pytest.mark.parametrize("unit_id", UNIT_IDS)
 async def test_settings_are_correct(ops_test: OpsTest, unit_id: int):
     password = await get_password(ops_test)
@@ -123,6 +123,7 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int):
     assert settings["postgresql"]["use_pg_rewind"]
 
 
+@pytest.mark.charm_tests
 async def test_cluster_is_stable_after_leader_deletion(ops_test: OpsTest) -> None:
     """Tests that the cluster maintains a primary after the primary is deleted."""
     # Find the current primary unit.
@@ -146,6 +147,7 @@ async def test_cluster_is_stable_after_leader_deletion(ops_test: OpsTest) -> Non
     assert await get_primary(ops_test, other_unit_id) != "None"
 
 
+@pytest.mark.charm_tests
 async def test_scale_down_and_up(ops_test: OpsTest):
     """Test data is replicated to new units after a scale up."""
     # Ensure the initial number of units in the application.
@@ -171,6 +173,7 @@ async def test_scale_down_and_up(ops_test: OpsTest):
     await scale_application(ops_test, APP_NAME, initial_scale)
 
 
+@pytest.mark.charm_tests
 async def test_persist_data_through_graceful_restart(ops_test: OpsTest):
     """Test data persists through a graceful restart."""
     primary = await get_primary(ops_test)
@@ -199,6 +202,7 @@ async def test_persist_data_through_graceful_restart(ops_test: OpsTest):
             connection.cursor().execute("SELECT * FROM gracetest;")
 
 
+@pytest.mark.charm_tests
 async def test_persist_data_through_failure(ops_test: OpsTest):
     """Test data persists through a failure."""
     primary = await get_primary(ops_test)
@@ -239,6 +243,7 @@ async def test_persist_data_through_failure(ops_test: OpsTest):
             connection.cursor().execute("SELECT * FROM failtest;")
 
 
+@pytest.mark.charm_tests
 async def test_automatic_failover_after_leader_issue(ops_test: OpsTest) -> None:
     """Tests that an automatic failover is triggered after an issue happens in the leader."""
     # Find the current primary unit.
@@ -256,6 +261,7 @@ async def test_automatic_failover_after_leader_issue(ops_test: OpsTest) -> None:
     assert await get_primary(ops_test) != "None"
 
 
+@pytest.mark.charm_tests
 async def test_application_removal(ops_test: OpsTest) -> None:
     # Remove the application to trigger some hooks (like peer relation departed).
     await ops_test.model.applications[APP_NAME].remove()
@@ -281,7 +287,7 @@ async def test_application_removal(ops_test: OpsTest) -> None:
     assert APP_NAME not in ops_test.model.applications
 
 
-@pytest.mark.charm
+@pytest.mark.charm_tests
 async def test_redeploy_charm_same_model(ops_test: OpsTest):
     """Redeploy the charm in the same model to test that it works."""
     charm = await ops_test.build_charm(".")

diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
-
+import pytest as pytest
 from pytest_operator.plugin import OpsTest
 
 from tests.helpers import METADATA
@@ -18,6 +18,7 @@
 DATABASE_UNITS = 3
 
 
+@pytest.mark.db_relation_tests
 async def test_finos_waltz_db(ops_test: OpsTest) -> None:
     """Deploy Finos Waltz to test the 'db' relation.