diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0190961d6f..e4195a9b7f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,6 +13,7 @@ jobs: run: python3 -m pip install tox - name: Run linters run: tox -e lint + unit-test: name: Unit tests runs-on: ubuntu-latest @@ -23,8 +24,69 @@ jobs: run: python -m pip install tox - name: Run tests run: tox -e unit - integration-test-microk8s: - name: Integration tests (microk8s) + + integration-test-microk8s-charm: + name: Integration tests for charm deployment (microk8s) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e charm-integration + + integration-test-microk8s-database-relation: + name: Integration tests for database relation (microk8s) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e database-relation-integration + + integration-test-microk8s-db-relation: + name: Integration tests for db relation (microk8s) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e db-relation-integration + + integration-test-microk8s-db-admin-relation: + name: Integration tests for db-admin relation (microk8s) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e db-admin-relation-integration + + integration-test-microk8s-password-rotation: + name: Integration tests for password rotation (microk8s) runs-on: ubuntu-latest steps: - name: Checkout @@ -36,4 +98,4 @@ jobs: # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. bootstrap-options: "--agent-version 2.9.29" - name: Run integration tests - run: tox -e integration + run: tox -e password-rotation-integration diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 070edb4ede..38d1f71a66 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -42,8 +42,8 @@ jobs: - name: Run tests run: tox -e unit - integration-test: - name: Integration tests + integration-test-charm: + name: Integration tests for charm deployment runs-on: ubuntu-latest steps: - name: Checkout @@ -55,7 +55,67 @@ jobs: # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. bootstrap-options: "--agent-version 2.9.29" - name: Run integration tests - run: tox -e integration + run: tox -e charm-integration + + integration-test-database-relation: + name: Integration tests for database relation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e database-relation-integration + + integration-test-db-relation: + name: Integration tests for db relation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e db-relation-integration + + integration-test-db-admin-relation: + name: Integration tests for db-admin relation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e db-admin-relation-integration + + integration-test-password-rotation: + name: Integration tests for password rotation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed. + bootstrap-options: "--agent-version 2.9.29" + - name: Run integration tests + run: tox -e password-rotation-integration release-to-charmhub: name: Release to CharmHub @@ -63,7 +123,11 @@ jobs: - lib-check - lint - unit-test - - integration-test + - integration-test-charm + - integration-test-database-relation + - integration-test-db-relation + - integration-test-db-admin-relation + - integration-test-password-rotation runs-on: ubuntu-latest steps: - name: Checkout diff --git a/pyproject.toml b/pyproject.toml index 9a931cf8aa..d873e96cb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,14 @@ show_missing = true [tool.pytest.ini_options] minversion = "6.0" log_cli_level = "INFO" +asyncio_mode = "auto" +markers = [ + "charm_tests", + "database_relation_tests", + "db_relation_tests", + "db_admin_relation_tests", + "password_rotation_tests", +] # Formatting tools configuration [tool.black] diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations.py index 67a7080b37..12883613c4 100644 --- a/tests/integration/new_relations/test_new_relations.py +++ b/tests/integration/new_relations/test_new_relations.py @@ -31,6 +31,7 @@ @pytest.mark.abort_on_fail +@pytest.mark.database_relation_tests async def test_deploy_charms(ops_test: OpsTest, application_charm, database_charm): """Deploy both charms (application and database) to use in the tests.""" # Deploy both charms (multiple units for each application to test that later they correctly @@ -68,7 +69,7 @@ async def test_deploy_charms(ops_test: OpsTest, application_charm, database_char await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", wait_for_units=1) -@pytest.mark.abort_on_fail +@pytest.mark.database_relation_tests async def test_database_relation_with_charm_libraries(ops_test: OpsTest): """Test basic functionality of database relation interface.""" # Relate the charms and wait for them exchanging some connection data. @@ -122,6 +123,7 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest): cursor.execute("DROP TABLE test;") +@pytest.mark.database_relation_tests async def test_user_with_extra_roles(ops_test: OpsTest): """Test superuser actions and the request for more permissions.""" # Get the connection string to connect to the database. @@ -142,6 +144,7 @@ async def test_user_with_extra_roles(ops_test: OpsTest): connection.close() +@pytest.mark.database_relation_tests async def test_two_applications_doesnt_share_the_same_relation_data( ops_test: OpsTest, application_charm ): @@ -176,6 +179,7 @@ async def test_two_applications_doesnt_share_the_same_relation_data( assert application_connection_string != another_application_connection_string +@pytest.mark.database_relation_tests async def test_an_application_can_connect_to_multiple_database_clusters( ops_test: OpsTest, database_charm ): @@ -208,6 +212,7 @@ async def test_an_application_can_connect_to_multiple_database_clusters( assert application_connection_string != another_application_connection_string +@pytest.mark.database_relation_tests async def test_an_application_can_connect_to_multiple_aliased_database_clusters( ops_test: OpsTest, database_charm ): @@ -243,6 +248,7 @@ async def test_an_application_can_connect_to_multiple_aliased_database_clusters( assert application_connection_string != another_application_connection_string +@pytest.mark.database_relation_tests async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, application_charm): """Test that an application can request additional databases using the same interface.""" # Relate the charms using another relation and wait for them exchanging some connection data. @@ -263,6 +269,7 @@ async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, assert first_database_connection_string != second_database_connection_string +@pytest.mark.database_relation_tests async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): """Test that there is no read-only endpoint in a standalone cluster.""" async with ops_test.fast_forward(): @@ -280,6 +287,7 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): ) +@pytest.mark.database_relation_tests async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest): """Test that there is read-only endpoint in a scaled up cluster.""" async with ops_test.fast_forward(): diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 75d2d85db8..e98ca06171 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -3,7 +3,6 @@ # See LICENSE file for licensing details. import logging -import os import psycopg2 import pytest @@ -31,11 +30,9 @@ UNIT_IDS = [0, 1, 2] -@pytest.mark.skipif( - os.environ.get("PYTEST_SKIP_DEPLOY", False), - reason="skipping deploy, model expected to be provided.", -) @pytest.mark.abort_on_fail +@pytest.mark.charm_tests +@pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest): """Build the charm-under-test and deploy it. @@ -54,7 +51,7 @@ async def test_build_and_deploy(ops_test: OpsTest): assert ops_test.model.applications[APP_NAME].units[unit_id].workload_status == "active" -@pytest.mark.charm +@pytest.mark.charm_tests async def test_application_created_required_resources(ops_test: OpsTest) -> None: # Compare the k8s resources that the charm and Patroni should create with # the currently created k8s resources. @@ -64,6 +61,7 @@ async def test_application_created_required_resources(ops_test: OpsTest) -> None assert set(existing_resources) == set(expected_resources) +@pytest.mark.charm_tests @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_labels_consistency_across_pods(ops_test: OpsTest, unit_id: int) -> None: model = ops_test.model.info @@ -75,6 +73,7 @@ async def test_labels_consistency_across_pods(ops_test: OpsTest, unit_id: int) - assert pod.metadata.labels["cluster-name"] == f"patroni-{APP_NAME}" +@pytest.mark.charm_tests @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_database_is_up(ops_test: OpsTest, unit_id: int): # Query Patroni REST API and check the status that indicates @@ -84,6 +83,7 @@ async def test_database_is_up(ops_test: OpsTest, unit_id: int): assert result.status_code == 200 +@pytest.mark.charm_tests @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): password = await get_password(ops_test) @@ -123,6 +123,7 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): assert settings["postgresql"]["use_pg_rewind"] +@pytest.mark.charm_tests async def test_cluster_is_stable_after_leader_deletion(ops_test: OpsTest) -> None: """Tests that the cluster maintains a primary after the primary is deleted.""" # Find the current primary unit. @@ -146,6 +147,7 @@ async def test_cluster_is_stable_after_leader_deletion(ops_test: OpsTest) -> Non assert await get_primary(ops_test, other_unit_id) != "None" +@pytest.mark.charm_tests async def test_scale_down_and_up(ops_test: OpsTest): """Test data is replicated to new units after a scale up.""" # Ensure the initial number of units in the application. @@ -171,6 +173,7 @@ async def test_scale_down_and_up(ops_test: OpsTest): await scale_application(ops_test, APP_NAME, initial_scale) +@pytest.mark.charm_tests async def test_persist_data_through_graceful_restart(ops_test: OpsTest): """Test data persists through a graceful restart.""" primary = await get_primary(ops_test) @@ -199,6 +202,7 @@ async def test_persist_data_through_graceful_restart(ops_test: OpsTest): connection.cursor().execute("SELECT * FROM gracetest;") +@pytest.mark.charm_tests async def test_persist_data_through_failure(ops_test: OpsTest): """Test data persists through a failure.""" primary = await get_primary(ops_test) @@ -239,6 +243,7 @@ async def test_persist_data_through_failure(ops_test: OpsTest): connection.cursor().execute("SELECT * FROM failtest;") +@pytest.mark.charm_tests async def test_automatic_failover_after_leader_issue(ops_test: OpsTest) -> None: """Tests that an automatic failover is triggered after an issue happens in the leader.""" # Find the current primary unit. @@ -256,6 +261,7 @@ async def test_automatic_failover_after_leader_issue(ops_test: OpsTest) -> None: assert await get_primary(ops_test) != "None" +@pytest.mark.charm_tests async def test_application_removal(ops_test: OpsTest) -> None: # Remove the application to trigger some hooks (like peer relation departed). await ops_test.model.applications[APP_NAME].remove() @@ -281,7 +287,7 @@ async def test_application_removal(ops_test: OpsTest) -> None: assert APP_NAME not in ops_test.model.applications -@pytest.mark.charm +@pytest.mark.charm_tests async def test_redeploy_charm_same_model(ops_test: OpsTest): """Redeploy the charm in the same model to test that it works.""" charm = await ops_test.build_charm(".") diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py index d7cdd4dd48..fde5f4d078 100644 --- a/tests/integration/test_db.py +++ b/tests/integration/test_db.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. - +import pytest as pytest from pytest_operator.plugin import OpsTest from tests.helpers import METADATA @@ -18,6 +18,7 @@ DATABASE_UNITS = 3 +@pytest.mark.db_relation_tests async def test_finos_waltz_db(ops_test: OpsTest) -> None: """Deploy Finos Waltz to test the 'db' relation. diff --git a/tests/integration/test_db_admin.py b/tests/integration/test_db_admin.py index 00eabd6b17..2371c26834 100644 --- a/tests/integration/test_db_admin.py +++ b/tests/integration/test_db_admin.py @@ -22,6 +22,7 @@ @pytest.mark.abort_on_fail +@pytest.mark.db_admin_relation_tests async def test_build_and_deploy(ops_test: OpsTest): """Build the charm-under-test and deploy it. @@ -56,6 +57,7 @@ async def test_build_and_deploy(ops_test: OpsTest): ) +@pytest.mark.db_admin_relation_tests async def test_discourse(ops_test: OpsTest): # Test the first Discourse charm. # Add both relations to Discourse (PostgreSQL and Redis) @@ -80,6 +82,7 @@ async def test_discourse(ops_test: OpsTest): await check_database_users_existence(ops_test, discourse_users, [], admin=True) +@pytest.mark.db_admin_relation_tests async def test_discourse_from_discourse_charmers(ops_test: OpsTest): # Test the second Discourse charm. diff --git a/tests/integration/test_password_rotation.py b/tests/integration/test_password_rotation.py index 0b5f225950..c02ac835de 100644 --- a/tests/integration/test_password_rotation.py +++ b/tests/integration/test_password_rotation.py @@ -18,7 +18,7 @@ @pytest.mark.abort_on_fail -@pytest.mark.password_rotation +@pytest.mark.password_rotation_tests @pytest.mark.skip_if_deployed async def test_deploy_active(ops_test: OpsTest): """Build the charm and deploy it.""" @@ -36,7 +36,7 @@ async def test_deploy_active(ops_test: OpsTest): await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", timeout=1000) -@pytest.mark.password_rotation +@pytest.mark.password_rotation_tests async def test_password_rotation(ops_test: OpsTest): """Test password rotation action.""" # Get the initial passwords set for the system users. diff --git a/tox.ini b/tox.ini index 799484d225..a199c75932 100644 --- a/tox.ini +++ b/tox.ini @@ -65,8 +65,63 @@ commands = -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} coverage report +[testenv:charm-integration] +description = Run charm integration tests +deps = + juju + pytest + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m charm_tests + +[testenv:database-relation-integration] +description = Run database relation integration tests +deps = + juju + pytest + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m database_relation_tests + +[testenv:db-relation-integration] +description = Run db relation integration tests +deps = + juju + pytest + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m db_relation_tests + +[testenv:db-admin-relation-integration] +description = Run db-admin relation integration tests +deps = + juju + pytest + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m db_admin_relation_tests + +[testenv:password-rotation-integration] +description = Run password rotation integration tests +deps = + juju + pytest + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m password_rotation_tests + [testenv:integration] -description = Run integration tests +description = Run all integration tests deps = juju pytest @@ -74,4 +129,4 @@ deps = psycopg2-binary -r{toxinidir}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 --asyncio-mode=auto + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0