Merge branch 'main' into docs/admin-note-token-scopes

.github/workflows/e2e_test.yaml

@@ -452,6 +452,7 @@ jobs:
     uses: ./.github/workflows/e2e_test_run.yaml
     with:
       runner-tag: "pr-${{ needs.run-id.outputs.run-id }}${{ github.run_attempt}}"
+      runner-virt-type: "lxd"
 
   required_status_checks:
     name: Required E2E Test Status Checks

.github/workflows/e2e_test_run.yaml

@@ -12,16 +12,29 @@ on:
         description: The e2e test runner tag to run the workflow on.
         type: string
         required: true
+      runner-virt-type:
+        description: The e2e test runner virtualization type. E.g. lxd, or openstack.
+        # workflow_call does not support choice type.
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       runner-tag:
         description: The e2e test runner tag to run the workflow on.
         type: string
         required: true
+      runner-virt-type:
+        description: The e2e test runner virtualization type.
+        type: choice
+        required: true
+        options:
+          - lxd
+          - openstack
+
 jobs:
   e2e-test:
     name: End-to-End Test Run
-    runs-on: [self-hosted, linux, x64, "${{ inputs.runner-tag }}"]
+    runs-on: [self-hosted, linux, "${{ inputs.runner-tag }}"]
     steps:
       # Snapd can have some issues in privileged LXD containers without setting
       # security.nesting=True and this.
@@ -89,8 +102,10 @@ jobs:
       - name: test check-jsonschema
         run: check-jsonschema --version
       - name: Test Firewall
+        if: "${{ github.event.inputs.runner-virt-type == 'lxd' }}"
         run: |
           HOST_IP=$(ip route | grep default | cut -f 3 -d" ")
           [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ]
       - name: Test sctp support
+        if: "${{ github.event.inputs.runner-virt-type == 'lxd' }}"
         run: sudo apt-get install lksctp-tools -yq && checksctp

.github/workflows/integration_test.yaml

@@ -15,7 +15,7 @@ jobs:
       pre-run-script: scripts/pre-integration-test.sh
       provider: lxd
       test-tox-env: integration-juju2.9
-      modules: '["test_charm_base_image", "test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh", "test_charm_upgrade"]'
+      modules: '["test_charm_base_image", "test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_lxd_runner", "test_charm_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh", "test_charm_upgrade"]'
   integration-tests:
     name: Integration test with juju 3.1
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
@@ -25,24 +25,17 @@ jobs:
       pre-run-script: scripts/pre-integration-test.sh
       provider: lxd
       test-tox-env: integration-juju3.1
-      modules: '["test_charm_base_image", "test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh", "test_charm_upgrade"]'
-  # openstack tests use microstack, whose setup is kind of special
-  # - due to the huge resource requirements, we use self-hosted runners for these tests
-  # - microstack requires juju 3.2 and microk8s 1.26
-  # - microk8s needs to be preconfigured with the actions operator to work properly with the dockerhub cache
-  # - we need to disable the rbac addon for microk8s, otherwise the setup will fail
-  integration-tests-microstack:
-    name: Integration test using microstack
+      modules: '["test_charm_base_image", "test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_lxd_runner", "test_charm_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh", "test_charm_upgrade"]'
+  openstack-integration-tests-private-endpoint:
+    name: Integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     secrets: inherit
     with:
       juju-channel: 3.2/stable
-      pre-run-script: scripts/setup-microstack.sh
-      provider: microk8s
-      channel: 1.26-strict/stable
-      microk8s-addons: "dns ingress hostpath-storage"
+      pre-run-script: scripts/setup-lxd.sh
+      provider: lxd
       test-tox-env: integration-juju3.2
-      modules: '["test_openstack_base_image", "test_openstack_one_runner"]'
+      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner", "test_e2e"]'
+      extra-arguments: "-m openstack --openstack-flavor-name=builder-cpu4-ram8-disk50 --http-proxy=http://squid.internal:3128 --https-proxy=http://squid.internal:3128 --no-proxy=keystone.ps6.canonical.com,glance.ps6.canonical.com,nova.ps6.canonical.com,neutron.ps6.canonical.com"
       self-hosted-runner: true
-      self-hosted-runner-label: two-xlarge
-      tmate-debug: true
+      self-hosted-runner-label: stg-private-endpoint

.github/workflows/push_test.yaml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   push-event-tests:
-    runs-on: [self-hosted, linux, x64, "${{ github.ref_name }}"]
+    runs-on: [self-hosted, linux, "${{ github.ref_name }}"]
     steps:
       - name: Echo runner name
         run: |

.github/workflows/schedule_issues_test.yaml

@@ -19,7 +19,7 @@ jobs:
 
   issues-schedule-event-tests:
     needs: [ extract-runner-name ]
-    runs-on: [ self-hosted, linux, x64, "${{ needs.extract-runner-name.outputs.name }}" ]
+    runs-on: [ self-hosted, linux, "${{ needs.extract-runner-name.outputs.name }}" ]
     steps:
       - name: Echo runner name
         run: |

.github/workflows/workflow_dispatch_crash_test.yaml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   workflow-dispatch-tests:
-    runs-on: [self-hosted, linux, x64, "${{ inputs.runner }}"]
+    runs-on: [self-hosted, linux, "${{ inputs.runner }}"]
     steps:
       - name: Sleep for an hour to give the integration test some time to crash the runner.
         run: |

.github/workflows/workflow_dispatch_failure_test.yaml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   workflow-dispatch-tests:
-    runs-on: [self-hosted, linux, x64, "${{ inputs.runner }}"]
+    runs-on: [self-hosted, linux, "${{ inputs.runner }}"]
     steps:
       - name: Echo input variable and message
         run: |

.github/workflows/workflow_dispatch_ssh_debug.yaml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   workflow-dispatch-tests:
-    runs-on: [self-hosted, linux, x64, "${{ inputs.runner }}"]
+    runs-on: [self-hosted, linux, "${{ inputs.runner }}"]
     steps:
     - name: Setup tmate session
       uses: canonical/action-tmate@chore/env_var_change

.github/workflows/workflow_dispatch_test.yaml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   workflow-dispatch-tests:
-    runs-on: [self-hosted, linux, x64, "${{ inputs.runner }}"]
+    runs-on: [self-hosted, linux, "${{ inputs.runner }}"]
     steps:
       - name: Echo input variable and message
         run: |

.github/workflows/workflow_dispatch_wait_test.yaml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   workflow-dispatch-tests:
-    runs-on: [self-hosted, linux, x64, "${{ inputs.runner }}"]
+    runs-on: [self-hosted, linux, "${{ inputs.runner }}"]
     steps:
       - name: Echo input variable and message
         run: |

.gitignore

@@ -5,6 +5,11 @@ placeholders/
 *.charm
 build/
 .coverage
+
+# testing artifacts
 lxd-profile.*
 tinyproxy.conf
+
+# development artifacts
 clouds.yaml
+.vscode/

config.yaml

@@ -7,7 +7,7 @@ options:
     default: "jammy"
     description: >-
       The base ubuntu OS image to use for the runners. Codename (e.g. "jammy") or version tag
-      (e.g. 22.04) is supported as input. Currently only supports LTS versions of jammy and higher, 
+      (e.g. 22.04) is supported as input. Currently only supports LTS versions of jammy and higher,
       i.e. jammy, noble.
   denylist:
     type: string
@@ -20,26 +20,48 @@ options:
     type: string
     default: ""
     description: >-
-      The private docker registry configured as dockerhub mirror to be used by the runners. If set 
-      a message will be displayed prior to job execution on self-hosted runner informing users to 
-      use the provided registry.
-  experimental-openstack-clouds-yaml:
+      The URL to the private docker registry configured as the dockerhub mirror to be used by the
+      runners. If set a message will be displayed prior to job execution on self-hosted runner
+      informing users to use the provided registry. You must use https:// as the protocol.
+  openstack-clouds-yaml:
     type: string
     default: ""
     description: >-
-      (Experimental, under development) The openstack clouds.yaml
+      Providing a valid clouds.yaml will enable OpenStack integration. Setting the
+      openstack-clouds-yaml would enable spawning runners on OpenStack, and disable usage of
+      local LXD for runners. The format for the clouds.yaml is described in the docs:
       (https://docs.openstack.org/python-openstackclient/pike/configuration/index.html#clouds-yaml).
-      Setting the openstack-clouds-yaml would enable spawning runners on OpenStack, instead of LXD
-      vms. This feature is not yet ready and is under development.
+  openstack-flavor:
+    type: string
+    default: ""
+    description: |
+      The openstack flavor to use to spawn virtual machines. This controls the hardware resource
+      of the virtual machines. See docs for details:
+      (https://docs.openstack.org/nova/rocky/user/flavors.html).
+  openstack-network:
+    type: string
+    default: ""
+    description: |
+      The openstack network for virtual machines hosting the runners. The network must allow this
+      application to be able to access the virtual machine though the IP assigned by the openstack
+      network. See docs for details:
+      (https://docs.openstack.org/neutron/latest/admin/intro-os-networking.html).
+  experimental-openstack-image-build-unit:
+    type: int
+    default: -1
+    description: >-
+      (Experimental, tmp workaround, will be removed in the future) The unit that performs the
+      OpenStack image build. All other units will have the image build disabled. Cannot be changed
+      after deployment. Will only take effect with OpenStack integration enabled.
   experimental-use-aproxy:
     type: boolean
     default: false
     description: >-
       (Experimental, may be removed) When set to true, aproxy (https://github.com/canonical/aproxy)
-      will be installed within the runners. It will forward all HTTP(S) traffic to standard ports 
-      (80, 443) to a proxy server configured by the juju model config 'juju-http-proxy' 
+      will be installed within the runners. It will forward all HTTP(S) traffic to standard ports
+      (80, 443) to a proxy server configured by the juju model config 'juju-http-proxy'
       (or, if this is not set, 'juju-https-proxy' will be used).
-      This is useful when the charm is deployed in a network that requires a proxy to access the 
+      This is useful when the charm is deployed in a network that requires a proxy to access the
       internet.
       Note that you should not specify a proxy server listening on port 80 or 443, as all traffic
       to these ports is relayed to aproxy, which would cause an infinite loop.
@@ -54,32 +76,44 @@ options:
     default: ""
     description: >-
       Additional comma separated labels to attach to self-hosted runners. By default, the labels
-      "self-hosted", application name (default: "github-runner"), 
+      "self-hosted", application name (default: "github-runner"),
       architecture (i.e. "x64", "arm64"), os (i.e. "linux"), os-flavor (i.e. "jammy") are set.
       Any labels provided via this configuration will be appended to the default values.
   path:
     type: string
     default: ""
     description: >-
-      The organization or the repository to register the self-hosted runners. For organization the 
-      path would be the name of the organization. For repository the path should be in the 
+      The organization or the repository to register the self-hosted runners. For organization the
+      path would be the name of the organization. For repository the path should be in the
       "{owner}/{repo}" format.
   reconcile-interval:
     type: int
     default: 10
     description: >-
-      Minutes between each reconciliation of the current runners state and their targeted state. 
-      On reconciliation, the charm polls the state of runners and see if actions are needed. A 
-      random delay of 25% is added. Changes to this time interval will apply when next interval is 
-      triggered. Prior to reconciliation, any packages, and services used by the software will be 
+      Minutes between each reconciliation of the current runners state and their targeted state.
+      On reconciliation, the charm polls the state of runners and see if actions are needed. A
+      random delay of 25% is added. Changes to this time interval will apply when next interval is
+      triggered. Prior to reconciliation, any packages, and services used by the software will be
       updated.
+  repo-policy-compliance-token:
+    type: string
+    description: >-
+      The token to authenticate with the repository-policy-compliance service in order to
+      generate one-time-tokens. This option requires the repo-policy-compliance-url to be set.
+      If not set, the repository-policy-compliance service will not be used.
+  repo-policy-compliance-url:
+    type: string
+    description: >-
+      The URL to the repository-policy-compliance service. This option requires the
+      repo-policy-compliance-token to be set. If not set, the repository-policy-compliance service
+      will not be used. This option is only supported when using OpenStack Cloud.
   runner-storage:
     type: string
     default: "juju-storage"
     description: >-
-      The storage for runner LXD instance to mount as disk. Can be "memory" or "juju-storage". If 
-      set to "memory", memory-based storage created with tmpfs will be used as disk for LXD 
-      instances. If set to "juju-storage", storage provided by juju will be used as disk for LXD 
+      The storage for runner LXD instance to mount as disk. Can be "memory" or "juju-storage". If
+      set to "memory", memory-based storage created with tmpfs will be used as disk for LXD
+      instances. If set to "juju-storage", storage provided by juju will be used as disk for LXD
       instances. This configuration cannot be changed after deployment. The charm needs to be
       redeployed to change the storage option.
   test-mode:

docs/how-to/openstack-runner.md

@@ -0,0 +1,45 @@
+# How to spawn OpenStack runner
+
+The charm can be configured to use OpenStack cloud for creating runners.
+The charm must be deployed with the correct configuration and once the OpenStack integration is
+enabled the charm cannot be changed to use other virtualization methods.
+
+## Configuration
+
+There are three configuration that the charm needs to be deployed with to enable OpenStack integration: `openstack-clouds-yaml`, `openstack-flavor`, and `openstack-network`.
+
+### OpenStack clouds.yaml
+
+The `openstack-clouds-yaml` configuration contains the authorization information needed for the charm to log in to the openstack cloud.
+The first cloud in the `clouds.yaml` is used by the charm.
+
+Here is a sample of the `clouds.yaml`:
+
+```yaml
+clouds:
+  cloud:
+    auth:
+      auth_url: https://keystone.cloud.com:5000/v3
+      project_name: github-runner
+      username: github-runner
+      password: PASSWORD
+      user_domain_name: Default
+      project_domain_name: Default
+    region_name: cloud
+```
+
+The `clouds.yaml` documentation is [here](https://docs.openstack.org/python-openstackclient/pike/configuration/index.html#clouds-yaml).
+
+### OpenStack Flavor
+
+The `openstack-flavor` configuration sets the flavor used to create the OpenStack virtual machine when spawning new runners.
+The flavor is tied with the vCPU, memory, and storage.
+The flavors documentation is [here](https://docs.openstack.org/nova/rocky/user/flavors.html).
+
+### OpenStack Network
+
+The  `openstack-network` configuration sets the network used to create the OpenStack virtual machine when spawning new runners.
+
+Note that the network should be configured to allow traffic from the charm deployment (juju machine) to the OpenStack virtual machine, and traffic from the OpenStack virtual machine to GitHub.
+
+The network documentation is [here](https://docs.openstack.org/neutron/latest/admin/intro-os-networking.html).

pyproject.toml

@@ -12,9 +12,13 @@ branch = true
 omit = [
     # Contains interface for calling LXD. Tested in integration tests and end to end tests.
     "src/lxd.py",
-    # Contains interface for calling repo policy compliance service. Tested in integration test 
+    # Contains interface for calling repo policy compliance service. Tested in integration test
     # and end to end tests.
     "src/repo_policy_compliance_client.py",
+    # 2024/04/17: These files are pending a major refactor. The refactor includes a RunnerManager
+    # interface class which will include a complete re-organization of the code in these files.
+    "src/runner.py",
+    "src/runner_manager.py",
 ]
 
 [tool.coverage.report]
@@ -24,6 +28,9 @@ show_missing = true
 [tool.pytest.ini_options]
 minversion = "6.0"
 log_cli_level = "INFO"
+markers = [
+    "openstack: mark a test as applicable for openstack cloud",
+]
 
 [tool.pylint.'MESSAGES CONTROL']
 extension-pkg-whitelist = "pydantic"

requirements.txt

@@ -1,5 +1,6 @@
 ghapi
 jinja2
+fabric >=3,<4
 openstacksdk>=3,<4
 ops>=2.8
 pylxd @ git+https://github.com/canonical/pylxd