Implement /resend-pending-builds endpoint (#48)

* Queue builds even when builder is unavailable

softpack_core/schemas/environment.py

@@ -292,15 +292,13 @@ def iter(cls) -> Iterable["Environment"]:
         )
 
         for env in environment_objects:
+            env.avg_wait_secs = avg_wait_secs
             status = status_map.get(str(Path(env.path, env.name)))
             if not status:
-                if env.state == State.queued:
-                    env.state = State.failed
                 continue
             env.requested = status.requested
             env.build_start = status.build_start
             env.build_done = status.build_done
-            env.avg_wait_secs = avg_wait_secs
 
         return environment_objects
 
@@ -360,7 +358,29 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
 
             version += 1
 
-        # Send build request
+        response = cls.submit_env_to_builder(env)
+        if response is not None:
+            return response
+
+        return CreateEnvironmentSuccess(
+            message="Successfully scheduled environment creation"
+        )
+
+    @classmethod
+    def submit_env_to_builder(
+        cls, env: EnvironmentInput
+    ) -> Union[None, BuilderError, InvalidInputError]:
+        """Submit an environment to the builder."""
+        try:
+            m = re.fullmatch(r"^(.*)-(\d+)$", env.name)
+            if not m:
+                raise Exception
+            versionless_name, version = m.groups()
+        except Exception:
+            return InvalidInputError(
+                message=f"could not parse version from name: {env.name!r}"
+            )
+
         try:
             host = app.settings.builder.host
             port = app.settings.builder.port
@@ -383,15 +403,11 @@ def create(cls, env: EnvironmentInput) -> CreateResponse:  # type: ignore
             )
             r.raise_for_status()
         except Exception as e:
-            cls.delete(env.name, env.path)
             return BuilderError(
                 message="Connection to builder failed: "
                 + "".join(format_exception_only(type(e), e))
             )
-
-        return CreateEnvironmentSuccess(
-            message="Successfully scheduled environment creation"
-        )
+        return None
 
     @classmethod
     def create_new_env(

softpack_core/service.py

@@ -10,14 +10,16 @@
 
 import typer
 import uvicorn
-from fastapi import APIRouter, Request, UploadFile
+from fastapi import APIRouter, Request, Response, UploadFile
 from typer import Typer
 from typing_extensions import Annotated
 
+from softpack_core.artifacts import State
 from softpack_core.schemas.environment import (
     CreateEnvironmentSuccess,
     Environment,
     EnvironmentInput,
+    PackageInput,
     WriteArtifactSuccess,
 )
 
@@ -92,3 +94,35 @@ async def upload_artifacts(  # type: ignore[no-untyped-def]
             raise Exception(resp)
 
         return resp
+
+    @staticmethod
+    @router.post("/resend-pending-builds")
+    async def resend_pending_builds(  # type: ignore[no-untyped-def]
+        response: Response,
+    ):
+        """Resubmit any pending builds to the builder."""
+        successes = 0
+        failures = 0
+        for env in Environment.iter():
+            if env.state != State.queued:
+                continue
+            result = Environment.submit_env_to_builder(
+                EnvironmentInput(
+                    name=env.name,
+                    path=env.path,
+                    description=env.description,
+                    packages=[PackageInput(**vars(p)) for p in env.packages],
+                )
+            )
+            if result is None:
+                successes += 1
+            else:
+                failures += 1
+
+        if failures == 0:
+            message = "Successfully triggered resends"
+        else:
+            response.status_code = 500
+            message = "Failed to trigger all resends"
+
+        return {"message": message, "successes": successes, "failures": failures}

tests/integration/test_environment.py

@@ -14,7 +14,7 @@
 import yaml
 from fastapi import UploadFile
 
-from softpack_core.artifacts import Artifacts, app
+from softpack_core.artifacts import Artifacts
 from softpack_core.schemas.environment import (
     BuilderError,
     CreateEnvironmentSuccess,
@@ -29,7 +29,7 @@
     UpdateEnvironmentSuccess,
     WriteArtifactSuccess,
 )
-from tests.integration.utils import file_in_remote
+from tests.integration.utils import builder_called_correctly, file_in_remote
 
 pytestmark = pytest.mark.repo
 
@@ -142,7 +142,7 @@ def test_create_path_invalid_disallowed(httpx_post, testable_env_input, path):
     assert isinstance(result, InvalidInputError)
 
 
-def test_create_cleans_up_after_builder_failure(
+def test_create_does_not_clean_up_after_builder_failure(
     httpx_post, testable_env_input
 ):
     httpx_post.side_effect = Exception('could not contact builder')
@@ -156,33 +156,8 @@ def test_create_cleans_up_after_builder_failure(
     )
     builtPath = dir / Environment.artifacts.built_by_softpack_file
     ymlPath = dir / Environment.artifacts.environments_file
-    assert not file_in_remote(builtPath)
-    assert not file_in_remote(ymlPath)
-
-
-def builder_called_correctly(
-    post_mock, testable_env_input: EnvironmentInput
-) -> None:
-    # TODO: don't mock this; actually have a real builder service to test with?
-    host = app.settings.builder.host
-    port = app.settings.builder.port
-    post_mock.assert_called_with(
-        f"http://{host}:{port}/environments/build",
-        json={
-            "name": f"{testable_env_input.path}/{testable_env_input.name}",
-            "version": "1",
-            "model": {
-                "description": testable_env_input.description,
-                "packages": [
-                    {
-                        "name": pkg.name,
-                        "version": pkg.version,
-                    }
-                    for pkg in testable_env_input.packages
-                ],
-            },
-        },
-    )
+    assert file_in_remote(builtPath)
+    assert file_in_remote(ymlPath)
 
 
 def test_delete(httpx_post, testable_env_input) -> None:
@@ -307,8 +282,8 @@ def test_iter_no_statuses(testable_env_input, mocker):
     assert envs[0].build_start is None
     assert envs[0].build_done is None
     assert envs[0].avg_wait_secs is None
-    assert envs[0].state == State.failed
-    assert envs[1].state == State.failed
+    assert envs[0].state == State.queued
+    assert envs[1].state == State.queued
 
 
 @pytest.mark.asyncio

tests/integration/test_resend_builds.py

@@ -0,0 +1,59 @@
+"""Copyright (c) 2024 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+
+import pytest
+from fastapi.testclient import TestClient
+
+from softpack_core.app import app
+from softpack_core.schemas.environment import (
+    CreateEnvironmentSuccess,
+    Environment,
+    EnvironmentInput,
+)
+from softpack_core.service import ServiceAPI
+from tests.integration.utils import builder_called_correctly
+
+pytestmark = pytest.mark.repo
+
+
+def test_resend_pending_builds(
+    httpx_post, testable_env_input: EnvironmentInput
+):
+    Environment.delete("test_environment", "users/test_user")
+    Environment.delete("test_environment", "groups/test_group")
+    ServiceAPI.register()
+    client = TestClient(app.router)
+
+    orig_name = testable_env_input.name
+    testable_env_input.name += "-1"
+    r = Environment.create_new_env(
+        testable_env_input, Environment.artifacts.built_by_softpack_file
+    )
+    assert isinstance(r, CreateEnvironmentSuccess)
+    testable_env_input.name = orig_name
+
+    httpx_post.assert_not_called()
+
+    resp = client.post(
+        url="/resend-pending-builds",
+    )
+    assert resp.status_code == 200
+    assert resp.json().get("message") == "Successfully triggered resends"
+    assert resp.json().get("successes") == 1
+    assert resp.json().get("failures") == 0
+
+    httpx_post.assert_called_once()
+    builder_called_correctly(httpx_post, testable_env_input)
+
+    httpx_post.side_effect = Exception('could not contact builder')
+    resp = client.post(
+        url="/resend-pending-builds",
+    )
+    assert resp.status_code == 500
+    assert resp.json().get("message") == "Failed to trigger all resends"
+    assert resp.json().get("successes") == 0
+    assert resp.json().get("failures") == 1

tests/integration/test_spack.py

@@ -45,7 +45,7 @@ def test_spack_packages():
     else:
         assert len(packages) > len(pkgs)
 
-        spack = Spack(custom_repo = app.settings.spack.repo)
+        spack = Spack(custom_repo=app.settings.spack.repo)
 
         spack.packages()
 

tests/integration/utils.py

@@ -12,6 +12,7 @@
 import pytest
 
 from softpack_core.artifacts import Artifacts, app
+from softpack_core.schemas.environment import EnvironmentInput
 
 artifacts_dict = dict[
     str,
@@ -199,3 +200,28 @@ def file_in_repo(
         current = current[part]
 
     return current
+
+
+def builder_called_correctly(
+    post_mock, testable_env_input: EnvironmentInput
+) -> None:
+    # TODO: don't mock this; actually have a real builder service to test with?
+    host = app.settings.builder.host
+    port = app.settings.builder.port
+    post_mock.assert_called_with(
+        f"http://{host}:{port}/environments/build",
+        json={
+            "name": f"{testable_env_input.path}/{testable_env_input.name}",
+            "version": "1",
+            "model": {
+                "description": testable_env_input.description,
+                "packages": [
+                    {
+                        "name": pkg.name,
+                        "version": pkg.version,
+                    }
+                    for pkg in testable_env_input.packages
+                ],
+            },
+        },
+    )