From 15e370e5f3b8d7c16f5310b903d3b64ff56e1c4e Mon Sep 17 00:00:00 2001 From: Michael Woolnough Date: Tue, 10 Sep 2024 10:17:59 +0100 Subject: [PATCH 1/3] Cache expensive data collection. --- softpack_core/schemas/environment.py | 13 ++++++++++++- softpack_core/schemas/package_collection.py | 8 ++++++-- softpack_core/spack.py | 4 ++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index c25697e..7290e3b 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -322,6 +322,8 @@ class Environment: state: Optional[State] tags: list[str] hidden: bool + cachedEnvs: list["Environment"] = None + envsUpdates: bool = True requested: Optional[datetime.datetime] = None build_start: Optional[datetime.datetime] = None @@ -335,6 +337,9 @@ def iter(cls) -> list["Environment"]: Returns: Iterable[Environment]: An iterator of Environment objects. """ + if not cls.envsUpdates: + return cls.cachedEnvs + statuses = BuildStatus.get_all() if isinstance(statuses, BuilderError): statuses = [] @@ -366,6 +371,9 @@ def iter(cls) -> list["Environment"]: env.build_start = status.build_start env.build_done = status.build_done + cls.cachedEnvs = environment_objects + cls.envsUpdates = False + return environment_objects @classmethod @@ -541,6 +549,7 @@ def create_new_env( ], True, ) + cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "create environment folder") except RuntimeError as e: return InvalidInputError( @@ -640,7 +649,7 @@ def store_metadata(cls, environment_path: Path, metadata: Box) -> None: metadata.to_yaml(), overwrite=True, ) - + cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "update metadata") @classmethod @@ -677,6 +686,7 @@ def delete(cls, name: str, path: str) -> DeleteResponse: # type: ignore """ if artifacts.get(Path(path), name): tree_oid = artifacts.delete_environment(name, path) + cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "delete environment") return DeleteEnvironmentSuccess( message="Successfully deleted the environment" @@ -846,6 +856,7 @@ async def write_artifacts( tree_oid = artifacts.create_files( Path(folder_path), new_files, overwrite=True ) + cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "write artifact") return WriteArtifactSuccess( message="Successfully written artifact(s)", diff --git a/softpack_core/schemas/package_collection.py b/softpack_core/schemas/package_collection.py index 6b0a3f2..84b9c2d 100644 --- a/softpack_core/schemas/package_collection.py +++ b/softpack_core/schemas/package_collection.py @@ -26,14 +26,18 @@ class PackageCollection: packages: list[PackageMultiVersion] @classmethod - def iter(cls) -> Iterable["PackageMultiVersion"]: + def iter(cls) -> list["PackageMultiVersion"]: """Get an iterator over PackageCollection objects. Returns: Iterable[PackageCollection]: An iterator of PackageCollection objects. """ - return map(cls.from_package, app.spack.packages()) + if app.spack.packagesUpdated: + cls.packages = list(map(cls.from_package, app.spack.packages())) + app.spack.packagesUpdated = False + + return cls.packages @classmethod def from_package(cls, package: Package) -> PackageMultiVersion: diff --git a/softpack_core/spack.py b/softpack_core/spack.py index 0edc4fb..09fbd81 100644 --- a/softpack_core/spack.py +++ b/softpack_core/spack.py @@ -30,6 +30,9 @@ class Package(PackageBase): class Spack: """Spack interface class.""" + packages: list[Package] + packagesUpdated: bool = True + def __init__( self, spack_exe: str = "spack", @@ -97,6 +100,7 @@ def store_packages_from_spack( json.loads(jsonData), ) ) + self.packagesUpdated = True def __readPackagesFromCacheOnce(self) -> Tuple[bytes, bool]: if len(self.stored_packages) > 0 or self.cacheDir == "": From ae7fb75c294e1520e13a2fd73d3efcb9cce68201 Mon Sep 17 00:00:00 2001 From: Michael Woolnough Date: Tue, 10 Sep 2024 10:25:51 +0100 Subject: [PATCH 2/3] Delint'd. --- softpack_core/schemas/environment.py | 4 ++-- softpack_core/schemas/package_collection.py | 1 - softpack_core/spack.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 7290e3b..82d9a1a 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -8,7 +8,7 @@ import io import re import statistics -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from traceback import format_exception_only from typing import List, Optional, Tuple, Union, cast @@ -322,7 +322,7 @@ class Environment: state: Optional[State] tags: list[str] hidden: bool - cachedEnvs: list["Environment"] = None + cachedEnvs: list["Environment"] = field(default_factory=list) envsUpdates: bool = True requested: Optional[datetime.datetime] = None diff --git a/softpack_core/schemas/package_collection.py b/softpack_core/schemas/package_collection.py index 84b9c2d..2338c75 100644 --- a/softpack_core/schemas/package_collection.py +++ b/softpack_core/schemas/package_collection.py @@ -5,7 +5,6 @@ """ from dataclasses import dataclass -from typing import Iterable import strawberry diff --git a/softpack_core/spack.py b/softpack_core/spack.py index 09fbd81..be72f15 100644 --- a/softpack_core/spack.py +++ b/softpack_core/spack.py @@ -30,7 +30,6 @@ class Package(PackageBase): class Spack: """Spack interface class.""" - packages: list[Package] packagesUpdated: bool = True def __init__( From ad9bce0cf98acfe75007163dd9cf6a551889e460 Mon Sep 17 00:00:00 2001 From: Michael Woolnough Date: Tue, 10 Sep 2024 13:04:13 +0100 Subject: [PATCH 3/3] Keep track of when artefacts updates, not the environments list. --- softpack_core/artifacts.py | 2 ++ softpack_core/schemas/environment.py | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/softpack_core/artifacts.py b/softpack_core/artifacts.py index 96fae0d..6f9681e 100644 --- a/softpack_core/artifacts.py +++ b/softpack_core/artifacts.py @@ -81,6 +81,7 @@ class Artifacts: users_folder_name = "users" groups_folder_name = "groups" credentials_callback = None + updated = True @dataclass class Object: @@ -408,6 +409,7 @@ def commit_and_push( ) remote = self.repo.remotes[0] remote.push([self.repo.head.name], callbacks=self.credentials_callback) + self.updated = True return oid def build_tree( diff --git a/softpack_core/schemas/environment.py b/softpack_core/schemas/environment.py index 82d9a1a..f036d84 100644 --- a/softpack_core/schemas/environment.py +++ b/softpack_core/schemas/environment.py @@ -323,7 +323,6 @@ class Environment: tags: list[str] hidden: bool cachedEnvs: list["Environment"] = field(default_factory=list) - envsUpdates: bool = True requested: Optional[datetime.datetime] = None build_start: Optional[datetime.datetime] = None @@ -337,7 +336,7 @@ def iter(cls) -> list["Environment"]: Returns: Iterable[Environment]: An iterator of Environment objects. """ - if not cls.envsUpdates: + if not artifacts.updated: return cls.cachedEnvs statuses = BuildStatus.get_all() @@ -372,7 +371,7 @@ def iter(cls) -> list["Environment"]: env.build_done = status.build_done cls.cachedEnvs = environment_objects - cls.envsUpdates = False + artifacts.updated = False return environment_objects @@ -549,7 +548,6 @@ def create_new_env( ], True, ) - cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "create environment folder") except RuntimeError as e: return InvalidInputError( @@ -649,7 +647,6 @@ def store_metadata(cls, environment_path: Path, metadata: Box) -> None: metadata.to_yaml(), overwrite=True, ) - cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "update metadata") @classmethod @@ -686,7 +683,6 @@ def delete(cls, name: str, path: str) -> DeleteResponse: # type: ignore """ if artifacts.get(Path(path), name): tree_oid = artifacts.delete_environment(name, path) - cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "delete environment") return DeleteEnvironmentSuccess( message="Successfully deleted the environment" @@ -856,7 +852,6 @@ async def write_artifacts( tree_oid = artifacts.create_files( Path(folder_path), new_files, overwrite=True ) - cls.envsUpdates = True artifacts.commit_and_push(tree_oid, "write artifact") return WriteArtifactSuccess( message="Successfully written artifact(s)",