MLTE-552 - Custom List File System Store

demo/scenarios/1_requirements.ipynb

@@ -232,7 +232,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv",
+   "display_name": "mlte-python-8nDiemWG-py3.9",
    "language": "python",
    "name": "python3"
   },
@@ -246,7 +246,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.9.20"
   }
  },
  "nbformat": 4,

demo/simple/2_requirements.ipynb

@@ -60,6 +60,10 @@
     "from mlte.qa_category.costs.training_memory_cost import TrainingMemoryCost\n",
     "from mlte.qa_category.costs.training_compute_cost import TrainingComputeCost\n",
     "from mlte.qa_category.functionality.task_efficacy import TaskEfficacy\n",
+    "from mlte.qa_category.costs.storage_cost import StorageCost\n",
+    "from mlte.qa_category.costs.training_memory_cost import TrainingMemoryCost\n",
+    "from mlte.qa_category.costs.training_compute_cost import TrainingComputeCost\n",
+    "from mlte.qa_category.functionality.task_efficacy import TaskEfficacy\n",
     "\n",
     "\n",
     "from mlte.measurement.storage import LocalObjectSize\n",

mlte/_private/reflection.py

@@ -1,5 +1,11 @@
+from __future__ import annotations
+
 import importlib
-from typing import Any, Type
+import importlib.resources
+import json
+import os
+from types import ModuleType
+from typing import Any, Generator, Type
 
 
 def load_class(class_path: str) -> Type[Any]:
@@ -24,3 +30,14 @@ def load_class(class_path: str) -> Type[Any]:
         )
 
     return class_type
+
+
+def get_json_resources(package: ModuleType) -> Generator[Any, None, None]:
+    """Load set of json files represented as a module and return a generator of their data."""
+    resources = importlib.resources.files(package)
+    with importlib.resources.as_file(resources) as resources_path:
+        with os.scandir(resources_path) as files:
+            for file in files:
+                if file.is_file() and file.name.endswith("json"):
+                    with open(file.path) as open_file:
+                        yield json.load(open_file)

mlte/artifact/model.py

@@ -13,7 +13,7 @@
 from mlte.negotiation.model import NegotiationCardModel
 from mlte.report.model import ReportModel
 from mlte.spec.model import SpecModel
-from mlte.store.query import Filtrable
+from mlte.store.query import Filterable
 from mlte.validation.model import ValidatedSpecModel
 from mlte.value.model import ValueModel
 
@@ -36,7 +36,7 @@ class ArtifactHeaderModel(BaseModel):
     model_config = ConfigDict(use_enum_values=True)
 
 
-class ArtifactModel(Filtrable):
+class ArtifactModel(Filterable):
     """The base model for all MLTE artifacts."""
 
     header: ArtifactHeaderModel

mlte/backend/core/state.py

@@ -10,6 +10,7 @@
 from mlte.store.artifact.store import ArtifactStore
 from mlte.store.catalog.catalog_group import CatalogStoreGroup
 from mlte.store.catalog.store import CatalogStore
+from mlte.store.custom_list.store import CustomListStore
 from mlte.store.user.store import UserStore
 
 
@@ -30,6 +31,9 @@ def reset(self):
         self._catalog_stores: CatalogStoreGroup = CatalogStoreGroup()
         """The list of catalog store instances maintained by the state object."""
 
+        self._custom_list_store: Optional[CustomListStore] = None
+        """The custom list store instance maintained by the state object."""
+
         self._jwt_secret_key: str = ""
         """Secret key used to sign authentication tokens."""
 
@@ -38,7 +42,7 @@ def set_artifact_store(self, store: ArtifactStore):
         self._artifact_store = store
 
     def set_user_store(self, store: UserStore):
-        """Set the globally-configured backend artifact store."""
+        """Set the globally-configured backend user store."""
         self._user_store = store
 
     def add_catalog_store(
@@ -53,6 +57,10 @@ def add_catalog_store_from_uri(
         """Adds to the the globally-configured backend list of catalog stores."""
         self._catalog_stores.add_catalog_from_uri(id, store_uri, overwite)
 
+    def set_custom_list_store(self, store: CustomListStore):
+        """Set the globally-configured backend custom list store."""
+        self._custom_list_store = store
+
     def set_token_key(self, token_key: str):
         """Sets the globally used token secret key."""
         self._jwt_secret_key = token_key

mlte/backend/main.py

@@ -18,6 +18,7 @@
 from mlte.store.artifact import factory as artifact_store_factory
 from mlte.store.base import StoreType, StoreURI
 from mlte.store.catalog.sample_catalog import SampleCatalog
+from mlte.store.custom_list.initial_custom_lists import InitialCustomLists
 from mlte.store.user import factory as user_store_factory
 
 # Application exit codes
@@ -97,6 +98,13 @@ def run(
         )
         state.add_catalog_store_from_uri(uri, id)
 
+    # Initialize the backing custom list store instance. Assume same store as artifact one for now.
+    # TODO: allow for separate config of uri here
+    custom_list_store = InitialCustomLists.setup_custom_list_store(
+        stores_uri=artifact_store.uri
+    )
+    state.set_custom_list_store(custom_list_store)
+
     # Set the token signing key.
     state.set_token_key(jwt_secret)
 

mlte/catalog/model.py

@@ -9,7 +9,7 @@
 from strenum import StrEnum
 
 from mlte.model import BaseModel
-from mlte.store.query import Filtrable
+from mlte.store.query import Filterable
 
 
 class CatalogEntryType(StrEnum):
@@ -44,7 +44,7 @@ class CatalogEntryHeader(BaseModel):
     """The id of the catalog this entry came from."""
 
 
-class CatalogEntry(Filtrable):
+class CatalogEntry(Filterable):
     """The base model for MLTE catalog entries."""
 
     header: CatalogEntryHeader

mlte/custom_list/custom_list_names.py

@@ -0,0 +1,16 @@
+"""
+mlte/custom_list/custom_list_names.py
+
+Enum of the predefined custom list names.
+"""
+
+from __future__ import annotations
+
+from strenum import StrEnum
+
+
+class CustomListName(StrEnum):
+    """Custom lists names."""
+
+    QA_CATEGORIES = "qa_categories"
+    QUALITY_ATTRIBUTES = "quality_attributes"

mlte/custom_list/model.py

@@ -0,0 +1,31 @@
+"""
+mlte/custom_list/model.py
+
+Model implementation for a custom list.
+"""
+from __future__ import annotations
+
+from typing import List
+
+from mlte.custom_list.custom_list_names import CustomListName
+from mlte.model import BaseModel
+
+
+class CustomListModel(BaseModel):
+    """A model class representing a custom list."""
+
+    name: CustomListName
+    """An name to uniquely identify the list."""
+
+    entries: List[CustomListEntryModel] = []
+    """A list of entries in the list."""
+
+
+class CustomListEntryModel(BaseModel):
+    """A model class representing a custom list entry."""
+
+    name: str
+    """A name to uniquely identify the entry."""
+
+    description: str
+    """A description of the the entry."""

mlte/frontend/nuxt-app/layouts/base-layout.vue

@@ -114,7 +114,7 @@
         <slot name="default" />
         <footer>
           <p class="footer-text-left">
-            <b>MLTE - 2024</b>
+            <b>MLTE - {{currentDate.getFullYear()}}</b>
           </p>
           <div class="footer-text-right">
             <a
@@ -155,6 +155,8 @@ const token = useCookie("token");
 const user = useCookie("user");
 const userRole = useCookie("userRole");
 const version = config.public.version;
+
+const currentDate = new Date();
 </script>
 
 <style>

mlte/store/artifact/underlying/fs.py

@@ -26,7 +26,7 @@ class LocalFileSystemStore(ArtifactStore):
     """A local file system implementation of the MLTE artifact store."""
 
     BASE_MODELS_FOLDER = "models"
-    """Base fodler to store models in."""
+    """Base folder to store models in."""
 
     def __init__(self, uri: StoreURI) -> None:
         super().__init__(uri=uri)

mlte/store/base.py

@@ -174,7 +174,10 @@ class ResourceMapper:
     DEFAULT_LIST_LIMIT = 100
     """Default limit for lists."""
 
-    def create(self, new_resource: Any) -> Any:
+    def create(
+        self,
+        new_resource: Any,
+    ) -> Any:
         """
         Create a new resource.
         :param new_resource: The data to create the resource

mlte/store/catalog/sample_catalog.py

@@ -6,12 +6,11 @@
 
 from __future__ import annotations
 
-import importlib.resources
-import json
 import os
 from typing import Optional
 
 import mlte.store.catalog.sample as sample_entries
+from mlte._private.reflection import get_json_resources
 from mlte.catalog.model import CatalogEntry
 from mlte.store.base import StoreType, StoreURI
 from mlte.store.catalog.factory import create_catalog_store
@@ -83,17 +82,9 @@ def reset_catalog(catalog_store: CatalogStore) -> None:
     def _populate_catalog(catalog_session: CatalogStoreSession) -> None:
         """Load all entry files from entry folder and put them into sample catalog."""
         num_entries = 0
-        resources = importlib.resources.files(sample_entries)
-        with importlib.resources.as_file(resources) as resources_path:
-            with os.scandir(resources_path) as files:
-                print("Loading sample catalog entries.")
-                for file in files:
-                    if file.is_file() and file.name.endswith("json"):
-                        with open(file.path) as open_file:
-                            entry = CatalogEntry(**json.load(open_file))
-                            entry.header.catalog_id = (
-                                SampleCatalog.SAMPLE_CATALOG_ID
-                            )
-                            catalog_session.entry_mapper.create(entry)
-                            num_entries += 1
-                print(f"Loaded {num_entries} entries for sample catalog.")
+        for json_data in get_json_resources(sample_entries):
+            entry = CatalogEntry(**json_data)
+            entry.header.catalog_id = SampleCatalog.SAMPLE_CATALOG_ID
+            catalog_session.entry_mapper.create(entry)
+            num_entries += 1
+        print(f"Loaded {num_entries} entries for sample catalog.")

mlte/store/catalog/underlying/fs.py

@@ -19,15 +19,15 @@
 from mlte.store.common.fs_storage import FileSystemStorage
 
 # -----------------------------------------------------------------------------
-# LocalFileSystemStore
+# FileSystemCatalogStore
 # -----------------------------------------------------------------------------
 
 
 class FileSystemCatalogStore(CatalogStore):
     """A local file system implementation of the MLTE catalog store."""
 
     BASE_CATALOGS_FOLDER = "catalogs"
-    """Base fodler to store catalog entries in."""
+    """Base folder to store catalog entries in."""
 
     DEFAULT_CATALOG_FOLDER = "catalog"
     """A default name for a catalog folder."""
@@ -56,7 +56,7 @@ def session(self) -> FileSystemCatalogStoreSession:
 
 
 # -----------------------------------------------------------------------------
-# LocalFileSystemStoreSession
+# FileSystemCatalogStoreSession
 # -----------------------------------------------------------------------------
 
 
@@ -99,7 +99,7 @@ def __init__(self, storage: FileSystemStorage) -> None:
         self.storage.set_base_path(
             Path(self.storage.sub_folder, self.ENTRIES_FOLDER)
         )
-        """Set the subfodler for this resrouce."""
+        """Set the subfolder for this resource."""
 
     def create(self, entry: CatalogEntry) -> CatalogEntry:
         self.storage.ensure_resource_does_not_exist(entry.header.identifier)
@@ -127,6 +127,6 @@ def _read_entry(self, entry_id: str) -> CatalogEntry:
         return CatalogEntry(**self.storage.read_resource(entry_id))
 
     def _write_entry(self, entry: CatalogEntry) -> CatalogEntry:
-        """Writes a entry to storage."""
+        """Writes an entry to storage."""
         self.storage.write_resource(entry.header.identifier, entry.model_dump())
         return self._read_entry(entry.header.identifier)

mlte/store/custom_list/factory.py

@@ -0,0 +1,36 @@
+"""
+mlte/store/custom_list/factory.py
+
+Top-level functions for custom list store creation.
+"""
+
+from mlte.store.base import StoreURI
+from mlte.store.custom_list.store import CustomListStore
+from mlte.store.custom_list.underlying.fs import FileSystemCustomListStore
+
+# from mlte.store.custom_list.underlying.memory import InMemoryCustomListStore
+# from mlte.store.custom_list.underlying.rdbs.store import RelationalDBCustomListStore
+# from mlte.store.custom_list.underlying.rdbs.store import HttpCustomListStore
+
+
+def create_custom_list_store(uri: str) -> CustomListStore:
+    """
+    Create a MLTE custom list store instance.
+    :param uri: The URI for the store instance
+    :return: The store instance
+    """
+    parsed_uri = StoreURI.from_string(uri)
+    return FileSystemCustomListStore(parsed_uri)
+
+    # if parsed_uri.type == StoreType.LOCAL_MEMORY:
+    # return InMemoryCustomListStore(parsed_uri)
+    #   elif parsed_uri.type == StoreType.RELATIONAL_DB:
+    # return RelationalDBCustomListStore(parsed_uri)
+    #   elif parsed_uri.type == StoreType.LOCAL_FILESYSTEM:
+    # return FileSystemCustomListStore(parsed_uri)
+    #   elif parsed_uri.type == StoreType.REMOTE_HTTP:
+    # return HttpCustomListStore(uri=parsed_uri)
+    # else:
+    #     raise Exception(
+    #         f"Store can't be created, unknown or unsupported URI prefix received for uri {parsed_uri}"
+    #     )