Merge branch 'main' into 716-updating-pydantic-version

README.md

@@ -113,6 +113,35 @@ Unless your focus is developing the VM-Connector, using the Docker image is easi
 ## Testing
 See [Testinc doc](./TESTING.md)
 
+## Code Formatting and Linting
+
+To help maintain a clean and consistent codebase, we provide automated tools for formatting and style checks.
+To ensure your code is properly **formatted** according to project standards, you can use:
+
+```bash
+hatch linting:fmt
+```
+
+**Typing** helps ensure your code adheres to expected type annotations, improving reliability and clarity. To validate
+typing in your code, use:
+```bash
+hatch linting:typing
+```
+
+These checks are also validated in Continuous Integration (CI) alongside unit tests. To ensure a smooth workflow, we 
+recommend running these commands before committing changes.
+
+**Linting** checks for potential errors, coding style violations, and patterns that may lead to bugs or reduce code
+quality (e.g., unused variables, incorrect imports, or inconsistent naming). While linting is not currently enforced in
+Continuous Integration (CI), it is considered a best practice to check linting manually to maintain high-quality code.
+You can manually lint your code by running:
+
+```bash
+hatch fmt
+```
+
+Following these best practices can help streamline code reviews and improve overall project quality.
+
 # Architecture
 
 ![Aleph im VM - Details](https://user-images.githubusercontent.com/404665/127126908-3225a633-2c36-4129-8766-9810f2fcd7d6.png)

examples/example_django/manage.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 """Django's command-line utility for administrative tasks."""
+
 import os
 import sys
 

pyproject.toml

@@ -31,7 +31,7 @@ dynamic = [ "version" ]
 # Upon adding or updating dependencies, update `packaging/Makefile` for the Debian package
 dependencies = [
   "aiodns==3.1",
-  "aiohttp==3.9.5",
+  "aiohttp==3.10.11",
   "aiohttp-cors~=0.7.0",
   "aioredis==1.3.1",
   "aiosqlite==0.19",
@@ -119,7 +119,6 @@ python = [ "3.10", "3.11", "3.12" ]
 [tool.hatch.envs.linting]
 detached = true
 dependencies = [
-  "black==24.3.0",
   "mypy==1.8.0",
   "ruff==0.4.6",
   "isort==5.13.2",
@@ -129,16 +128,16 @@ dependencies = [
 ]
 [tool.hatch.envs.linting.scripts]
 typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"
+# Check
 style = [
-  #  "ruff {args:.}",
-  "black --check --diff {args:.}",
+  "ruff format --diff {args:.}",
   "isort --check-only --profile black {args:.}",
   "yamlfix --check .",
   "pyproject-fmt --check pyproject.toml",
 ]
+# Do modification
 fmt = [
-  "black {args:.}",
-  #  "ruff --fix {args:.}",
+  "ruff format {args:.}",
   "isort --profile black {args:.}",
   "yamlfix .",
   "pyproject-fmt pyproject.toml",
@@ -157,6 +156,7 @@ line-length = 120
 [tool.ruff]
 target-version = "py310"
 line-length = 120
+src = [ "src" ]
 lint.select = [
   "A",
   "ARG",
@@ -185,6 +185,8 @@ lint.select = [
   "YTT",
 ]
 lint.ignore = [
+  "ISC001",
+  # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/#single-line-implicit-string-concatenation-isc001
   #  # Allow non-abstract empty methods in abstract base classes
   #  "B027",
   #  # Allow boolean positional values in function calls, like `dict.get(... True)`
@@ -202,11 +204,14 @@ lint.ignore = [
 #  # Don't touch unused imports
 #  "F401",
 #]
-#lint.isort = [ "aleph.vm" ]
 
 # Tests can use magic values, assertions, and relative imports
 lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ]
 
+[tool.isort]
+profile = "black"
+extra_standard_library = [ "packaging" ]
+
 [tool.pytest.ini_options]
 pythonpath = [
   "src",

runtimes/aleph-debian-12-python/init1.py

@@ -558,6 +558,7 @@ async def main() -> None:
 
     class ServerReference:
         "Reference used to close the server from within `handle_instruction"
+
         server: asyncio.AbstractServer
 
     server_reference = ServerReference()

src/aleph/vm/controllers/__main__.py

@@ -53,6 +53,7 @@ def parse_args(args):
         help="set loglevel to DEBUG",
         action="store_const",
         const=logging.DEBUG,
+        default=logging.INFO,
     )
     return parser.parse_args(args)
 

src/aleph/vm/controllers/firecracker/executable.py

@@ -13,6 +13,7 @@
 from aiohttp import ClientResponseError
 from aleph_message.models import ExecutableContent, ItemHash
 from aleph_message.models.execution.environment import MachineResources
+from aleph_message.models.execution.volume import PersistentVolume
 
 from aleph.vm.conf import settings
 from aleph.vm.controllers.configuration import (
@@ -114,7 +115,13 @@ async def download_kernel(self):
     async def download_volumes(self):
         volumes = []
         # TODO: Download in parallel
-        for volume in self.message_content.volumes:
+        for i, volume in enumerate(self.message_content.volumes):
+            # only persistant volume has name and mount
+            if isinstance(volume, PersistentVolume):
+                if not volume.name:
+                    volume.name = f"unamed_volume_{i}"
+                if not volume.mount:
+                    volume.mount = f"/mnt/{volume.name}"
             volumes.append(
                 HostVolume(
                     mount=volume.mount,

src/aleph/vm/guest_api/__main__.py

@@ -24,8 +24,15 @@
 
 async def get_redis(address: str = REDIS_ADDRESS) -> aioredis.Redis:
     global _redis
-    if _redis is None:
+    # Ensure the redis connection is still up before returning it
+    if _redis:
+        try:
+            await _redis.ping()
+        except aioredis.ConnectionClosedError:
+            _redis = None
+    if not _redis:
         _redis = await aioredis.create_redis(address=address)
+
     return _redis
 
 

src/aleph/vm/hypervisors/qemu/qemuvm.py

@@ -102,6 +102,13 @@ async def start(
             # Tell to put the output to std fd, so we can include them in the log
             "-serial",
             "stdio",
+            # nographics. Seems redundant with -serial stdio but without it the boot process is not displayed on stdout
+            "-nographic",
+            # Boot
+            # order=c only first hard drive
+            # reboot-timeout in combination with -no-reboot, makes it so qemu stop if there is no bootable device
+            "-boot",
+            "order=c,reboot-timeout=1",
             # Uncomment for debug
             # "-serial", "telnet:localhost:4321,server,nowait",
             # "-snapshot",  # Do not save anything to disk

src/aleph/vm/hypervisors/qemu_confidential/qemuvm.py

@@ -12,7 +12,6 @@
 
 
 class QemuConfidentialVM(QemuVM):
-
     sev_policy: str = hex(AMDSEVPolicy.NO_DBG)
     sev_dh_cert_file: Path  # "vm_godh.b64"
     sev_session_file: Path  # "vm_session.b64"
@@ -88,12 +87,18 @@ async def start(
             "-qmp",
             f"unix:{self.qmp_socket_path},server,nowait",
             # Tell to put the output to std fd, so we can include them in the log
-            "-nographic",
             "-serial",
             "stdio",
-            "--no-reboot",  # Rebooting from inside the VM shuts down the machine
-            "-S",
+            # nographics. Seems redundant with -serial stdio but without it the boot process is not displayed on stdout
+            "-nographic",
+            # Boot
+            # order=c only first hard drive
+            # reboot-timeout in combination with -no-reboot, makes it so qemu stop if there is no bootable device
+            "-boot",
+            "order=c,reboot-timeout=1",
             # Confidential options
+            # Do not start CPU at startup, we will start it via QMP after injecting the secret
+            "-S",
             "-object",
             f"sev-guest,id=sev0,policy={self.sev_policy},cbitpos={sev_info.c_bit_position},"
             f"reduced-phys-bits={sev_info.phys_addr_reduction},"

src/aleph/vm/network/interfaces.py

@@ -61,18 +61,18 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Inter
         logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
-def delete_ip_address(ipr: IPRoute, device_name: str, ip: IPv4Interface | IPv6Interface):
+def delete_ip_address(ipr: IPRoute, device_name: str, ip: str, mask: int):
     """Delete an IP address to the given interface."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
     if not interface_index:
         msg = f"Interface {device_name} does not exist, can't delete address {ip} to it."
         raise MissingInterfaceError(msg)
     try:
-        ipr.addr("del", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
+        ipr.addr("del", index=interface_index[0], address=ip, mask=mask)
     except NetlinkError as e:
-        logger.exception(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+        logger.exception(f"Unknown exception while deleting address {ip}/{mask} to interface {device_name}: {e}")
     except OSError as e:
-        logger.exception(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+        logger.exception(f"Unknown exception while deleting address {ip}/{mask} to interface {device_name}: {e}")
 
 
 def set_link_up(ipr: IPRoute, device_name: str):
@@ -170,6 +170,11 @@ async def delete(self) -> None:
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
         with IPRoute() as ipr:
-            delete_ip_address(ipr, self.device_name, self.host_ip)
-            delete_ip_address(ipr, self.device_name, self.host_ipv6)
+            interface_index: list[int] = ipr.link_lookup(ifname=self.device_name)
+            for addr in ipr.get_addr(index=interface_index):
+                # The order of attributes in the attrs field comes from the Netlink protocol
+                attrs = dict(addr["attrs"])
+                ip_addr: str = attrs["IFA_ADDRESS"]
+                mask: int = addr["prefixlen"]
+                delete_ip_address(ipr, self.device_name, ip_addr, mask)
             delete_tap_interface(ipr, self.device_name)

src/aleph/vm/orchestrator/custom_logs.py

@@ -25,7 +25,6 @@ class InjectingFilter(logging.Filter):
     """
 
     def filter(self, record):
-
         vm_hash = ctx_current_execution_hash.get(None)
         if not vm_hash:
             vm_execution: VmExecution | None = ctx_current_execution.get(None)

src/aleph/vm/orchestrator/run.py

@@ -4,7 +4,7 @@
 from typing import Any
 
 import msgpack
-from aiohttp import web
+from aiohttp import ClientResponseError, web
 from aiohttp.web_exceptions import (
     HTTPBadGateway,
     HTTPBadRequest,
@@ -89,6 +89,12 @@ async def create_vm_execution_or_raise_http_error(vm_hash: ItemHash, pool: VmPoo
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)
         raise HTTPInternalServerError(reason="Host did not respond to ping") from error
+    except ClientResponseError as error:
+        logger.exception(error)
+        if error.status == 404:
+            raise HTTPInternalServerError(reason=f"Item hash {vm_hash} not found") from error
+        else:
+            raise HTTPInternalServerError(reason=f"Error downloading {vm_hash}") from error
     except Exception as error:
         logger.exception(error)
         pool.forget_vm(vm_hash=vm_hash)