Skip to content

Commit

Permalink
Fixed vm costs view (#527)
Browse files Browse the repository at this point in the history 
* Fix: Added cost tests and fixed issue with vm calculation.

* Fix: Changed test name to avoid duplication.

* Fix: Changed cost calculation on views and models, and added another test case.

* Fix: Fixed migration and tests.

* Fix: Fixed unit tests and type mismatching for float.

* Fix: Fixed last float value on test.

---------

Co-authored-by: Andres D. Molins <[email protected]>
  • Loading branch information
@nesitor
nesitor and Andres D. Molins authored Dec 11, 2023
1 parent 19df1e4 commit 64c09d3
Show file tree
Hide file tree
Showing 5 changed files with 295 additions and 8 deletions.
146 changes: 146 additions & 0 deletions deployment/migrations/versions/0020_e682fc8f9506_fix_vm_costs_view.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
"""fix_vm_costs_view
Revision ID: e682fc8f9506
Revises: 08602db6c78f
Create Date: 2023-11-27 18:19:35.782198
"""
from alembic import op
import sqlalchemy as sa


revision = 'e682fc8f9506'
down_revision = '3bf484f2cc95'
branch_labels = None
depends_on = None


def upgrade() -> None:
op.execute(
"""
create or replace view vm_costs_view as
SELECT vm_versions.vm_hash,
vm_versions.owner,
vms.resources_vcpus,
vms.resources_memory,
file_volumes_size.file_volumes_size,
other_volumes_size.other_volumes_size,
used_disk.required_disk_space,
cu.compute_units_required,
bcp.base_compute_unit_price,
m.compute_unit_price_multiplier,
cpm.compute_unit_price,
free_disk.included_disk_space,
additional_disk.additional_disk_space,
adp.disk_price,
tp.total_price
FROM vm_versions
JOIN vms ON vm_versions.current_version::text = vms.item_hash::text
JOIN (SELECT volume.vm_hash,
sum(files.size) AS file_volumes_size
FROM vm_volumes_files_view volume
LEFT JOIN files ON volume.volume_to_use::text = files.hash::text
GROUP BY volume.vm_hash) file_volumes_size
ON vm_versions.current_version::text = file_volumes_size.vm_hash::text
LEFT JOIN (SELECT instance_rootfs.instance_hash,
instance_rootfs.size_mib::bigint * 1024 * 1024 AS rootfs_size
FROM instance_rootfs) rootfs_size ON vm_versions.vm_hash::text = rootfs_size.instance_hash::text
JOIN (SELECT vm_machine_volumes.vm_hash,
sum(vm_machine_volumes.size_mib) * 1024 * 1024 AS other_volumes_size
FROM vm_machine_volumes
GROUP BY vm_machine_volumes.vm_hash) other_volumes_size
ON vm_versions.current_version::text = other_volumes_size.vm_hash::text,
LATERAL ( SELECT file_volumes_size.file_volumes_size +
other_volumes_size.other_volumes_size::numeric AS required_disk_space) used_disk,
LATERAL ( SELECT ceil(GREATEST(ceil((vms.resources_vcpus / 1)::double precision),
(vms.resources_memory / 2048)::double precision)) AS compute_units_required) cu,
LATERAL ( SELECT CASE
WHEN COALESCE(vms.persistent, true)
THEN '21474836480'::bigint::double precision * cu.compute_units_required
ELSE 2147483648::double precision * cu.compute_units_required
END AS included_disk_space) free_disk,
LATERAL ( SELECT GREATEST((file_volumes_size.file_volumes_size + rootfs_size.rootfs_size::numeric +
other_volumes_size.other_volumes_size::numeric)::double precision -
free_disk.included_disk_space,
0::double precision) AS additional_disk_space) additional_disk,
LATERAL ( SELECT CASE
WHEN COALESCE(vms.persistent, true) THEN 2000
ELSE 200
END AS base_compute_unit_price) bcp,
LATERAL ( SELECT CASE
WHEN COALESCE(vms.persistent, true) THEN 1 + vms.environment_internet::integer
ELSE 1
END AS compute_unit_price_multiplier) m,
LATERAL ( SELECT cu.compute_units_required * m.compute_unit_price_multiplier::double precision *
bcp.base_compute_unit_price::double precision AS compute_unit_price) cpm,
LATERAL ( SELECT additional_disk.additional_disk_space / 20::double precision /
(1024 * 1024)::double precision AS disk_price) adp,
LATERAL ( SELECT cpm.compute_unit_price + adp.disk_price AS total_price) tp
"""
)


def downgrade() -> None:
op.execute(
"""
create or replace view vm_costs_view as
SELECT vm_versions.vm_hash,
vm_versions.owner,
vms.resources_vcpus,
vms.resources_memory,
file_volumes_size.file_volumes_size,
other_volumes_size.other_volumes_size,
used_disk.required_disk_space,
cu.compute_units_required,
bcp.base_compute_unit_price,
m.compute_unit_price_multiplier,
cpm.compute_unit_price,
free_disk.included_disk_space,
additional_disk.additional_disk_space,
adp.disk_price,
tp.total_price
FROM vm_versions
JOIN vms ON vm_versions.current_version::text = vms.item_hash::text
JOIN (SELECT volume.vm_hash,
sum(files.size) AS file_volumes_size
FROM vm_volumes_files_view volume
LEFT JOIN files ON volume.volume_to_use::text = files.hash::text
GROUP BY volume.vm_hash) file_volumes_size
ON vm_versions.current_version::text = file_volumes_size.vm_hash::text
LEFT JOIN (SELECT instance_rootfs.instance_hash,
instance_rootfs.size_mib::bigint * 1024 * 1024 AS rootfs_size
FROM instance_rootfs) rootfs_size ON vm_versions.vm_hash::text = rootfs_size.instance_hash::text
JOIN (SELECT vm_machine_volumes.vm_hash,
sum(vm_machine_volumes.size_mib) * 1024 * 1024 AS other_volumes_size
FROM vm_machine_volumes
GROUP BY vm_machine_volumes.vm_hash) other_volumes_size
ON vm_versions.current_version::text = other_volumes_size.vm_hash::text,
LATERAL ( SELECT file_volumes_size.file_volumes_size +
other_volumes_size.other_volumes_size::numeric AS required_disk_space) used_disk,
LATERAL ( SELECT ceil(GREATEST(ceil((vms.resources_vcpus / 1)::double precision),
(vms.resources_memory / 2048)::double precision)) AS compute_units_required) cu,
LATERAL ( SELECT CASE
WHEN COALESCE(vms.persistent, true)
THEN '21474836480'::bigint::double precision * cu.compute_units_required
ELSE 2147483648::double precision * cu.compute_units_required
END AS included_disk_space) free_disk,
LATERAL ( SELECT GREATEST((file_volumes_size.file_volumes_size + rootfs_size.rootfs_size::numeric +
other_volumes_size.other_volumes_size::numeric)::double precision -
free_disk.included_disk_space,
0::double precision) AS additional_disk_space) additional_disk,
LATERAL ( SELECT CASE
WHEN COALESCE(vms.persistent, true) THEN 2000
ELSE 200
END AS base_compute_unit_price) bcp,
LATERAL ( SELECT 1 + vms.environment_internet::integer AS compute_unit_price_multiplier) m,
LATERAL ( SELECT cu.compute_units_required * m.compute_unit_price_multiplier::double precision *
bcp.base_compute_unit_price::double precision AS compute_unit_price) cpm,
LATERAL ( SELECT additional_disk.additional_disk_space * 20::double precision /
(1024 * 1024)::double precision AS disk_price) adp,
LATERAL ( SELECT cpm.compute_unit_price + adp.disk_price AS total_price) tp
"""
)
2 changes: 1 addition & 1 deletion src/aleph/services/cost.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def get_additional_storage_price(
additional_storage = max(
total_volume_size - (free_storage_per_compute_unit * nb_compute_units), 0
)
price = Decimal(additional_storage) * 20 / MiB
price = Decimal(additional_storage) / 20 / MiB
return price


Expand Down
2 changes: 1 addition & 1 deletion tests/api/test_balance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@ async def test_get_balance(
assert response.status == 200, await response.text()
data = await response.json()
assert data["balance"] == user_balance.balance
assert data["locked_amount"] == 2720.6666666666665
assert data["locked_amount"] == 2002.4666666666667
12 changes: 6 additions & 6 deletions tests/message_processing/test_process_instances.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def fixture_instance_message(session_factory: DbSessionFactory) -> PendingMessag
"aleph_api": False,
"shared_cache": False,
},
"resources": {"vcpus": 1, "memory": 128, "seconds": 30},
"resources": {"vcpus": 1, "memory": 2048, "seconds": 30},
"requirements": {"cpu": {"architecture": "x86_64"}},
"rootfs": {
"parent": {
Expand All @@ -70,7 +70,7 @@ def fixture_instance_message(session_factory: DbSessionFactory) -> PendingMessag
},
"persistence": "host",
"name": "test-rootfs",
"size_mib": 20 * 1024,
"size_mib": 20480,
},
"authorized_keys": [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGULT6A41Msmw2KEu0R9MvUjhuWNAsbdeZ0DOwYbt4Qt user@example",
Expand Down Expand Up @@ -462,7 +462,7 @@ async def test_get_additional_storage_price(
additional_price = get_additional_storage_price(
content=content, session=session
)
assert additional_price == Decimal("720")
assert additional_price == Decimal("1.8")


@pytest.mark.asyncio
Expand All @@ -477,7 +477,7 @@ async def test_get_compute_cost(
content = InstanceContent.parse_raw(fixture_instance_message.item_content)
with session_factory() as session:
price: Decimal = compute_cost(content=content, session=session)
assert price == Decimal("2720")
assert price == Decimal("2001.8")


@pytest.mark.asyncio
Expand All @@ -497,10 +497,10 @@ async def test_compare_cost_view_with_cost_function(

content = InstanceContent.parse_raw(fixture_instance_message.item_content)
with session_factory() as session:
cost_from_function = compute_cost(session=session, content=content)
cost_from_function: Decimal = compute_cost(session=session, content=content)
cost_from_view = session.execute(
text("SELECT total_price from vm_costs_view WHERE vm_hash = :vm_hash"),
{"vm_hash": fixture_instance_message.item_hash},
).scalar_one()

assert cost_from_view == cost_from_function
assert Decimal(str(cost_from_view)) == cost_from_function
141 changes: 141 additions & 0 deletions tests/services/test_cost_service.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import pytest
from aleph.types.db_session import DbSession

from aleph_message.models import ExecutableContent, InstanceContent

from aleph.services.cost import (
compute_cost,
get_additional_storage_price,
)

from unittest.mock import Mock


class StoredFileDb:
pass


@pytest.fixture
def fixture_instance_message() -> ExecutableContent:
content = {
"time": 1701099523.849,
"rootfs": {
"parent": {
"ref": "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595",
"use_latest": True
},
"size_mib": 20480,
"persistence": "host"
},
"address": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874",
"volumes": [],
"metadata": {
"name": "Test Debian 12"
},
"resources": {
"vcpus": 1,
"memory": 2048,
"seconds": 30
},
"allow_amend": False,
"environment": {
"internet": True,
"aleph_api": True,
"reproducible": False,
"shared_cache": False
},
"authorized_keys": [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y [email protected]"
]
}

return InstanceContent.parse_obj(content)

@pytest.fixture
def fixture_instance_message_complete() -> ExecutableContent:
content = {
"time": 1701099523.849,
"rootfs": {
"parent": {
"ref": "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595",
"use_latest": True
},
"size_mib": 20480,
"persistence": "host"
},
"address": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874",
"volumes": [
{
"comment": "Ephemeral storage, read-write but will not persist after the VM stops",
"mount": "/var/cache",
"ephemeral": True,
"size_mib": 50,
},
{
"comment": "Working data persisted on the VM supervisor, not available on other nodes",
"mount": "/var/lib/sqlite",
"name": "sqlite-data",
"persistence": "host",
"size_mib": 100,
},
{
"comment": "Working data persisted on the Aleph network. "
"New VMs will try to use the latest version of this volume, "
"with no guarantee against conflicts",
"mount": "/var/lib/statistics",
"name": "statistics",
"persistence": "store",
"size_mib": 100,
},
{
"comment": "Raw drive to use by a process, do not mount it",
"name": "raw-data",
"persistence": "host",
"size_mib": 100,
},
],
"metadata": {
"name": "Test Debian 12"
},
"resources": {
"vcpus": 1,
"memory": 2048,
"seconds": 30
},
"allow_amend": False,
"environment": {
"internet": True,
"aleph_api": True,
"reproducible": False,
"shared_cache": False
},
"authorized_keys": [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y [email protected]"
]
}

return InstanceContent.parse_obj(content)


def test_compute_cost(fixture_instance_message):
file_db = StoredFileDb()
mock = Mock()
mock.patch('_get_file_from_ref', return_value=file_db)
cost = compute_cost(content=fixture_instance_message, session=DbSession())
assert cost == 2000


def test_get_additional_storage_price(fixture_instance_message):
file_db = StoredFileDb()
mock = Mock()
mock.patch('_get_file_from_ref', return_value=file_db)
cost = get_additional_storage_price(content=fixture_instance_message, session=DbSession())
assert cost == 0


def test_compute_cost_complete(fixture_instance_message_complete):
file_db = StoredFileDb()
mock = Mock()
mock.patch('_get_file_from_ref', return_value=file_db)
cost = compute_cost(content=fixture_instance_message_complete, session=DbSession())
assert cost == 2017.50

0 comments on commit 64c09d3

Please sign in to comment.