diff --git a/src/spaceone/inventory_v2/conf/collector_conf.py b/src/spaceone/inventory_v2/conf/collector_conf.py index 50bd53c..076e5a8 100644 --- a/src/spaceone/inventory_v2/conf/collector_conf.py +++ b/src/spaceone/inventory_v2/conf/collector_conf.py @@ -7,16 +7,15 @@ ###################################################################### RESOURCE_MAP = { - "inventory.CloudService": ("CloudServiceService", "CloudServiceManager"), - "inventory.CloudServiceType": ( - "CloudServiceTypeService", - "CloudServiceTypeManager", + "inventory.Asset": ("AssetService", "AssetManager"), + "inventory.AssetType": ( + "AssetTypeService", + "AssetTypeManager", ), "inventory.Region": ("RegionService", "RegionManager"), "inventory.ErrorResource": ("CollectorService", "CollectingManager"), } - OP_MAP = {"=": "eq", ">=": "gte", "<=": "lte", ">": "gt", "<": "lt", "!=": "not"} DB_QUEUE_NAME = "db_q" diff --git a/src/spaceone/inventory_v2/connector/__init__.py b/src/spaceone/inventory_v2/connector/__init__.py new file mode 100644 index 0000000..2364a32 --- /dev/null +++ b/src/spaceone/inventory_v2/connector/__init__.py @@ -0,0 +1,9 @@ +from spaceone.inventory_v2.connector.collector_plugin_connector import ( + BaseCollectorPluginConnector, +) +from spaceone.inventory_v2.connector.collector_plugin_connector.collector_plugin_v1_connector import ( + CollectorPluginV1Connector, +) +from spaceone.inventory_v2.connector.collector_plugin_connector.collector_plugin_v2_connector import ( + CollectorPluginV2Connector, +) diff --git a/src/spaceone/inventory_v2/connector/collector_plugin_connector/__init__.py b/src/spaceone/inventory_v2/connector/collector_plugin_connector/__init__.py new file mode 100644 index 0000000..bee8792 --- /dev/null +++ b/src/spaceone/inventory_v2/connector/collector_plugin_connector/__init__.py @@ -0,0 +1,27 @@ +from spaceone.core.connector import BaseConnector +from spaceone.core.connector.space_connector import SpaceConnector + + +class BaseCollectorPluginConnector(BaseConnector): + collector_version = None + + @classmethod + def init_plugin(cls, endpoint: str, options: dict) -> dict: + plugin_connector = SpaceConnector(endpoint=endpoint, token="NO_TOKEN") + return plugin_connector.dispatch("Collector.init", {"options": options}) + + def verify_plugin(self, *args, **kwargs): + raise NotImplementedError() + + def get_tasks(self, *args, **kwargs): + raise NotImplementedError() + + def collect(self, *args, **kwargs): + raise NotImplementedError() + + @classmethod + def get_connector_by_collector_version(cls, collector_version: str): + for subclass in cls.__subclasses__(): + if subclass.collector_version == collector_version: + return subclass() + raise Exception(f"Not found collector plugin connector: {collector_version}") diff --git a/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py new file mode 100644 index 0000000..6e38391 --- /dev/null +++ b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v1_connector.py @@ -0,0 +1,114 @@ +import logging + +from typing import Generator + +from spaceone.core.connector.space_connector import SpaceConnector + +from spaceone.inventory_v2.connector.collector_plugin_connector import ( + BaseCollectorPluginConnector, +) + +_LOGGER = logging.getLogger(__name__) + + +class CollectorPluginV1Connector(BaseCollectorPluginConnector): + collector_version = "v1" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def verify_plugin(self, endpoint: str, secret_data: dict, options: dict) -> dict: + plugin_connector = SpaceConnector(endpoint=endpoint, token="NO_TOKEN") + params = {"options": options, "secret_data": secret_data} + return plugin_connector.dispatch("Collect.collect", params) + + def get_tasks(self, endpoint: str, options: dict, secret_data: dict) -> dict: + try: + plugin_connector = SpaceConnector(endpoint=endpoint, token="NO_TOKEN") + params = {"options": options, "secret_data": secret_data} + return plugin_connector.dispatch("Job.get_tasks", params) + except Exception as e: + return {"tasks": []} + + def collect( + self, + endpoint: str, + options: dict, + secret_data: dict, + task_options: dict = None, + ) -> Generator[dict, None, None]: + plugin_connector = SpaceConnector(endpoint=endpoint, token="NO_TOKEN") + + params = {"options": options, "secret_data": secret_data, "filter": {}} + + if task_options: + params["task_options"] = task_options + + for resource_data in plugin_connector.dispatch("Collector.collect", params): + yield self._convert_resource_data(resource_data) + + @staticmethod + def _convert_resource_data(resource_data: dict) -> dict: + _LOGGER.debug( + f"[_convert_resource_data] before convert resource_data: {resource_data}" + ) + + resource_type = resource_data.get("resource_type") + + if resource_type in ["inventory.CloudService", "inventory.CloudServiceType"]: + if resource_type == "inventory.CloudService": + resource_data["resource_type"] = "inventory.Asset" + resource_data["resource"]["asset_type_id"] = resource_data[ + "resource" + ].get("asset_type") + # resource_data["resource"]["asset_group_id"] = resource_data[ + # "resource" + # ].get("cloud_service_group") + elif resource_type == "inventory.CloudServiceType": + resource_data["resource_type"] = "inventory.AssetType" + + resource_type = resource_data.get("resource_type") + + # convert match rule + for rule_values in resource_data.get("match_rules", {}).values(): + for index, rule_value in enumerate(rule_values): + if rule_value == "cloud_service_id": + rule_values[index] = "asset_id" + elif rule_value == "cloud_service_type": + rule_values[index] = "asset_type_id" + elif rule_value == "cloud_service_group": + del rule_values[index] + elif rule_value == "reference.resource_id": + rule_values[index] = "resource_id" + elif rule_value == "group": + rule_values[index] = "asset_group_id" + + if _resource := resource_data.get("resource"): + _resource_v1 = {} + if "instance_size" in _resource: + _resource_v1["instance_size"] = _resource.pop("instance_size") + if "instance_type" in _resource: + _resource_v1["instance_type"] = _resource.pop("instance_type") + del _resource["metadata"] + + if resource_type == "inventory.Asset": + asset_type_id = f"{_resource['provider']}.{_resource['cloud_service_group']}.{_resource['cloud_service_type']}" + _resource["asset_type_id"] = asset_type_id + resource_data["asset_type_id"] = asset_type_id + elif resource_type == "inventory.AssetType": + + asset_type_id = f"at-{_resource['provider']}-{_resource['group']}-{_resource['name']}" + asset_groups = [ + f"ag-{_resource['provider']}-{_resource['group']}", + f"ag-{_resource['provider']}", + ] + _resource["asset_type_id"] = asset_type_id + resource_data["asset_type_id"] = asset_type_id + resource_data["asset_groups"] = asset_groups + resource_data["icon"] = _resource.get("tags", {}).get("icon", "") + + resource_data["resource"]["v1"] = _resource_v1 + + _LOGGER.debug(f"[_convert_resource_data] resource_data: {resource_data}") + + return resource_data diff --git a/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v2_connector.py b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v2_connector.py new file mode 100644 index 0000000..0503dda --- /dev/null +++ b/src/spaceone/inventory_v2/connector/collector_plugin_connector/collector_plugin_v2_connector.py @@ -0,0 +1,17 @@ +from spaceone.inventory_v2.connector import BaseCollectorPluginConnector + + +class CollectorPluginV2Connector(BaseCollectorPluginConnector): + collector_version = "v1" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def verify_plugin(self, endpoint, secret_data, options): + pass + + def get_tasks(self, params): + pass + + def collect(self, params): + pass diff --git a/src/spaceone/inventory_v2/error/__init__.py b/src/spaceone/inventory_v2/error/__init__.py index f4b15a8..e4099d8 100644 --- a/src/spaceone/inventory_v2/error/__init__.py +++ b/src/spaceone/inventory_v2/error/__init__.py @@ -1,2 +1,3 @@ from spaceone.inventory_v2.error.region import * from spaceone.inventory_v2.error.collector import * +from spaceone.inventory_v2.error.metric import * diff --git a/src/spaceone/inventory_v2/error/metric.py b/src/spaceone/inventory_v2/error/metric.py new file mode 100644 index 0000000..0ef24e2 --- /dev/null +++ b/src/spaceone/inventory_v2/error/metric.py @@ -0,0 +1,17 @@ +from spaceone.core.error import * + + +class ERROR_NOT_SUPPORT_RESOURCE_TYPE(ERROR_INVALID_ARGUMENT): + _message = "Not support resource type. (resource_type = {resource_type})" + + +class ERROR_INVALID_DATE_RANGE(ERROR_INVALID_ARGUMENT): + _message = "{reason}" + + +class ERROR_METRIC_QUERY_RUN_FAILED(ERROR_BASE): + _message = "Metric query run failed. (metric_id = {metric_id})" + + +class ERROR_WRONG_QUERY_OPTIONS(ERROR_INVALID_ARGUMENT): + _message = "Wrong query options. (query_options = {query_options})" diff --git a/src/spaceone/inventory_v2/interface/grpc/metric.py b/src/spaceone/inventory_v2/interface/grpc/metric.py index 8297ce9..8b8a366 100644 --- a/src/spaceone/inventory_v2/interface/grpc/metric.py +++ b/src/spaceone/inventory_v2/interface/grpc/metric.py @@ -1,6 +1,6 @@ from spaceone.core.pygrpc import BaseAPI from spaceone.api.inventory_v2.v1 import metric_pb2, metric_pb2_grpc -from spaceone.inventory.service.metric_service import MetricService +from spaceone.inventory_v2.service.metric_service import MetricService class Metric(BaseAPI, metric_pb2_grpc.MetricServicer): diff --git a/src/spaceone/inventory_v2/manager/__init__.py b/src/spaceone/inventory_v2/manager/__init__.py index 93bbe5b..ba0d6c9 100644 --- a/src/spaceone/inventory_v2/manager/__init__.py +++ b/src/spaceone/inventory_v2/manager/__init__.py @@ -1 +1,5 @@ +from spaceone.inventory_v2.manager.asset_manager import AssetManager +from spaceone.inventory_v2.manager.asset_type_manager import AssetTypeManager +from spaceone.inventory_v2.lib.resource_manager import ResourceManager from spaceone.inventory_v2.manager.region_manager import RegionManager +from spaceone.inventory_v2.manager.collecting_manager import CollectingManager diff --git a/src/spaceone/inventory_v2/manager/asset_manager.py b/src/spaceone/inventory_v2/manager/asset_manager.py index f59f1dc..dc73da0 100644 --- a/src/spaceone/inventory_v2/manager/asset_manager.py +++ b/src/spaceone/inventory_v2/manager/asset_manager.py @@ -9,6 +9,8 @@ from spaceone.core.manager import BaseManager from spaceone.core import utils +from spaceone.inventory_v2.lib.resource_manager import ResourceManager +from spaceone.inventory_v2.manager.identity_manager import IdentityManager from spaceone.inventory_v2.model.asset.database import Asset _LOGGER = logging.getLogger(__name__) @@ -38,7 +40,7 @@ } -class AssetManager(BaseManager): +class AssetManager(BaseManager, ResourceManager): resource_keys = ["asset_id"] query_method = "list_assets" @@ -53,6 +55,10 @@ def _rollback(vo: Asset): ) vo.terminate() + params["state"] = "ACTIVE" + if "asset_id" not in params: + params["asset_id"] = utils.generate_id("asset") + asset_vo: Asset = self.asset_model.create(params) self.transaction.add_rollback(_rollback, asset_vo) @@ -60,7 +66,7 @@ def _rollback(vo: Asset): def update_asset_by_vo(self, params: dict, asset_vo: Asset) -> Asset: def _rollback(old_data): - _LOGGER.info(f'[ROLLBACK] Revert Data : {old_data.get("cloud_service_id")}') + _LOGGER.info(f'[ROLLBACK] Revert Data : {old_data.get("asset_id")}') asset_vo.update(old_data) self.transaction.add_rollback(_rollback, asset_vo.to_dict()) @@ -89,6 +95,140 @@ def get_asset( return self.asset_model.get(**conditions) + def list_assets( + self, + query: dict, + target: str = None, + change_filter: bool = False, + domain_id: str = None, + reference_filter: dict = None, + ) -> Tuple[QuerySet, int]: + if change_filter: + query = self._change_filter_tags(query) + query = self._change_only_tags(query) + query = self._change_sort_tags(query) + query = self._change_filter_project_group_id(query, domain_id) + + # Append Query for DELETED filter (Temporary Logic) + query = self._append_state_query(query) + + return self.asset_model.query( + **query, target=target, reference_filter=reference_filter + ) + + def _change_filter_tags(self, query: dict) -> dict: + change_filter = [] + + for condition in query.get("filter", []): + key = condition.get("k", condition.get("key")) + value = condition.get("v", condition.get("value")) + operator = condition.get("o", condition.get("operator")) + + if key.startswith("tags."): + hashed_key = self._get_hashed_key(key) + + change_filter.append( + {"key": hashed_key, "value": value, "operator": operator} + ) + + else: + change_filter.append(condition) + + query["filter"] = change_filter + return query + + def _change_only_tags(self, query: dict) -> dict: + change_only_tags = [] + if "only" in query: + for key in query.get("only", []): + if key.startswith("tags."): + hashed_key = self._get_hashed_key(key, only=True) + change_only_tags.append(hashed_key) + else: + change_only_tags.append(key) + query["only"] = change_only_tags + + return query + + def _change_sort_tags(self, query: dict) -> dict: + if sort_conditions := query.get("sort"): + change_filter = [] + for condition in sort_conditions: + sort_key = condition.get("key", "") + desc = condition.get("desc", False) + + if sort_key.startswith("tags."): + hashed_key = self._get_hashed_key(sort_key) + change_filter.append({"key": hashed_key, "desc": desc}) + else: + change_filter.append({"key": sort_key, "desc": desc}) + + query["sort"] = change_filter + + return query + + def _change_filter_project_group_id(self, query: dict, domain_id: str) -> dict: + change_filter = [] + self.identity_mgr = None + + for condition in query.get("filter", []): + key = condition.get("k", condition.get("key")) + value = condition.get("v", condition.get("value")) + operator = condition.get("o", condition.get("operator")) + + if key == "project_group_id": + if self.identity_mgr is None: + self.identity_mgr = IdentityManager() + + project_groups_info = self.identity_mgr.list_project_groups( + { + "query": { + "only": ["project_group_id"], + "filter": [{"k": key, "v": value, "o": operator}], + } + }, + domain_id, + ) + + project_group_ids = [ + project_group_info["project_group_id"] + for project_group_info in project_groups_info.get("results", []) + ] + + project_ids = [] + + for project_group_id in project_group_ids: + projects_info = self.identity_mgr.get_projects_in_project_group( + project_group_id + ) + project_ids.extend( + [ + project_info["project_id"] + for project_info in projects_info.get("results", []) + ] + ) + + project_ids = list(set(project_ids)) + change_filter.append({"k": "project_id", "v": project_ids, "o": "in"}) + + else: + change_filter.append(condition) + + query["filter"] = change_filter + return query + + @staticmethod + def _get_hashed_key(key: str, only: bool = False) -> str: + if key.count(".") < 2: + return key + + prefix, provider, key = key.split(".", 2) + hash_key = utils.string_to_hash(key) + if only: + return f"{prefix}.{provider}.{hash_key}" + else: + return f"{prefix}.{provider}.{hash_key}.value" + @staticmethod def merge_data(new_data: dict, old_data: dict) -> dict: for key in MERGE_KEYS: @@ -113,3 +253,25 @@ def merge_data(new_data: dict, old_data: dict) -> dict: del new_data[key] return new_data + + @staticmethod + def _append_state_query(query: dict) -> dict: + state_default_filter = {"key": "state", "value": "ACTIVE", "operator": "eq"} + + show_deleted_resource = False + for condition in query.get("filter", []): + key = condition.get("k", condition.get("key")) + value = condition.get("v", condition.get("value")) + operator = condition.get("o", condition.get("operator")) + + if key == "state": + if operator == "eq" and value == "DELETED": + show_deleted_resource = True + elif operator in ["in", "contain_in"] and "DELETED" in value: + show_deleted_resource = True + + if not show_deleted_resource: + query["filter"] = query.get("filter", []) + query["filter"].append(state_default_filter) + + return query diff --git a/src/spaceone/inventory_v2/manager/asset_type_manager.py b/src/spaceone/inventory_v2/manager/asset_type_manager.py new file mode 100644 index 0000000..9992124 --- /dev/null +++ b/src/spaceone/inventory_v2/manager/asset_type_manager.py @@ -0,0 +1,76 @@ +import copy +import logging +from typing import Tuple + +from spaceone.core.model.mongo_model import QuerySet +from spaceone.core import utils +from spaceone.core.manager import BaseManager +from spaceone.inventory_v2.model.asset_type.database import AssetType +from spaceone.inventory_v2.lib.resource_manager import ResourceManager + +_LOGGER = logging.getLogger(__name__) + + +class AssetTypeManager(BaseManager, ResourceManager): + resource_keys = ["asset_type_id"] + query_method = "list_asset_types" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.asset_type_model = AssetType + + def create_asset_type(self, params: dict) -> AssetType: + def _rollback(vo: AssetType): + _LOGGER.info( + f"[ROLLBACK] Delete Asset Type : {vo.name} ({vo.asset_type_id})" + ) + vo.delete() + + if "asset_type_id" not in params: + params["asset_type_id"] = utils.generate_id("asset-type") + if is_managed := params.get("is_managed", False): + params["is_managed"] = is_managed + + asset_type_vo: AssetType = self.asset_type_model.create(params) + self.transaction.add_rollback(_rollback, asset_type_vo) + + return asset_type_vo + + def update_asset_type_by_vo(self, params: dict, asset_type_vo: AssetType): + def _rollback(old_data): + _LOGGER.info(f'[ROLLBACK] Revert Data : {old_data.get("asset_Type_id")}') + asset_type_vo.update(old_data) + + self.transaction.add_rollback(_rollback, asset_type_vo.to_dict()) + + return asset_type_vo.update(params) + + @staticmethod + def delete_asset_type_by_vo(asset_type_vo: AssetType) -> None: + asset_type_vo.delete() + + def get_asset_type( + self, + asset_Type_id: str, + domain_id: str, + workspace_id: str = None, + ) -> AssetType: + conditions = { + "asset_Type_id": asset_Type_id, + "domain_id": domain_id, + } + + if workspace_id: + conditions.update({"workspace_id": workspace_id}) + + return self.asset_type_model.get(**conditions) + + def filter_asset_types(self, **conditions) -> QuerySet: + return self.asset_type_model.filter(**conditions) + + def list_asset_types(self, query: dict) -> Tuple[QuerySet, int]: + asset_type_vos, total_count = self.asset_type_model.query(**query) + return asset_type_vos, total_count + + def stat_asset_types(self, query: dict) -> dict: + return self.asset_type_model.stat(**query) diff --git a/src/spaceone/inventory_v2/manager/cleanup_manager.py b/src/spaceone/inventory_v2/manager/cleanup_manager.py new file mode 100644 index 0000000..62577f3 --- /dev/null +++ b/src/spaceone/inventory_v2/manager/cleanup_manager.py @@ -0,0 +1,136 @@ +import logging +from typing import Tuple +from datetime import datetime, timedelta + +from spaceone.core import config +from spaceone.core.manager import BaseManager +from spaceone.inventory_v2.manager.collection_state_manager import ( + CollectionStateManager, +) +from spaceone.inventory_v2.manager.asset_manager import AssetManager +from spaceone.inventory_v2.conf.collector_conf import * + +_LOGGER = logging.getLogger(__name__) + + +class CleanupManager(BaseManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def update_disconnected_and_deleted_count( + self, collector_id: str, secret_id: str, job_task_id: str, domain_id: str + ) -> dict: + state_mgr = CollectionStateManager() + disconnected_count = self._increment_disconnected_count_by_collector( + state_mgr, collector_id, secret_id, job_task_id, domain_id + ) + deleted_count = self._delete_resources_by_collector( + state_mgr, collector_id, domain_id + ) + + return { + "disconnected_count": disconnected_count - deleted_count, + "deleted_count": deleted_count, + } + + def delete_resources_by_policy(self, resource_type, hour, domain_id): + updated_at = datetime.utcnow() - timedelta(hours=hour) + query = { + "filter": [ + {"k": "domain_id", "v": domain_id, "o": "eq"}, + {"k": "updated_at", "v": updated_at, "o": "lt"}, + ] + } + + if resource_type in ["inventory.CloudServiceType", "inventory.Region"]: + query["filter"].append({"k": "updated_by", "v": "manual", "o": "not"}) + + if resource_type not in RESOURCE_MAP: + _LOGGER.error(f"[delete_resources_by_policy] not found {resource_type}") + return 0 + + resource_manager = self.locator.get_manager(RESOURCE_MAP[resource_type][1]) + + try: + deleted_count = resource_manager.delete_resources(query) + if deleted_count > 0: + _LOGGER.debug( + f"[delete_resources_by_policy] {deleted_count} in {domain_id}" + ) + + return deleted_count + + except Exception as e: + _LOGGER.error(f"[delete_resources] {e}", exc_info=True) + return 0 + + @staticmethod + def _delete_resources_by_collector( + state_mgr: CollectionStateManager, collector_id: str, domain_id: str + ) -> int: + disconnected_count = config.get_global( + "DEFAULT_DISCONNECTED_STATE_DELETE_POLICY", 3 + ) + + query = { + "filter": [ + {"k": "collector_id", "v": collector_id, "o": "eq"}, + {"k": "disconnected_count", "v": disconnected_count, "o": "gte"}, + {"k": "domain_id", "v": domain_id, "o": "eq"}, + ], + "only": ["cloud_service_id"], + } + + state_vos, total_count = state_mgr.list_collection_states(query) + + asset_ids = [state_vo.asset_id for state_vo in state_vos] + total_deleted_count = 0 + + if len(asset_ids) > 0: + asset_mgr = AssetManager() + + query = { + "filter": [ + {"k": "asset_id", "v": asset_ids, "o": "in"}, + {"k": "domain_id", "v": domain_id, "o": "eq"}, + ] + } + + try: + deleted_count = asset_mgr.delete_resources(query) + _LOGGER.debug( + f"[_delete_resources_by_collector] delete asset {deleted_count} in {domain_id}" + ) + total_deleted_count = deleted_count + except Exception as e: + _LOGGER.error( + f"[_delete_resources_by_collector] delete cloud service error: {e}", + exc_info=True, + ) + + return total_deleted_count + + @staticmethod + def _increment_disconnected_count_by_collector( + state_mgr: CollectionStateManager, + collector_id: str, + secret_id: str, + job_task_id: str, + domain_id: str, + ): + updated_at = datetime.utcnow() - timedelta(hours=1) + + query = { + "filter": [ + {"k": "collector_id", "v": collector_id, "o": "eq"}, + {"k": "secret_id", "v": secret_id, "o": "eq"}, + {"k": "job_task_id", "v": job_task_id, "o": "not"}, + {"k": "domain_id", "v": domain_id, "o": "eq"}, + {"k": "updated_at", "v": updated_at, "o": "lt"}, + ] + } + + state_vos, total_count = state_mgr.list_collection_states(query) + state_vos.increment("disconnected_count") + + return total_count diff --git a/src/spaceone/inventory_v2/manager/collecting_manager.py b/src/spaceone/inventory_v2/manager/collecting_manager.py index 9b5ed6e..5a6c5ef 100644 --- a/src/spaceone/inventory_v2/manager/collecting_manager.py +++ b/src/spaceone/inventory_v2/manager/collecting_manager.py @@ -19,7 +19,6 @@ from spaceone.inventory_v2.error import * from spaceone.inventory_v2.lib import rule_matcher from spaceone.inventory_v2.conf.collector_conf import * -from spaceone.inventory_v2.service.asset_service import AssetService _LOGGER = logging.getLogger(__name__) @@ -51,6 +50,9 @@ def collecting_resources(self, params: dict) -> bool: 'token': 'str' } """ + from spaceone.core import model + + model.init_all(False) # set token to transaction meta token = params["token"] @@ -236,6 +238,9 @@ def _upsert_collecting_resources( # total_count -= 1 pass + elif resource_type in ["inventory.Region"]: + pass + else: upsert_result = self._upsert_resource( resource_data, params, job_task_vo @@ -377,6 +382,18 @@ def _upsert_resource( response = ERROR + if resource_type in [ + "inventory.AssetType", + "inventory.AssetGroup", + "inventory.NamespaceGroup", + "inventory.Namespace", + "inventory.Metric", + "inventory.Region", + ]: + request_data["resource_group"] = "DOMAIN" + request_data["workspace_id"] = "*" + request_data["is_managed"] = True + if resource_state == "FAILURE": error_message = resource_data.get("message", "Unknown error.") _LOGGER.error( @@ -453,6 +470,7 @@ def _upsert_resource( _LOGGER.error( f"[_upsert_resource] resource upsert error ({job_task_id}): {e.message}" ) + _LOGGER.error(request_data) additional = self._set_error_addition_info( resource_type, total_count, request_data ) @@ -513,7 +531,7 @@ def _get_resource_map(self, resource_type: str): if resource_type in self._service_and_manager_map: return self._service_and_manager_map[resource_type] - service: AssetService = self.locator.get_service(RESOURCE_MAP[resource_type][0]) + service = self.locator.get_service(RESOURCE_MAP[resource_type][0]) manager = self.locator.get_manager(RESOURCE_MAP[resource_type][1]) self._service_and_manager_map[resource_type] = service, manager @@ -559,7 +577,7 @@ def _query_with_match_rules( match_rules (list): e.g. {1:['reference.resource_id'], 2:['name']} Return: - match_resource (dict) : resource_id for update (e.g. {'cloud_service_id': 'cloud-svc-abcde12345'}) + match_resource (dict) : resource_id for update (e.g. {'asset_id': 'asset-abcde12345'}) total_count (int) : total count of matched resources """ diff --git a/src/spaceone/inventory_v2/manager/collection_state_manager.py b/src/spaceone/inventory_v2/manager/collection_state_manager.py index 2b7ef3d..41b0393 100644 --- a/src/spaceone/inventory_v2/manager/collection_state_manager.py +++ b/src/spaceone/inventory_v2/manager/collection_state_manager.py @@ -19,7 +19,7 @@ def __init__(self, *args, **kwargs): def create_collection_state(self, asset_id: str, domain_id: str) -> None: def _rollback(vo: CollectionState): _LOGGER.info( - f"[ROLLBACK] Delete collection state: cloud_service_id = {vo.cloud_service_id}, " + f"[ROLLBACK] Delete collection state: asset_id = {vo.asset_id}, " f"collector_id = {vo.collector_id}" ) vo.terminate() @@ -41,7 +41,7 @@ def update_collection_state_by_vo( ) -> CollectionState: def _rollback(old_data): _LOGGER.info( - f"[ROLLBACK] Revert collection state : cloud_service_id = {state_vo.cloud_service_id}, " + f"[ROLLBACK] Revert collection state : asset_id = {state_vo.asset_id}, " f"collector_id = {state_vo.collector_id}" ) state_vo.update(old_data) @@ -56,13 +56,13 @@ def reset_collection_state(self, state_vo: CollectionState) -> None: self.update_collection_state_by_vo(params, state_vo) def get_collection_state( - self, cloud_service_id: str, domain_id: str + self, asset_id: str, domain_id: str ) -> Union[CollectionState, None]: if self.collector_id and self.secret_id: state_vos = self.collection_state_model.filter( collector_id=self.collector_id, secret_id=self.secret_id, - cloud_service_id=cloud_service_id, + asset_id=asset_id, domain_id=domain_id, ) @@ -77,18 +77,16 @@ def filter_collection_states(self, **conditions) -> QuerySet: def list_collection_states(self, query: dict) -> Tuple[QuerySet, int]: return self.collection_state_model.query(**query) - def delete_collection_state_by_cloud_service_id( + def delete_collection_state_by_asset_id( self, resource_id: str, domain_id: str ) -> None: state_vos = self.filter_collection_states( - cloud_service_id=resource_id, domain_id=domain_id + asset_id=resource_id, domain_id=domain_id ) state_vos.delete() - def delete_collection_state_by_cloud_service_ids( - self, cloud_service_ids: List[str] - ) -> None: - state_vos = self.filter_collection_states(cloud_service_id=cloud_service_ids) + def delete_collection_state_by_asset_ids(self, asset_ids: List[str]) -> None: + state_vos = self.filter_collection_states(asset_id=asset_ids) state_vos.delete() def delete_collection_state_by_collector_id( diff --git a/src/spaceone/inventory_v2/manager/collector_plugin_manager.py b/src/spaceone/inventory_v2/manager/collector_plugin_manager.py index 9a10b81..4104b33 100644 --- a/src/spaceone/inventory_v2/manager/collector_plugin_manager.py +++ b/src/spaceone/inventory_v2/manager/collector_plugin_manager.py @@ -1,7 +1,10 @@ import logging from typing import Generator, Union from spaceone.core.manager import BaseManager -from spaceone.core.connector.space_connector import SpaceConnector + +from spaceone.inventory_v2.connector import ( + BaseCollectorPluginConnector as PluginConnector, +) __ALL__ = ["CollectorPluginManager"] @@ -9,18 +12,26 @@ class CollectorPluginManager(BaseManager): - def init_plugin(self, endpoint: str, options: dict) -> dict: - plugin_connector: SpaceConnector = self.locator.get_connector( - SpaceConnector, endpoint=endpoint, token="NO_TOKEN" - ) - return plugin_connector.dispatch("Collector.init", {"options": options}) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @staticmethod + def init_plugin(endpoint: str, options: dict) -> dict: + return PluginConnector.init_plugin(endpoint, options) def verify_plugin(self, endpoint: str, options: dict, secret_data: dict) -> None: - plugin_connector: SpaceConnector = self.locator.get_connector( - SpaceConnector, endpoint=endpoint, token="NO_TOKEN" + self.collector_version = options.get("collector_version", "v1") + collector_plugin_conn = PluginConnector.get_connector_by_collector_version( + self.collector_version + ) + collector_plugin_conn.verify_plugin(endpoint, options, secret_data) + + def get_tasks(self, endpoint: str, secret_data: dict, options: dict) -> dict: + self.collector_version = options.get("collector_version", "v1") + collector_plugin_conn: PluginConnector = ( + PluginConnector.get_connector_by_collector_version(self.collector_version) ) - params = {"options": options, "secret_data": secret_data} - plugin_connector.dispatch("Collector.verify", params) + return collector_plugin_conn.get_tasks(endpoint, secret_data, options) def collect( self, @@ -29,21 +40,10 @@ def collect( secret_data: dict, task_options: dict = None, ) -> Generator[dict, None, None]: - plugin_connector: SpaceConnector = self.locator.get_connector( - SpaceConnector, endpoint=endpoint, token="NO_TOKEN" + self.collector_version = options.get("collector_version", "v1") + collector_plugin_conn = PluginConnector.get_connector_by_collector_version( + self.collector_version ) - - params = {"options": options, "secret_data": secret_data, "filter": {}} - - if task_options: - params["task_options"] = task_options - - return plugin_connector.dispatch("Collector.collect", params) - - def get_tasks(self, endpoint: str, secret_data: dict, options: dict) -> dict: - plugin_connector: SpaceConnector = self.locator.get_connector( - SpaceConnector, endpoint=endpoint, token="NO_TOKEN" + return collector_plugin_conn.collect( + endpoint, options, secret_data, task_options ) - - params = {"options": options, "secret_data": secret_data} - return plugin_connector.dispatch("Job.get_tasks", params) diff --git a/src/spaceone/inventory_v2/manager/job_manager.py b/src/spaceone/inventory_v2/manager/job_manager.py index 7c0140e..2b623c4 100644 --- a/src/spaceone/inventory_v2/manager/job_manager.py +++ b/src/spaceone/inventory_v2/manager/job_manager.py @@ -5,13 +5,12 @@ from spaceone.core.manager import BaseManager from spaceone.core.model.mongo_model import QuerySet from spaceone.inventory_v2.error import * +from spaceone.inventory_v2.manager.metric_data_manager import MetricDataManager +from spaceone.inventory_v2.manager.metric_manager import MetricManager +from spaceone.inventory_v2.model import JobTask from spaceone.inventory_v2.model.collector.database import Collector from spaceone.inventory_v2.model.job.database import Job -# from spaceone.inventory_v2.model.job_task_model import JobTask -# from spaceone.inventory_v2.manager.metric_manager import MetricManager -# from spaceone.inventory_v2.manager.metric_data_manager import MetricDataManager - _LOGGER = logging.getLogger(__name__) @@ -19,6 +18,7 @@ class JobManager(BaseManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.job_model = Job + self.job_task_model = JobTask def create_job(self, collector_vo: Collector, params: dict) -> Job: """Create Job for collect method @@ -106,8 +106,7 @@ def decrease_remained_tasks_by_vo(self, job_vo: Job) -> None: self._run_metric_queries(job_vo.plugin_id, job_vo.domain_id) def _is_changed(self, job_vo: Job) -> bool: - job_task_model: JobTask = self.locator.get_model("JobTask") - job_task_vos: List[JobTask] = job_task_model.filter( + job_task_vos: List[JobTask] = self.job_task_model.filter( job_id=job_vo.job_id, domain_id=job_vo.domain_id ) is_changed = False diff --git a/src/spaceone/inventory_v2/manager/job_task_manager.py b/src/spaceone/inventory_v2/manager/job_task_manager.py index 0385194..f07d55e 100644 --- a/src/spaceone/inventory_v2/manager/job_task_manager.py +++ b/src/spaceone/inventory_v2/manager/job_task_manager.py @@ -9,6 +9,7 @@ from spaceone.core.scheduler.task_schema import SPACEONE_TASK_SCHEMA from spaceone.core.model.mongo_model import QuerySet +from spaceone.inventory_v2.manager.cleanup_manager import CleanupManager from spaceone.inventory_v2.manager.job_manager import JobManager # from spaceone.inventory.manager.cleanup_manager import CleanupManager @@ -190,21 +191,22 @@ def _update_collecting_count_info( if isinstance(value, int) and value > 0: job_task_vo.increment(key, value) - # def _update_disconnected_and_deleted_count(self, job_task_vo: JobTask) -> dict: - # try: - # cleanup_mgr: CleanupManager = self.locator.get_manager(CleanupManager) - # return cleanup_mgr.update_disconnected_and_deleted_count( - # job_task_vo.collector_id, - # job_task_vo.secret_id, - # job_task_vo.job_task_id, - # job_task_vo.domain_id, - # ) - # except Exception as e: - # _LOGGER.error(f"[_update_collection_state] failed: {e}") - # return { - # "disconnected_count": 0, - # "deleted_count": 0, - # } + @staticmethod + def _update_disconnected_and_deleted_count(job_task_vo: JobTask) -> dict: + try: + cleanup_mgr = CleanupManager() + return cleanup_mgr.update_disconnected_and_deleted_count( + job_task_vo.collector_id, + job_task_vo.secret_id, + job_task_vo.job_task_id, + job_task_vo.domain_id, + ) + except Exception as e: + _LOGGER.error(f"[_update_collection_state] failed: {e}") + return { + "disconnected_count": 0, + "deleted_count": 0, + } @staticmethod def delete_job_task_by_vo(job_task_vo: JobTask) -> None: diff --git a/src/spaceone/inventory_v2/manager/managed_resource_manager.py b/src/spaceone/inventory_v2/manager/managed_resource_manager.py new file mode 100644 index 0000000..b3a269c --- /dev/null +++ b/src/spaceone/inventory_v2/manager/managed_resource_manager.py @@ -0,0 +1,37 @@ +import logging +import os +from typing import List + +from spaceone.core import utils +from spaceone.core.manager import BaseManager + +_LOGGER = logging.getLogger(__name__) +CURRENT_DIR = os.path.dirname(__file__) +_NAMESPACE_DIR = os.path.join(CURRENT_DIR, "../managed_resource/namespace/") +_METRIC_DIR = os.path.join(CURRENT_DIR, "../managed_resource/metric/") + + +class ManagedResourceManager(BaseManager): + def get_managed_namespaces(self) -> dict: + namespace_map = {} + for namespace_info in self._load_managed_resources(_NAMESPACE_DIR): + namespace_map[namespace_info["namespace_id"]] = namespace_info + + return namespace_map + + def get_managed_metrics(self) -> dict: + metric_map = {} + for metric_info in self._load_managed_resources(_METRIC_DIR): + metric_map[metric_info["metric_id"]] = metric_info + + return metric_map + + @staticmethod + def _load_managed_resources(dir_path: str) -> List[dict]: + managed_resources = [] + for filename in os.listdir(dir_path): + if filename.endswith(".yaml"): + file_path = os.path.join(dir_path, filename) + managed_resource_info = utils.load_yaml_from_file(file_path) + managed_resources.append(managed_resource_info) + return managed_resources diff --git a/src/spaceone/inventory_v2/manager/metric_data_manager.py b/src/spaceone/inventory_v2/manager/metric_data_manager.py index b4593b5..6c4a486 100644 --- a/src/spaceone/inventory_v2/manager/metric_data_manager.py +++ b/src/spaceone/inventory_v2/manager/metric_data_manager.py @@ -6,12 +6,12 @@ from spaceone.core.model.mongo_model import QuerySet from spaceone.core.manager import BaseManager from spaceone.core import utils, cache -from spaceone.inventory.model.metric_data.database import ( +from spaceone.inventory_v2.model.metric_data.database import ( MetricData, MonthlyMetricData, MetricQueryHistory, ) -from spaceone.inventory.error.metric import ( +from spaceone.inventory_v2.error.metric import ( ERROR_INVALID_DATE_RANGE, ERROR_INVALID_PARAMETER_TYPE, ) diff --git a/src/spaceone/inventory_v2/manager/metric_manager.py b/src/spaceone/inventory_v2/manager/metric_manager.py index 440bb29..6555f9d 100644 --- a/src/spaceone/inventory_v2/manager/metric_manager.py +++ b/src/spaceone/inventory_v2/manager/metric_manager.py @@ -18,7 +18,7 @@ from spaceone.inventory_v2.manager.managed_resource_manager import ( ManagedResourceManager, ) -from spaceone.inventory_v2.manager.cloud_service_manager import CloudServiceManager +from spaceone.inventory_v2.manager.asset_manager import AssetManager from spaceone.inventory_v2.manager.metric_data_manager import MetricDataManager _LOGGER = logging.getLogger(__name__) @@ -331,8 +331,8 @@ def _analyze_cloud_service( query["select"][group_by_key] = group_by_key _LOGGER.debug(f"[_analyze_cloud_service] Analyze Query: {query}") - cloud_svc_mgr = CloudServiceManager() - response = cloud_svc_mgr.analyze_cloud_services( + asset_mgr = AssetManager() + response = asset_mgr.analyze_assets( query, change_filter=True, domain_id=domain_id ) return response.get("results", []) diff --git a/src/spaceone/inventory_v2/model/__init__.py b/src/spaceone/inventory_v2/model/__init__.py index cdd606b..ef7405a 100644 --- a/src/spaceone/inventory_v2/model/__init__.py +++ b/src/spaceone/inventory_v2/model/__init__.py @@ -1,3 +1,5 @@ +from spaceone.inventory_v2.model.asset.database import Asset +from spaceone.inventory_v2.model.asset_type.database import AssetType from spaceone.inventory_v2.model.region.region_model import Region from spaceone.inventory_v2.model.collector.database import Collector from spaceone.inventory_v2.model.collector_rule.database import CollectorRule diff --git a/src/spaceone/inventory_v2/model/asset/database.py b/src/spaceone/inventory_v2/model/asset/database.py index 9c26c2a..d21f4a8 100644 --- a/src/spaceone/inventory_v2/model/asset/database.py +++ b/src/spaceone/inventory_v2/model/asset/database.py @@ -9,7 +9,7 @@ class Asset(MongoModel): - asset_id = StringField(max_length=40, generate_id="asset", unique=True) + asset_id = StringField(max_length=40, unique=True) name = StringField(default=None, null=True) state = StringField(max_length=20, choices=("ACTIVE", "DELETED"), default="ACTIVE") resource_id = StringField(max_length=255, default=None, null=True) diff --git a/src/spaceone/inventory_v2/model/asset/request.py b/src/spaceone/inventory_v2/model/asset/request.py index e36bcea..57b05ad 100644 --- a/src/spaceone/inventory_v2/model/asset/request.py +++ b/src/spaceone/inventory_v2/model/asset/request.py @@ -4,12 +4,55 @@ __all__ = [ "AssetCreateRequest", "AssetUpdateRequest", + "AssetGetRequest", + "AssetSearchQueryRequest", ] class AssetCreateRequest(BaseModel): - pass + asset_code: Union[str, None] + name: Union[str, None] + provider: str + asset_type: str + ipaddresses: Union[List[str], None] + account: Union[str, None] + instance_type: Union[str, None] + instance_size: Union[float, None] + data: dict + metadata: Union[dict, None] + tags: Union[dict, None] + region: Union[str, None] + project_id: str + workspace_id: str + domain_id: str class AssetUpdateRequest(BaseModel): - pass + asset_id: str + name: Union[str, None] + account: Union[str, None] + instance_type: Union[str, None] + instance_size: Union[float, None] + ip_addresses: Union[List[str], None] + data: dict + metadata: Union[dict, None] + tags: Union[dict, None] + region: Union[str, None] + user_projects: Union[List[str], None] + project_id: Union[str, None] + workspace_id: str + domain_id: str + + +class AssetGetRequest(BaseModel): + asset_id: str + user_projects: List[str] + workspace_id: str + domain_id: str + + +class AssetSearchQueryRequest(BaseModel): + query: Union[dict, None] = None + user_projects: List[str] + workspace_id: Union[str, None] = None + domain_id: str diff --git a/src/spaceone/inventory_v2/model/asset/response.py b/src/spaceone/inventory_v2/model/asset/response.py index e459f63..7563421 100644 --- a/src/spaceone/inventory_v2/model/asset/response.py +++ b/src/spaceone/inventory_v2/model/asset/response.py @@ -1,5 +1,32 @@ +from datetime import datetime +from typing import Union, List + from pydantic import BaseModel +from spaceone.core import utils class AssetResponse(BaseModel): - pass + asset_id: Union[str, None] + asset_code: Union[str, None] + name: Union[str, None] + provider: Union[str, None] + ipaddresses: Union[List[str], None] + account: Union[str, None] + instance_type: Union[str, None] + instance_size: Union[float, None] + data: Union[dict, None] + metadata: Union[dict, None] + tags: Union[dict, None] + region: Union[str, None] + asset_type_id: Union[str, None] + project_id: Union[str, None] + workspace_id: Union[str, None] + domain_id: Union[str, None] + created_at: Union[datetime, None] + updated_at: Union[datetime, None] + + def dict(self, *args, **kwargs): + data = super().dict(*args, **kwargs) + data["created_at"] = utils.datetime_to_iso8601(data["created_at"]) + data["updated_at"] = utils.datetime_to_iso8601(data["updated_at"]) + return data diff --git a/src/spaceone/inventory_v2/model/asset_type/database.py b/src/spaceone/inventory_v2/model/asset_type/database.py index fa69222..5118d8f 100644 --- a/src/spaceone/inventory_v2/model/asset_type/database.py +++ b/src/spaceone/inventory_v2/model/asset_type/database.py @@ -4,22 +4,21 @@ class AssetType(MongoModel): - asset_type_id = StringField(max_length=40, generate_id="asset-type", unique=True) + asset_type_id = StringField(max_length=40, unique=True) name = StringField( max_length=255, - unique_with=["provider", "asset_group_id", "workspace_id", "domain_id"], + unique_with=["provider", "workspace_id", "domain_id"], ) + description = StringField(max_length=255, default=None, null=True) + icon = StringField(max_length=255, default=None, null=True) provider = StringField(max_length=255) - asset_grou_id = StringField(max_length=255) - # cloud_service_type_key = StringField(max_length=255) - # ref_cloud_service_type = StringField(max_length=255) - service_code = StringField(max_length=255, default=None, null=True) - # is_primary = BooleanField(default=False) - # is_major = BooleanField(default=False) - resource_type = StringField(max_length=255) - labels = ListField(StringField(max_length=255)) metadata = DictField() tags = DictField() + is_managed = BooleanField(default=None, null=True) + resource_group = StringField( + max_length=255, required=True, choices=("DOMAIN", "WORKSPACE") + ) + asset_groups = StringField(max_length=255) workspace_id = StringField(max_length=40) domain_id = StringField(max_length=40) updated_by = StringField(default=None, null=True) @@ -28,28 +27,20 @@ class AssetType(MongoModel): meta = { "updatable_fields": [ - # "cloud_service_type_key", - "service_code", - # "is_primary", - # "is_major", - "resource_type", + "name", + "description", + "icon", "metadata", - "labels", "tags", + "asset_groups", "updated_by", "updated_at", ], - "minimal_fields": [ - "asset_type_id", - "name", - "provider", - "group", - "service_code", - # "is_primary", - # "is_major", - "resource_type", - ], - "ordering": ["provider", "group", "name"], + "minimal_fields": ["asset_type_id", "name", "provider", "asset_groups"], + "change_query_keys": { + "asset_groups": "asset_group_id", + }, + "ordering": ["provider", "name", "resource_group"], "indexes": [ { "fields": ["domain_id", "-updated_at", "updated_by"], @@ -64,9 +55,8 @@ class AssetType(MongoModel): "domain_id", "workspace_id", "provider", - "group", + "asset_groups", "name", - "is_primary", ], "name": "COMPOUND_INDEX_FOR_SEARCH_2", }, diff --git a/src/spaceone/inventory_v2/model/asset_type/request.py b/src/spaceone/inventory_v2/model/asset_type/request.py new file mode 100644 index 0000000..a8000e1 --- /dev/null +++ b/src/spaceone/inventory_v2/model/asset_type/request.py @@ -0,0 +1,65 @@ +from typing import Union, Literal, List +from pydantic import BaseModel + +__all__ = [ + "AsseTypeCreateRequest", + "AsseTypeUpdateRequest", + "AssetTypeAddAssetGroupRequest", + "AssetTypeRemoveAssetGroupRequest", + "AssetTypeDeleteRequest", + "AssetTypeGetRequest", + "AssetTypeSearchQueryRequest", +] + +ResourceGroup = Literal["DOMAIN", "WORKSPACE"] + + +class AsseTypeCreateRequest(BaseModel): + asset_type_id: Union[str, None] + name: str + description: Union[str, None] + icon: Union[str, None] + provider: str + metadata: Union[dict, None] + tags: Union[dict, None] + asset_groups: Union[List[str], str] + resource_group: ResourceGroup + workspace_id: Union[str, None] + domain_id: str + + +class AsseTypeUpdateRequest(BaseModel): + name: Union[str, None] + description: Union[str, None] + icon: Union[str, None] + provider: str + metadata: Union[dict, None] + tags: Union[dict, None] + workspace_id: Union[str, None] + domain_id: str + + +class AssetTypeAddAssetGroupRequest(BaseModel): + pass + + +class AssetTypeRemoveAssetGroupRequest(BaseModel): + pass + + +class AssetTypeDeleteRequest(BaseModel): + asset_type_id: str + + +class AssetTypeGetRequest(BaseModel): + asset_type_id: str + workspace_id: Union[str, None] + domain_id: str + + +class AssetTypeSearchQueryRequest(BaseModel): + query: Union[dict, None] = None + exists_only: Union[bool, None] = None + users_projects: Union[List[str], None] = None + workspace_id: Union[str, None] = None + domain_id: str diff --git a/src/spaceone/inventory_v2/model/asset_type/response.py b/src/spaceone/inventory_v2/model/asset_type/response.py new file mode 100644 index 0000000..bc1f6a5 --- /dev/null +++ b/src/spaceone/inventory_v2/model/asset_type/response.py @@ -0,0 +1,10 @@ +from typing import Union, Literal, List +from pydantic import BaseModel + +__all__ = ["AssetTypeResponse"] + +ResourceGroup = Literal["DOMAIN", "WORKSPACE"] + + +class AssetTypeResponse(BaseModel): + pass diff --git a/src/spaceone/inventory_v2/model/collector/database.py b/src/spaceone/inventory_v2/model/collector/database.py index dbb7aeb..e5db366 100644 --- a/src/spaceone/inventory_v2/model/collector/database.py +++ b/src/spaceone/inventory_v2/model/collector/database.py @@ -51,6 +51,7 @@ class Collector(MongoModel): schedule = EmbeddedDocumentField(Scheduled, default=None, null=False) secret_filter = EmbeddedDocumentField(SecretFilter, default=None, null=True) tags = DictField() + created_by = StringField(max_length=255) resource_group = StringField(max_length=40, choices=("DOMAIN", "WORKSPACE")) workspace_id = StringField(max_length=40) domain_id = StringField(max_length=40) @@ -72,6 +73,7 @@ class Collector(MongoModel): "name", "provider", "plugin_info", + "created_by", "resource_group", ], "change_query_keys": { diff --git a/src/spaceone/inventory_v2/model/collector/response.py b/src/spaceone/inventory_v2/model/collector/response.py index 0c31422..f7904a6 100644 --- a/src/spaceone/inventory_v2/model/collector/response.py +++ b/src/spaceone/inventory_v2/model/collector/response.py @@ -20,6 +20,7 @@ class CollectorResponse(BaseModel): plugin_info: Union[dict, None] = None schedule: Union[dict, None] = None tags: Union[dict, None] = None + created_by: Union[str, None] = None resource_group: ResourceGroup workspace_id: Union[str, None] = None domain_id: Union[str, None] = None diff --git a/src/spaceone/inventory_v2/model/metric/request.py b/src/spaceone/inventory_v2/model/metric/request.py index 7a8c1a6..3210c18 100644 --- a/src/spaceone/inventory_v2/model/metric/request.py +++ b/src/spaceone/inventory_v2/model/metric/request.py @@ -75,6 +75,7 @@ class MetricSearchQueryRequest(BaseModel): metric_type: Union[MetricType, None] = None resource_type: Union[str, None] = None is_managed: Union[bool, None] = None + exists_only: Union[bool, None] = None namespace_id: Union[str, None] = None workspace_id: Union[str, list, None] = None domain_id: str diff --git a/src/spaceone/inventory_v2/service/__init__.py b/src/spaceone/inventory_v2/service/__init__.py index 02d7275..607f444 100644 --- a/src/spaceone/inventory_v2/service/__init__.py +++ b/src/spaceone/inventory_v2/service/__init__.py @@ -1 +1,3 @@ +from spaceone.inventory_v2.service.asset_service import AssetService +from spaceone.inventory_v2.service.asset_type_service import AssetTypeService from spaceone.inventory_v2.service.region_service import RegionService diff --git a/src/spaceone/inventory_v2/service/asset_service.py b/src/spaceone/inventory_v2/service/asset_service.py index 5d8b3f8..f2dcd80 100644 --- a/src/spaceone/inventory_v2/service/asset_service.py +++ b/src/spaceone/inventory_v2/service/asset_service.py @@ -8,6 +8,9 @@ from spaceone.core import utils from spaceone.inventory_v2.manager.asset_manager import AssetManager +from spaceone.inventory_v2.manager.collection_state_manager import ( + CollectionStateManager, +) from spaceone.inventory_v2.manager.collector_rule_manager import CollectorRuleManager from spaceone.inventory_v2.manager.identity_manager import IdentityManager from spaceone.inventory_v2.model.asset.database import Asset @@ -16,7 +19,7 @@ from spaceone.inventory_v2.error import * _KEYWORD_FILTER = [ - "cloud_service_id", + "asset_id", "name", "ip_addresses", "cloud_service_group", @@ -32,12 +35,14 @@ @mutation_handler @event_handler class AssetService(BaseService): - resource = "Asset" + resource = "AssetType" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.asset_mgr = AssetManager() self.collector_rule_mgr = CollectorRuleManager() + self.state_mgr = CollectionStateManager() + self.identity_mgr = IdentityManager() self.collector_id = self.transaction.get_meta("collector_id") self.job_id = self.transaction.get_meta("job_id") @@ -76,6 +81,7 @@ def create(self, params: AssetCreateRequest) -> Union[AssetResponse, dict]: cloud_service_vo (object) """ + # check if asset type is last asset_vo = self.create_resource(params.dict()) return AssetResponse(**asset_vo.to_dict()) @@ -130,11 +136,11 @@ def create_resource(self, params: dict) -> Asset: elif secret_project_id: params["project_id"] = secret_project_id - params["ref_cloud_service_type"] = self._make_cloud_service_type_key(params) + # params["ref_cloud_service_type"] = self._make_cloud_service_type_key(params) - if "region_code" in params: + if "region_id" in params: params["ref_region"] = self._make_region_key( - domain_id, workspace_id, provider, params["region_code"] + domain_id, provider, params["region_code"] ) if "metadata" in params: @@ -157,6 +163,7 @@ def create_resource(self, params: dict) -> Asset: permission="inventory-v2:CloudService.write", role_types=["WORKSPACE_OWNER", "WORKSPACE_MEMBER"], ) + @convert_model def update(self, params: AssetUpdateRequest) -> Union[AssetResponse, dict]: """ Args: @@ -170,7 +177,7 @@ def update(self, params: AssetUpdateRequest) -> Union[AssetResponse, dict]: 'data': 'dict', 'json_data': 'dict', 'metadata': 'dict', - 'reference': 'dict', + # 'reference': 'dict', 'tags': 'list or dict', 'region_code': 'str', 'project_id': 'str', @@ -182,10 +189,11 @@ def update(self, params: AssetUpdateRequest) -> Union[AssetResponse, dict]: Returns: cloud_service_vo (object) """ + # check if asset type is last asset_vo = self.update_resource(params.dict()) return AssetResponse(**asset_vo.to_dict()) - @check_required(["cloud_service_id", "workspace_id", "domain_id"]) + @check_required(["asset_id", "workspace_id", "domain_id"]) def update_resource(self, params: dict) -> Asset: # ch_mgr: ChangeHistoryManager = self.locator.get_manager("ChangeHistoryManager") @@ -209,7 +217,7 @@ def update_resource(self, params: dict) -> Asset: secret_project_id = self.transaction.get_meta("secret.project_id") - cloud_service_id = params["cloud_service_id"] + asset_id = params["asset_id"] workspace_id = params["workspace_id"] user_projects = params.get("user_projects") domain_id = params["domain_id"] @@ -232,7 +240,7 @@ def update_resource(self, params: dict) -> Asset: ) asset_vo: Asset = self.asset_mgr.get_asset( - cloud_service_id, domain_id, workspace_id, user_projects + asset_id, domain_id, workspace_id, user_projects ) if "project_id" in params: @@ -243,7 +251,6 @@ def update_resource(self, params: dict) -> Asset: if "region_code" in params: params["ref_region"] = self._make_region_key( asset_vo.domain_id, - asset_vo.workspace_id, asset_vo.provider, params["region_code"], ) @@ -286,14 +293,104 @@ def update_resource(self, params: dict) -> Asset: # ch_mgr.add_update_history(asset_vo, params, old_asset_data) # Update Collection History - state_vo = self.state_mgr.get_collection_state(cloud_service_id, domain_id) + state_vo = self.state_mgr.get_collection_state(asset_id, domain_id) if state_vo: self.state_mgr.reset_collection_state(state_vo) else: - self.state_mgr.create_collection_state(cloud_service_id, domain_id) + self.state_mgr.create_collection_state(asset_id, domain_id) return asset_vo + @transaction( + permission="inventory-v2:Asset.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"], + ) + @convert_model + def get(self, params: AssetGetRequest) -> Union[AssetResponse, dict]: + """ + Args: + params (dict): { + 'asset_id': 'str', # required + 'user_projects': 'list' # injected from auth + 'workspace_id': 'str', # injected from auth + 'domain_id': 'str', # injected from auth (required) + } + + Returns: + cloud_service_vo (object) + + """ + + asset_vo = self.asset_mgr.get_asset( + params.asset_id, params.domain_id, params.workspace_id, params.user_projects + ) + return AssetResponse(**asset_vo.to_dict()) + + @transaction( + permission="inventory-v2:Asset.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"], + ) + @append_query_filter( + [ + "asset_id", + "name", + "state", + "ip_address", + "account", + "instance_type", + "asset_type_id", + "cloud_service_group", + "provider", + "region_code", + "project_id", + "project_group_id", + "workspace_id", + "domain_id", + "user_projects", + ] + ) + @append_keyword_filter(_KEYWORD_FILTER) + @set_query_page_limit(1000) + @convert_model + def list(self, params: AssetSearchQueryRequest): + """ + Args: + params (dict): { + 'query': 'dict (spaceone.api.core.v1.Query)', + 'asset_id': 'str', + 'name': 'str', + 'state': 'str', + 'ip_address': 'str', + 'account': 'str', + 'instance_type': 'str', + 'cloud_service_type': 'str', + 'cloud_service_group': 'str', + 'provider': 'str', + 'region_code': 'str', + 'project_id': 'str', + 'project_group_id': 'str', + 'workspace_id': 'str', # injected from auth + 'domain_id': 'str', # injected from auth (required) + 'user_projects': 'list', # injected from auth + } + + Returns: + results (list) + total_count (int) + """ + + domain_id = params.domain_id + workspace_id = params.workspace_id + query = params.query or {} + reference_filter = {"domain_id": domain_id, "workspace_id": workspace_id} + + return self.asset_mgr.list_assets( + query, + change_filter=True, + domain_id=domain_id, + reference_filter=reference_filter, + ) + @staticmethod def _make_cloud_service_type_key(resource_data: dict) -> str: return ( @@ -302,10 +399,8 @@ def _make_cloud_service_type_key(resource_data: dict) -> str: ) @staticmethod - def _make_region_key( - domain_id: str, workspace_id: str, provider: str, region_code: str - ) -> str: - return f"{domain_id}.{workspace_id}.{provider}.{region_code}" + def _make_region_key(domain_id: str, provider: str, region_code: str) -> str: + return f"{domain_id}.{provider}.{region_code}" @staticmethod def _convert_metadata(metadata: dict, provider: str) -> dict: diff --git a/src/spaceone/inventory_v2/service/asset_type_service.py b/src/spaceone/inventory_v2/service/asset_type_service.py new file mode 100644 index 0000000..23904c4 --- /dev/null +++ b/src/spaceone/inventory_v2/service/asset_type_service.py @@ -0,0 +1,278 @@ +import logging +import copy +import pytz +from datetime import datetime +from typing import List, Union, Tuple + +from mongoengine import QuerySet +from spaceone.core.service import * +from spaceone.core import utils + +from spaceone.inventory_v2.manager import AssetTypeManager +from spaceone.inventory_v2.manager.asset_manager import AssetManager +from spaceone.inventory_v2.manager.collection_state_manager import ( + CollectionStateManager, +) +from spaceone.inventory_v2.manager.collector_rule_manager import CollectorRuleManager +from spaceone.inventory_v2.manager.identity_manager import IdentityManager +from spaceone.inventory_v2.model.asset.database import Asset +from spaceone.inventory_v2.model.asset.request import * +from spaceone.inventory_v2.model.asset.response import * +from spaceone.inventory_v2.error import * +from spaceone.inventory_v2.model.asset_type.database import AssetType +from spaceone.inventory_v2.model.asset_type.response import AssetTypeResponse + +_KEYWORD_FILTER = [ + "asset_id", + "name", + "ip_addresses", + "cloud_service_group", + "cloud_service_type", + "reference.resource_id", +] + +_LOGGER = logging.getLogger(__name__) + +_KEYWORD_FILTER = ["asset_type_id", "name", "asset_group_id", "service_code"] + + +@authentication_handler +@authorization_handler +@mutation_handler +@event_handler +class AssetTypeService(BaseService): + resource = "AssetType" + + def __init__(self, metadata): + super().__init__(metadata) + self.asset_type_mgr = AssetTypeManager() + + @transaction( + permission="inventory-v2:CloudServiceType.write", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER"], + ) + @convert_model + def create(self, params: AssetGetRequest) -> AssetTypeResponse: + """ + Args: + params (dict): { + 'name': 'str', # required + 'group': 'str', # required + 'provider': 'str', # required + 'service_code': 'str', + 'is_primary': 'bool', + 'is_major': 'bool', + 'resource_type': 'str', + 'json_metadata': 'str', + 'metadata': 'dict', + 'labels': 'list, + 'tags': 'dict', + 'workspace_id': 'str', # injected from auth (required) + 'domain_id': 'str' # injected from auth (required) + } + + Returns: + asset_type_vo (object) + """ + + asset_type_vo = self.create_resource(params.dict()) + return AssetTypeResponse(**asset_type_vo.to_dict()) + + @check_required(["name", "provider", "resource_group", "domain_id"]) + def create_resource(self, params: dict) -> AssetType: + if json_metadata := params.get("json_metadata"): + params["metadata"] = utils.load_json(json_metadata) + if not isinstance(params["metadata"], dict): + raise ERROR_INVALID_PARAMETER_TYPE( + key="json_metadata", type=type(params["metadata"]) + ) + + del params["json_metadata"] + + if "tags" in params: + if isinstance(params["tags"], list): + params["tags"] = utils.tags_to_dict(params["tags"]) + + params["updated_by"] = self.transaction.get_meta("collector_id") or "manual" + + provider = params.get("provider", self.transaction.get_meta("secret.provider")) + + if provider: + params["provider"] = provider + + params["resource_type"] = params.get("resource_type", "inventory.Asset") + + return self.asset_type_mgr.create_asset_type(params) + + @transaction( + permission="inventory:CloudServiceType.write", + role_types=["WORKSPACE_OWNER"], + ) + def update(self, params: dict) -> AssetType: + """ + Args: + params (dict): { + 'cloud_service_type_id': 'str', # required + 'service_code': 'str', + 'is_primary': 'bool', + 'is_major': 'bool', + 'resource_type': 'str', + 'json_metadata': 'str', + 'metadata': 'dict', + 'labels': 'list', + 'tags': 'dict', + 'workspace_id': 'str', # injected from auth (required) + 'domain_id': 'str' # injected from auth (required) + } + + Returns: + cloud_service_type_vo (object) + """ + + return self.update_resource(params) + + @check_required(["cloud_service_type_id", "workspace_id", "domain_id"]) + def update_resource(self, params: dict) -> AssetType: + if json_metadata := params.get("json_metadata"): + params["metadata"] = utils.load_json(json_metadata) + if not isinstance(params["metadata"], dict): + raise ERROR_INVALID_PARAMETER_TYPE( + key="json_metadata", type=type(params["metadata"]) + ) + + del params["json_metadata"] + + if "tags" in params: + if isinstance(params["tags"], list): + params["tags"] = utils.tags_to_dict(params["tags"]) + + params["updated_by"] = self.transaction.get_meta("collector_id") or "manual" + domain_id = params["domain_id"] + + cloud_svc_type_vo = self.asset_type_mgr.get_asset_type( + params["asset_type_id"], domain_id + ) + + return self.asset_type_mgr.update_asset_type_by_vo(params, cloud_svc_type_vo) + + @transaction( + permission="inventory:AssetType.write", + role_types=["WORKSPACE_OWNER"], + ) + def delete(self, params: dict) -> None: + """ + Args: + params (dict): { + 'cloud_service_type_id': 'str', # required + 'workspace_id': 'str', # injected from auth (required) + 'domain_id': 'str' # injected from auth (required) + } + Returns: + None + """ + + self.delete_resource(params) + + @check_required(["cloud_service_type_id", "workspace_id", "domain_id"]) + def delete_resource(self, params: dict) -> None: + asset_type_vo = self.asset_type_mgr.get_asset_type( + params["cloud_service_type_id"], params["domain_id"], params["workspace_id"] + ) + + self.asset_type_mgr.delete_asset_type_by_vo(asset_type_vo) + + @transaction( + permission="inventory:AssetType.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"], + ) + @check_required(["cloud_service_type_id", "domain_id"]) + def get(self, params: dict) -> AssetType: + """ + Args: + params (dict): { + 'cloud_service_type_id': 'str', # required + 'workspace_id': 'str', # injected from auth + 'domain_id': 'str', # injected from auth (required) + } + + Returns: + cloud_service_type_vo (object) + + """ + + return self.asset_type_mgr.get_asset_type( + params["asset_type_id"], + params["domain_id"], + params.get("workspace_id"), + ) + + @transaction( + permission="inventory:AssetType.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"], + ) + @check_required(["domain_id"]) + @append_query_filter( + [ + "cloud_service_type_id", + "name", + "provider", + "group", + "cloud_service_type_key", + "service_code", + "is_primary", + "is_major", + "resource_type", + "workspace_id", + "domain_id", + ] + ) + @append_keyword_filter(_KEYWORD_FILTER) + def list(self, params: dict) -> Tuple[QuerySet, int]: + """ + Args: + params (dict): { + 'query': 'dict (spaceone.api.core.v1.Query)', + 'cloud_service_type_id': 'str', + 'name': 'str', + 'group': 'str', + 'provider': 'str', + 'cloud_service_type_key': 'str', + 'service_code': 'str', + 'is_primary': 'str', + 'is_major': 'str', + 'resource_type': 'str', + 'workspace_id': 'str', # injected from auth + 'domain_id': 'str', # injected from auth (required) + } + + Returns: + results (list) + total_count (int) + + """ + + return self.asset_type_mgr.list_asset_types(params.get("query", {})) + + @transaction( + permission="inventory:AssetType.read", + role_types=["DOMAIN_ADMIN", "WORKSPACE_OWNER", "WORKSPACE_MEMBER"], + ) + @check_required(["query", "domain_id"]) + @append_query_filter(["workspace_id", "domain_id"]) + @append_keyword_filter(_KEYWORD_FILTER) + def stat(self, params: dict) -> dict: + """ + Args: + params (dict): { + 'query': 'dict (spaceone.api.core.v1.StatisticsQuery)', + 'workspace_id': 'str', # injected from auth + 'domain_id': 'str', # injected from auth (required) + } + + Returns: + values (list) : 'list of statistics data' + + """ + + query = params.get("query", {}) + return self.asset_type_mgr.stat_asset_types(query) diff --git a/src/spaceone/inventory_v2/service/collector_service.py b/src/spaceone/inventory_v2/service/collector_service.py index e079b40..459e113 100644 --- a/src/spaceone/inventory_v2/service/collector_service.py +++ b/src/spaceone/inventory_v2/service/collector_service.py @@ -1,4 +1,5 @@ import logging +import os from typing import Union, Tuple from mongoengine import QuerySet @@ -609,9 +610,7 @@ def _get_tasks( collector_workspace_id: str = None, ) -> list: secret_mgr: SecretManager = self.locator.get_manager(SecretManager) - collector_plugin_mgr: CollectorPluginManager = self.locator.get_manager( - CollectorPluginManager - ) + collector_plugin_mgr = CollectorPluginManager() tasks = [] secret_ids = self._get_secret_ids_from_filter( @@ -632,18 +631,13 @@ def _get_tasks( "domain_id": domain_id, } - try: - response = collector_plugin_mgr.get_tasks( - endpoint, - secret_data.get("data", {}), - plugin_info.get("options", {}), - ) - _LOGGER.debug(f"[get_tasks] sub tasks({collector_id}): {response}") - _task["sub_tasks"] = response.get("tasks", []) - - except Exception as e: - pass - + response = collector_plugin_mgr.get_tasks( + endpoint, + secret_data.get("data", {}), + plugin_info.get("options", {}), + ) + _LOGGER.debug(f"[get_tasks] sub tasks({collector_id}): {response}") + _task["sub_tasks"] = response.get("tasks", []) tasks.append(_task) return tasks