Skip to content

Commit

Permalink
Add --selected-only flag to release
Browse files Browse the repository at this point in the history
  • Loading branch information
@rehoumir
rehoumir committed Aug 12, 2024
1 parent b8753a7 commit 7272485
Show file tree
Hide file tree
Showing 10 changed files with 258 additions and 89 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ dist

dummy_project
temp_project
.vscode
.vscode
.DS_Store
166 changes: 105 additions & 61 deletions project_rossum_deploy/commands/migrate/helpers.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
import asyncio
import copy
from typing import Callable
from rich.progress import Progress
from rossum_api import ElisAPIClient
from rossum_api.api_client import Resource

from project_rossum_deploy.utils.consts import settings
from project_rossum_deploy.utils.consts import (
display_warning,
settings,
MAPPING_SELECTED_ATTRIBUTE,
)
from project_rossum_deploy.utils.functions import (
PauseProgress,
extract_id_from_url,
)
from project_rossum_deploy.utils.functions import find_object_by_id
Expand All @@ -15,6 +21,10 @@ def is_first_time_migration(submapping: dict):
return not submapping.get("target_object", None)


def check_if_selected(submapping: dict):
return bool(submapping.get(MAPPING_SELECTED_ATTRIBUTE, None))


# TODO: not used for release
async def should_upload_object(
client: ElisAPIClient, target_id: int, target_objects: list[dict]
Expand Down Expand Up @@ -42,72 +52,101 @@ def replace_dependency_url(
dependency: str,
source_id_target_pairs: dict[int, list],
target_object: dict = None,
progress: Progress = None,
):
if isinstance(object[dependency], list):
new_urls = []
for source_dependency_url in object[dependency]:
source_id = extract_id_from_url(source_dependency_url)
target_dependency_objects = source_id_target_pairs.get(source_id, [])
# The object was ignored during release, no target equivalents exist
if not len(target_dependency_objects):
continue
# Assume each object should have its own dependency
elif len(target_dependency_objects) == target_objects_count:
new_url = source_dependency_url.replace(
str(source_id),
str(target_dependency_objects[target_index]["id"]),
)
new_urls.append(new_url)
# All objects will have the same dependency
else:
new_url = source_dependency_url.replace(
str(source_id), str(target_dependency_objects[0]["id"])
)
new_urls.append(new_url)

# Target queues can have 'dangling' hooks that exist only on target, these should not be overwritten.
if target_object:
for target_dependency_url in target_object[dependency]:
# Target ID was found in the new list as well
if target_dependency_url in new_urls:
continue

target_has_source = False
# Check if this target has a source. If not, it is a dangling target and we need to add it back.
target_id = extract_id_from_url(target_dependency_url)
for _, targets in source_id_target_pairs.items():
for target in targets:
if target.get("id", "") == target_id:
target_has_source = True
break

if not target_has_source:
new_urls.append(target_dependency_url)

object[dependency] = new_urls
replace_list_of_dependency_urls(
object=object,
target_index=target_index,
target_objects_count=target_objects_count,
dependency=dependency,
source_id_target_pairs=source_id_target_pairs,
target_object=target_object,
progress=progress,
)
else:
source_dependency_url = object[dependency]
source_id = extract_id_from_url(source_dependency_url)
target_dependency_objects = source_id_target_pairs.get(source_id)

# The object was ignored during release, no target equivalents exist
if not len(target_dependency_objects):
return
target_dependency_objects = source_id_target_pairs.get(source_id, [])

# Assume each object should have its own dependency
if len(target_dependency_objects) == target_objects_count:
new_url = source_dependency_url.replace(
str(source_id),
str(target_dependency_objects[target_index]["id"]),
)
target_id_str = str(target_dependency_objects[target_index]["id"])
# All objects will have the same dependency
else:
new_url = source_dependency_url.replace(
str(source_id), str(target_dependency_objects[0]["id"])
)
target_id_str = str(target_dependency_objects[0]["id"])

source_id_str = str(source_id)
new_url = source_dependency_url.replace(
source_id_str,
target_id_str,
)
object[dependency] = new_url

if source_id_str == target_id_str:
if settings.IS_PROJECT_IN_SAME_ORG and dependency == "organization":
return

with PauseProgress(progress):
display_warning(
f'Dependency "{dependency}" for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.'
)


# TODO: refactor to use the same replace URL function
def replace_list_of_dependency_urls(
object: dict,
target_index: int,
target_objects_count: int,
dependency: str,
source_id_target_pairs: dict[int, list],
target_object: dict = None,
progress: Progress = None,
):
new_urls = []
for source_index, source_dependency_url in enumerate(object[dependency]):
source_id = extract_id_from_url(source_dependency_url)
target_dependency_objects = source_id_target_pairs.get(source_id, [])

# Assume each object should have its own dependency
if len(target_dependency_objects) == target_objects_count:
target_id_str = str(target_dependency_objects[target_index]["id"])
# All objects will have the same dependency
else:
target_id_str = str(target_dependency_objects[0]["id"])

source_id_str = str(source_id)
new_url = source_dependency_url.replace(source_id_str, target_id_str)

new_urls.append(new_url)

if source_id_str == target_id_str:
with PauseProgress(progress):
display_warning(
f'Dependency "{dependency}"[{source_index}] for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.'
)

# Target queues can have 'dangling' hooks that exist only on target, these should not be overwritten.
if target_object:
for target_dependency_url in target_object[dependency]:
# Target ID was found in the new list as well
if target_dependency_url in new_urls:
continue

target_has_source = False
# Check if this target has a source. If not, it is a dangling target and we need to add it back.
target_id = extract_id_from_url(target_dependency_url)
for _, targets in source_id_target_pairs.items():
for target in targets:
if target.get("id", "") == target_id:
target_has_source = True
break

if not target_has_source:
new_urls.append(target_dependency_url)

object[dependency] = new_urls


async def get_token_owner(client: ElisAPIClient):
async for user in client.list_all_users(username=client._http_client.username):
Expand Down Expand Up @@ -160,7 +199,9 @@ async def migrate_object_to_multiple_targets(
results = await asyncio.gather(*requests)
# asyncio.gather returns results in the same order as they were put in
for index, result in enumerate(results):
if result and result.get("id", None):
# In case of planning and selected_only mode, the same objects are returned
# They should target themselves
if result and (target_id := result.get("id", None)) != submapping.get("id", ""):
submapping.get("targets", [])[index]["target_id"] = result.get("id", None)

return list(filter(lambda x: x, results))
Expand All @@ -175,15 +216,18 @@ async def simulate_migrate_object(
target_index: int = 0,
target_objects_count: int = None,
source_id_target_pairs: dict[int, list] = None,
silent: bool = False,
):
object_counter = f"({target_index +1}/{target_objects_count if target_objects_count is not None else 1})"
if target_id:
print(
f'UPDATE source {target_object_type} "{source_object.get('id', None)} {source_object.get('name', '')}" -> target "{target_id}" {object_counter}.'
)
if not silent:
print(
f'UPDATE source {target_object_type} "{source_object.get('id', None)} {source_object.get('name', '')}" -> target "{target_id}" {object_counter}.'
)
return await client._http_client.fetch_one(target_object_type, target_id)
else:
print(
f'CREATE source {target_object_type} "{source_object.get('id', None)} {source_object.get('name', '')}" -> target {object_counter}.'
)
if not silent:
print(
f'CREATE source {target_object_type} "{source_object.get('id', None)} {source_object.get('name', '')}" -> target {object_counter}.'
)
return copy.deepcopy(source_object)
12 changes: 8 additions & 4 deletions project_rossum_deploy/commands/migrate/hooks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from rich.prompt import Prompt

from project_rossum_deploy.commands.migrate.helpers import (
check_if_selected,
get_token_owner,
migrate_object_to_multiple_targets,
simulate_migrate_object,
Expand Down Expand Up @@ -36,6 +37,7 @@ async def migrate_hooks(
sources_by_source_id_map: dict,
progress: Progress,
plan_only: bool = False,
selected_only: bool = False,
target_objects: list[dict] = [],
errors: dict = {},
force: bool = False,
Expand Down Expand Up @@ -73,18 +75,20 @@ async def migrate_hook(hook_path: Path):
)

hook_mapping = find_mapping_of_object(mapping["organization"]["hooks"], id)
if hook_mapping.get("ignore", None):
progress.update(task, advance=1)
return

skip_migration = hook_mapping.get("ignore", None) or (
selected_only and not check_if_selected(hook_mapping)
)

await update_hook_code(hook_path, hook)

if plan_only:
if plan_only or skip_migration:
partial_upload_hook = functools.partial(
simulate_migrate_object,
client=client,
source_object=hook,
target_object_type=Resource.Hook,
silent=skip_migration,
)
else:
partial_upload_hook = functools.partial(
Expand Down
33 changes: 30 additions & 3 deletions project_rossum_deploy/commands/migrate/migrate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@
default=False,
is_flag=True,
)
@click.option(
"--selected-only",
"-so",
default=False,
is_flag=True,
help="Releases only objects with a `selected: true` attribute in mapping.yaml and ignores the rest even without an ignore flag. Unlike ignore, this flag is not recursive = you have to put it on sub-objects (e.g., both queue AND inbox).",
)
@click.option(
"--force",
"-f",
Expand All @@ -75,17 +82,22 @@
)
@coro
async def migrate_project_wrapper(
plan_only: bool, force: bool, commit: bool, message: str
plan_only: bool, selected_only: bool, force: bool, commit: bool, message: str
):
await migrate_project(
plan_only=plan_only, force=force, commit=commit, commit_message=message
plan_only=plan_only,
selected_only=selected_only,
force=force,
commit=commit,
commit_message=message,
)


async def migrate_project(
client: ElisAPIClient = None,
org_path: Path = None,
plan_only: bool = False,
selected_only: bool = False,
force: bool = False,
commit: bool = False,
commit_message: str = "",
Expand All @@ -103,6 +115,11 @@ async def migrate_project(
f'Detected "target_object" for organization. Please run "prd {settings.MIGRATE_MAPPING_COMMAND_NAME}" to have the correct mapping format.'
)

if "ignore" in mapping["organization"]:
raise click.ClickException(
"Cannot ignore the whole organization, please remove the attribute from mapping."
)

source_organization_id = mapping["organization"].get("id", "")
target_organization_id = (
target_organizations[0].get("target_id", None)
Expand Down Expand Up @@ -160,6 +177,7 @@ async def migrate_project(
target_organization_id=target_organization_id,
progress=progress,
plan_only=plan_only,
selected_only=selected_only,
target_objects=target_objects,
errors=errors_by_target_id,
force=force,
Expand All @@ -173,6 +191,7 @@ async def migrate_project(
sources_by_source_id_map=sources_by_source_id_map,
progress=progress,
plan_only=plan_only,
selected_only=selected_only,
target_objects=target_objects,
errors=errors_by_target_id,
force=force,
Expand All @@ -185,6 +204,7 @@ async def migrate_project(
sources_by_source_id_map=sources_by_source_id_map,
progress=progress,
plan_only=plan_only,
selected_only=selected_only,
target_objects=target_objects,
errors=errors_by_target_id,
force=force,
Expand All @@ -197,6 +217,7 @@ async def migrate_project(
sources_by_source_id_map=sources_by_source_id_map,
progress=progress,
plan_only=plan_only,
selected_only=selected_only,
target_objects=target_objects,
errors=errors_by_target_id,
force=force,
Expand Down Expand Up @@ -231,6 +252,7 @@ async def migrate_project(
lookup_table=lookup_table,
errors=errors_by_target_id,
plan_only=plan_only,
selected_only=selected_only,
)

if plan_only:
Expand All @@ -255,7 +277,7 @@ async def migrate_project(
commit=commit,
commit_message=commit_message,
)
print(Panel(f"Finished {settings.MIGRATE_COMMAND_NAME}."))

hints = """
! attention !
The following was not migrated - queue.dedicated_engine, queue.generic_engine, queue.users, hook.secrets, queue.workflows.
Expand All @@ -267,6 +289,11 @@ async def migrate_project(
This applies only for newly created objects. Once these attributes are set on the target object, subsequent release commands keep the values.
"""
print(Panel(f"{hints}"))
print(
Panel(
f"Finished {settings.MIGRATE_COMMAND_NAME}. Please check all messages printed during the process."
)
)
except PrdVersionException as e:
print(Panel(f"Unexpected error while migrating objects: {e}"))
return
Expand Down
Loading

0 comments on commit 7272485

Please sign in to comment.