Various release fixes

install.sh

@@ -9,6 +9,11 @@ git clone "$PRD_GIT_URL"
 
 cd deployment-manager
 
+if [ -n "$1" ]
+then
+    git checkout "$1"
+fi
+
 if ! command -v pipx &> /dev/null
 then
     echo 'pipx not found, attempting install...'

project_rossum_deploy/commands/download/download.py

@@ -14,7 +14,6 @@
     should_write_object,
 )
 from project_rossum_deploy.commands.download.hooks import download_hooks
-from project_rossum_deploy.common.migrate_config import migrate_config
 from project_rossum_deploy.common.client import create_and_validate_client
 from project_rossum_deploy.common.mapping import (
     create_update_mapping,
@@ -74,7 +73,6 @@
 async def download_project_wrapper(
     destination: str, commit: bool = False, message: str = "", all: bool = False
 ):
-    await migrate_config()
     await download_project(
         destination=destination, commit_message=message, commit=commit, download_all=all
     )

project_rossum_deploy/commands/download/helpers.py

@@ -430,6 +430,8 @@ async def get_all_objects_for_destination(org_path: Path, destination: str) -> t
                 queues.append(object)
             case Resource.Inbox:
                 inboxes.append(object)
+            case Resource.EmailTemplate:
+                ...
             case _:
                 display_warning(f"Unrecognized type '{type}' - skipping.")
                 continue

project_rossum_deploy/commands/migrate/helpers.py

@@ -67,7 +67,10 @@ def replace_dependency_url(
         source_id = extract_id_from_url(source_dependency_url)
         target_dependency_objects = source_id_target_pairs.get(source_id, [])
 
-        # Assume each object should have its own dependency
+        # Dependency object has no target equivalents (e.g., when ignored)
+        if not len(target_dependency_objects):
+            return
+        # There are multiple objects released (e.g., queues) and their number is the same as the number of their dependencies (e.g., hooks) -> assume that each object should have its own dependency
         if len(target_dependency_objects) == target_objects_count:
             target_id_str = str(target_dependency_objects[target_index]["id"])
         # All objects will have the same dependency
@@ -104,7 +107,10 @@ def replace_list_of_dependency_urls(
         source_id = extract_id_from_url(source_dependency_url)
         target_dependency_objects = source_id_target_pairs.get(source_id, [])
 
-        # Assume each object should have its own dependency
+        # Dependency object has no target equivalents (e.g., when ignored)
+        if not len(target_dependency_objects):
+            continue
+        # There are multiple objects released (e.g., queues) and their number is the same as the number of their dependencies (e.g., hooks) -> assume that each object should have its own dependency
         if len(target_dependency_objects) == target_objects_count:
             target_id_str = str(target_dependency_objects[target_index]["id"])
         # All objects will have the same dependency
@@ -114,12 +120,12 @@ def replace_list_of_dependency_urls(
         source_id_str = str(source_id)
         new_url = source_dependency_url.replace(source_id_str, target_id_str)
 
-        new_urls.append(new_url)
-
         if source_id_str == target_id_str:
             display_warning(
-                f'Dependency "{dependency}"[{source_index}] for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.'
+                f'Dependency "{dependency}"[{source_index}] for object "{object.get('id', 'no-ID')}" was not changed to a target counterpart because none was found.'
             )
+        else:
+            new_urls.append(new_url)
 
     # Target queues can have 'dangling' hooks that exist only on target, these should not be overwritten.
     if target_object:
@@ -236,8 +242,4 @@ async def skip_migrate_object(
     target_index: int = 0,
     target_objects_count: int = None,
 ):
-    object_type = determine_object_type_from_url(source_object["url"])
-    print(
-        f'Skipping {settings.MIGRATE_COMMAND_NAME} of {object_type} "{source_object['id']} {source_object['name']}".'
-    )
-    return copy.deepcopy(source_object)
+    return None

project_rossum_deploy/commands/migrate/hooks.py

@@ -46,7 +46,7 @@ async def migrate_hooks(
     hook_paths = [
         hook_path
         async for hook_path in (source_path / "hooks").iterdir()
-        if await hook_path.is_file()
+        if await hook_path.is_file() and hook_path.suffix == ".json"
     ]
 
     target_token_owner_id = ""
@@ -60,9 +60,6 @@ async def migrate_hooks(
             target_token_owner_id = target_org_token_owner.id
 
     async def migrate_hook(hook_path: Path):
-        if hook_path.suffix != ".json":
-            return
-
         try:
             _, id = detemplatize_name_id(hook_path.stem)
             hook = await read_json(hook_path)
@@ -132,7 +129,7 @@ async def migrate_hook(hook_path: Path):
     if plan_only:
         return
 
-    await migrate_hook_dependency_graph(client, source_path, source_id_target_pairs)
+    await migrate_hook_dependency_graph(client, hook_paths, source_id_target_pairs)
 
     print(
         Panel(
@@ -155,61 +152,90 @@ async def update_hook_code(hook_path: Path, hook: dict):
 
 
 async def migrate_hook_dependency_graph(
-    client: ElisAPIClient, source_path: Path, source_id_target_pairs: dict[int, list]
+    client: ElisAPIClient,
+    hook_paths: list[Path],
+    source_id_target_pairs: dict[int, list],
 ):
-    async for hook_path in (source_path / "hooks").iterdir():
-        if hook_path.suffix != ".json":
-            continue
-
+    for hook_path in hook_paths:
         try:
-            _, old_hook_id = detemplatize_name_id(hook_path.stem)
-            old_hook = await read_json(hook_path)
-            new_hooks = source_id_target_pairs.get(old_hook_id, None)
+            _, source_hook_id = detemplatize_name_id(hook_path.stem)
+            source_hook = await read_json(hook_path)
+            target_hooks = source_id_target_pairs.get(source_hook_id, [])
 
             # The hook was ignored, it has no targets equivalent
-            if not new_hooks or not len(new_hooks):
+            if not len(target_hooks):
                 continue
 
-            for new_hook_index, new_hook in enumerate(new_hooks):
-                new_run_after = []
-                for predecessor_url in old_hook["run_after"]:
-                    predecessor_id = extract_id_from_url(predecessor_url)
-                    target_objects = source_id_target_pairs.get(predecessor_id, [])
-                    # The hook was ignored, it has no targets equivalent
-                    if not len(target_objects):
-                        continue
-                    # Assume each hook should have its own run_after
-                    elif len(new_hooks) == len(target_objects):
-                        new_url = predecessor_url.replace(
-                            str(predecessor_id),
-                            str(target_objects[new_hook_index]["id"]),
-                        )
-                        new_run_after.append(new_url)
-                    # All hooks will have the same single run_after
-                    elif len(target_objects) == 1:
-                        new_url = predecessor_url.replace(
-                            str(predecessor_id), str(target_objects[0]["id"])
-                        )
-                        new_run_after.append(new_url)
-                    else:
-                        new_url = predecessor_url.replace(
-                            str(predecessor_id), str(target_objects[0]["id"])
-                        )
-                        new_run_after.append(new_url)
-                        new_hook_ids = "".join(list(map(lambda x: x["id"], new_hooks)))
-                        print(
-                            Panel(
-                                f'Could not determine new predecessors for migrated hooks "{new_hook_ids}". The source predecessor ID is "{predecessor_id}". Assigning everything to target predecessor "{target_objects[0]["id"]}"'
-                            )
-                        )
-                        continue
-
+            for target_hook_index, target_hook in enumerate(target_hooks):
+                target_run_after = await migrate_target_hook_run_after(
+                    client=client,
+                    target_hook_index=target_hook_index,
+                    target_hook_count=len(target_hooks),
+                    source_run_after=source_hook.get("run_after", []),
+                    source_id_target_pairs=source_id_target_pairs,
+                )
                 await client._http_client.update(
-                    Resource.Hook, id_=new_hook["id"], data={"run_after": new_run_after}
+                    Resource.Hook,
+                    id_=target_hook["id"],
+                    data={"run_after": target_run_after},
                 )
         except Exception as e:
             display_error(
-                f"Error while migrating dependency graph for hook '{source_path}': {e}",
+                f"Error while migrating dependency graph for hook '{hook_path}':",
                 e,
             )
-            raise e
+
+
+async def migrate_target_hook_run_after(
+    client: ElisAPIClient,
+    source_run_after: dict,
+    target_hook_index: int,
+    target_hook_count: int,
+    source_id_target_pairs: dict[int, list],
+):
+    async def find_missing_hook_run_after(predecessor_id: int):
+        # The predecessor hook was ignored, it has no targets equivalent
+        # Take the predecessor's source and find its predecessor (if none, stop)
+        # Find the predecessors' target and put that into run_after for this hook
+        # If there is no target, repeat from line one
+        try:
+            predecessor = await client.retrieve_hook(predecessor_id)
+        except Exception as e:
+            display_error(
+                f'Error while finding predecessor hook with ID "{predecessor_id}" in Rossum.',
+                e,
+            )
+            return []
+
+        return await migrate_target_hook_run_after(
+            client=client,
+            source_run_after=predecessor.run_after,
+            target_hook_index=target_hook_index,
+            target_hook_count=target_hook_count,
+            source_id_target_pairs=source_id_target_pairs,
+        )
+
+    target_run_after = []
+
+    for predecessor_url in source_run_after:
+        predecessor_id = extract_id_from_url(predecessor_url)
+        predecessor_target_objects = source_id_target_pairs.get(predecessor_id, [])
+
+        if not len(predecessor_target_objects):
+            target_run_after += await find_missing_hook_run_after(predecessor_id)
+        # Assume each newly created hook should have its own run_after
+        elif target_hook_count == len(predecessor_target_objects):
+            new_url = predecessor_url.replace(
+                str(predecessor_id),
+                str(predecessor_target_objects[target_hook_index]["id"]),
+            )
+            target_run_after.append(new_url)
+        # All hooks will have the same single run_after
+        else:
+            new_url = predecessor_url.replace(
+                str(predecessor_id),
+                str(predecessor_target_objects[0]["id"]),
+            )
+            target_run_after.append(new_url)
+
+    return target_run_after

project_rossum_deploy/commands/migrate/migrate.py

@@ -11,7 +11,6 @@
     check_ids_exclusively_source_or_target,
 )
 from project_rossum_deploy.commands.migrate.organization import migrate_organization
-from project_rossum_deploy.common.migrate_config import migrate_config
 from project_rossum_deploy.common.attribute_override import (
     override_migrated_objects_attributes,
     validate_override_migrated_objects_attributes,
@@ -30,6 +29,7 @@
 
 from project_rossum_deploy.common.read_write import read_json
 from project_rossum_deploy.utils.consts import (
+    MAPPING_SELECTED_ATTRIBUTE,
     PrdVersionException,
     display_error,
     settings,
@@ -61,7 +61,7 @@
     "-so",
     default=False,
     is_flag=True,
-    help="Releases only objects with a `selected: true` attribute in mapping.yaml and ignores the rest even without an ignore flag. Unlike ignore, this flag is not recursive = you have to put it on sub-objects (e.g., both queue AND inbox).",
+    help=f"Releases only objects with a `{MAPPING_SELECTED_ATTRIBUTE}: true` attribute in mapping.yaml and ignores the rest even without an ignore flag. Unlike ignore, this flag is not recursive = you have to put it on sub-objects (e.g., both queue AND inbox).",
 )
 @click.option(
     "--force",
@@ -87,7 +87,6 @@
 async def migrate_project_wrapper(
     plan_only: bool, selected_only: bool, force: bool, commit: bool, message: str
 ):
-    await migrate_config()
     await migrate_project(
         plan_only=plan_only,
         selected_only=selected_only,
@@ -288,9 +287,10 @@ async def migrate_project(
                 2. set hook.secrets for migrated hooks
                 3. assign users to queues
                 4. assign workflows to queues
+                5. add datasets (if the target organization is different)
             This applies only for newly created objects. Once these attributes are set on the target object, subsequent release commands keep the values.
             """
-            print(Panel(f"{hints}"))
+            print(Panel(f"{hints}", style="yellow"))
             print(
                 Panel(
                     f"Finished {settings.MIGRATE_COMMAND_NAME}. Please check all messages printed during the process."

project_rossum_deploy/commands/migrate/schemas.py

@@ -105,7 +105,7 @@ async def migrate_schema(schema_path: Path):
         except PrdVersionException as e:
             raise e
         except Exception as e:
-            display_error(f"Error while migrating schema: {e}", e)
+            display_error(f"Error while migrating schema {schema_path}:", e)
 
     if plan_only:
         print(Panel("Simulating schemas."))

project_rossum_deploy/commands/migrate/upload_helpers.py

@@ -280,10 +280,19 @@ async def create_hook_based_on_template(hook: dict, client: ElisAPIClient):
         )
 
         # In case the hook became private, remove conflicting fields
-        if (hook_config := created_hook.get("config", {})).get("private", False):
-            fields_to_remove = ["code", "third_part_library_pack", "runtime"]
+        if created_hook.get("config", {}).get("private", False):
+            fields_to_remove = [
+                "code",
+                "third_party_library_pack",
+                "runtime",
+                "private",
+            ]
+            hook_config = hook.get("config", {})
             for field in fields_to_remove:
-                hook_config.pop("code", field)
+                hook_config.pop(field, None)
+
+        # Do not try patching the type of extension in case it changed (e.g., SF to lambda)
+        hook.pop("type", None)
 
         return await client._http_client.update(
             resource=Resource.Hook, id_=created_hook["id"], data=hook

project_rossum_deploy/commands/purge/purge.py

@@ -7,7 +7,6 @@
 from rossum_api import ElisAPIClient
 
 from project_rossum_deploy.commands.download.download import download_project
-from project_rossum_deploy.common.migrate_config import migrate_config
 from project_rossum_deploy.commands.migrate_mapping import migrate_mapping
 from project_rossum_deploy.commands.purge.delete_objects import (
     delete_all_objects_with_ids,
@@ -37,7 +36,6 @@
 )
 @coro
 async def purge_project_wrapper(destination):
-    await migrate_config()
     # To be able to run the command progammatically without the CLI decorators
     await purge_project(
         destination=destination,

project_rossum_deploy/commands/upload/upload.py

@@ -8,7 +8,6 @@
 from rossum_api import ElisAPIClient
 
 from project_rossum_deploy.commands.download.download import download_project
-from project_rossum_deploy.common.migrate_config import migrate_config
 from project_rossum_deploy.commands.upload.dependencies import (
     evaluate_create_dependencies,
     merge_formula_changes,
@@ -83,7 +82,6 @@
 async def upload_project_wrapper(
     destination, all, force, indexed_only, commit, message
 ):
-    await migrate_config()
     # To be able to run the command progammatically without the CLI decorators
     await upload_project(
         destination=destination,

project_rossum_deploy/common/attribute_override.py

@@ -185,10 +185,21 @@ async def replace_ids_in_settings(
         if not re.search(source_id_regex, stringified_dict):
             continue
 
-        if len(target_ids) != 1 and num_targets != len(target_ids):
+        basic_error_message = (
+            f"Could not override source '{source_id}' in settings of '{object_id}'."
+        )
+        if not target_ids:
+            print(
+                Panel(
+                    f"{basic_error_message} No target IDs found.",
+                    style="yellow",
+                ),
+            )
+            continue
+        elif num_targets != len(target_ids):
             print(
                 Panel(
-                    f"Could not override source '{source_id}' in settings of '{object_id}'. There are multiple target IDs. Please do the attribute_override explicitly.",
+                    f"{basic_error_message} There are multiple target IDs that could be assigned. Please do the attribute_override explicitly.",
                     style="yellow",
                 ),
             )