Fix/code file changes (#3)

* Fix hook and formula code changes being ignored

* Add test

* Fix lint

project_rossum_deploy/commands/download/download.py

@@ -10,6 +10,7 @@
 from project_rossum_deploy.commands.download.helpers import (
     get_all_objects_for_destination,
     remove_local_nonexistent_objects,
+    replace_code_paths,
     should_write_object,
 )
 from project_rossum_deploy.commands.download.hooks import download_hooks
@@ -93,6 +94,7 @@ async def download_project(
         *get_changed_file_paths(settings.TARGET_DIRNAME),
     ]
     changed_files = list(map(lambda x: x[1], changed_files))
+    changed_files = replace_code_paths(changed_files)
 
     try:
         if settings.IS_PROJECT_IN_SAME_ORG:

project_rossum_deploy/commands/download/helpers.py

@@ -53,6 +53,22 @@ class MissingParentObjectException(Exception):
     status_code = 404
 
 
+def replace_code_paths(file_paths: list[Path]):
+    """Since only .json files are compared when pulling, this flags the json file as changed if its code (.py/.js) file has changed."""
+    replaced_paths = []
+
+    for path in file_paths:
+        if path.suffix in [".py", ".js"]:
+            if path.parent.name == "hooks":
+                path = path.with_suffix(".json")
+            elif settings.FORMULA_DIR_PREFIX in path.parent.name:
+                only_schema_name = path.parent.name.split(":")[1]
+                path = path.parent.with_name(only_schema_name).with_suffix(".json")
+        replaced_paths.append(path)
+
+    return replaced_paths
+
+
 async def should_write_object(path: Path, remote_object: Any, changed_files: list):
     if await path.exists():
         local_file = await read_json(path)

project_rossum_deploy/commands/upload/dependencies.py

@@ -3,9 +3,14 @@
 from rossum_api import ElisAPIClient
 from project_rossum_deploy.commands.migrate.schemas import find_schema_id
 from project_rossum_deploy.common.read_write import (
+    create_custom_hook_code_path,
+    create_formula_directory_path,
+    create_formula_file,
+    find_formula_fields_in_schema,
     read_formula_file,
     read_json,
     write_json,
+    write_str,
 )
 from project_rossum_deploy.utils.consts import GIT_CHARACTERS, display_error, settings
 
@@ -21,22 +26,20 @@ def is_change_existing(change, changes):
     return False
 
 
-async def merge_formula_changes(changes):
+async def merge_formula_changes(changes: list[tuple[str, Path]]):
     merged_changes = []
     for change in changes:
-        op: str
-        path: Path
         op, path = change
-        str_path = str(path)
 
         if (
-            (
-                op == GIT_CHARACTERS.UPDATED
-                or op == GIT_CHARACTERS.CREATED
-                or op == GIT_CHARACTERS.CREATED_STAGED
-            )
+            op
+            in [
+                GIT_CHARACTERS.UPDATED,
+                GIT_CHARACTERS.CREATED,
+                GIT_CHARACTERS.CREATED_STAGED,
+            ]
+            and "schemas" in path.parent.parent.name
             and (path.suffix == ".py")
-            and "schemas" in str_path
         ):
             formula_code = await read_formula_file(path)
             formula_name = path.stem
@@ -51,38 +54,92 @@ async def merge_formula_changes(changes):
             schema_id["formula"] = formula_code
 
             await write_json(schema_path, schema)
-            new_change = ("M", schema_path)
+            new_change = (GIT_CHARACTERS.UPDATED, schema_path)
             if not is_change_existing(new_change, merged_changes):
                 merged_changes.append(new_change)
         elif not is_change_existing(change, merged_changes):
             merged_changes.append(change)
+
+    # If code file was not among the changes, the JSON schemas file already has the new code thanks to the for loop above and no change is technically actually made.
+    # In case code of a schema was changed directly in the JSON file, update the code file as well.
+    for change in merged_changes:
+        op, path = change
+        if (
+            op
+            in [
+                GIT_CHARACTERS.UPDATED,
+                GIT_CHARACTERS.CREATED,
+                GIT_CHARACTERS.CREATED_STAGED,
+            ]
+            and "schemas" in path.parent.name
+        ) and path.suffix == ".json":
+            schema = await read_json(path)
+
+            formula_fields = find_formula_fields_in_schema(schema["content"])
+            if formula_fields:
+                formula_directory_path = create_formula_directory_path(
+                    path, schema.get("name", ""), schema.get("id", "")
+                )
+                for field_id, code in formula_fields:
+                    await create_formula_file(
+                        formula_directory_path / f"{field_id}.py", code
+                    )
+
     return merged_changes
 
 
-async def merge_hook_changes(changes, org_path):
+async def merge_hook_changes(changes: list[tuple[str, Path]], org_path: Path):
     merged_changes = []
     for change in changes:
         op, path = change
-        path = str(path)
         if (
-            op == GIT_CHARACTERS.UPDATED
-            or op == GIT_CHARACTERS.CREATED
-            or op == GIT_CHARACTERS.CREATED_STAGED
-        ) and (path.endswith("py") and "hooks" in path):
+            op
+            in [
+                GIT_CHARACTERS.UPDATED,
+                GIT_CHARACTERS.CREATED,
+                GIT_CHARACTERS.CREATED_STAGED,
+            ]
+            and path.parent.name == "hooks"
+            and path.suffix in [".py", ".js"]
+        ):
+            # Overwrite the code property in the JSON hook file with the code from the file.
+            # If the JSON hook file also had changed code, it will get overwritten!
             with open(path, "r") as file:
                 code_str = file.read()
                 object_path = org_path / (
                     Path(str(path).removesuffix(".py").removesuffix(".js") + ".json")
                 )
-                hook_object = await read_json(object_path)
-                hook_object["config"]["code"] = code_str
-                await write_json(object_path, hook_object)
-                new_change = ("M", object_path)
+                hook = await read_json(object_path)
+                hook["config"]["code"] = code_str
+                await write_json(object_path, hook)
+                new_change = (GIT_CHARACTERS.UPDATED, object_path)
                 exists = is_change_existing(new_change, merged_changes)
                 if not exists:
                     merged_changes.append(new_change)
         elif not is_change_existing(change, merged_changes):
             merged_changes.append(change)
+
+    # If code file was not among the changes, the JSON hook file already has the new code thanks to the for loop above and no change is technically actually made.
+    # In case code of a hook was changed directly in the JSON file, update the code file as well.
+    for change in merged_changes:
+        op, path = change
+        if (
+            op
+            in [
+                GIT_CHARACTERS.UPDATED,
+                GIT_CHARACTERS.CREATED,
+                GIT_CHARACTERS.CREATED_STAGED,
+            ]
+            and path.parent.name == "hooks"
+        ) and path.suffix == ".json":
+            hook = await read_json(path)
+
+            code_path = create_custom_hook_code_path(Path(path), hook)
+            if not code_path:
+                continue
+
+            await write_str(code_path, hook.get("config", {}).get("code", None))
+
     return merged_changes
 
 

project_rossum_deploy/common/git.py

@@ -7,7 +7,7 @@
 
 def get_changed_file_paths(
     destination: str, indexed_only=False
-) -> list[tuple[str, str]]:
+) -> list[tuple[str, Path]]:
     # The -s flag is there to show a simplified list of changes
     # The -u flag is there to show each individual file (and not a subdir)
     # The change in git config is because of potential 'unusual' (non-ASCII) characters in paths
@@ -34,11 +34,7 @@ def get_changed_file_paths(
         change = change.strip(" ")
         op, path = tuple(change.split(" ", maxsplit=1))
 
-        # The code file has changed, but only .json files are compared when pulling
-        # Use the .json path to prevent code changes being lost
         path = Path(path.strip().strip('"'))
-        if path.suffix in [".py", ".js"]:
-            path = path.with_suffix(".json")
 
         changes.append((op, path))
     return changes

tests/test_upload.py

@@ -15,7 +15,12 @@
 )
 from project_rossum_deploy.common.mapping import read_mapping, write_mapping
 from project_rossum_deploy.commands.upload.upload import upload_project
-from project_rossum_deploy.common.read_write import read_json, write_json
+from project_rossum_deploy.common.read_write import (
+    create_custom_hook_code_path,
+    read_json,
+    write_json,
+    write_str,
+)
 from project_rossum_deploy.utils.consts import settings
 from project_rossum_deploy.utils.functions import (
     templatize_name_id,
@@ -68,7 +73,8 @@ async def updated_file_tuple(client: ElisAPIClient, tmp_path):
 
 @pytest.mark.asyncio
 async def test_push_updated_file(
-    client: ElisAPIClient, updated_file_tuple, monkeypatch: pytest.MonkeyPatch
+    client: ElisAPIClient,
+    updated_file_tuple,
 ):
     tmp_path, updated_file_path, updated_file = updated_file_tuple
 
@@ -81,7 +87,8 @@ async def test_push_updated_file(
 
 @pytest.mark.asyncio
 async def test_push_ignored_non_staged_file(
-    client: ElisAPIClient, updated_file_tuple, monkeypatch: pytest.MonkeyPatch
+    client: ElisAPIClient,
+    updated_file_tuple,
 ):
     tmp_path, updated_file_path, updated_file = updated_file_tuple
 
@@ -121,7 +128,8 @@ async def updated_file_name_tuple(client: ElisAPIClient, tmp_path):
 
 @pytest.mark.asyncio
 async def test_push_updated_file_name_changed(
-    client: ElisAPIClient, updated_file_name_tuple, monkeypatch: pytest.MonkeyPatch
+    client: ElisAPIClient,
+    updated_file_name_tuple,
 ):
     tmp_path, previous_file_path, updated_file = updated_file_name_tuple
 
@@ -138,7 +146,7 @@ async def test_push_updated_file_name_changed(
 
 
 @pytest_asyncio.fixture(scope="function")
-async def source_and_target_schema(client: ElisAPIClient, tmp_path, monkeypatch):
+async def source_and_target_schema(client: ElisAPIClient, tmp_path):
     source_schema = await client.create_new_schema(
         {"name": "source_schema", "content": []}
     )
@@ -178,7 +186,6 @@ async def source_and_target_schema(client: ElisAPIClient, tmp_path, monkeypatch)
 async def test_push_ignores_file_from_target(
     client: ElisAPIClient,
     tmp_path: Path,
-    monkeypatch: pytest.MonkeyPatch,
     source_and_target_schema,
 ):
     source_schema, target_schema = source_and_target_schema
@@ -216,7 +223,6 @@ async def test_push_ignores_file_from_target(
 async def test_push_ignores_file_from_source(
     client: ElisAPIClient,
     tmp_path: Path,
-    monkeypatch: pytest.MonkeyPatch,
     source_and_target_schema,
 ):
     source_schema, target_schema = source_and_target_schema
@@ -248,3 +254,54 @@ async def test_push_ignores_file_from_source(
 
     assert is_object_equal(redownloaded_target_schema, updated_target_schema)
     assert dataclasses.asdict(redownloaded_source_schema)["metadata"] == {}
+
+
+@pytest_asyncio.fixture(scope="function")
+async def custom_code_hook(client: ElisAPIClient, tmp_path):
+    custom_code_hook = await client.create_new_hook(
+        {
+            "name": "custom_code_hook",
+            "type": "function",
+            "events": ["invocation.manual"],
+            "queues": [],
+            "config": {"runtime": "python3.12", "code": "print('hello')"},
+        }
+    )
+
+    await download_organization_combined_source_target(client, org_path=tmp_path)
+
+    await setup_project(client, tmp_path)
+
+    settings.IS_PROJECT_IN_SAME_ORG = True
+    # The command must be run from the directory because it is running 'git status' internally
+    os.chdir(tmp_path)
+
+    yield custom_code_hook
+
+    # Cleanup
+    await client._http_client.delete(Resource.Hook, custom_code_hook.id)
+
+
+@pytest.mark.asyncio
+async def test_push_uses_change_in_code_file(
+    client: ElisAPIClient, tmp_path, custom_code_hook
+):
+    hook_path = (
+        tmp_path
+        / settings.SOURCE_DIRNAME
+        / "hooks"
+        / f"{templatize_name_id(custom_code_hook.name, custom_code_hook.id)}.json"
+    )
+    hook_code_path = create_custom_hook_code_path(
+        hook_path, dataclasses.asdict(custom_code_hook)
+    )
+
+    NEW_CODE = 'print("The world has changed.")'
+    await write_str(hook_code_path, NEW_CODE)
+
+    await upload_project(destination=settings.SOURCE_DIRNAME, client=client)
+    redownloaded_hook = await read_json(hook_path)
+    redownloaded_code = await hook_code_path.read_text()
+
+    assert redownloaded_hook.get("config", {}).get("code") == NEW_CODE
+    assert redownloaded_code == NEW_CODE