combine the two REMOVE_IF_MISSING & REMOVE_ID_NULL

USERGUIDE.md

@@ -313,15 +313,18 @@ To define a new substitution the following steps have to be performed:
    > required since this can overwrite mandatory Cumulocity attributes, e.g. <code>source.id</code>. This can result in API calls that are rejected by the Cumulocity backend!
 
 1. Press the button "Add substitution". In the next modal dialog the following details can be specified: 1. Select option `Expand Array` if the result of the source expression is an array and you want to generate any of the following substitutions:
-_ `multi-device-single-value`
-_ `multi-device-multi-value`
-_ `single-device-multi-value`\
- Otherwise an extracted array is treated as a single value, see [Different type of substitutions](#different-type-of-substitutions). 1. Select option `Resolve to externalId` if you want to resolve system Cumulocity Id to externalId using externalIdType. This can only be used for OUTBOUND mappings. 1. Select a `Reapir Strategy` that determines how the mapping is applied:
-_ `DEFAULT`: Map the extracted values to the attribute addressed on right side
-_ `USE_FIRST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the 1. item in the array and map this to the right side
-_ `USE_LAST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the last item in the array and map this to the right side
-_ `REMOVE_IF_MISSING`: When the left side of the mapping returns no result (not NULL), then delete the attribute (that is addressed in mapping) in the target on the right side. This avoids empty attribute, e.d. `airsensor: undefined`
-_ `REMOVE_IF_NULL`: When the left side of the mapping returns `null`, then delete the attribute (that is addressed in mapping) in the target on the right side. This avoids empty attribute, e.d. `airsensor: undefined`
+    * `multi-device-single-value`
+    * `multi-device-multi-value`
+    * `single-device-multi-value`\
+\
+Otherwise an extracted array is treated as a single value, see [Different type of substitutions](#different-type-of-substitutions). 
+4. Select option `Resolve to externalId` if you want to resolve system Cumulocity Id to externalId using externalIdType. This can only be used for OUTBOUND mappings. 
+5. Select a `Repair Strategy` that determines how the mapping is applied:
+    * `DEFAULT`: Map the extracted values to the attribute addressed on right side
+    * `USE_FIRST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the 1. item in the array and map this to the right side
+    * `USE_LAST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the last item in the array and map this to the right side
+    * `REMOVE_IF_MISSING_OR_NULL`: When the left side of the mapping returns no result (not NULL), then delete the attribute (that is addressed in mapping) in the target on the right side. This avoids empty attribute, e.g. `airsensor: undefined`
+
 <p align="center">
 <img src="resources/image/Dynamic_Mapper_Mapping_Stepper_Edit_Modal.png"  style="width: 70%;" />
 </p>

dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionCustomMeasurement.java

@@ -70,7 +70,7 @@ public void extractFromSource(ProcessingContext<byte[]> context)
                 new ArrayList<MappingSubstitution.SubstituteValue>(Arrays.asList(
                         new MappingSubstitution.SubstituteValue(null,
                                 MappingSubstitution.SubstituteValue.TYPE.TEXTUAL,
-                                RepairStrategy.REMOVE_IF_NULL))));
+                                RepairStrategy.REMOVE_IF_MISSING_OR_NULL))));
 
         postProcessingCache.put("c8y_Temperature",
                 new ArrayList<MappingSubstitution.SubstituteValue>(Arrays.asList(

dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingSubstitution.java

@@ -133,10 +133,9 @@ public static String toJsonString(Object obj) {
     public static void substituteValueInPayload(MappingType type, MappingSubstitution.SubstituteValue sub,
             DocumentContext jsonObject, String keys)
             throws JSONException {
-        boolean subValueMissing = sub.value == null;
+        boolean subValueMissingOrNull = sub == null || sub.value == null;
         // TOFDO fix this, we have to differentiate between {"nullField": null } and
         // "nonExisting"
-        boolean subValueNull = subValueMissing;
         try {
             if ("$".equals(keys)) {
                 Object replacement = sub;
@@ -148,8 +147,7 @@ public static void substituteValueInPayload(MappingType type, MappingSubstitutio
                     }
                 }
             } else {
-                if ((sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_MISSING) && subValueMissing) ||
-                        (sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_NULL) && subValueNull)) {
+                if ((sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_MISSING_OR_NULL) && subValueMissingOrNull)) {
                     jsonObject.delete(keys);
                 } else if (sub.repairStrategy.equals(RepairStrategy.CREATE_IF_MISSING)) {
                     jsonObject.set("$." + keys, sub.value);

dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/RepairStrategy.java

@@ -26,7 +26,6 @@ public enum RepairStrategy {
     USE_FIRST_VALUE_OF_ARRAY,
     USE_LAST_VALUE_OF_ARRAY,
     IGNORE,
-    REMOVE_IF_MISSING, // remove the node in the target if it the evaluation of the source expression returns undefined, empty. This allows for using mapping with dynamic content
-    REMOVE_IF_NULL,    // remove the node in the target if it the evaluation of the source expression returns null. This allows for using mapping with dynamic content
+    REMOVE_IF_MISSING_OR_NULL, // remove the node in the target if it the evaluation of the source expression returns undefined, empty. This allows for using mapping with dynamic content
     CREATE_IF_MISSING, // create the node in the target if it doesn't exist. This allows for using mapping with dynamic content
 }

dynamic-mapping-ui/src/mapping/core/processor/payload-processor-inbound.service.ts

@@ -165,7 +165,7 @@ export abstract class PayloadProcessorInbound {
                   externalIdType: mapping.externalIdType,
                   request,
                   targetAPI: API.INVENTORY.name,
-                  hide: true
+                  hidden: true
                 });
                 const response = await this.c8yClient.upsertDevice(
                   {

dynamic-mapping-ui/src/mapping/core/processor/payload-processor-outbound.service.ts

@@ -163,20 +163,15 @@ export abstract class PayloadProcessorOutbound {
     jsonObject: JSON,
     keys: string
   ) {
-    const subValueMissing: boolean = sub.value == null;
-    const subValueNull: boolean =
+    const subValueMissingOrNull: boolean =
       sub.value == null || (sub.value != null && sub.value != undefined);
 
     if (keys == '$') {
       Object.keys(getTypedValue(sub)).forEach((key) => {
         jsonObject[key] = getTypedValue(sub)[key as keyof unknown];
       });
     } else {
-      if (
-        (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING &&
-          subValueMissing) ||
-        (sub.repairStrategy == RepairStrategy.REMOVE_IF_NULL && subValueNull)
-      ) {
+      if (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING_OR_NULL && subValueMissingOrNull) {
         _.unset(jsonObject, keys);
       } else if (sub.repairStrategy == RepairStrategy.CREATE_IF_MISSING) {
         // const pathIsNested: boolean = keys.includes('.') || keys.includes('[');

dynamic-mapping-ui/src/mapping/core/processor/processor.model.ts

@@ -29,7 +29,7 @@ export interface C8YRequest {
   response?: any;
   targetAPI?: string;
   error?: string;
-  hide?:boolean;
+  hidden?:boolean;
 }
 
 export interface ProcessingContext {

dynamic-mapping-ui/src/mapping/core/processor/util.ts

@@ -167,18 +167,16 @@ export function substituteValueInPayload(
   jsonObject: JSON,
   keys: string
 ) {
-  const subValueMissing: boolean = !sub || sub.value == null;
-  const subValueNull: boolean = subValueMissing;
+  const subValueMissingOrNull: boolean = !sub || sub.value == null;
 
   if (keys == '$') {
     Object.keys(getTypedValue(sub)).forEach((key) => {
       jsonObject[key] = getTypedValue(sub)[key as keyof unknown];
     });
   } else {
     if (
-      (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING &&
-        subValueMissing) ||
-      (sub.repairStrategy == RepairStrategy.REMOVE_IF_NULL && subValueNull)
+      (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING_OR_NULL &&
+        subValueMissingOrNull)
     ) {
       _.unset(jsonObject, keys);
     } else if (sub.repairStrategy == RepairStrategy.CREATE_IF_MISSING) {

dynamic-mapping-ui/src/mapping/step-testing/mapping-testing.component.ts

@@ -212,7 +212,7 @@ export class MappingStepTestingComponent implements OnInit, OnDestroy {
         let nextIndex = currentIndex;
         do {
             nextIndex = (nextIndex >= results.length - 1) ? 0 : nextIndex + 1;
-            if (!results[nextIndex].hide) {
+            if (!results[nextIndex].hidden) {
                 return nextIndex;
             }
         } while (nextIndex !== currentIndex);

dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.ts

@@ -35,7 +35,7 @@ import { AlertService, C8yStepper } from '@c8y/ngx-components';
 import { FormlyFieldConfig } from '@ngx-formly/core';
 import * as _ from 'lodash';
 import { BsModalService } from 'ngx-bootstrap/modal';
-import { BehaviorSubject, Subject } from 'rxjs';
+import { BehaviorSubject, filter, Subject, take } from 'rxjs';
 import { Content, Mode } from 'vanilla-jsoneditor';
 import { ExtensionService } from '../../extension';
 import {
@@ -860,92 +860,130 @@ export class MappingStepperComponent implements OnInit, OnDestroy {
     this.expertMode = !this.expertMode;
   }
 
-  onUpdateSubstitution() {
-    if (this.selectedSubstitution != -1) {
-      const selected = this.selectedSubstitution;
-      const initialState = {
-        substitution: _.clone(this.mapping.substitutions[selected]),
-        mapping: this.mapping,
-        stepperConfiguration: this.stepperConfiguration,
-        isUpdate: true
-      };
-      if (
-        this.substitutionModel.sourceExpression.valid &&
-        this.substitutionModel.targetExpression.valid
-      ) {
-        initialState.substitution.pathSource =
-          this.substitutionModel.pathSource;
-        initialState.substitution.pathTarget =
-          this.substitutionModel.pathTarget;
-      }
-      const modalRef = this.bsModalService.show(EditSubstitutionComponent, {
-        initialState
-      });
-      modalRef.content.closeSubject.subscribe((editedSub) => {
-        if (editedSub) {
-          this.mapping.substitutions[selected] = editedSub;
-          this.updateSubstitutionValid();
-          //this.substitutionModel = editedSub; not needed
-        }
-      });
+  onUpdateSubstitution(): void {
+    const { selectedSubstitution, mapping, stepperConfiguration, substitutionModel } = this;
+
+    // Early return if no substitution is selected
+    if (selectedSubstitution === -1) {
+        return;
+    }
 
+    // Prepare initial state
+    const initialState = {
+        substitution: { ...mapping.substitutions[selectedSubstitution] },
+        mapping,
+        stepperConfiguration,
+        isUpdate: true
+    };
 
-      // console.log("Updated subs I:", this.mapping.substitutions);
+    // Update paths if expressions are valid
+    const { sourceExpression, targetExpression, pathSource, pathTarget } = substitutionModel;
+    if (sourceExpression.valid && targetExpression.valid) {
+        initialState.substitution = {
+            ...initialState.substitution,
+            pathSource,
+            pathTarget
+        };
     }
-  }
 
-  private addSubstitution(ns: MappingSubstitution) {
-    const sub: MappingSubstitution = _.clone(ns);
-    let duplicateSubstitutionIndex = -1;
-    let duplicate;
-    this.mapping.substitutions.forEach((s, index) => {
-      if (sub.pathTarget == s.pathTarget) {
-        duplicateSubstitutionIndex = index;
-        duplicate = this.mapping.substitutions[index];
-      }
-    });
-    const isDuplicate = duplicateSubstitutionIndex != -1;
+    // Show modal and handle response
+    const modalRef = this.bsModalService.show(EditSubstitutionComponent, { initialState });
+
+    modalRef.content.closeSubject
+        .pipe(
+            take(1), // Automatically unsubscribe after first emission
+            filter(Boolean) // Only proceed if we have valid data
+        )
+        .subscribe({
+            next: (editedSubstitution: MappingSubstitution) => {
+                try {
+                    mapping.substitutions[selectedSubstitution] = editedSubstitution;
+                    this.updateSubstitutionValid();
+                } catch (error) {
+                    console.log('Failed to update substitution', error);
+                }
+            },
+            error: (error) => console.log('Error in modal operation', error)
+        });
+}
+
+  private addSubstitution(newSubstitution: MappingSubstitution): void {
+    const substitution = { ...newSubstitution };
+    const { mapping, stepperConfiguration, expertMode } = this;
+
+    // Find duplicate substitution
+    const duplicateIndex = mapping.substitutions.findIndex(
+        sub => sub.pathTarget === substitution.pathTarget
+    );
+
+    const isDuplicate = duplicateIndex !== -1;
+    const duplicate = isDuplicate ? mapping.substitutions[duplicateIndex] : undefined;
+
     const initialState = {
-      isDuplicate,
-      duplicate,
-      duplicateSubstitutionIndex,
-      substitution: sub,
-      mapping: this.mapping,
-      stepperConfiguration: this.stepperConfiguration
+        isDuplicate,
+        duplicate,
+        duplicateSubstitutionIndex: duplicateIndex,
+        substitution,
+        mapping,
+        stepperConfiguration
     };
-    if (this.expertMode || isDuplicate) {
-      const modalRef = this.bsModalService.show(EditSubstitutionComponent, {
-        initialState
-      });
 
-      modalRef.content.closeSubject.subscribe((newSub: MappingSubstitution) => {
-        if (newSub && !isDuplicate) {
-          this.mapping.substitutions.push(newSub);
-        } else if (newSub && isDuplicate) {
-          this.mapping.substitutions[duplicateSubstitutionIndex] = newSub;
-        }
+    // Handle simple case first (non-expert mode, no duplicates)
+    if (!expertMode && !isDuplicate) {
+        mapping.substitutions.push(substitution);
         this.updateSubstitutionValid();
-      });
-    } else {
-      this.mapping.substitutions.push(sub);
-      this.updateSubstitutionValid();
+        return;
     }
-  }
+
+    // Handle expert mode or duplicates
+    const modalRef = this.bsModalService.show(EditSubstitutionComponent, {
+        initialState
+    });
+
+    modalRef.content.closeSubject
+        .pipe(
+            take(1) // Automatically unsubscribe after first emission
+        )
+        .subscribe((updatedSubstitution: MappingSubstitution) => {
+            if (!updatedSubstitution) return;
+
+            if (isDuplicate) {
+                mapping.substitutions[duplicateIndex] = updatedSubstitution;
+            } else {
+                mapping.substitutions.push(updatedSubstitution);
+            }
+
+            this.updateSubstitutionValid();
+        });
+}
 
   async onSelectSubstitution(selected: number) {
-    if (selected < this.mapping.substitutions.length && selected > -1) {
-      this.selectedSubstitution = selected;
-      this.substitutionModel = _.clone(this.mapping.substitutions[selected]);
-      this.substitutionModel.stepperConfiguration = this.stepperConfiguration;
-      await this.editorSourceStepSubstitution.setSelectionToPath(
-        this.substitutionModel.pathSource
-      );
-      await this.editorTargetStepSubstitution.setSelectionToPath(
-        this.substitutionModel.pathTarget
-      );
+    const { mapping, stepperConfiguration } = this;
+    const { substitutions } = mapping;
+
+    // Early return if selection is out of bounds
+    if (selected < 0 || selected >= substitutions.length) {
+        return;
     }
-    // console.log("Updated subs II:", this.mapping.substitutions);
-  }
+
+    this.selectedSubstitution = selected;
+
+    // Create substitution model
+    this.substitutionModel = {
+        ...substitutions[selected],
+        stepperConfiguration
+    };
+
+    // Parallel execution of path selections
+    await Promise.all([
+        this.editorSourceStepSubstitution.setSelectionToPath(
+            this.substitutionModel.pathSource
+        ),
+        this.editorTargetStepSubstitution.setSelectionToPath(
+            this.substitutionModel.pathTarget
+        )
+    ]);
+}
 
   ngOnDestroy() {
     this.countDeviceIdentifiers$.complete();

dynamic-mapping-ui/src/shared/mapping/shared.model.ts

@@ -89,8 +89,7 @@ export enum RepairStrategy {
   USE_FIRST_VALUE_OF_ARRAY = 'USE_FIRST_VALUE_OF_ARRAY',
   USE_LAST_VALUE_OF_ARRAY = 'USE_LAST_VALUE_OF_ARRAY',
   IGNORE = 'IGNORE',
-  REMOVE_IF_MISSING = 'REMOVE_IF_MISSING',
-  REMOVE_IF_NULL = 'REMOVE_IF_NULL',
+  REMOVE_IF_MISSING_OR_NULL = 'REMOVE_IF_MISSING_OR_NULL',
   CREATE_IF_MISSING = 'CREATE_IF_MISSING'
 }
 

resources/samples/mappings-INBOUND.json

@@ -374,7 +374,7 @@
       {
         "pathSource": "model",
         "pathTarget": "customProperties",
-        "repairStrategy": "REMOVE_IF_MISSING",
+        "repairStrategy": "REMOVE_IF_MISSING_OR_NULL",
         "expandArray": false
       }
     ],
@@ -701,13 +701,13 @@
       {
         "pathSource": " Measurementname = \"Airsensor\" ? {Seriesname:{\"value\": value, \"unit\": unit}} : null",
         "pathTarget": "Airsensor",
-        "repairStrategy": "REMOVE_IF_NULL",
+        "repairStrategy": "REMOVE_IF_MISSING_OR_NULL",
         "expandArray": false
       },
       {
         "pathSource": "Measurementname = \"Liquidsensor\" ? {Seriesname:{\"value\": value, \"unit\": unit}} : null",
         "pathTarget": "Liquidsensor",
-        "repairStrategy": "REMOVE_IF_NULL",
+        "repairStrategy": "REMOVE_IF_MISSING_OR_NULL",
         "expandArray": false
       },
       {
@@ -977,7 +977,7 @@
       {
         "pathSource": "(oil?{\"Motor\": {\"value\":oil, \"unit\":\"l\"}}:null)",
         "pathTarget": "c8y_OilMeasurement",
-        "repairStrategy": "REMOVE_IF_NULL",
+        "repairStrategy": "REMOVE_IF_MISSING_OR_NULL",
         "expandArray": false
       }
     ],