diff --git a/USERGUIDE.md b/USERGUIDE.md index 553c8f72..0146cdc5 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -313,15 +313,18 @@ To define a new substitution the following steps have to be performed: > required since this can overwrite mandatory Cumulocity attributes, e.g. source.id. This can result in API calls that are rejected by the Cumulocity backend! 1. Press the button "Add substitution". In the next modal dialog the following details can be specified: 1. Select option `Expand Array` if the result of the source expression is an array and you want to generate any of the following substitutions: -_ `multi-device-single-value` -_ `multi-device-multi-value` -_ `single-device-multi-value`\ - Otherwise an extracted array is treated as a single value, see [Different type of substitutions](#different-type-of-substitutions). 1. Select option `Resolve to externalId` if you want to resolve system Cumulocity Id to externalId using externalIdType. This can only be used for OUTBOUND mappings. 1. Select a `Reapir Strategy` that determines how the mapping is applied: -_ `DEFAULT`: Map the extracted values to the attribute addressed on right side -_ `USE_FIRST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the 1. item in the array and map this to the right side -_ `USE_LAST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the last item in the array and map this to the right side -_ `REMOVE_IF_MISSING`: When the left side of the mapping returns no result (not NULL), then delete the attribute (that is addressed in mapping) in the target on the right side. This avoids empty attribute, e.d. `airsensor: undefined` -_ `REMOVE_IF_NULL`: When the left side of the mapping returns `null`, then delete the attribute (that is addressed in mapping) in the target on the right side. This avoids empty attribute, e.d. `airsensor: undefined` + * `multi-device-single-value` + * `multi-device-multi-value` + * `single-device-multi-value`\ +\ +Otherwise an extracted array is treated as a single value, see [Different type of substitutions](#different-type-of-substitutions). +4. Select option `Resolve to externalId` if you want to resolve system Cumulocity Id to externalId using externalIdType. This can only be used for OUTBOUND mappings. +5. Select a `Repair Strategy` that determines how the mapping is applied: + * `DEFAULT`: Map the extracted values to the attribute addressed on right side + * `USE_FIRST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the 1. item in the array and map this to the right side + * `USE_LAST_VALUE_OF_ARRAY`: When the left side of the mapping returns an array, only use the last item in the array and map this to the right side + * `REMOVE_IF_MISSING_OR_NULL`: When the left side of the mapping returns no result (not NULL), then delete the attribute (that is addressed in mapping) in the target on the right side. This avoids empty attribute, e.g. `airsensor: undefined` +

diff --git a/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionCustomMeasurement.java b/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionCustomMeasurement.java index 04f0245b..561078cf 100644 --- a/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionCustomMeasurement.java +++ b/dynamic-mapping-extension/src/main/java/dynamic/mapping/processor/extension/external/ProcessorExtensionCustomMeasurement.java @@ -70,7 +70,7 @@ public void extractFromSource(ProcessingContext context) new ArrayList(Arrays.asList( new MappingSubstitution.SubstituteValue(null, MappingSubstitution.SubstituteValue.TYPE.TEXTUAL, - RepairStrategy.REMOVE_IF_NULL)))); + RepairStrategy.REMOVE_IF_MISSING_OR_NULL)))); postProcessingCache.put("c8y_Temperature", new ArrayList(Arrays.asList( diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingSubstitution.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingSubstitution.java index 6e6dd3a1..66a9242d 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingSubstitution.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/model/MappingSubstitution.java @@ -133,10 +133,9 @@ public static String toJsonString(Object obj) { public static void substituteValueInPayload(MappingType type, MappingSubstitution.SubstituteValue sub, DocumentContext jsonObject, String keys) throws JSONException { - boolean subValueMissing = sub.value == null; + boolean subValueMissingOrNull = sub == null || sub.value == null; // TOFDO fix this, we have to differentiate between {"nullField": null } and // "nonExisting" - boolean subValueNull = subValueMissing; try { if ("$".equals(keys)) { Object replacement = sub; @@ -148,8 +147,7 @@ public static void substituteValueInPayload(MappingType type, MappingSubstitutio } } } else { - if ((sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_MISSING) && subValueMissing) || - (sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_NULL) && subValueNull)) { + if ((sub.repairStrategy.equals(RepairStrategy.REMOVE_IF_MISSING_OR_NULL) && subValueMissingOrNull)) { jsonObject.delete(keys); } else if (sub.repairStrategy.equals(RepairStrategy.CREATE_IF_MISSING)) { jsonObject.set("$." + keys, sub.value); diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/RepairStrategy.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/RepairStrategy.java index 5ecb44c5..73389e79 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/RepairStrategy.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/processor/model/RepairStrategy.java @@ -26,7 +26,6 @@ public enum RepairStrategy { USE_FIRST_VALUE_OF_ARRAY, USE_LAST_VALUE_OF_ARRAY, IGNORE, - REMOVE_IF_MISSING, // remove the node in the target if it the evaluation of the source expression returns undefined, empty. This allows for using mapping with dynamic content - REMOVE_IF_NULL, // remove the node in the target if it the evaluation of the source expression returns null. This allows for using mapping with dynamic content + REMOVE_IF_MISSING_OR_NULL, // remove the node in the target if it the evaluation of the source expression returns undefined, empty. This allows for using mapping with dynamic content CREATE_IF_MISSING, // create the node in the target if it doesn't exist. This allows for using mapping with dynamic content } diff --git a/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-inbound.service.ts b/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-inbound.service.ts index e6968bf9..309e2944 100644 --- a/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-inbound.service.ts +++ b/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-inbound.service.ts @@ -165,7 +165,7 @@ export abstract class PayloadProcessorInbound { externalIdType: mapping.externalIdType, request, targetAPI: API.INVENTORY.name, - hide: true + hidden: true }); const response = await this.c8yClient.upsertDevice( { diff --git a/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-outbound.service.ts b/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-outbound.service.ts index d5e2e9bf..7ad28f9e 100644 --- a/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-outbound.service.ts +++ b/dynamic-mapping-ui/src/mapping/core/processor/payload-processor-outbound.service.ts @@ -163,8 +163,7 @@ export abstract class PayloadProcessorOutbound { jsonObject: JSON, keys: string ) { - const subValueMissing: boolean = sub.value == null; - const subValueNull: boolean = + const subValueMissingOrNull: boolean = sub.value == null || (sub.value != null && sub.value != undefined); if (keys == '$') { @@ -172,11 +171,7 @@ export abstract class PayloadProcessorOutbound { jsonObject[key] = getTypedValue(sub)[key as keyof unknown]; }); } else { - if ( - (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING && - subValueMissing) || - (sub.repairStrategy == RepairStrategy.REMOVE_IF_NULL && subValueNull) - ) { + if (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING_OR_NULL && subValueMissingOrNull) { _.unset(jsonObject, keys); } else if (sub.repairStrategy == RepairStrategy.CREATE_IF_MISSING) { // const pathIsNested: boolean = keys.includes('.') || keys.includes('['); diff --git a/dynamic-mapping-ui/src/mapping/core/processor/processor.model.ts b/dynamic-mapping-ui/src/mapping/core/processor/processor.model.ts index 38f646c4..4c6ea107 100644 --- a/dynamic-mapping-ui/src/mapping/core/processor/processor.model.ts +++ b/dynamic-mapping-ui/src/mapping/core/processor/processor.model.ts @@ -29,7 +29,7 @@ export interface C8YRequest { response?: any; targetAPI?: string; error?: string; - hide?:boolean; + hidden?:boolean; } export interface ProcessingContext { diff --git a/dynamic-mapping-ui/src/mapping/core/processor/util.ts b/dynamic-mapping-ui/src/mapping/core/processor/util.ts index 4dea7dbd..5df3e965 100644 --- a/dynamic-mapping-ui/src/mapping/core/processor/util.ts +++ b/dynamic-mapping-ui/src/mapping/core/processor/util.ts @@ -167,8 +167,7 @@ export function substituteValueInPayload( jsonObject: JSON, keys: string ) { - const subValueMissing: boolean = !sub || sub.value == null; - const subValueNull: boolean = subValueMissing; + const subValueMissingOrNull: boolean = !sub || sub.value == null; if (keys == '$') { Object.keys(getTypedValue(sub)).forEach((key) => { @@ -176,9 +175,8 @@ export function substituteValueInPayload( }); } else { if ( - (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING && - subValueMissing) || - (sub.repairStrategy == RepairStrategy.REMOVE_IF_NULL && subValueNull) + (sub.repairStrategy == RepairStrategy.REMOVE_IF_MISSING_OR_NULL && + subValueMissingOrNull) ) { _.unset(jsonObject, keys); } else if (sub.repairStrategy == RepairStrategy.CREATE_IF_MISSING) { diff --git a/dynamic-mapping-ui/src/mapping/step-testing/mapping-testing.component.ts b/dynamic-mapping-ui/src/mapping/step-testing/mapping-testing.component.ts index 41ac3d8c..b5f28b9a 100644 --- a/dynamic-mapping-ui/src/mapping/step-testing/mapping-testing.component.ts +++ b/dynamic-mapping-ui/src/mapping/step-testing/mapping-testing.component.ts @@ -212,7 +212,7 @@ export class MappingStepTestingComponent implements OnInit, OnDestroy { let nextIndex = currentIndex; do { nextIndex = (nextIndex >= results.length - 1) ? 0 : nextIndex + 1; - if (!results[nextIndex].hide) { + if (!results[nextIndex].hidden) { return nextIndex; } } while (nextIndex !== currentIndex); diff --git a/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.ts b/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.ts index 2157f2e0..84d6225f 100644 --- a/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.ts +++ b/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.ts @@ -35,7 +35,7 @@ import { AlertService, C8yStepper } from '@c8y/ngx-components'; import { FormlyFieldConfig } from '@ngx-formly/core'; import * as _ from 'lodash'; import { BsModalService } from 'ngx-bootstrap/modal'; -import { BehaviorSubject, Subject } from 'rxjs'; +import { BehaviorSubject, filter, Subject, take } from 'rxjs'; import { Content, Mode } from 'vanilla-jsoneditor'; import { ExtensionService } from '../../extension'; import { @@ -860,92 +860,130 @@ export class MappingStepperComponent implements OnInit, OnDestroy { this.expertMode = !this.expertMode; } - onUpdateSubstitution() { - if (this.selectedSubstitution != -1) { - const selected = this.selectedSubstitution; - const initialState = { - substitution: _.clone(this.mapping.substitutions[selected]), - mapping: this.mapping, - stepperConfiguration: this.stepperConfiguration, - isUpdate: true - }; - if ( - this.substitutionModel.sourceExpression.valid && - this.substitutionModel.targetExpression.valid - ) { - initialState.substitution.pathSource = - this.substitutionModel.pathSource; - initialState.substitution.pathTarget = - this.substitutionModel.pathTarget; - } - const modalRef = this.bsModalService.show(EditSubstitutionComponent, { - initialState - }); - modalRef.content.closeSubject.subscribe((editedSub) => { - if (editedSub) { - this.mapping.substitutions[selected] = editedSub; - this.updateSubstitutionValid(); - //this.substitutionModel = editedSub; not needed - } - }); + onUpdateSubstitution(): void { + const { selectedSubstitution, mapping, stepperConfiguration, substitutionModel } = this; + + // Early return if no substitution is selected + if (selectedSubstitution === -1) { + return; + } + // Prepare initial state + const initialState = { + substitution: { ...mapping.substitutions[selectedSubstitution] }, + mapping, + stepperConfiguration, + isUpdate: true + }; - // console.log("Updated subs I:", this.mapping.substitutions); + // Update paths if expressions are valid + const { sourceExpression, targetExpression, pathSource, pathTarget } = substitutionModel; + if (sourceExpression.valid && targetExpression.valid) { + initialState.substitution = { + ...initialState.substitution, + pathSource, + pathTarget + }; } - } - private addSubstitution(ns: MappingSubstitution) { - const sub: MappingSubstitution = _.clone(ns); - let duplicateSubstitutionIndex = -1; - let duplicate; - this.mapping.substitutions.forEach((s, index) => { - if (sub.pathTarget == s.pathTarget) { - duplicateSubstitutionIndex = index; - duplicate = this.mapping.substitutions[index]; - } - }); - const isDuplicate = duplicateSubstitutionIndex != -1; + // Show modal and handle response + const modalRef = this.bsModalService.show(EditSubstitutionComponent, { initialState }); + + modalRef.content.closeSubject + .pipe( + take(1), // Automatically unsubscribe after first emission + filter(Boolean) // Only proceed if we have valid data + ) + .subscribe({ + next: (editedSubstitution: MappingSubstitution) => { + try { + mapping.substitutions[selectedSubstitution] = editedSubstitution; + this.updateSubstitutionValid(); + } catch (error) { + console.log('Failed to update substitution', error); + } + }, + error: (error) => console.log('Error in modal operation', error) + }); +} + + private addSubstitution(newSubstitution: MappingSubstitution): void { + const substitution = { ...newSubstitution }; + const { mapping, stepperConfiguration, expertMode } = this; + + // Find duplicate substitution + const duplicateIndex = mapping.substitutions.findIndex( + sub => sub.pathTarget === substitution.pathTarget + ); + + const isDuplicate = duplicateIndex !== -1; + const duplicate = isDuplicate ? mapping.substitutions[duplicateIndex] : undefined; + const initialState = { - isDuplicate, - duplicate, - duplicateSubstitutionIndex, - substitution: sub, - mapping: this.mapping, - stepperConfiguration: this.stepperConfiguration + isDuplicate, + duplicate, + duplicateSubstitutionIndex: duplicateIndex, + substitution, + mapping, + stepperConfiguration }; - if (this.expertMode || isDuplicate) { - const modalRef = this.bsModalService.show(EditSubstitutionComponent, { - initialState - }); - modalRef.content.closeSubject.subscribe((newSub: MappingSubstitution) => { - if (newSub && !isDuplicate) { - this.mapping.substitutions.push(newSub); - } else if (newSub && isDuplicate) { - this.mapping.substitutions[duplicateSubstitutionIndex] = newSub; - } + // Handle simple case first (non-expert mode, no duplicates) + if (!expertMode && !isDuplicate) { + mapping.substitutions.push(substitution); this.updateSubstitutionValid(); - }); - } else { - this.mapping.substitutions.push(sub); - this.updateSubstitutionValid(); + return; } - } + + // Handle expert mode or duplicates + const modalRef = this.bsModalService.show(EditSubstitutionComponent, { + initialState + }); + + modalRef.content.closeSubject + .pipe( + take(1) // Automatically unsubscribe after first emission + ) + .subscribe((updatedSubstitution: MappingSubstitution) => { + if (!updatedSubstitution) return; + + if (isDuplicate) { + mapping.substitutions[duplicateIndex] = updatedSubstitution; + } else { + mapping.substitutions.push(updatedSubstitution); + } + + this.updateSubstitutionValid(); + }); +} async onSelectSubstitution(selected: number) { - if (selected < this.mapping.substitutions.length && selected > -1) { - this.selectedSubstitution = selected; - this.substitutionModel = _.clone(this.mapping.substitutions[selected]); - this.substitutionModel.stepperConfiguration = this.stepperConfiguration; - await this.editorSourceStepSubstitution.setSelectionToPath( - this.substitutionModel.pathSource - ); - await this.editorTargetStepSubstitution.setSelectionToPath( - this.substitutionModel.pathTarget - ); + const { mapping, stepperConfiguration } = this; + const { substitutions } = mapping; + + // Early return if selection is out of bounds + if (selected < 0 || selected >= substitutions.length) { + return; } - // console.log("Updated subs II:", this.mapping.substitutions); - } + + this.selectedSubstitution = selected; + + // Create substitution model + this.substitutionModel = { + ...substitutions[selected], + stepperConfiguration + }; + + // Parallel execution of path selections + await Promise.all([ + this.editorSourceStepSubstitution.setSelectionToPath( + this.substitutionModel.pathSource + ), + this.editorTargetStepSubstitution.setSelectionToPath( + this.substitutionModel.pathTarget + ) + ]); +} ngOnDestroy() { this.countDeviceIdentifiers$.complete(); diff --git a/dynamic-mapping-ui/src/shared/mapping/shared.model.ts b/dynamic-mapping-ui/src/shared/mapping/shared.model.ts index ab996b0d..53a5a709 100644 --- a/dynamic-mapping-ui/src/shared/mapping/shared.model.ts +++ b/dynamic-mapping-ui/src/shared/mapping/shared.model.ts @@ -89,8 +89,7 @@ export enum RepairStrategy { USE_FIRST_VALUE_OF_ARRAY = 'USE_FIRST_VALUE_OF_ARRAY', USE_LAST_VALUE_OF_ARRAY = 'USE_LAST_VALUE_OF_ARRAY', IGNORE = 'IGNORE', - REMOVE_IF_MISSING = 'REMOVE_IF_MISSING', - REMOVE_IF_NULL = 'REMOVE_IF_NULL', + REMOVE_IF_MISSING_OR_NULL = 'REMOVE_IF_MISSING_OR_NULL', CREATE_IF_MISSING = 'CREATE_IF_MISSING' } diff --git a/resources/samples/mappings-INBOUND.json b/resources/samples/mappings-INBOUND.json index 0d32ed1f..06532ed6 100644 --- a/resources/samples/mappings-INBOUND.json +++ b/resources/samples/mappings-INBOUND.json @@ -374,7 +374,7 @@ { "pathSource": "model", "pathTarget": "customProperties", - "repairStrategy": "REMOVE_IF_MISSING", + "repairStrategy": "REMOVE_IF_MISSING_OR_NULL", "expandArray": false } ], @@ -701,13 +701,13 @@ { "pathSource": " Measurementname = \"Airsensor\" ? {Seriesname:{\"value\": value, \"unit\": unit}} : null", "pathTarget": "Airsensor", - "repairStrategy": "REMOVE_IF_NULL", + "repairStrategy": "REMOVE_IF_MISSING_OR_NULL", "expandArray": false }, { "pathSource": "Measurementname = \"Liquidsensor\" ? {Seriesname:{\"value\": value, \"unit\": unit}} : null", "pathTarget": "Liquidsensor", - "repairStrategy": "REMOVE_IF_NULL", + "repairStrategy": "REMOVE_IF_MISSING_OR_NULL", "expandArray": false }, { @@ -977,7 +977,7 @@ { "pathSource": "(oil?{\"Motor\": {\"value\":oil, \"unit\":\"l\"}}:null)", "pathTarget": "c8y_OilMeasurement", - "repairStrategy": "REMOVE_IF_NULL", + "repairStrategy": "REMOVE_IF_MISSING_OR_NULL", "expandArray": false } ], diff --git a/resources/script/mgmt/dynamic-mapping-mgmt.sh b/resources/script/mgmt/dynamic-mapping-mgmt.sh index e757edb3..75b822a9 100755 --- a/resources/script/mgmt/dynamic-mapping-mgmt.sh +++ b/resources/script/mgmt/dynamic-mapping-mgmt.sh @@ -39,7 +39,7 @@ function show_usage() { echo " Step 4 create transformed mappings in tenant" echo " deleteMappings: Delete mappings" echo " deleteConnectors: Delete connectors" - echo " importMappingsAsMO: [filename]: Import mappings from file as managedObjects, see output format generated by exportMappingsAsMO" + echo " importMappingsAsMO: [filename]: Import mappings from file as managedObjects, see output format generated by exportMappingsAsMO" echo " exportMappingsAsMO: [filename]: Export mappings to file as managedObjects" echo "" echo "If filename is not provided, defaults to 'mappings-all.json'" @@ -115,6 +115,11 @@ function migrate_mappings() { . + {pathSource: "_IDENTITY_.externalId"} else . + end | + if (.repairStrategy == "REMOVE_IF_MISSING" or .repairStrategy == "REMOVE_IF_NULL") then + . + {repairStrategy: "REMOVE_IF_MISSING_OR_NULL"} + else + . end )) } else {} end) @@ -135,6 +140,11 @@ function migrate_mappings() { . + {pathTarget: "_IDENTITY_.externalId"} else . + end | + if (.repairStrategy == "REMOVE_IF_MISSING" or .repairStrategy == "REMOVE_IF_NULL") then + . + {repairStrategy: "REMOVE_IF_MISSING_OR_NULL"} + else + . end )) } else {} end) @@ -188,6 +198,23 @@ function check_prerequisites() { fi } + +function import_mappings_from_ui_export() { + local missing_programs=() + local filename="${1:-mappings-all.json}" # Default to mappings-all.json if no argument provided + + if [ ! -f "$filename" ]; then + echo "Error: File '$filename' not found!" + exit 1 + fi + + cat "$filename" | jq -c 'to_entries[] | { + name: "Mapping - " + ((.key + 1) | tostring), + type: "d11r_mapping", + d11r_mapping: .value +}' | c8y inventory create --template "input.value" +} + # Check if argument is provided if [ $# -ne 1 ]; then show_usage @@ -208,6 +235,9 @@ deleteConnectors) importMappingsAsMO) import_mappings_as_mo "${2:-}" ;; +importMappingsFromUIExport) + import_mappings_from_ui_export "${2:-}" + ;; exportMappingsAsMO) export_mappings_as_mo "${2:-}" ;;