diff --git a/common.props b/common.props index 0421cb61cb..13c352e712 100644 --- a/common.props +++ b/common.props @@ -34,7 +34,7 @@ - + true @@ -44,8 +44,8 @@ - - + + diff --git a/deploy/iotedge/eflow-setup.ps1 b/deploy/iotedge/eflow-setup.ps1 index ed2090a726..cbd1f90b45 100644 --- a/deploy/iotedge/eflow-setup.ps1 +++ b/deploy/iotedge/eflow-setup.ps1 @@ -40,6 +40,8 @@ param( [switch] $NoCleanup ) +#Requires -RunAsAdministrator + $eflowMsiUri = "https://aka.ms/AzEFLOWMSI_1_4_LTS_X64" $ErrorActionPreference = "Stop" diff --git a/deploy/k3s/docker-compose.yaml b/deploy/k3s/docker-compose.yaml deleted file mode 100644 index ccd8c88691..0000000000 --- a/deploy/k3s/docker-compose.yaml +++ /dev/null @@ -1,19 +0,0 @@ -services: - nginx-proxy: - image: nginxproxy/nginx-proxy - container_name: nginx-proxy - ports: - - "80:80" - volumes: - - /var/run/docker.sock:/tmp/docker.sock:ro - autok3s: - image: cnrancher/autok3s:v0.9.1 - init: true - ports: - - 8080 - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - $HOME/.autok3s/:$HOME/.autok3s/ - environment: - - AUTOK3S_CONFIG=$HOME/.autok3s/ - - VIRTUAL_HOST=autok3s.vcap.me \ No newline at end of file diff --git a/deploy/scripts/deploy.ps1 b/deploy/scripts/deploy.ps1 index 6312dd4058..2e0e47495e 100644 --- a/deploy/scripts/deploy.ps1 +++ b/deploy/scripts/deploy.ps1 @@ -37,29 +37,14 @@ .PARAMETER tenantId The Azure Active Directory tenant tied to the subscription(s) - that should be listed as options. - - .PARAMETER authTenantId - Specifies an Azure Active Directory tenant for authentication - that is different from the one tied to the subscription. - - .PARAMETER accountName - The account name to use if not to use default. + that should be listed as options. .PARAMETER applicationName The name of the application, if not local deployment. - .PARAMETER aadConfig - The aad configuration object (use aad-register.ps1 to create - object). If not provided, calls aad-register.ps1. - .PARAMETER context A previously created az context to be used for authentication. - .PARAMETER aadApplicationName - The application name to use when registering aad application. - If not set, uses applicationName. - .PARAMETER containerRegistryServer The container registry server to use to pull images @@ -111,10 +96,31 @@ Suggestion: use VM with at least 1 core and 2 GB of memory. Must Support Generation 1. + .PARAMETER noAadAppRegistration + Do not deploy service with Azure Active Directory authentication + support. Do not use in production!. + + .PARAMETER authTenantId + Specifies an Azure Active Directory tenant for authentication + that is different from the one tied to the subscription. + + .PARAMETER aadConfig + The aad configuration object (use aad-register.ps1 to create + object). If not provided, calls aad-register.ps1. + + .PARAMETER aadApplicationName + The application name to use when registering aad application. + If not set, uses applicationName. + .PARAMETER credentials Use these credentials to log in. If not provided you are prompted to provide credentials + .PARAMETER disableRbacAuthorization + Disable using Azure RBAC authorization using role assignments + to the managed identity and use legacy style keys and shared + access tokens to access services. + .PARAMETER isServicePrincipal The credentials provided are service principal credentials. @@ -136,10 +142,7 @@ param( [string] $resourceGroupLocation, [string] $subscriptionName, [string] $subscriptionId, - [string] $accountName, [string] $tenantId, - [string] $authTenantId, - [string] $aadApplicationName, [string] $containerRegistryServer, [string] $containerRegistryUsername, [securestring] $containerRegistryPassword, @@ -156,6 +159,10 @@ param( [pscredential] $credentials, [secureString] $accessToken, [switch] $isServicePrincipal, + [switch] $noAadAppRegistration, + [switch] $disableRbacAuthorization, + [string] $authTenantId, + [string] $aadApplicationName, [object] $aadConfig, [object] $context, [string] $environmentName = "AzureCloud", @@ -785,6 +792,14 @@ Function New-Deployment() { $templateParameters.Add("templateUrl", $templateUrl) } + if ($script:disableRbacAuthorization.IsPresent) { + Write-Host "Deploying without Azure RBAC role based authorization." + $templateParameters.Add("enableRbacAuthorization", $false) + } + else { + $templateParameters.Add("enableRbacAuthorization", $true) + } + # Select an application name if (($script:type -eq "local") -or ($script:type -eq "simulation")) { if ([string]::IsNullOrEmpty($script:applicationName) ` @@ -998,33 +1013,35 @@ Write-Warning "Standard_D4s_v4 VM with Nested virtualization for IoT Edge Eflow $aadAddReplyUrls = $false if (!$script:aadConfig) { - if ([string]::IsNullOrEmpty($script:aadApplicationName)) { - $script:aadApplicationName = $script:applicationName - } + if (!$script:noAadAppRegistration.IsPresent) { + if ([string]::IsNullOrEmpty($script:aadApplicationName)) { + $script:aadApplicationName = $script:applicationName + } - # register aad application - Write-Host - Write-Host "Registering client and services AAD applications in your tenant..." - $aadRegisterContext = $context - - # Use context of auth tenant - if (![string]::IsNullOrEmpty($authTenantId)) { - Write-Host "Connecting to AAD tenant $($authTenantId)..." - Connect-AzAccount -Tenant $authTenantId -ContextName AuthTenantId -Force - $aadRegisterContext = Select-AzContext AuthTenantId - } + # register aad application + Write-Host + Write-Host "Registering client and services AAD applications in your tenant..." + $aadRegisterContext = $context - $script:aadConfig = & (Join-Path $script:ScriptDir "aad-register.ps1") ` - -Context $aadRegisterContext -Name $script:aadApplicationName + # Use context of auth tenant + if (![string]::IsNullOrEmpty($authTenantId)) { + Write-Host "Connecting to AAD tenant $($authTenantId)..." + Connect-AzAccount -Tenant $authTenantId -ContextName AuthTenantId -Force + $aadRegisterContext = Select-AzContext AuthTenantId + } - Write-Host "Client and services AAD applications registered..." - Write-Host - $aadAddReplyUrls = $true + $script:aadConfig = & (Join-Path $script:ScriptDir "aad-register.ps1") ` + -Context $aadRegisterContext -Name $script:aadApplicationName - # Restore AD context - if (![string]::IsNullOrEmpty($authTenantId)) { - Write-Host "Switching to AAD tenant $($context.Tenant)..." - Set-AzContext -Context $context + Write-Host "Client and services AAD applications registered..." + Write-Host + $aadAddReplyUrls = $true + + # Restore AD context + if (![string]::IsNullOrEmpty($authTenantId)) { + Write-Host "Switching to AAD tenant $($context.Tenant)..." + Set-AzContext -Context $context + } } } elseif (($script:aadConfig -is [string]) -and (Test-Path $script:aadConfig)) { @@ -1060,16 +1077,16 @@ Write-Warning "Standard_D4s_v4 VM with Nested virtualization for IoT Edge Eflow # Register current aad user to access keyvault if (![string]::IsNullOrEmpty($script:aadConfig.UserPrincipalId)) { - $templateParameters.Add("keyVaultPrincipalId", $script:aadConfig.UserPrincipalId) + $templateParameters.Add("userPrincipalId", $script:aadConfig.UserPrincipalId) } else { $userPrincipalId = (Get-AzADUser -UserPrincipalName (Get-AzContext).Account.Id).Id if (![string]::IsNullOrEmpty($userPrincipalId)) { - $templateParameters.Add("keyVaultPrincipalId", $userPrincipalId) + $templateParameters.Add("userPrincipalId", $userPrincipalId) } else { - $templateParameters.Add("keyVaultPrincipalId", $script:aadConfig.FallBackPrincipalId) + $templateParameters.Add("userPrincipalId", $script:aadConfig.FallBackPrincipalId) } } @@ -1125,16 +1142,15 @@ Write-Warning "Standard_D4s_v4 VM with Nested virtualization for IoT Edge Eflow # # Add reply urls # - $replyUrls = New-Object System.Collections.Generic.List[System.String] - if ($aadAddReplyUrls) { + if ($aadAddReplyUrls -and ![string]::IsNullOrEmpty($script:aadConfig.WebAppId)) { + $replyUrls = New-Object System.Collections.Generic.List[System.String] + # retrieve existing urls $app = Get-AzADApplication -ApplicationId $script:aadConfig.WebAppId if ($app.ReplyUrls -and ($app.ReplyUrls.Count -ne 0)) { $replyUrls = $app.ReplyUrls; } - } - if ($aadAddReplyUrls -and ![string]::IsNullOrEmpty($script:aadConfig.WebAppId)) { $serviceUri = $deployment.Outputs["serviceUrl"].Value if (![string]::IsNullOrEmpty($serviceUri)) { @@ -1145,9 +1161,7 @@ Write-Warning "Standard_D4s_v4 VM with Nested virtualization for IoT Edge Eflow $replyUrls.Add("http://localhost:5000/signin-oidc") $replyUrls.Add("https://localhost:5001/signin-oidc") - } - if ($aadAddReplyUrls) { # register reply urls in web application registration Write-Host Write-Host "Registering reply urls for $($script:aadConfig.WebAppId)..." diff --git a/deploy/templates/azuredeploy.json b/deploy/templates/azuredeploy.json index 323b73b0e2..fee016f858 100644 --- a/deploy/templates/azuredeploy.json +++ b/deploy/templates/azuredeploy.json @@ -37,11 +37,18 @@ "description": "A client application identifier (GUID) in your Azure Active Directory tenant for public application such as CLI." } }, - "keyVaultPrincipalId": { + "enableRbacAuthorization": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Whether to use azure rbac." + } + }, + "userPrincipalId": { "type": "string", "defaultValue": "", "metadata": { - "description": "Specifies the object ID of a principal in your Azure Active Directory tenant to access keyvault." + "description": "Specifies the object ID of a principal in your Azure Active Directory tenant to access the deployed services." } }, "authorityUri": { @@ -222,6 +229,13 @@ "description": "The storage SKU to use." } }, + "storageAccountKeyEnabled": { + "type": "bool", + "defaultValue": "[not(parameters('enableRbacAuthorization'))]", + "metadata": { + "description": "Whether to use storage account key instead of RBAC access via managed identity." + } + }, "iotHubName": { "type": "string", "defaultValue": "[concat('iothub-', take(uniqueString(subscription().subscriptionId, resourceGroup().id), 6))]", @@ -274,11 +288,11 @@ "description": "The Azure IoT Hub default message retention in days." } }, - "iotHubRoleNameGuid": { - "type": "string", - "defaultValue": "[newGuid()]", + "iotHubSharedAccessKeyEnabled": { + "type": "bool", + "defaultValue": true, "metadata": { - "description": "A new GUID used to identify the IoTHub contrib role assignment" + "description": "Whether to use shared access key instead of RBAC access via managed identity." } }, "dpsName": { @@ -332,6 +346,20 @@ "description": "The KeyVault SKU to use." } }, + "keyVaultSoftDeleteEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Whether to enable soft delete for the key vault." + } + }, + "keyVaultUseAccessPolicies": { + "type": "bool", + "defaultValue": "[not(parameters('enableRbacAuthorization'))]", + "metadata": { + "description": "Whether to use classic auth roles in key vault." + } + }, "tags": { "type": "object", "defaultValue": {}, @@ -350,13 +378,29 @@ "simulationConfigurationResourceName": "[concat(deployment().name, '.simulation.configuration')]", "iotHubResourceId": "[resourceId('Microsoft.Devices/Iothubs', parameters('iotHubName'))]", "iotHubKeyName": "iothubowner", - "iotHubContributorRoleId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', '4fc6c259-987e-4a07-842e-c321cc9d413f')]", + "iotHubContributorRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4fc6c259-987e-4a07-842e-c321cc9d413f')]", + "iotHubPrincipalRoleAssignment": "[guid(parameters('iotHubName'), parameters('userPrincipalId'))]", + "iotHubRoleAssignment": "[guid(parameters('iotHubName'), parameters('managedIdentityName'))]", + "iotHubRoleAssignmentResourceId": "[resourceId('Microsoft.Resources/deployments', variables('iotHubRoleAssignment'))]", "iotHubKeyResource": "[resourceId('Microsoft.Devices/Iothubs/Iothubkeys', parameters('iotHubName'), variables('iotHubKeyName'))]", "iothubTelemetryConsumerGroup": "telemetry", "iothubEventsConsumerGroup": "events", "identityResourceId": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]", "storageResourceId": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageName'))]", + "storageBlobDataOwnerRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b')]", + "storageAccountContributorRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '17d1049b-9a84-46fb-8f53-869881c3d3ab')]", + "storagePrincipalBlobDataOwnerRoleAssignment": "[guid(parameters('storageName'), 'StorageBlobDataOwner', parameters('userPrincipalId'))]", + "storagePrincipalStorageAccountContributorRoleAssignment": "[guid(parameters('storageName'), 'StorageAccountContributor', parameters('userPrincipalId'))]", + "storageRoleAssignment": "[guid(parameters('storageName'), parameters('managedIdentityName'))]", + "storageRoleAssignmentResourceId": "[resourceId('Microsoft.Resources/deployments', variables('storageRoleAssignment'))]", "keyVaultResourceId": "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]", + "keyVaultSecretUserRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", + "keyVaultPrincipalRoleAssignment": "[guid(parameters('keyVaultName'), parameters('userPrincipalId'))]", + "keyVaultPrincipalAccessPolicy": { "tenantId": "[subscription().tenantId]", "objectId": "[parameters('userPrincipalId')]", "permissions": { "secrets": [ "all" ] }}, + "keyVaultAccessPolicies": "[if(and(not(empty(parameters('userPrincipalId'))), parameters('keyVaultUseAccessPolicies')), createArray(variables('keyVaultPrincipalAccessPolicy')), createArray())]", + "keyVaultSecretOfficerRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7')]", + "keyVaultRoleAssignment": "[guid(parameters('keyVaultName'), parameters('managedIdentityName'))]", + "keyVaultRoleAssignmentResourceId": "[resourceId('Microsoft.Resources/deployments', variables('keyVaultRoleAssignment'))]", "configurationResourceName": "[concat(deployment().name, '.configuration')]", "configurationResourceId": "[resourceId('Microsoft.Resources/deployments', variables('configurationResourceName'))]", "dpsResourceId": "[resourceId('Microsoft.Devices/provisioningServices', parameters('dpsName'))]", @@ -367,7 +411,7 @@ "comments": "Managed identity to access keyvault.", "name": "[parameters('managedIdentityName')]", "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", + "apiVersion": "2023-01-31", "location": "[resourceGroup().location]", "tags": "[parameters('tags')]" }, @@ -375,89 +419,86 @@ "comments": "KeyVault for secrets and certificate store.", "type": "Microsoft.KeyVault/vaults", "name": "[parameters('keyVaultName')]", - "apiVersion": "2016-10-01", + "apiVersion": "2023-07-01", "location": "[resourceGroup().location]", "tags": "[parameters('tags')]", "properties": { "enabledForDeployment": false, "enabledForTemplateDeployment": false, "enabledForVolumeEncryption": false, + "enableSoftDelete": "[parameters('keyVaultSoftDeleteEnabled')]", + "enableRbacAuthorization": "[not(parameters('keyVaultUseAccessPolicies'))]", + "accessPolicies": "[variables('keyVaultAccessPolicies')]", "tenantId": "[reference(variables('identityResourceId'), '2018-11-30').tenantId]", "sku": { "name": "[parameters('keyVaultSkuName')]", "family": "A" - }, - "accessPolicies": [ - { - "tenantId": "[reference(variables('identityResourceId'), '2018-11-30').tenantId]", - "objectId": "[reference(variables('identityResourceId'), '2018-11-30').principalId]", - "permissions": { - "keys": [ - "get", - "list", - "sign", - "unwrapKey", - "wrapKey", - "create" - ], - "secrets": [ - "get", - "list", - "set", - "delete" - ], - "certificates": [ - "get", - "list", - "update", - "create", - "import" - ] - } - } - ] + } }, "dependsOn": [ "[variables('identityResourceId')]" ] }, { - "comments": "Optional KeyVault principal access permissions for configuration secrets.", - "condition": "[not(empty(parameters('keyVaultPrincipalId')))]", + "comments": "Add get and list permissions to key vault secrets for our managed identity.", + "condition": "[parameters('keyVaultUseAccessPolicies')]", "type": "Microsoft.KeyVault/vaults/accessPolicies", "name": "[concat(parameters('keyVaultName'), '/add')]", - "apiVersion": "2016-10-01", + "apiVersion": "2023-07-01", "tags": "[parameters('tags')]", "properties": { "accessPolicies": [ { - "tenantId": "[subscription().tenantId]", - "objectId": "[parameters('keyVaultPrincipalId')]", + "tenantId": "[reference(variables('identityResourceId'), '2018-11-30').tenantId]", + "objectId": "[reference(variables('identityResourceId'), '2018-11-30').principalId]", "permissions": { - "keys": [ - "get", - "list", - "sign", - "unwrapKey", - "wrapKey", - "create" - ], "secrets": [ "get", - "list", - "set", - "delete" + "list" ] } } ] }, "dependsOn": [ + "[variables('keyVaultResourceId')]", + "[variables('identityResourceId')]" + ] + }, + { + "comments": "Or assign access to key vault to our managed identity using rbac.", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('keyVaultRoleAssignment')]", + "condition": "[not(parameters('keyVaultUseAccessPolicies'))]", + "scope": "[variables('keyVaultResourceId')]", + "properties": { + "roleDefinitionId": "[variables('keyVaultSecretUserRoleId')]", + "principalId": "[reference(variables('identityResourceId'), '2018-11-30').principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[variables('identityResourceId')]", "[variables('keyVaultResourceId')]" ] }, { - "comments": "Azure IoT Hub", + "comments": "And optionally the secret officer role to the KeyVault principal when access policies are not used.", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('keyVaultPrincipalRoleAssignment')]", + "condition": "[and(not(parameters('keyVaultUseAccessPolicies')), not(empty(parameters('userPrincipalId'))))]", + "scope": "[variables('keyVaultResourceId')]", + "properties": { + "roleDefinitionId": "[variables('keyVaultSecretOfficerRoleId')]", + "principalId": "[parameters('userPrincipalId')]" + }, + "dependsOn": [ + "[variables('keyVaultResourceId')]" + ] + }, + { + "comments": "Create an Azure IoT Hub.", "apiVersion": "2023-06-30", "type": "Microsoft.Devices/Iothubs", "name": "[parameters('iotHubName')]", @@ -541,14 +582,16 @@ ] }, { - "comments": "Assign access to IoT Hub to our managed identity", + "comments": "And assign access to IoT Hub to our managed identity.", "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2022-04-01", - "name": "[parameters('iotHubRoleNameGuid')]", + "condition": "[not(parameters('iotHubSharedAccessKeyEnabled'))]", + "name": "[variables('iotHubRoleAssignment')]", "scope": "[variables('iotHubResourceId')]", "properties": { "roleDefinitionId": "[variables('iotHubContributorRoleId')]", - "principalId": "[reference(variables('identityResourceId'), '2018-11-30').principalId]" + "principalId": "[reference(variables('identityResourceId'), '2018-11-30').principalId]", + "principalType": "ServicePrincipal" }, "dependsOn": [ "[variables('identityResourceId')]", @@ -556,7 +599,22 @@ ] }, { - "comments": "Telemetry Consumer Group in IoT Hub", + "comments": "And optionally the iot hub owner role to the user principal.", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('iotHubPrincipalRoleAssignment')]", + "condition": "[and(not(parameters('iotHubSharedAccessKeyEnabled')), not(empty(parameters('userPrincipalId'))))]", + "scope": "[variables('keyVaultResourceId')]", + "properties": { + "roleDefinitionId": "[variables('iotHubContributorRoleId')]", + "principalId": "[parameters('userPrincipalId')]" + }, + "dependsOn": [ + "[variables('iotHubResourceId')]" + ] + }, + { + "comments": "Create a Telemetry Consumer Group in IoT Hub.", "apiVersion": "2019-03-22", "name": "[concat(parameters('iotHubName'), '/events/', variables('iothubTelemetryConsumerGroup'))]", "type": "Microsoft.Devices/Iothubs/eventhubEndpoints/ConsumerGroups", @@ -566,7 +624,7 @@ ] }, { - "comments": "Edge Events Consumer Group in IoT Hub", + "comments": "And a edge Events Consumer Group in the hub.", "apiVersion": "2019-03-22", "name": "[concat(parameters('iotHubName'), '/events/', variables('iothubEventsConsumerGroup'))]", "type": "Microsoft.Devices/Iothubs/eventhubEndpoints/ConsumerGroups", @@ -576,10 +634,10 @@ ] }, { - "comments": "Blob storage account", + "comments": "Create blob storage account for event processing offset snapshots.", "type": "Microsoft.Storage/storageAccounts", "name": "[parameters('storageName')]", - "apiVersion": "2019-04-01", + "apiVersion": "2023-04-01", "location": "[resourceGroup().location]", "tags": "[parameters('tags')]", "kind": "StorageV2", @@ -593,6 +651,8 @@ "ipRules": [], "defaultAction": "Allow" }, + "defaultToOAuthAuthentication": "[not(parameters('storageAccountKeyEnabled'))]", + "allowSharedKeyAccess": "[parameters('storageAccountKeyEnabled')]", "supportsHttpsTrafficOnly": true, "encryption": { "services": { @@ -608,7 +668,54 @@ } }, { - "comments": "Applications monitoring instance", + "comments": "Assign access to own blobs in the storage to our managed identity.", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "condition": "[not(parameters('storageAccountKeyEnabled'))]", + "name": "[variables('storageRoleAssignment')]", + "scope": "[variables('storageResourceId')]", + "properties": { + "roleDefinitionId": "[variables('storageBlobDataOwnerRoleId')]", + "principalId": "[reference(variables('identityResourceId'), '2018-11-30').principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[variables('identityResourceId')]", + "[variables('storageResourceId')]" + ] + }, + { + "comments": "And optionally the storage blob data owner role...", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('storagePrincipalBlobDataOwnerRoleAssignment')]", + "condition": "[and(not(parameters('storageAccountKeyEnabled')), not(empty(parameters('userPrincipalId'))))]", + "scope": "[variables('storageResourceId')]", + "properties": { + "roleDefinitionId": "[variables('storageBlobDataOwnerRoleId')]", + "principalId": "[parameters('userPrincipalId')]" + }, + "dependsOn": [ + "[variables('storageResourceId')]" + ] + }, + { + "comments": "... as well as the storage account contributor role to the user principal.", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('storagePrincipalStorageAccountContributorRoleAssignment')]", + "condition": "[and(not(parameters('storageAccountKeyEnabled')), not(empty(parameters('userPrincipalId'))))]", + "scope": "[variables('storageResourceId')]", + "properties": { + "roleDefinitionId": "[variables('storageAccountContributorRoleId')]", + "principalId": "[parameters('userPrincipalId')]" + }, + "dependsOn": [ + "[variables('storageResourceId')]" + ] + }, + { + "comments": "Create a Azure monitor and app insights instance.", "type": "Microsoft.Insights/components", "name": "[parameters('appInsightsName')]", "apiVersion": "2015-05-01", @@ -685,10 +792,6 @@ "key": "PCS_IMAGES_NAMESPACE", "value": "[parameters('imagesNamespace')]" }, - { - "key": "PCS_IOTHUB_CONNSTRING", - "value": "[concat('HostName=', reference(variables('iotHubResourceId')).hostName, ';SharedAccessKeyName=', variables('iotHubKeyName'), ';SharedAccessKey=', listkeys(variables('iotHubKeyResource'), '2018-04-01').primaryKey)]" - }, { "key": "PCS_IOTHUB_EVENTHUBENDPOINT", "value": "[reference(variables('iotHubResourceId')).eventHubEndpoints.events.endpoint]" @@ -702,12 +805,12 @@ "value": "[variables('iothubEventsConsumerGroup')]" }, { - "key": "PCS_STORAGE_CONNSTRING", - "value": "[concat('DefaultEndpointsProtocol=https', ';EndpointSuffix=', environment().suffixes.storage, ';AccountName=', parameters('storageName'), ';AccountKey=', listKeys(variables('storageResourceId'), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).keys[0].value)]" + "key": "PCS_IOTHUB_CONNSTRING", + "value": "[concat('HostName=', reference(variables('iotHubResourceId')).hostName, if(parameters('iotHubSharedAccessKeyEnabled'), concat(';SharedAccessKeyName=', variables('iotHubKeyName'), ';SharedAccessKey=', listkeys(variables('iotHubKeyResource'), '2018-04-01').primaryKey), ''))]" }, { - "key": "PCS_STORAGE_KEY", - "value": "[listKeys(variables('storageResourceId'), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).keys[0].value]" + "key": "PCS_STORAGE_CONNSTRING", + "value": "[concat('DefaultEndpointsProtocol=https', ';EndpointSuffix=', environment().suffixes.storage, ';AccountName=', parameters('storageName'), if(parameters('storageAccountKeyEnabled'), concat(';AccountKey=', listKeys(variables('storageResourceId'), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).keys[0].value), ''))]" }, { "key": "PCS_KEYVAULT_URL", @@ -790,7 +893,10 @@ } }, "dependsOn": [ - "[variables('configurationResourceId')]" + "[variables('configurationResourceId')]", + "[variables('iotHubRoleAssignmentResourceId')]", + "[variables('keyVaultRoleAssignmentResourceId')]", + "[variables('storageRoleAssignmentResourceId')]" ] }, { @@ -827,11 +933,11 @@ ] }, { - "comments": "Azure Device Provisioning service.", + "comments": "Create Azure Device Provisioning service.", "type": "Microsoft.Devices/provisioningServices", "name": "[parameters('dpsName')]", "condition": "[variables('simulationDeployment')]", - "apiVersion": "2018-01-22", + "apiVersion": "2022-12-12", "location": "[resourceGroup().location]", "tags": "[parameters('tags')]", "sku": { @@ -882,7 +988,7 @@ "value": "[parameters('edgeVmSize')]" }, "simulationVmSize": { - "value": "[parameters('simulationVmSize')]" + "value": "[parameters('simulationVmSize')]" }, "edgeUserName": { "value": "[parameters('edgeUserName')]" diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/Discovery/DiscoveryTestTheory.cs b/e2e-tests/IIoTPlatform-E2E-Tests/Discovery/DiscoveryTestTheory.cs index 267aea7514..41417ea5e6 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/Discovery/DiscoveryTestTheory.cs +++ b/e2e-tests/IIoTPlatform-E2E-Tests/Discovery/DiscoveryTestTheory.cs @@ -36,7 +36,6 @@ public async Task TestPrepareAsync() { // Get OAuth token var token = await TestHelper.GetTokenAsync(_context, _cancellationTokenSource.Token); - Assert.NotEmpty(token); } [Fact, PriorityOrder(1)] diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/IIoTPlatform-E2E-Tests.csproj b/e2e-tests/IIoTPlatform-E2E-Tests/IIoTPlatform-E2E-Tests.csproj index cefea19410..a168c92099 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/IIoTPlatform-E2E-Tests.csproj +++ b/e2e-tests/IIoTPlatform-E2E-Tests/IIoTPlatform-E2E-Tests.csproj @@ -17,7 +17,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -40,9 +40,4 @@ Always - - - - - diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/Orchestrated/A_PublishSingleNodeOrchestratedTestTheory.cs b/e2e-tests/IIoTPlatform-E2E-Tests/Orchestrated/A_PublishSingleNodeOrchestratedTestTheory.cs index eb3059198d..40f4f6125e 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/Orchestrated/A_PublishSingleNodeOrchestratedTestTheory.cs +++ b/e2e-tests/IIoTPlatform-E2E-Tests/Orchestrated/A_PublishSingleNodeOrchestratedTestTheory.cs @@ -46,7 +46,6 @@ public async Task TestCollectOAuthToken() { using var cts = new CancellationTokenSource(TestConstants.MaxTestTimeoutMilliseconds); var token = await TestHelper.GetTokenAsync(_context, cts.Token); - Assert.NotEmpty(token); } [Fact, PriorityOrder(2)] diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs b/e2e-tests/IIoTPlatform-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs index 9d9f65fd14..7facc59f98 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs +++ b/e2e-tests/IIoTPlatform-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs @@ -258,10 +258,6 @@ public void LogEnvironment(ITestOutputHelper output) Log("PCS_SUBSCRIPTION_ID"); Log("PCS_RESOURCE_GROUP"); Log("PCS_SERVICE_URL"); - Log("PCS_AUTH_TENANT"); - Log("PCS_AUTH_CLIENT_APPID"); - Log("PCS_AUTH_CLIENT_SECRET"); - Log("PCS_AUTH_SERVICE_APPID"); Log("PLC_SIMULATION_URLS"); Log("IOT_EDGE_VERSION"); Log("IOT_EDGE_DEVICE_ID"); diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Registry.cs b/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Registry.cs index 408bee564e..4e418f7f71 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Registry.cs +++ b/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Registry.cs @@ -52,7 +52,10 @@ public static async Task RegisterServerAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { discoveryUrl }; @@ -82,7 +85,10 @@ public static async Task GetApplicationIdAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var response = await client.ExecuteAsync(request, ct).ConfigureAwait(false); Assert.NotNull(response); @@ -145,7 +151,10 @@ public static async Task UnregisterServerAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var response = await client.ExecuteAsync(request, ct).ConfigureAwait(false); Assert.NotNull(response); diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Twin.cs b/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Twin.cs index 09b8c5248f..ed4e6b22a2 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Twin.cs +++ b/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.Twin.cs @@ -104,7 +104,10 @@ public static class Twin request.AddQueryParameter("continuationToken", continuationToken); } - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var response = await client.ExecuteAsync(request, ct).ConfigureAwait(false); @@ -252,7 +255,10 @@ public static async Task GetMethodMetadataAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { @@ -293,7 +299,10 @@ public static async Task ReadNodeAttributesAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { attributes }; @@ -333,7 +342,10 @@ public static async Task WriteNodeAttributesAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { attributes }; @@ -376,7 +388,10 @@ public static async Task CallMethodAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { @@ -421,7 +436,10 @@ public static async Task GetBrowseNodePathAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { @@ -466,7 +484,10 @@ public static async Task GetBrowseNodePathAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { nodeId }; @@ -520,7 +541,10 @@ public static async Task WriteNodeValueAsync( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var body = new { nodeId, value, dataType }; diff --git a/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.cs b/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.cs index c3947fa19f..f68d7f3e83 100644 --- a/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.cs +++ b/e2e-tests/IIoTPlatform-E2E-Tests/TestHelper.cs @@ -52,6 +52,10 @@ public static async Task GetTokenAsync( CancellationToken ct = default ) { + if (string.IsNullOrWhiteSpace(context.IIoTPlatformConfigHubConfig.AuthClientId)) + { + return null; + } if (context.Token != null && (DateTime.UtcNow + TimeSpan.FromSeconds(10) < context.TokenExpiration)) { return context.Token; @@ -268,7 +272,10 @@ public static async Task CallRestApi( { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } if (body != null) { @@ -833,7 +840,10 @@ private static async Task GetEndpointsInternalAsync(IIoTPlatformTestCon { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var response = await client.ExecuteAsync(request, ct).ConfigureAwait(false); Assert.NotNull(response); @@ -855,7 +865,10 @@ private static async Task GetApplicationsInternalAsync(IIoTPlatformTest { Timeout = TimeSpan.FromMilliseconds(TestConstants.DefaultTimeoutInMilliseconds) }; - request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + if (accessToken != null) + { + request.AddHeader(TestConstants.HttpHeaderNames.Authorization, accessToken); + } var response = await client.ExecuteAsync(request, ct).ConfigureAwait(false); Assert.NotNull(response); diff --git a/e2e-tests/OpcPublisher-E2E-Tests/OpcPublisher-AE-E2E-Tests.csproj b/e2e-tests/OpcPublisher-E2E-Tests/OpcPublisher-AE-E2E-Tests.csproj index f72e1d3846..d8e11ff03c 100644 --- a/e2e-tests/OpcPublisher-E2E-Tests/OpcPublisher-AE-E2E-Tests.csproj +++ b/e2e-tests/OpcPublisher-E2E-Tests/OpcPublisher-AE-E2E-Tests.csproj @@ -10,12 +10,12 @@ - - + + - + @@ -46,9 +46,4 @@ PreserveNewest - - - - - diff --git a/e2e-tests/OpcPublisher-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs b/e2e-tests/OpcPublisher-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs index aec56c0df8..0e3d30b610 100644 --- a/e2e-tests/OpcPublisher-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs +++ b/e2e-tests/OpcPublisher-E2E-Tests/TestExtensions/IIoTPlatformTestContext.cs @@ -268,10 +268,6 @@ public void LogEnvironment(ITestOutputHelper output) Log("PCS_SUBSCRIPTION_ID"); Log("PCS_RESOURCE_GROUP"); Log("PCS_SERVICE_URL"); - Log("PCS_AUTH_TENANT"); - Log("PCS_AUTH_CLIENT_APPID"); - Log("PCS_AUTH_CLIENT_SECRET"); - Log("PCS_AUTH_SERVICE_APPID"); Log("PLC_SIMULATION_URLS"); Log("IOT_EDGE_VERSION"); Log("IOT_EDGE_DEVICE_ID"); diff --git a/samples/mqtt/MqttSamples.sln b/samples/Mqtt/MqttSamples.sln similarity index 100% rename from samples/mqtt/MqttSamples.sln rename to samples/Mqtt/MqttSamples.sln diff --git a/samples/mqtt/readme.md b/samples/Mqtt/readme.md similarity index 100% rename from samples/mqtt/readme.md rename to samples/Mqtt/readme.md diff --git a/samples/Netcap/src/Netcap.cs b/samples/Netcap/src/Netcap.cs index 0046c0bfb4..35844ca62f 100644 --- a/samples/Netcap/src/Netcap.cs +++ b/samples/Netcap/src/Netcap.cs @@ -640,7 +640,7 @@ private async ValueTask RunAsIoTHubConnectedModuleAsync( { // NOTE: This is for local testing against IoT Hub string deviceId; - var ncModuleId = "netcap"; + const string ncModuleId = "netcap"; _run ??= new RunOptions(); var edgeHubConnectionString = Environment.GetEnvironmentVariable("EdgeHubConnectionString"); if (!string.IsNullOrWhiteSpace(edgeHubConnectionString)) diff --git a/samples/Netcap/src/Netcap.csproj b/samples/Netcap/src/Netcap.csproj index 52285e5c33..0ad8f4f9a7 100644 --- a/samples/Netcap/src/Netcap.csproj +++ b/samples/Netcap/src/Netcap.csproj @@ -18,14 +18,14 @@ - - + + - - + + - + diff --git a/samples/Netcap/src/Pcap.cs b/samples/Netcap/src/Pcap.cs index d447b00629..2981582adf 100644 --- a/samples/Netcap/src/Pcap.cs +++ b/samples/Netcap/src/Pcap.cs @@ -97,6 +97,7 @@ public static void Merge(string folder, string outputFile) /// /// Get file path /// + /// /// /// private static string GetFilePath(string folder, int index) diff --git a/samples/Netcap/src/Publisher.cs b/samples/Netcap/src/Publisher.cs index 040d3bc814..289e1d9f3f 100644 --- a/samples/Netcap/src/Publisher.cs +++ b/samples/Netcap/src/Publisher.cs @@ -83,7 +83,7 @@ public Pcap.CaptureConfiguration GetCaptureConfiguration( return new Pcap.CaptureConfiguration(itf, "ip and tcp", maxPcapFileSize, maxPcapDuration); } - var filter = "src or dst host " + ((addresses.Count == 1) ? addresses.First() : + var filter = "src or dst host " + ((addresses.Count == 1) ? addresses[0] : ("(" + string.Join(" or ", addresses.Select(a => $"{a}")) + ")")); return new Pcap.CaptureConfiguration(itf, filter, maxPcapFileSize, maxPcapDuration); diff --git a/samples/Netcap/src/Storage.cs b/samples/Netcap/src/Storage.cs index dc0cf2d950..d25db240ab 100644 --- a/samples/Netcap/src/Storage.cs +++ b/samples/Netcap/src/Storage.cs @@ -24,11 +24,11 @@ internal sealed class Storage /// /// Create capture sync /// - /// - /// - /// - /// - /// + /// + /// + /// + /// + /// public Storage(string deviceId, string moduleId, string connectionString, ILogger logger, string? runName = null) { @@ -43,8 +43,8 @@ public Storage(string deviceId, string moduleId, string connectionString, /// /// Download files /// - /// - /// + /// + /// /// public async Task DownloadAsync(string path, CancellationToken ct = default) { @@ -138,8 +138,8 @@ await queueClient.DeleteMessageAsync(message.Value.MessageId, /// /// Upload file /// - /// - /// + /// + /// /// public async ValueTask UploadAsync(string file, CancellationToken ct = default) { @@ -193,7 +193,7 @@ await containerClient.CreateIfNotExistsAsync(PublicAccessType.None, /// /// Delete storage /// - /// + /// /// public async Task DeleteAsync(CancellationToken ct) { @@ -218,8 +218,8 @@ public async Task DeleteAsync(CancellationToken ct) /// /// Ensure queue exists and can be used /// - /// - /// + /// + /// /// private async Task EnsureQueueAsync(QueueClient queueClient, CancellationToken ct) { diff --git a/src/Azure.IIoT.OpcUa.Publisher.Models/src/Azure.IIoT.OpcUa.Publisher.Models.csproj b/src/Azure.IIoT.OpcUa.Publisher.Models/src/Azure.IIoT.OpcUa.Publisher.Models.csproj index 4a9ebbd880..f7955c6c78 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Models/src/Azure.IIoT.OpcUa.Publisher.Models.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Models/src/Azure.IIoT.OpcUa.Publisher.Models.csproj @@ -8,7 +8,7 @@ enable - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Models/src/ExtensionFieldModel.cs b/src/Azure.IIoT.OpcUa.Publisher.Models/src/ExtensionFieldModel.cs index 24769979ad..5f08f272eb 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Models/src/ExtensionFieldModel.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Models/src/ExtensionFieldModel.cs @@ -15,13 +15,6 @@ namespace Azure.IIoT.OpcUa.Publisher.Models [DataContract] public record ExtensionFieldModel { - /// - /// Field index of this variable in the dataset. - /// - [DataMember(Name = "fieldIndex", Order = 0, - EmitDefaultValue = false)] - public int FieldIndex { get; init; } - /// /// Field name or display name of the published variable /// @@ -44,10 +37,11 @@ public record ExtensionFieldModel public Guid DataSetClassFieldId { get; init; } /// - /// Unique Identifier of variable in the dataset. + /// Description for the field as it should show up in + /// the data set meta data. /// - [DataMember(Name = "id", Order = 4, + [DataMember(Name = "dataSetFieldDescription", Order = 4, EmitDefaultValue = false)] - public string? Id { get; set; } + public string? DataSetFieldDescription { get; set; } } } diff --git a/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedDataSetModel.cs b/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedDataSetModel.cs index e81e805cb4..90a2b95f62 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedDataSetModel.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedDataSetModel.cs @@ -5,7 +5,6 @@ namespace Azure.IIoT.OpcUa.Publisher.Models { - using Furly.Extensions.Serializers; using System.Collections.Generic; using System.Runtime.Serialization; @@ -41,7 +40,7 @@ public sealed record class PublishedDataSetModel /// [DataMember(Name = "extensionFields", Order = 3, EmitDefaultValue = false)] - public IDictionary? ExtensionFields { get; set; } + public IReadOnlyList? ExtensionFields { get; set; } /// /// Send keep alive messages for the data set diff --git a/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedNodeExpansionModel.cs b/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedNodeExpansionModel.cs index c067a6b9e6..044fae3e57 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedNodeExpansionModel.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Models/src/PublishedNodeExpansionModel.cs @@ -105,19 +105,19 @@ public sealed record class PublishedNodeExpansionModel public uint? MaxDepth { get; init; } /// - /// If true, treats instance nodes found just like + /// If false, treats instance nodes found just like /// objects that need to be expanded. In case of a - /// companion spec object type this should be set to - /// false, flattening the structure into a single + /// companion spec object type this could be set to + /// true, flattening the structure into a single /// writer that represents the object in its entirety. /// However, when using generic interfaces that can /// be implemented across objects in the address /// space and only its variables are important, it - /// might be useful to set this to true. + /// might be useful to set this to false. /// - [DataMember(Name = "doNotFlattenTypeInstance", Order = 7, + [DataMember(Name = "flattenTypeInstance", Order = 7, EmitDefaultValue = false)] - public bool DoNotFlattenTypeInstance { get; init; } + public bool FlattenTypeInstance { get; init; } /// /// Errors are silently discarded and only diff --git a/src/Azure.IIoT.OpcUa.Publisher.Models/tests/Azure.IIoT.OpcUa.Publisher.Models.Tests.csproj b/src/Azure.IIoT.OpcUa.Publisher.Models/tests/Azure.IIoT.OpcUa.Publisher.Models.Tests.csproj index 5dff748f98..72c24ba465 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Models/tests/Azure.IIoT.OpcUa.Publisher.Models.Tests.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Models/tests/Azure.IIoT.OpcUa.Publisher.Models.Tests.csproj @@ -4,7 +4,7 @@ - + all @@ -17,9 +17,9 @@ all runtime; build; native; contentfiles; analyzers - - - + + + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Azure.IIoT.OpcUa.Publisher.Module.Cli.csproj b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Azure.IIoT.OpcUa.Publisher.Module.Cli.csproj index 9141d11d39..eb974914e7 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Azure.IIoT.OpcUa.Publisher.Module.Cli.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Azure.IIoT.OpcUa.Publisher.Module.Cli.csproj @@ -7,8 +7,8 @@ true - - + + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/MachineTools.init b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/MachineTools.init index 9c564b5bc1..f0e2eb1e76 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/MachineTools.init +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/MachineTools.init @@ -19,6 +19,7 @@ ExpandAndCreateOrUpdateDataSetWriterEntries_V2 ] }, "request": { + "flattenTypeInstance": true } } diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Machinery.init b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Machinery.init index 47a6b71564..b69ae90d7b 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Machinery.init +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Machinery.init @@ -10,6 +10,9 @@ ExpandAndCreateOrUpdateDataSetWriterEntries "OpcNodes": [ { "Id": "nsu=http://opcfoundation.org/UA/Machinery/;i=1001" } ] + }, + "request": { + "flattenTypeInstance": true } } diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Objects.init b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Objects.init index 466fe6da59..4c873f30bc 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Objects.init +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Objects.init @@ -17,5 +17,6 @@ ExpandAndCreateOrUpdateDataSetWriterEntries_V2 ### # @on-error Shutdown_V2 + true ### diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Variables.init b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Variables.init index b66e94b162..ce54ed33e6 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Variables.init +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/cli/Initfiles/Variables.init @@ -22,5 +22,6 @@ ExpandAndCreateOrUpdateDataSetWriterEntries_V2 ### # @on-error Shutdown_V2 + true ### diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/src/Azure.IIoT.OpcUa.Publisher.Module.csproj b/src/Azure.IIoT.OpcUa.Publisher.Module/src/Azure.IIoT.OpcUa.Publisher.Module.csproj index 2bc360bd42..f6bd9f0d20 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/src/Azure.IIoT.OpcUa.Publisher.Module.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/src/Azure.IIoT.OpcUa.Publisher.Module.csproj @@ -33,15 +33,15 @@ - - - - - - - + + + + + + + - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Azure.IIoT.OpcUa.Publisher.Module.Tests.csproj b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Azure.IIoT.OpcUa.Publisher.Module.Tests.csproj index 10dfc657c0..96a47a06d5 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Azure.IIoT.OpcUa.Publisher.Module.Tests.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Azure.IIoT.OpcUa.Publisher.Module.Tests.csproj @@ -4,11 +4,11 @@ - + - + all diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Clients/FileSystemServicesRestClient.cs b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Clients/FileSystemServicesRestClient.cs index 64560be199..9c2c7703d7 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Clients/FileSystemServicesRestClient.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Clients/FileSystemServicesRestClient.cs @@ -466,7 +466,7 @@ private async Task StartAsync(CancellationToken ct) if (errorInfo != null) { // Error response - this.Result = new ServiceResponse + Result = new ServiceResponse { ErrorInfo = _serializer.Deserialize(errorInfo) }; diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Fixtures/PublisherModule.cs b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Fixtures/PublisherModule.cs index e71441b774..cabf7c4413 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Fixtures/PublisherModule.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Fixtures/PublisherModule.cs @@ -202,7 +202,7 @@ public PublisherModule(IMessageSink messageSink, IEnumerable de var register = ClientContainer.Resolve>(); _telemetry = new IoTHubTelemetryHandler(); _handler1 = register.Register(_telemetry); - Target = HubResource.Format(null, device.Id, device.ModuleId); + Target = Furly.Azure.HubResource.Format(null, device.Id, device.ModuleId); } else { diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Resources/ExtensionFields.json b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Resources/ExtensionFields.json new file mode 100644 index 0000000000..2e6db12c2b --- /dev/null +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Resources/ExtensionFields.json @@ -0,0 +1,22 @@ +[ + { + "EndpointUrl": "{{EndpointUrl}}", + "UseSecurity": false, + "DataSetWriterGroup": "{{DataSetWriterGroup}}", + "DataSetExtensionFields": { + "EngineeringUnits": "mm/sec", + "AssetId": 5, + "Important": false, + "Variance": 12.3465 + }, + "OpcNodes": [ + { + "Id": "ns=23;i=1259", + "OpcSamplingInterval": 200, + "OpcPublishingInterval": 200, + "DisplayName": "Output", + "SkipFirst": true + } + ] + } +] diff --git a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Sdk/ReferenceServer/BasicPubSubIntegrationTests.cs b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Sdk/ReferenceServer/BasicPubSubIntegrationTests.cs index e6f49b3b9a..cc48374dea 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Sdk/ReferenceServer/BasicPubSubIntegrationTests.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Module/tests/Sdk/ReferenceServer/BasicPubSubIntegrationTests.cs @@ -420,6 +420,44 @@ public async Task CanSendPendingConditionsToIoTHubTest() Assert.NotNull(metadata); } + [Fact] + public async Task CanSendExtensionFieldsToIoTHubTest() + { + // Arrange + // Act + var (metadata, result) = await ProcessMessagesAndMetadataAsync( + nameof(CanSendExtensionFieldsToIoTHubTest), "./Resources/ExtensionFields.json", + messageType: "ua-data", arguments: new string[] { "--mm=FullNetworkMessages", "--dm=false" }); + + Assert.Single(result); + + var messages = result + .SelectMany(x => x.Message.GetProperty("Messages").EnumerateArray()) + .ToArray(); + + // Assert + Assert.NotEmpty(messages); + Assert.All(messages, m => + { + var payload = m.GetProperty("Payload"); + Assert.False(payload.GetProperty("Important").GetProperty("Value").GetBoolean()); + Assert.Equal(5, payload.GetProperty("AssetId").GetProperty("Value").GetInt16()); + Assert.Equal("mm/sec", payload.GetProperty("EngineeringUnits").GetProperty("Value").GetString()); + Assert.Equal(12.3465, payload.GetProperty("Variance").GetProperty("Value").GetDouble(), 6); + Assert.NotEmpty(payload.GetProperty("EndpointUrl").GetProperty("Value").GetString()); + Assert.NotEmpty(payload.GetProperty("ApplicationUri").GetProperty("Value").GetString()); + }); + + Assert.NotNull(metadata); + var metadataFields = metadata.Value.Message.GetProperty("MetaData").GetProperty("Fields"); + Assert.Equal(JsonValueKind.Array, metadataFields.ValueKind); + var fieldNames = metadataFields.EnumerateArray().Select(v => v.GetProperty("Name").GetString()).ToHashSet(); + + var expectedNames = new[] { "Output", "EndpointUrl", "ApplicationUri", "EngineeringUnits", "AssetId", "Important", "Variance" }; + Assert.Equal(expectedNames.Length, fieldNames.Count); + Assert.All(expectedNames, n => fieldNames.Contains(n)); + } + [Fact] public async Task CanSendKeyFramesWithExtensionFieldsToIoTHubTest() { @@ -515,7 +553,7 @@ public async Task CanSendKeyFramesWithExtensionFieldsToIoTHubTestJsonReversible( Assert.False(payload.GetProperty("Important").GetProperty("Value").GetProperty("Body").GetBoolean()); Assert.Equal("5", payload.GetProperty("AssetId").GetProperty("Value").GetProperty("Body").GetString()); Assert.Equal("mm/sec", payload.GetProperty("EngineeringUnits").GetProperty("Value").GetProperty("Body").GetString()); - Assert.Equal(12.3465, payload.GetProperty("Variance").GetProperty("Value").GetProperty("Body").GetDouble()); + Assert.Equal(12.3465f, payload.GetProperty("Variance").GetProperty("Value").GetProperty("Body").GetSingle()); Assert.NotNull(metadata); var metadataFields = metadata.Value.Message.GetProperty("MetaData").GetProperty("Fields"); diff --git a/src/Azure.IIoT.OpcUa.Publisher.Sdk/src/Azure.IIoT.OpcUa.Publisher.Sdk.csproj b/src/Azure.IIoT.OpcUa.Publisher.Sdk/src/Azure.IIoT.OpcUa.Publisher.Sdk.csproj index 042317fde2..da53cbfe5c 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Sdk/src/Azure.IIoT.OpcUa.Publisher.Sdk.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Sdk/src/Azure.IIoT.OpcUa.Publisher.Sdk.csproj @@ -8,9 +8,9 @@ enable - - - + + + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/cli/Azure.IIoT.OpcUa.Publisher.Service.Cli.csproj b/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/cli/Azure.IIoT.OpcUa.Publisher.Service.Cli.csproj index c859ac047b..579493a655 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/cli/Azure.IIoT.OpcUa.Publisher.Service.Cli.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/cli/Azure.IIoT.OpcUa.Publisher.Service.Cli.csproj @@ -15,8 +15,8 @@ - - + + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk.csproj b/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk.csproj index 03cd73d327..a2fdf271eb 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk/src/Azure.IIoT.OpcUa.Publisher.Service.Sdk.csproj @@ -12,10 +12,10 @@ - - - - + + + + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi.csproj b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi.csproj index ed13add15a..5364a1f2aa 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi.csproj @@ -20,7 +20,7 @@ - + @@ -28,10 +28,10 @@ - - - - + + + + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Runtime/Security.cs b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Runtime/Security.cs index 1aa6797970..cdac6b4b1b 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Runtime/Security.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Runtime/Security.cs @@ -7,12 +7,15 @@ namespace Azure.IIoT.OpcUa.Publisher.Service.WebApi { using Furly.Extensions.Configuration; using Microsoft.AspNetCore.Authentication.JwtBearer; + using Microsoft.AspNetCore.Authorization; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Identity.Web; using System; using System.Linq; + using System.Threading.Tasks; /// /// Service auth configuration @@ -20,14 +23,39 @@ namespace Azure.IIoT.OpcUa.Publisher.Service.WebApi public static class Security { /// - /// Helper to add jwt bearer authentication + /// Add authentication /// /// - public static MicrosoftIdentityWebApiAuthenticationBuilder AddMicrosoftIdentityWebApiAuthentication( - this IServiceCollection services) + /// + /// + public static IServiceCollection AddAuthentication(this IServiceCollection services, + IConfiguration configuration) { + // + // Detect Microsoft.Identity.Web detects EasyAuth and register authentication if so. + // + if (AppServicesAuthenticationInformation.IsAppServicesAadAuthenticationEnabled) + { + services.AddAuthentication(AppServicesAuthenticationDefaults.AuthenticationScheme) + .AddMicrosoftIdentityWebApi(configuration, + subscribeToJwtBearerMiddlewareDiagnosticsEvents: false) + ; + return services; + } + + // + // Otherwise we check if a client id was configured and allow anonymous access if not. + // + var options = new MicrosoftIdentityOptions(); + new MicrosoftIdentity(configuration).Configure(options); + if (string.IsNullOrEmpty(options.ClientId)) + { + return services.AddSingleton(); + } + services.AddTransient, MicrosoftIdentity>(); services.AddTransient, MicrosoftIdentity>(); + services.AddTransient>(context => { // Support 2.8 where the audience does not contain api:// @@ -36,10 +64,47 @@ public static MicrosoftIdentityWebApiAuthenticationBuilder AddMicrosoftIdentityW options => options.TokenValidationParameters.ValidAudiences = clientId == null ? Enumerable.Empty() : clientId.YieldReturn()); }); - return services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme) + services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme) .AddMicrosoftIdentityWebApi(_ => { }, _ => { }, subscribeToJwtBearerMiddlewareDiagnosticsEvents: false) ; + return services; + } + + /// + /// This authoriuation handler will bypass all requirements + /// + internal sealed class AllowAnonymous : IAuthorizationHandler + { + /// + public AllowAnonymous(ILogger logger) + { + _logger = logger; + } + + /// + public Task HandleAsync(AuthorizationHandlerContext context) + { + // Simply pass all requirements + var authorized = false; + foreach (var requirement in context.PendingRequirements.ToList()) + { + authorized = true; + context.Succeed(requirement); + } + if (authorized) + { + _logger.LogWarning(@" + + An anonyomous user was authorized because of missing configuration. + !!! Do not use in production !!! + Configure app service authentication (see http://aka.ms/easyauth) +"); + } + return Task.CompletedTask; + } + + private readonly ILogger _logger; } /// diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Startup.cs b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Startup.cs index 586cd3be21..ba46e18b23 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Startup.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/src/Startup.cs @@ -89,15 +89,12 @@ public void ConfigureServices(IServiceCollection services) services.AddHttpClient(); services.AddExceptionSummarization(); - services.AddMicrosoftIdentityWebApiAuthentication(); + services.AddAuthentication(Configuration); services.AddAuthorization(); services.AddAuthorizationBuilder() - .AddPolicy(Policies.CanRead, - options => options.RequireAuthenticatedUser()) - .AddPolicy(Policies.CanWrite, - options => options.RequireAuthenticatedUser()) - .AddPolicy(Policies.CanPublish, - options => options.RequireAuthenticatedUser()); + .AddPolicy(Policies.CanRead, options => options.RequireAuthenticatedUser()) + .AddPolicy(Policies.CanWrite, options => options.RequireAuthenticatedUser()) + .AddPolicy(Policies.CanPublish, options => options.RequireAuthenticatedUser()); // Add controllers as services so they'll be resolved. services.AddControllers() @@ -140,7 +137,6 @@ public void Configure(IApplicationBuilder app, IHostApplicationLifetime appLifet app.UseRouting(); app.UseResponseCompression(); app.UseCors(); - app.UseAuthentication(); app.UseAuthorization(); app.UseSwagger(); diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Azure.IIoT.OpcUa.Publisher.Service.WebApi.Tests.csproj b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Azure.IIoT.OpcUa.Publisher.Service.WebApi.Tests.csproj index c081a38050..98a89d39fe 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Azure.IIoT.OpcUa.Publisher.Service.WebApi.Tests.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Azure.IIoT.OpcUa.Publisher.Service.WebApi.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Fixtures/PublisherModule.cs b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Fixtures/PublisherModule.cs index bfe886ab26..33c516f255 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Fixtures/PublisherModule.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Service.WebApi/tests/Fixtures/PublisherModule.cs @@ -104,7 +104,7 @@ public PublisherModule(ILifetimeScope serviceContainer) _config = configBuilder.Build(); _ = Server; // Ensure server is created - Target = HubResource.Format(null, device.Id, device.ModuleId); + Target = Furly.Azure.HubResource.Format(null, device.Id, device.ModuleId); } /// diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Azure.IIoT.OpcUa.Publisher.Service.csproj b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Azure.IIoT.OpcUa.Publisher.Service.csproj index 8654f5c0d8..4020e4928b 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Azure.IIoT.OpcUa.Publisher.Service.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Azure.IIoT.OpcUa.Publisher.Service.csproj @@ -6,7 +6,7 @@ enable - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/MonitoredItemMessageHandler.cs b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/MonitoredItemMessageHandler.cs index a4e8d34914..929f4b1b32 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/MonitoredItemMessageHandler.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/MonitoredItemMessageHandler.cs @@ -60,7 +60,7 @@ public async ValueTask HandleAsync(string deviceId, string? moduleId, ReadOnlySe { var type = BuiltInType.Null; var codec = _encoder.Create(context); - var extensionFields = message.ExtensionFields?.ToDictionary(k => k.Key, v => v.Value); + var extensionFields = message.ExtensionFields?.ToDictionary(k => k.DataSetFieldName, v => v.Value); var sample = new MonitoredItemMessageModel { PublisherId = (extensionFields != null && diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageJsonHandler.cs b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageJsonHandler.cs index f58c66b191..65ef5fd681 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageJsonHandler.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageJsonHandler.cs @@ -71,12 +71,12 @@ public async ValueTask HandleAsync(string deviceId, string? moduleId, ReadOnlySe Timestamp = dataSetMessage.Timestamp, Payload = new Dictionary() }; - foreach (var datapoint in dataSetMessage.Payload) + foreach (var datapoint in dataSetMessage.Payload.DataSetFields) { var codec = _encoder.Create(context); var type = BuiltInType.Null; var dataValue = datapoint.Value; - dataset.Payload[datapoint.Key] = dataValue == null ? null : new DataValueModel + dataset.Payload[datapoint.Name] = dataValue == null ? null : new DataValueModel { Value = codec.Encode(dataValue.WrappedValue, out type), DataType = type == BuiltInType.Null diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageUadpHandler.cs b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageUadpHandler.cs index 5d47ab0e64..4a6ae5e786 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageUadpHandler.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Service/src/Handlers/NetworkMessageUadpHandler.cs @@ -71,12 +71,12 @@ public async ValueTask HandleAsync(string deviceId, string? moduleId, ReadOnlySe Timestamp = dataSetMessage.Timestamp, Payload = new Dictionary() }; - foreach (var datapoint in dataSetMessage.Payload) + foreach (var datapoint in dataSetMessage.Payload.DataSetFields) { var codec = _encoder.Create(context); var type = BuiltInType.Null; var dataValue = datapoint.Value; - dataset.Payload[datapoint.Key] = dataValue == null ? null : new DataValueModel + dataset.Payload[datapoint.Name] = dataValue == null ? null : new DataValueModel { Value = codec.Encode(dataValue.WrappedValue, out type), DataType = type == BuiltInType.Null diff --git a/src/Azure.IIoT.OpcUa.Publisher.Service/tests/Azure.IIoT.OpcUa.Publisher.Service.Tests.csproj b/src/Azure.IIoT.OpcUa.Publisher.Service/tests/Azure.IIoT.OpcUa.Publisher.Service.Tests.csproj index e91cf7db09..55b48a9ccd 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Service/tests/Azure.IIoT.OpcUa.Publisher.Service.Tests.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Service/tests/Azure.IIoT.OpcUa.Publisher.Service.Tests.csproj @@ -11,7 +11,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Testing/src/Azure.IIoT.OpcUa.Publisher.Testing.Servers.csproj b/src/Azure.IIoT.OpcUa.Publisher.Testing/src/Azure.IIoT.OpcUa.Publisher.Testing.Servers.csproj index 2cc9421ec3..b64ef23107 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Testing/src/Azure.IIoT.OpcUa.Publisher.Testing.Servers.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Testing/src/Azure.IIoT.OpcUa.Publisher.Testing.Servers.csproj @@ -3,7 +3,7 @@ net8.0 Contains several test servers to run tests against true - true + true false false disable @@ -58,7 +58,7 @@ - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Azure.IIoT.OpcUa.Publisher.Testing.csproj b/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Azure.IIoT.OpcUa.Publisher.Testing.csproj index 9d872cdca0..dae299388d 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Azure.IIoT.OpcUa.Publisher.Testing.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Azure.IIoT.OpcUa.Publisher.Testing.csproj @@ -5,12 +5,12 @@ enable - - - + + + - + diff --git a/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests1.cs b/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests1.cs index 62ad7fe430..5d275c3987 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests1.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests1.cs @@ -309,7 +309,7 @@ public async Task ExpandBaseObjectTypeTest1Async(CancellationToken ct = default) DiscardErrors = false, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, - DoNotFlattenTypeInstance = true, + FlattenTypeInstance = false, CreateSingleWriter = false }, ct).ToListAsync(ct).ConfigureAwait(false); @@ -339,7 +339,7 @@ public async Task ExpandBaseObjectTypeTest2Async(CancellationToken ct = default) DiscardErrors = false, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, - DoNotFlattenTypeInstance = false, + FlattenTypeInstance = true, CreateSingleWriter = false }, ct).ToListAsync(ct).ConfigureAwait(false); @@ -379,7 +379,7 @@ public async Task ExpandBaseObjectsAndObjectTypesTestAsync(CancellationToken ct { DiscardErrors = false, ExcludeRootIfInstanceNode = false, - DoNotFlattenTypeInstance = true, + FlattenTypeInstance = false, NoSubTypesOfTypeNodes = false, CreateSingleWriter = false }, ct).ToListAsync(ct).ConfigureAwait(false); @@ -438,7 +438,7 @@ public async Task ExpandVariablesAndObjectsTest1Async(CancellationToken ct = def DiscardErrors = false, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, - DoNotFlattenTypeInstance = true, + FlattenTypeInstance = false, CreateSingleWriter = false }, ct).ToListAsync(ct).ConfigureAwait(false); @@ -472,7 +472,7 @@ public async Task ExpandVariableTypesTest1Async(CancellationToken ct = default) var result = Assert.Single(results); Assert.Null(result.ErrorInfo); Assert.NotNull(result.Result); - Assert.Equal(Opc.Ua.VariableTypeIds.PropertyType + "/Variables", result.Result.DataSetWriterId); + Assert.Equal(Opc.Ua.VariableTypeIds.PropertyType + "/PropertyType", result.Result.DataSetWriterId); Assert.NotNull(result.Result.OpcNodes); Assert.Equal(675, result.Result.OpcNodes.Count); } @@ -497,7 +497,7 @@ public async Task ExpandVariableTypesTest2Async(CancellationToken ct = default) var result = Assert.Single(results); Assert.Null(result.ErrorInfo); Assert.NotNull(result.Result); - Assert.Equal(Opc.Ua.VariableTypeIds.DataItemType + "/Variables", result.Result.DataSetWriterId); + Assert.Equal(Opc.Ua.VariableTypeIds.DataItemType + "/DataItemType", result.Result.DataSetWriterId); Assert.NotNull(result.Result.OpcNodes); Assert.Equal(96, result.Result.OpcNodes.Count); } @@ -527,7 +527,6 @@ public async Task ExpandVariableTypesTest3Async(CancellationToken ct = default) Assert.Null(r.ErrorInfo); Assert.NotNull(r.Result); Assert.NotNull(r.Result.OpcNodes); - Assert.EndsWith("/Variables", r.Result.DataSetWriterId, StringComparison.InvariantCulture); total += r.Result.OpcNodes.Count; }); Assert.Equal(96 + 675, total); diff --git a/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests2.cs b/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests2.cs index 35ec58671e..02da0225a4 100644 --- a/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests2.cs +++ b/src/Azure.IIoT.OpcUa.Publisher.Testing/tests/Tests/TestData/ConfigurationTests2.cs @@ -447,7 +447,7 @@ public async Task ConfigureFromBaseObjectTypeTest1Async(CancellationToken ct = d new PublishedNodeExpansionModel { DiscardErrors = false, - DoNotFlattenTypeInstance = true, + FlattenTypeInstance = false, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, CreateSingleWriter = false @@ -480,7 +480,7 @@ public async Task ConfigureFromBaseObjectTypeTest2Async(CancellationToken ct = d new PublishedNodeExpansionModel { DiscardErrors = false, - DoNotFlattenTypeInstance = false, + FlattenTypeInstance = true, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, CreateSingleWriter = false @@ -524,7 +524,7 @@ public async Task ConfigureFromBaseObjectsAndObjectTypesTestAsync(CancellationTo new PublishedNodeExpansionModel { DiscardErrors = false, - DoNotFlattenTypeInstance = true, + FlattenTypeInstance = false, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, CreateSingleWriter = false @@ -588,7 +588,7 @@ public async Task ConfigureFromVariablesAndObjectsTest1Async(CancellationToken c new PublishedNodeExpansionModel { DiscardErrors = false, - DoNotFlattenTypeInstance = true, + FlattenTypeInstance = false, ExcludeRootIfInstanceNode = false, NoSubTypesOfTypeNodes = false, CreateSingleWriter = false @@ -627,7 +627,7 @@ public async Task ConfigureFromVariableTypesTest1Async(CancellationToken ct = de var result = Assert.Single(results); Assert.Null(result.ErrorInfo); Assert.NotNull(result.Result); - Assert.Equal(Opc.Ua.VariableTypeIds.PropertyType + "/Variables", result.Result.DataSetWriterId); + Assert.Equal(Opc.Ua.VariableTypeIds.PropertyType + "/PropertyType", result.Result.DataSetWriterId); Assert.NotNull(result.Result.OpcNodes); Assert.Equal(675, result.Result.OpcNodes.Count); _publishedNodesServices.Verify(); @@ -655,7 +655,7 @@ public async Task ConfigureFromVariableTypesTest2Async(CancellationToken ct = de var result = Assert.Single(results); Assert.Null(result.ErrorInfo); Assert.NotNull(result.Result); - Assert.Equal(Opc.Ua.VariableTypeIds.DataItemType + "/Variables", result.Result.DataSetWriterId); + Assert.Equal(Opc.Ua.VariableTypeIds.DataItemType + "/DataItemType", result.Result.DataSetWriterId); Assert.NotNull(result.Result.OpcNodes); Assert.Equal(96, result.Result.OpcNodes.Count); _publishedNodesServices.Verify(); @@ -688,7 +688,6 @@ public async Task ConfigureFromVariableTypesTest3Async(CancellationToken ct = de Assert.Null(r.ErrorInfo); Assert.NotNull(r.Result); Assert.NotNull(r.Result.OpcNodes); - Assert.EndsWith("/Variables", r.Result.DataSetWriterId, StringComparison.InvariantCulture); total += r.Result.OpcNodes.Count; }); Assert.Equal(96 + 675, total); diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Azure.IIoT.OpcUa.Publisher.csproj b/src/Azure.IIoT.OpcUa.Publisher/src/Azure.IIoT.OpcUa.Publisher.csproj index 65261299ac..1d6b82d848 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Azure.IIoT.OpcUa.Publisher.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Azure.IIoT.OpcUa.Publisher.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Extensions/PublishedDataSetSourceModelEx.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Extensions/PublishedDataSetSourceModelEx.cs index c251b890c6..756d55c501 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Extensions/PublishedDataSetSourceModelEx.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Extensions/PublishedDataSetSourceModelEx.cs @@ -50,11 +50,9 @@ public static SubscriptionModel ToSubscriptionModel( /// /// /// - /// /// public static IReadOnlyList ToMonitoredItems( - this PublishedDataSetSourceModel dataSetSource, NamespaceFormat namespaceFormat, - IDictionary? extensionFields = null) + this PublishedDataSetSourceModel dataSetSource, NamespaceFormat namespaceFormat) { var monitoredItems = Enumerable.Empty(); if (dataSetSource.PublishedVariables?.PublishedData != null) @@ -69,11 +67,6 @@ public static IReadOnlyList ToMonitoredItems( .Concat(dataSetSource.PublishedEvents .ToMonitoredItems(dataSetSource.SubscriptionSettings, namespaceFormat)); } - if (extensionFields != null) - { - monitoredItems = monitoredItems - .Concat(extensionFields.ToMonitoredItems()); - } return monitoredItems.ToList(); } @@ -103,24 +96,6 @@ internal static IEnumerable ToMonitoredItems( } } - /// - /// Convert to extension field items - /// - /// - /// - internal static IEnumerable ToMonitoredItems( - this IDictionary extensionFields) - { - foreach (var extensionField in extensionFields) - { - var item = extensionField.ToMonitoredItemTemplate(); - if (item != null) - { - yield return item; - } - } - } - /// /// Convert to monitored items /// @@ -221,26 +196,6 @@ internal static IEnumerable ToMonitoredItems( }; } - /// - /// Convert to monitored item - /// - /// - /// - internal static ExtensionFieldItemModel? ToMonitoredItemTemplate( - this KeyValuePair extensionField) - { - if (string.IsNullOrEmpty(extensionField.Key)) - { - return null; - } - return new ExtensionFieldItemModel - { - DataSetFieldName = extensionField.Key, - Value = extensionField.Value, - StartNodeId = string.Empty - }; - } - /// /// Convert published dataset variable to monitored item /// diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Models/DataSetWriterContext.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Models/DataSetWriterContext.cs index 473a4581f7..f298988cc4 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Models/DataSetWriterContext.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Models/DataSetWriterContext.cs @@ -6,7 +6,9 @@ namespace Azure.IIoT.OpcUa.Publisher.Models { using Furly.Extensions.Messaging; + using Furly.Extensions.Serializers; using System; + using System.Collections.Generic; /// /// Context to add to notification to convey data required for @@ -59,6 +61,11 @@ public record class DataSetWriterContext /// public required PublishedDataSetMessageSchemaModel? MetaData { get; init; } + /// + /// Extension fields + /// + public required IReadOnlyList<(string, Opc.Ua.DataValue?)> ExtensionFields { get; init; } + /// /// Sequence number inside the writer /// diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Services/AsyncEnumerableBrowser.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Services/AsyncEnumerableBrowser.cs index af363e61eb..d2e7c1f03f 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Services/AsyncEnumerableBrowser.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Services/AsyncEnumerableBrowser.cs @@ -248,12 +248,10 @@ private IEnumerable MatchReferences(BrowseFrame frame, ServiceCallContext con var matching = refs .Where(reference => reference.NodeClass == _matchClass && (reference.NodeId?.ServerIndex ?? 1u) == 0) - .Where(reference => _typeDefinitionId == null || - reference.TypeDefinition == _typeDefinitionId || - (_includeTypeDefinitionSubtypes && context.Session.TypeTree - .IsTypeOf(reference.TypeDefinition, _typeDefinitionId))) + .Where(reference => MatchTypeDefinitionId(context.Session, reference.TypeDefinition)) .Select(reference => new BrowseFrame((NodeId)reference.NodeId, - reference.BrowseName, reference.DisplayName?.Text, frame)) + reference.BrowseName, reference.DisplayName?.Text, + reference.TypeDefinition, reference.NodeClass, frame)) .ToList(); if (_stopWhenFound && matching.Count != 0) @@ -265,7 +263,8 @@ private IEnumerable MatchReferences(BrowseFrame frame, ServiceCallContext con if (!stop.Contains((NodeId)reference.NodeId)) { Push(reference.NodeId, reference.BrowseName, - reference.DisplayName?.Text, frame); + reference.DisplayName?.Text, reference.TypeDefinition, + reference.NodeClass, frame); } } } @@ -275,7 +274,8 @@ private IEnumerable MatchReferences(BrowseFrame frame, ServiceCallContext con foreach (var reference in refs) { Push(reference.NodeId, reference.BrowseName, - reference.DisplayName?.Text, frame); + reference.DisplayName?.Text, reference.TypeDefinition, + reference.NodeClass, frame); } } @@ -286,6 +286,23 @@ private IEnumerable MatchReferences(BrowseFrame frame, ServiceCallContext con // Pass matching on return HandleMatching(context, matching); + + // Helper to match type definition to desired type definition id + bool MatchTypeDefinitionId(IOpcUaSession session, ExpandedNodeId typeDefinition) + { + if (typeDefinition == _typeDefinitionId || _typeDefinitionId == null) + { + return true; + } + if (_includeTypeDefinitionSubtypes && !Opc.Ua.NodeId.IsNull(typeDefinition)) + { + var typeDefinitionId = ExpandedNodeId.ToNodeId(typeDefinition, + session.MessageContext.NamespaceUris); + return session.NodeCache.IsTypeOf(typeDefinitionId, _typeDefinitionId); + } + return false; + + } } /// @@ -295,7 +312,7 @@ private void Start() { // Initialize _visited.Clear(); - _browseStack.Push(new BrowseFrame(_root, null, null)); + _browseStack.Push(new BrowseFrame(_root)); Push(context => BrowseAsync(context)); } @@ -305,9 +322,12 @@ private void Start() /// /// /// + /// + /// /// private void Push(ExpandedNodeId nodeId, QualifiedName? browseName, - string? displayName, BrowseFrame? parent) + string? displayName, ExpandedNodeId typeDefinition, + Opc.Ua.NodeClass nodeClass, BrowseFrame? parent) { if ((nodeId?.ServerIndex ?? 1u) != 0) { @@ -316,7 +336,8 @@ private void Push(ExpandedNodeId nodeId, QualifiedName? browseName, var local = (NodeId)nodeId; if (!NodeId.IsNull(local) && !_visited.Contains(local)) { - var frame = new BrowseFrame(local, browseName, displayName, parent); + var frame = new BrowseFrame(local, browseName, displayName, + typeDefinition, nodeClass, parent); if (_maxDepth.HasValue && frame.Depth >= _maxDepth.Value) { return; @@ -347,9 +368,12 @@ private void Push(ExpandedNodeId nodeId, QualifiedName? browseName, /// /// /// + /// + /// /// - protected record class BrowseFrame(NodeId NodeId, QualifiedName? BrowseName, - string? DisplayName, BrowseFrame? Parent = null) + protected record class BrowseFrame(NodeId NodeId, QualifiedName? BrowseName = null, + string? DisplayName = null, ExpandedNodeId? TypeDefinitionId = null, + Opc.Ua.NodeClass? NodeClass = null, BrowseFrame? Parent = null) { /// /// Current depth of this frame diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Services/ConfigurationServices.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Services/ConfigurationServices.cs index 3303998eb4..f2a6857fe9 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Services/ConfigurationServices.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Services/ConfigurationServices.cs @@ -5,6 +5,7 @@ namespace Azure.IIoT.OpcUa.Publisher.Services { + using Azure.Core; using Azure.IIoT.OpcUa.Publisher; using Azure.IIoT.OpcUa.Publisher.Config.Models; using Azure.IIoT.OpcUa.Publisher.Models; @@ -15,6 +16,7 @@ namespace Azure.IIoT.OpcUa.Publisher.Services using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Opc.Ua; + using Opc.Ua.Client; using Opc.Ua.Extensions; using System; using System.Buffers; @@ -23,6 +25,7 @@ namespace Azure.IIoT.OpcUa.Publisher.Services using System.IO; using System.Linq; using System.Runtime.CompilerServices; + using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -122,7 +125,7 @@ async IAsyncEnumerable> CoreAsync( var browser = new ConfigBrowser(entry with { // Named object in the address space of the server. - OpcNodes = new List { new OpcNodeModel { Id = AssetsEx.Root } } + OpcNodes = new List { new () { Id = AssetsEx.Root } } }, expansion, _options, null, _logger, _timeProvider, true); // Browse and swap the data set writer id and data set name to make an asset entry. @@ -234,7 +237,7 @@ public async Task> CreateOrUpdateAsset } errorInfo = await context.Session.WriteAsync( request.Header.ToRequestHeader(_timeProvider), nodeId, - fileHandle.Value, buffer.AsMemory().Slice(0, read), + fileHandle.Value, buffer.AsMemory()[..read], context.Ct).ConfigureAwait(false); if (errorInfo != null) { @@ -553,8 +556,7 @@ async Task ProcessAsync(ObjectToExpand currentObject, ServiceCallContext context { DataSetName = currentObject.CreateDataSetName( context.Session.MessageContext), - DataSetWriterId = currentObject.CreateWriterId( - context.Session.MessageContext), + DataSetWriterId = currentObject.CreateWriterId(), OpcNodes = currentObject .GetOpcNodeModels( currentObject.OriginalNode.NodeFromConfiguration, @@ -589,19 +591,78 @@ private async ValueTask>> var nodeId = await context.Session.ResolveNodeIdAsync(_request.Header, node.Id, node.BrowsePath, nameof(node.BrowsePath), TimeProvider, context.Ct).ConfigureAwait(false); - _expanded.Add(new NodeToExpand(node, nodeId)); - } - // Resolve node classes - var results = await context.Session.ReadAttributeAsync( - _request.Header.ToRequestHeader(TimeProvider), - _expanded.Select(r => r.NodeId ?? NodeId.Null), - (uint)NodeAttribute.NodeClass, context.Ct).ConfigureAwait(false); - foreach (var result in results.Zip(_expanded)) - { - result.Second.AddErrorInfo(result.First.Item2); - result.Second.NodeClass = (uint)result.First.Item1; + var readValueIds = new ReadValueIdCollection + { + new ReadValueId + { + NodeId = nodeId, + AttributeId = Attributes.NodeClass + }, + new ReadValueId + { + NodeId = nodeId, + AttributeId = Attributes.BrowseName + }, + new ReadValueId + { + NodeId = nodeId, + AttributeId = Attributes.DisplayName + }, + new ReadValueId + { + NodeId = nodeId, + AttributeId = Attributes.EventNotifier + } + }; + var response = await context.Session.Services.ReadAsync( + _request.Header.ToRequestHeader(TimeProvider), 0, + Opc.Ua.TimestampsToReturn.Neither, readValueIds, + context.Ct).ConfigureAwait(false); + + var readResults = response.Validate(response.Results, + s => s.StatusCode, response.DiagnosticInfos, readValueIds); + + var errorInfo = readResults.ErrorInfo ?? + readResults[0].ErrorInfo; + var nodeClass = errorInfo != null ? Opc.Ua.NodeClass.Unspecified : + readResults[0].Result.GetValueOrDefault(); + var browseName = errorInfo != null ? null : + readResults[1].Result.GetValueOrDefault(); + var displayName = errorInfo != null ? null : + readResults[2].Result.GetValueOrDefault(); + var eventNotifier = errorInfo != null ? (byte)0 : + readResults[3].Result.GetValueOrDefault(); + + ExpandedNodeId? typeDefinitionId = null; + if (errorInfo == null) + { + switch (nodeClass) + { + case Opc.Ua.NodeClass.ObjectType: + case Opc.Ua.NodeClass.VariableType: + typeDefinitionId = nodeId; + break; + case Opc.Ua.NodeClass.Object: + var (results, errorInfo2) = await context.Session.FindAsync( + _request.Header.ToRequestHeader(TimeProvider), + nodeId.YieldReturn(), ReferenceTypeIds.HasTypeDefinition, + nodeClassMask: (uint)Opc.Ua.NodeClass.ObjectType, + ct: context.Ct).ConfigureAwait(false); + errorInfo = errorInfo2; + if (errorInfo != null) + { + break; + } + Debug.Assert(results.Count == 1); + typeDefinitionId = results[0].Node; + break; + } + } + _expanded.Add(new NodeToExpand(node, nodeId, nodeClass, + browseName, displayName, eventNotifier, typeDefinitionId, errorInfo)); } + if (!TryMoveToNextNode()) { // Complete @@ -616,7 +677,6 @@ private async ValueTask>> /// /// /// - /// private async ValueTask>> EndAsync( ServiceCallContext context) { @@ -683,7 +743,7 @@ private bool TryMoveToNextNode() { // Add root CurrentNode.AddObjectsOrVariables( - new BrowseFrame(CurrentNode.NodeId!, null, null).YieldReturn()); + new BrowseFrame(CurrentNode.NodeId!).YieldReturn()); if (_request.MaxDepth == 0) { @@ -703,7 +763,7 @@ private bool TryMoveToNextNode() CurrentNode.NodeClass == (uint)Opc.Ua.NodeClass.ObjectType ? Opc.Ua.NodeClass.Object : Opc.Ua.NodeClass.Variable; var stopWhenFound = instanceClass == Opc.Ua.NodeClass.Variable || - !_request.DoNotFlattenTypeInstance; + _request.FlattenTypeInstance; Restart(ObjectIds.ObjectsFolder, maxDepth: _request.MaxDepth, typeDefinitionId: CurrentNode.NodeId, stopWhenFound: stopWhenFound, @@ -715,7 +775,7 @@ private bool TryMoveToNextNode() { // Add root CurrentNode.AddObjectsOrVariables( - new BrowseFrame(CurrentNode.NodeId!, null, null).YieldReturn()); + new BrowseFrame(CurrentNode.NodeId!).YieldReturn()); if (_request.MaxLevelsToExpand == 0) { @@ -760,7 +820,7 @@ private bool TryMoveToNextObject() var maxDepth = _request.MaxLevelsToExpand == 0 ? (uint?)null : _request.MaxLevelsToExpand; if (_currentObject.OriginalNode.NodeClass == (uint)Opc.Ua.NodeClass.ObjectType - && !_request.DoNotFlattenTypeInstance) + && _request.FlattenTypeInstance) { nodeClass |= Opc.Ua.NodeClass.Object; maxDepth = null; @@ -817,7 +877,7 @@ private NodeToExpand CurrentNode /// /// Node that should be expanded /// - private record class NodeToExpand + private class NodeToExpand { public IEnumerable ErrorInfos => _errorInfos; @@ -840,21 +900,42 @@ private record class NodeToExpand /// /// Node class of the node /// - public uint NodeClass { get; internal set; } + public uint NodeClass { get; } + + /// + /// Event Notifier of the node + /// + public byte EventNotifier { get; } /// /// Create node to expand /// /// /// - public NodeToExpand(OpcNodeModel nodeFromConfiguration, NodeId? nodeId) + /// + /// + /// + /// + /// + /// + public NodeToExpand(OpcNodeModel nodeFromConfiguration, NodeId? nodeId, + Opc.Ua.NodeClass nodeClass, QualifiedName? browseName, LocalizedText? displayName, + byte eventNotifier, ExpandedNodeId? typeDefinitionId, ServiceResultModel? errorInfo) { NodeFromConfiguration = nodeFromConfiguration; NodeId = nodeId; + NodeClass = (uint)nodeClass; + EventNotifier = eventNotifier; + + if (errorInfo != null) + { + AddErrorInfo(errorInfo); + } // Hold variables resolved from a variable or variable type - Variables = new ObjectToExpand( - new BrowseFrame(nodeId ?? NodeId.Null, "Variables", "Variables"), this); + Variables = new ObjectToExpand(new BrowseFrame( + nodeId ?? NodeId.Null, browseName ?? "Variables", + displayName?.Text ?? "Variables", typeDefinitionId, nodeClass), this); } /// @@ -971,11 +1052,23 @@ public bool TryGetNextObject(out ObjectToExpand? obj) /// /// The object to expand /// - /// - /// - private record class ObjectToExpand(BrowseFrame ObjectFromBrowse, - NodeToExpand OriginalNode) + private class ObjectToExpand { + public BrowseFrame ObjectFromBrowse { get; } + public NodeToExpand OriginalNode { get; } + + /// + /// Create object to expand + /// + /// + /// + public ObjectToExpand(BrowseFrame objectFromBrowse, + NodeToExpand originalNode) + { + ObjectFromBrowse = objectFromBrowse; + OriginalNode = originalNode; + } + public bool EntriesAlreadyReturned { get; internal set; } public bool ContainsVariables => _variables.Count > 0; @@ -990,7 +1083,22 @@ public bool AddVariables(IEnumerable frames) var duplicates = false; foreach (var frame in frames) { - duplicates |= _variables.Add(frame); + duplicates |= !_variables.Add(frame); + } + return duplicates; + } + + /// + /// Add events + /// + /// + /// + public bool AddEvents(IEnumerable frames) + { + var duplicates = false; + foreach (var frame in frames) + { + duplicates |= !_events.Add(frame); } return duplicates; } @@ -1037,9 +1145,8 @@ string CreateUniqueId(BrowseFrame frame) /// /// Create writer id for the object /// - /// /// - public string CreateWriterId(IServiceMessageContext context) + public string CreateWriterId() { var sb = new StringBuilder(); if (OriginalNode.NodeFromConfiguration.DataSetFieldId != null) @@ -1063,6 +1170,7 @@ public string CreateDataSetName(IServiceMessageContext context) } private readonly HashSet _variables = new(); + private readonly HashSet _events = new(); } private int _nodeIndex = -1; diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Services/DataSetWriter.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Services/DataSetWriter.cs index 17c7cd6794..087788a154 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Services/DataSetWriter.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Services/DataSetWriter.cs @@ -10,7 +10,9 @@ namespace Azure.IIoT.OpcUa.Publisher.Services using Azure.IIoT.OpcUa.Publisher.Stack; using Azure.IIoT.OpcUa.Publisher.Stack.Models; using Azure.IIoT.OpcUa.Encoders.PubSub; + using Azure.IIoT.OpcUa.Encoders; using Furly.Extensions.Messaging; + using Furly.Extensions.Serializers; using Microsoft.Extensions.Logging; using Nito.AsyncEx; using System; @@ -21,8 +23,6 @@ namespace Azure.IIoT.OpcUa.Publisher.Services using System.Text; using System.Threading; using System.Threading.Tasks; - using Avro.Generic; - using Irony.Parsing.Construction; public sealed partial class WriterGroupDataSource { @@ -166,7 +166,8 @@ public static IEnumerable GetDataSetWriters(WriterGroupDataSource { foreach (var (p, item) in data.SelectMany(d => d.Select(i => (d.Key, i)))) { - var id = $"{dataSetWriter.Id}_{item.Id ?? item.GetHashCode().ToString(CultureInfo.InvariantCulture)}"; + var id = $"{dataSetWriter.Id}_{item.Id + ?? item.GetHashCode().ToString(CultureInfo.InvariantCulture)}"; yield return CreateDataSetWriter(id, p, new[] { item }); } } @@ -206,9 +207,6 @@ DataSetWriter CreateDataSetWriter(string id, DataSet = dataset with { DataSetMetaData = dataset.DataSetMetaData.Clone(), - ExtensionFields = dataset.ExtensionFields? - .ToDictionary(k => k.Key, v => v.Value), - DataSetSource = source with { Connection = source.Connection.Clone(), @@ -236,9 +234,6 @@ DataSetWriter CreateEventWriter(string id, DataSet = dataset with { DataSetMetaData = dataset.DataSetMetaData.Clone(), - ExtensionFields = dataset.ExtensionFields? - .ToDictionary(k => k.Key, v => v.Value), - DataSetSource = source with { Connection = source.Connection.Clone(), @@ -438,8 +433,9 @@ private DataSetWriterSubscription(WriterGroupDataSource group, DataSetWriter wri _group._writerGroup.MessageSettings?.NamespaceFormat ?? _group._options.Value.DefaultNamespaceFormat ?? NamespaceFormat.Uri; - MonitoredItems = _writer.Source.ToMonitoredItems(namespaceFormat, - _writer.DataSet.ExtensionFields); + MonitoredItems = _writer.Source.ToMonitoredItems(namespaceFormat); + _extensionFields = new ExtensionFields(_group._serializer, + _writer.DataSet.ExtensionFields, _writer.Writer.DataSetFieldContentMask); _template = _writer.Source.SubscriptionSettings.ToSubscriptionModel( _writer.Routing != DataSetRoutingMode.None, _group._options.Value.IgnoreConfiguredPublishingIntervals); @@ -500,8 +496,9 @@ public async ValueTask UpdateAsync(DataSetWriter dataSetWriter, HashSet _group._writerGroup.MessageSettings?.NamespaceFormat ?? _group._options.Value.DefaultNamespaceFormat ?? NamespaceFormat.Uri; - MonitoredItems = _writer.Source.ToMonitoredItems(namespaceFormat, - _writer.DataSet.ExtensionFields); + MonitoredItems = _writer.Source.ToMonitoredItems(namespaceFormat); + _extensionFields = new ExtensionFields(_group._serializer, + _writer.DataSet.ExtensionFields, _writer.Writer.DataSetFieldContentMask); var template = _writer.Source.SubscriptionSettings.ToSubscriptionModel( _writer.Routing != DataSetRoutingMode.None, _group._options.Value.IgnoreConfiguredPublishingIntervals); @@ -885,6 +882,7 @@ DataSetWriterContext CreateMessageContext(string topic, QoS? qos, bool? retain, DataSetWriterId = (ushort)Index, MetaData = metadata, Writer = _writer.Writer, + ExtensionFields = _extensionFields.GetExtensionFieldData(notification), WriterName = Name, NextWriterSequenceNumber = sequenceNumber, WriterGroup = writerGroup, @@ -1075,7 +1073,10 @@ internal async Task UpdateMetaDataAsync(CancellationToken ct = default) var msgMask = _writer._writer.Writer.MessageSettings?.DataSetMessageContentMask; MetaData = new PublishedDataSetMessageSchemaModel { - MetaData = metaData, + MetaData = metaData with + { + Fields = _writer._extensionFields.AddMetadata(metaData.Fields) + }, TypeName = null, DataSetFieldContentFlags = fieldMask, DataSetMessageContentFlags = msgMask @@ -1089,6 +1090,179 @@ internal async Task UpdateMetaDataAsync(CancellationToken ct = default) private readonly DataSetWriterSubscription _writer; } + /// + /// Extension fields of the writer + /// + private sealed class ExtensionFields + { + /// + /// Get extension fields as configured + /// + /// + public IReadOnlyList? Fields + { + get + { + if ((_fieldMask & (DataSetFieldContentFlags.EndpointUrl | + DataSetFieldContentFlags.ApplicationUri)) == 0) + { + return _extensionFields; + } + var extensionFields = _extensionFields?.ToList() ?? new List(); + if ((_fieldMask & DataSetFieldContentFlags.EndpointUrl) != 0 && + !extensionFields + .Any(f => f.DataSetFieldName == nameof(DataSetFieldContentFlags.EndpointUrl))) + { + extensionFields.Add(new ExtensionFieldModel + { + DataSetFieldName = nameof(DataSetFieldContentFlags.EndpointUrl), + Value = "{{EndpointUrl}}", + DataSetFieldDescription = "Endpoint Url of the data source." + }); + } + if ((_fieldMask & DataSetFieldContentFlags.ApplicationUri) != 0 && + !extensionFields + .Any(f => f.DataSetFieldName == nameof(DataSetFieldContentFlags.ApplicationUri))) + { + extensionFields.Add(new ExtensionFieldModel + { + DataSetFieldName = nameof(DataSetFieldContentFlags.ApplicationUri), + Value = "{{ApplicationUri}}", + DataSetFieldDescription = "Application Uri of the data source." + }); + } + return extensionFields; + } + } + + /// + /// Create extension fields + /// + /// + /// + /// + public ExtensionFields(IJsonSerializer serializer, + IReadOnlyList? extensionFields, + DataSetFieldContentFlags? dataSetFieldContentMask) + { + _serializer = serializer; + _fieldMask = dataSetFieldContentMask ?? 0; + _extensionFields = extensionFields; + _data = GenerateExtensionFieldData(); + } + + /// + /// Get extension field data + /// + /// + /// + public IReadOnlyList<(string, Opc.Ua.DataValue?)> GetExtensionFieldData( + OpcUaSubscriptionNotification notification) + { + if ((_fieldMask & (DataSetFieldContentFlags.EndpointUrl | + DataSetFieldContentFlags.ApplicationUri)) == 0) + { + return _data; + } + return _data + .Select(f => f.Item1 switch + { + nameof(DataSetFieldContentFlags.EndpointUrl) => + (f.Item1, new Opc.Ua.DataValue(notification.EndpointUrl)), + nameof(DataSetFieldContentFlags.ApplicationUri) => + (f.Item1, new Opc.Ua.DataValue(notification.ApplicationUri)), + _ => f + }) + .ToList(); + } + + /// + /// Add extension field metadata to the end of the metadata fields + /// + /// + /// + public IReadOnlyList AddMetadata( + IReadOnlyList metadataFields) + { + var extensionFields = Fields; + if (extensionFields == null || extensionFields.Count == 0) + { + return metadataFields; + } + var fields = new List(metadataFields); + foreach (var field in extensionFields) + { + var builtInType = GetBuiltInType(field.Value); + fields.Add(new PublishedFieldMetaDataModel + { + Flags = 0, // Set to 1 << 1 for PromotedField fields. + Name = field.DataSetFieldName, + Id = field.DataSetClassFieldId, + Description = field.DataSetFieldDescription, + ValueRank = (int)(field.Value.IsArray ? + NodeValueRank.OneDimension : NodeValueRank.Scalar), + ArrayDimensions = null, + MaxStringLength = 0, + // If the Property is EngineeringUnits, the unit of the Field Value + // shall match the unit of the FieldMetaData. + Properties = null, // TODO: Add engineering units etc. to properties + BuiltInType = (byte)builtInType + }); + } + return fields; + } + + /// + /// Generate extension field data values + /// + /// + private IReadOnlyList<(string, Opc.Ua.DataValue?)> GenerateExtensionFieldData() + { + var extensionFields = Fields; + if (extensionFields == null || extensionFields.Count == 0) + { + return Array.Empty<(string, Opc.Ua.DataValue?)>(); + } + var extensions = new List<(string, Opc.Ua.DataValue?)>(); + var encoder = new JsonVariantEncoder(new Opc.Ua.ServiceMessageContext(), + _serializer); + foreach (var field in extensionFields) + { + extensions.Add((field.DataSetFieldName, + new Opc.Ua.DataValue(encoder.Decode(field.Value, + (Opc.Ua.BuiltInType)GetBuiltInType(field.Value))))); + } + return extensions; + } + + private static byte GetBuiltInType(VariantValue value) + { + return value.GetTypeCode() switch + { + TypeCode.Empty => 0, + TypeCode.Boolean => 1, + TypeCode.SByte => 2, + TypeCode.Byte => 3, + TypeCode.Int16 => 4, + TypeCode.UInt16 => 5, + TypeCode.Int32 => 6, + TypeCode.UInt32 => 7, + TypeCode.Int64 => 8, + TypeCode.UInt64 => 9, + TypeCode.Single => 10, + TypeCode.Double => 11, + TypeCode.String or TypeCode.Char => 12, + TypeCode.DateTime => 13, + _ => 24 + }; + } + + private readonly IJsonSerializer _serializer; + private readonly DataSetFieldContentFlags _fieldMask; + private readonly IReadOnlyList? _extensionFields; + private readonly IReadOnlyList<(string, Opc.Ua.DataValue?)> _data; + } + private readonly WriterGroupDataSource _group; private readonly ILogger _logger; private readonly object _lock = new(); @@ -1099,6 +1273,7 @@ internal async Task UpdateMetaDataAsync(CancellationToken ct = default) private SubscriptionModel _template; private ConnectionIdentifier _connection; private DataSetWriter _writer; + private ExtensionFields _extensionFields; private readonly Lazy _metaDataLoader; private uint _dataSetSequenceNumber; private uint _metadataSequenceNumber; diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Services/NetworkMessageEncoder.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Services/NetworkMessageEncoder.cs index 5cecfb5bbd..14f57b3292 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Services/NetworkMessageEncoder.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Services/NetworkMessageEncoder.cs @@ -303,7 +303,12 @@ static PublishingQueueSettingsModel GetQueue(DataSetWriterContext context, if (!PubSubMessage.TryCreateDataSetMessage(encoding, GetDataSetWriterName(Notification, Context), Context.DataSetWriterId, dataSetMessageContentMask, - MessageType.KeepAlive, new DataSet(), + MessageType.KeepAlive, +#if KA_WITH_EX_FIELDS + new DataSet(Context.ExtensionFields, dataSetFieldContentMask), +#else + new DataSet(), +#endif GetTimestamp(Notification), Context.NextWriterSequenceNumber(), standardsCompliant, Notification.EndpointUrl, Notification.ApplicationUri, Context.MetaData?.MetaData, @@ -348,8 +353,10 @@ static PublishingQueueSettingsModel GetQueue(DataSetWriterContext context, if (!PubSubMessage.TryCreateDataSetMessage(encoding, GetDataSetWriterName(Notification, Context), Context.DataSetWriterId, dataSetMessageContentMask, Notification.MessageType, - new DataSet(orderedNotifications.ToDictionary( - s => s.DataSetFieldName!, s => s.Value), dataSetFieldContentMask), + new DataSet(orderedNotifications + .Select(s => (s.DataSetFieldName!, s.Value)) + .Concat(Context.ExtensionFields) + .ToList(), dataSetFieldContentMask), GetTimestamp(Notification), Context.NextWriterSequenceNumber(), standardsCompliant, Notification.EndpointUrl, Notification.ApplicationUri, Context.MetaData?.MetaData, out var dataSetMessage)) diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Services/WriterGroupDataSource.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Services/WriterGroupDataSource.cs index c77bed2056..d3a232e666 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Services/WriterGroupDataSource.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Services/WriterGroupDataSource.cs @@ -21,6 +21,7 @@ namespace Azure.IIoT.OpcUa.Publisher.Services using System.Linq; using System.Threading; using System.Threading.Tasks; + using Furly.Extensions.Serializers; /// /// Triggers dataset writer messages on subscription changes @@ -44,18 +45,20 @@ public sealed partial class WriterGroupDataSource : IMessageSource, IDisposable, /// /// /// + /// /// /// /// /// public WriterGroupDataSource(IOpcUaClientManager clients, - WriterGroupModel writerGroup, IOptions options, - IMetricsContext? metrics, ILoggerFactory loggerFactory, - TimeProvider? timeProvider = null) + WriterGroupModel writerGroup, IJsonSerializer serializer, + IOptions options, IMetricsContext? metrics, + ILoggerFactory loggerFactory, TimeProvider? timeProvider = null) { ArgumentNullException.ThrowIfNull(writerGroup, nameof(writerGroup)); _loggerFactory = loggerFactory; + _serializer = serializer; _options = options; _logger = loggerFactory.CreateLogger(); _timeProvider = timeProvider ?? TimeProvider.System; @@ -584,6 +587,7 @@ private void InitializeMetrics() private readonly ConcurrentDictionary _writers = new(); private readonly Meter _meter = Diagnostics.NewMeter(); private readonly ILoggerFactory _loggerFactory; + private readonly IJsonSerializer _serializer; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; private readonly long _startTime; diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Models/ExtensionFieldItemModel.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Models/ExtensionFieldItemModel.cs deleted file mode 100644 index 4b9adbc810..0000000000 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Models/ExtensionFieldItemModel.cs +++ /dev/null @@ -1,20 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -namespace Azure.IIoT.OpcUa.Publisher.Stack.Models -{ - using Furly.Extensions.Serializers; - - /// - /// Extension field template - /// - public sealed record class ExtensionFieldItemModel : BaseMonitoredItemModel - { - /// - /// Value of the extension field to inject - /// - public required VariantValue Value { get; set; } - } -} diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.Subscription.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.Subscription.cs index 7091cbdf06..3e52d6bd64 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.Subscription.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.Subscription.cs @@ -192,14 +192,16 @@ internal async Task SyncAsync(OpcUaSubscription subscription, // Get the max item per subscription as well as max var caps = await session.GetServerCapabilitiesAsync( NamespaceFormat.Uri, ct).ConfigureAwait(false); - await subscription.SyncAsync(caps.MaxMonitoredItemsPerSubscription, + var delay = await subscription.SyncAsync(caps.MaxMonitoredItemsPerSubscription, caps.OperationLimits, ct).ConfigureAwait(false); + RescheduleSynchronization(delay); } catch (Exception ex) { _logger.LogError(ex, "{Client}: Error trying to sync subscription {Subscription}", this, subscription); + RescheduleSynchronization(TimeSpan.FromMinutes(1)); } finally { @@ -242,6 +244,7 @@ await EnsureSessionIsReadyForSubscriptionsAsync(session, NamespaceFormat.Uri, ct).ConfigureAwait(false); var maxMonitoredItems = caps.MaxMonitoredItemsPerSubscription; var limits = caps.OperationLimits; + var delay = Timeout.InfiniteTimeSpan; // // Take the subscription lock here! - we hold it all the way until we @@ -292,7 +295,7 @@ await Task.WhenAll(existing.Keys })).ConfigureAwait(false); // Add new subscription for items with subscribers - await Task.WhenAll(s2r.Keys + var delays = await Task.WhenAll(s2r.Keys .Except(existing.Keys) .Select(async add => { @@ -305,8 +308,7 @@ await Task.WhenAll(s2r.Keys // #pragma warning disable CA2000 // Dispose objects before losing scope var subscription = new OpcUaSubscription(this, - add, _subscriptionOptions, CreateSessionTimeout, - _loggerFactory, + add, _subscriptionOptions, _loggerFactory, new OpcUaClientTagList(_connection, _metrics), null, _timeProvider); #pragma warning restore CA2000 // Dispose objects before losing scope @@ -315,48 +317,64 @@ await Task.WhenAll(s2r.Keys session.AddSubscription(subscription); // Sync the subscription which will get it to go live. - await subscription.SyncAsync(maxMonitoredItems, + var delay = await subscription.SyncAsync(maxMonitoredItems, caps.OperationLimits, ct).ConfigureAwait(false); Interlocked.Increment(ref additions); Debug.Assert(session == subscription.Session); s2r[add].ForEach(r => r.Dirty = false); + return delay; + } + catch (OperationCanceledException) + { + return Timeout.InfiniteTimeSpan; } - catch (OperationCanceledException) { } catch (Exception ex) { _logger.LogError(ex, "{Client}: Failed to add " + "subscription {Subscription} in session.", this, add); + return TimeSpan.FromMinutes(1); } })).ConfigureAwait(false); + delay = delays.DefaultIfEmpty(Timeout.InfiniteTimeSpan).Min(); // Update any items where subscriber signalled the item was updated - await Task.WhenAll(s2r.Keys.Intersect(existing.Keys) + delays = await Task.WhenAll(s2r.Keys.Intersect(existing.Keys) .Where(u => s2r[u].Any(b => b.Dirty)) .Select(async update => { try { var subscription = existing[update]; - await subscription.SyncAsync(maxMonitoredItems, + var delay = await subscription.SyncAsync(maxMonitoredItems, caps.OperationLimits, ct).ConfigureAwait(false); Interlocked.Increment(ref updates); Debug.Assert(session == subscription.Session); s2r[update].ForEach(r => r.Dirty = false); + return delay; + } + catch (OperationCanceledException) + { + return Timeout.InfiniteTimeSpan; } - catch (OperationCanceledException) { } catch (Exception ex) { _logger.LogError(ex, "{Client}: Failed to update " + "subscription {Subscription} in session.", this, update); + return TimeSpan.FromMinutes(1); } })).ConfigureAwait(false); + + var delay2 = delays.DefaultIfEmpty(Timeout.InfiniteTimeSpan).Min(); + RescheduleSynchronization(delay < delay2 ? delay : delay2); } catch (Exception ex) { _logger.LogError(ex, "{Client}: Error trying to sync subscriptions.", this); + var delay2 = TimeSpan.FromMinutes(1); + RescheduleSynchronization(delay < delay2 ? delay : delay2); } finally { @@ -371,7 +389,6 @@ await subscription.SyncAsync(maxMonitoredItems, { return; } - _logger.LogInformation("{Client}: Removed {Removals}, added {Additions}, " + "and updated {Updates} subscriptions (total: {Total}) took {Duration} ms.", this, removals, additions, updates, session.SubscriptionHandles.Count, @@ -408,6 +425,29 @@ private async Task EnsureSessionIsReadyForSubscriptionsAsync(OpcUaSession sessio } } + /// + /// Called under lock, schedule resynchronization of all subscriptions + /// after the specified delay + /// + /// + private void RescheduleSynchronization(TimeSpan delay) + { + Debug.Assert(delay <= Timeout.InfiniteTimeSpan, delay.ToString()); + Debug.Assert(_subscriptionLock.CurrentCount == 0, "Must be locked"); + + if (delay == Timeout.InfiniteTimeSpan) + { + return; + } + + var nextSync = _timeProvider.GetUtcNow() + delay; + if (nextSync <= _nextSync) + { + _nextSync = nextSync; + _resyncTimer.Change(delay, Timeout.InfiniteTimeSpan); + } + } + /// /// Subscription registration /// @@ -573,8 +613,10 @@ private void RemoveNoLockInternal() private readonly OpcUaClient _outer; } + private DateTimeOffset _nextSync; #pragma warning disable CA2213 // Disposable fields should be disposed private readonly SemaphoreSlim _subscriptionLock = new(1, 1); + private readonly ITimer _resyncTimer; #pragma warning restore CA2213 // Disposable fields should be disposed private readonly Dictionary _registrations = new(); private readonly ConcurrentDictionary> _s2r = new(); diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.cs index 3ea2db3511..a29c6cdce1 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaClient.cs @@ -283,6 +283,8 @@ public OpcUaClient(ApplicationConfiguration configuration, _disconnectLock = _lock.WriterLock(_cts.Token); _channelMonitor = _timeProvider.CreateTimer(_ => OnUpdateConnectionDiagnostics(), null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _resyncTimer = _timeProvider.CreateTimer(_ => TriggerSubscriptionSynchronization(), + null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _diagnosticsDumper = !DumpDiagnostics ? null : DumpDiagnosticsPeriodicallyAsync(_cts.Token); _sessionManager = ManageSessionStateMachineAsync(_cts.Token); @@ -476,9 +478,10 @@ internal async ValueTask CloseAsync(bool shutdown = false) } finally { - _subscriptionLock.Dispose(); _channelMonitor.Dispose(); + _resyncTimer.Dispose(); _cts.Dispose(); + _subscriptionLock.Dispose(); } } @@ -911,10 +914,7 @@ private async Task ManageSessionStateMachineAsync(CancellationToken ct) await SyncAsync(subscriptionToSync, ct).ConfigureAwait(false); break; case ConnectionEvent.SubscriptionSyncAll: - if (_session != null) - { - await SyncAsync(ct).ConfigureAwait(false); - } + await SyncAsync(ct).ConfigureAwait(false); break; case ConnectionEvent.StartReconnect: // sent by the keep alive timeout path switch (currentSessionState) @@ -928,6 +928,7 @@ private async Task ManageSessionStateMachineAsync(CancellationToken ct) _logger.LogInformation("{Client}: Reconnecting session {Session} due to {Reason}...", this, _sessionName, (context is ServiceResult sr) ? "error " + sr : "RESET"); + Debug.Assert(_session != null); var state = _reconnectHandler.BeginReconnect(_session, _reverseConnectManager, GetMinReconnectPeriod(), (sender, evt) => { @@ -957,7 +958,7 @@ private async Task ManageSessionStateMachineAsync(CancellationToken ct) case ConnectionEvent.ReconnectComplete: // if session recovered, Session property is not null - var reconnected = _reconnectHandler.Session; + var reconnected = _reconnectHandler.Session as OpcUaSession; switch (currentSessionState) { case SessionState.Reconnecting: @@ -1311,8 +1312,9 @@ private async ValueTask TryConnectAsync(CancellationToken ct) updateBeforeConnect: _reverseConnectManager != null, checkDomain: false, // Domain must match on connect _sessionName, (uint)sessionTimeout.TotalMilliseconds, - userIdentity, preferredLocales, ct).ConfigureAwait(false); + userIdentity, preferredLocales, ct).ConfigureAwait(false) as OpcUaSession; + Debug.Assert(session != null); session.RenewUserIdentity += (_, _) => userIdentity; // Assign the createdSubscriptions session @@ -1345,7 +1347,7 @@ private async ValueTask TryConnectAsync(CancellationToken ct) /// internal void Session_HandlePublishError(ISession session, PublishErrorEventArgs e) { - if (session == _session && session.Connected) + if (_disconnectLock == null && session == _session) { switch (e.Status.Code) { @@ -1367,11 +1369,16 @@ internal void Session_HandlePublishError(ISession session, PublishErrorEventArgs /// /// Feed back acknoledgements /// - /// + /// /// - internal void Session_PublishSequenceNumbersToAcknowledge(ISession session, + internal void Session_PublishSequenceNumbersToAcknowledge(ISession context, PublishSequenceNumbersToAcknowledgeEventArgs e) { + if (context is not OpcUaSession session) + { + return; + } + // Reset timeout counter _publishTimeoutCounter = 0; diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Condition.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Condition.cs index 574e355020..fd1e2cfd8b 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Condition.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Condition.cs @@ -172,6 +172,7 @@ protected override bool ProcessEventNotification(DateTimeOffset publishTime, else if (eventType == ObjectTypeIds.RefreshRequiredEventType) { var noErrorFound = true; + Debug.Assert(Subscription != null); // issue a condition refresh to make sure we are in a correct state _logger.LogInformation("{Item}: Issuing ConditionRefresh for " + @@ -180,7 +181,7 @@ protected override bool ProcessEventNotification(DateTimeOffset publishTime, Subscription.DisplayName); try { - Subscription.ConditionRefresh(); + Subscription.ConditionRefreshAsync(default).GetAwaiter().GetResult(); // TODO } catch (Exception e) { diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.CyclicRead.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.CyclicRead.cs index 33d6b507da..1aa1f23e35 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.CyclicRead.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.CyclicRead.cs @@ -162,6 +162,7 @@ public override bool MergeWith(OpcUaMonitoredItem item, IOpcUaSession session, } else { + Debug.Assert(Subscription != null); _subscriptionName = Subscription.DisplayName; Debug.Assert(MonitoringMode == MonitoringMode.Disabled); EnsureSamplerRunning(); diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.DataChange.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.DataChange.cs index 9de95929c5..cd0c0f8de4 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.DataChange.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.DataChange.cs @@ -268,6 +268,27 @@ public override bool AddTo(Subscription subscription, IOpcUaSession session, return base.AddTo(subscription, session, out metadataChanged); } + /// + public override bool TryCompleteChanges(Subscription subscription, ref bool applyChanges) + { + var msgContext = subscription.Session?.MessageContext; + if (Filter is AggregateFilter && + Status?.FilterResult is AggregateFilterResult afr && msgContext != null) + { + if (Status.Error != null && ServiceResult.IsNotGood(Status.Error)) + { + _logger.LogError("Aggregate filter applied with result {Result} for {Item}", + afr.AsJson(msgContext), this); + } + else + { + _logger.LogDebug("Aggregate filter applied with result {Result} for {Item}", + afr.AsJson(msgContext), this); + } + } + return base.TryCompleteChanges(subscription, ref applyChanges); + } + /// public override bool TryGetMonitoredItemNotifications( DateTimeOffset publishTime, IEncodeable evt, MonitoredItemNotifications notifications) diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Event.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Event.cs index 03eb829b46..e4040af7c5 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Event.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Event.cs @@ -106,7 +106,7 @@ protected Event(Event item, bool copyEventHandlers, : base(item, copyEventHandlers, copyClientHandle) { Fields = item.Fields; - RelativePath = item.RelativePath; + TheResolvedRelativePath = item.TheResolvedRelativePath; Template = item.Template; } @@ -290,6 +290,26 @@ public override bool MergeWith(OpcUaMonitoredItem item, IOpcUaSession session, return itemChange; } + /// + public override bool TryCompleteChanges(Subscription subscription, ref bool applyChanges) + { + var msgContext = subscription.Session?.MessageContext; + if (Status?.FilterResult is EventFilterResult evr && msgContext != null) + { + if (Status.Error != null && ServiceResult.IsNotGood(Status.Error)) + { + _logger.LogError("Event filter applied with result {Result} for {Item}", + evr.AsJson(msgContext), this); + } + else + { + _logger.LogDebug("Event filter applied with result {Result} for {Item}", + evr.AsJson(msgContext), this); + } + } + return base.TryCompleteChanges(subscription, ref applyChanges); + } + /// protected override bool OnSamplingIntervalOrQueueSizeRevised( bool samplingIntervalChanged, bool queueSizeChanged) diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Field.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Field.cs deleted file mode 100644 index 206e2c665c..0000000000 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Field.cs +++ /dev/null @@ -1,246 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See License.txt in the repo root for license information. -// ------------------------------------------------------------ - -namespace Azure.IIoT.OpcUa.Publisher.Stack.Services -{ - using Azure.IIoT.OpcUa.Publisher.Stack.Models; - using Azure.IIoT.OpcUa.Publisher.Models; - using Furly.Extensions.Serializers; - using Microsoft.Extensions.Logging; - using Opc.Ua; - using Opc.Ua.Client; - using Opc.Ua.Client.ComplexTypes; - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Runtime.Serialization; - using System.Threading; - using System.Threading.Tasks; - - internal abstract partial class OpcUaMonitoredItem - { - /// - /// Extension Field item - /// - [DataContract(Namespace = Namespaces.OpcUaXsd)] - [KnownType(typeof(DataChangeFilter))] - [KnownType(typeof(EventFilter))] - [KnownType(typeof(AggregateFilter))] - internal class Field : OpcUaMonitoredItem - { - /// - /// Item as extension field - /// - public ExtensionFieldItemModel Template { get; protected internal set; } - - /// - /// Create wrapper - /// - /// - /// - /// - /// - public Field(ISubscriber owner, ExtensionFieldItemModel template, - ILogger logger, TimeProvider timeProvider) : - base(owner, logger, template.StartNodeId, timeProvider) - { - Template = template; - } - - /// - /// Copy constructor - /// - /// - /// - /// - protected Field(Field item, bool copyEventHandlers, - bool copyClientHandle) - : base(item, copyEventHandlers, copyClientHandle) - { - Template = item.Template; - _fieldId = item._fieldId; - _value = item._value; - } - - /// - public override MonitoredItem CloneMonitoredItem( - bool copyEventHandlers, bool copyClientHandle) - { - return new Field(this, copyEventHandlers, copyClientHandle); - } - - /// - public override bool Equals(object? obj) - { - if (obj is not Field fieldItem) - { - return false; - } - if ((Template.DataSetFieldName ?? string.Empty) != - (fieldItem.Template.DataSetFieldName ?? string.Empty)) - { - return false; - } - if (Template.Value != fieldItem.Template.Value) - { - return false; - } - return true; - } - - /// - public override int GetHashCode() - { - var hashCode = 81523234; - hashCode = (hashCode * -1521134295) + - EqualityComparer.Default.GetHashCode( - Template.DataSetFieldName ?? string.Empty); - hashCode = (hashCode * -1521134295) + - Template.Value.GetHashCode(); - return hashCode; - } - - /// - public override string ToString() - { - return $"Field '{Template.DataSetFieldName}' with value {Template.Value}."; - } - - /// - public override ValueTask GetMetaDataAsync(IOpcUaSession session, - ComplexTypeSystem? typeSystem, List fields, - NodeIdDictionary dataTypes, CancellationToken ct) - { - return AddVariableFieldAsync(fields, dataTypes, session, typeSystem, - new VariableNode - { - DataType = GetBuiltInType(Template.Value), - ValueRank = Template.Value.IsArray ? - ValueRanks.OneDimension : ValueRanks.Scalar - }, Template.DisplayName, (Uuid)_fieldId, ct); - - static NodeId GetBuiltInType(VariantValue value) - { - return new NodeId((uint)(value.GetTypeCode() switch - { - TypeCode.Boolean => BuiltInType.Boolean, - TypeCode.SByte => BuiltInType.SByte, - TypeCode.Byte => BuiltInType.Byte, - TypeCode.Int16 => BuiltInType.Int16, - TypeCode.UInt16 => BuiltInType.UInt16, - TypeCode.Int32 => BuiltInType.Int32, - TypeCode.UInt32 => BuiltInType.UInt32, - TypeCode.Int64 => BuiltInType.Int64, - TypeCode.UInt64 => BuiltInType.UInt64, - TypeCode.Double => BuiltInType.Double, - TypeCode.String => BuiltInType.String, - TypeCode.DateTime => BuiltInType.DateTime, - TypeCode.Empty => BuiltInType.Null, - TypeCode.Object => BuiltInType.Variant, // Structure - TypeCode.Char => BuiltInType.String, - TypeCode.Single => BuiltInType.Float, - TypeCode.Decimal => BuiltInType.Variant, // Number - _ => BuiltInType.Variant - })); - } - } - - /// - public override bool AddTo(Subscription subscription, - IOpcUaSession session, out bool metadataChanged) - { - metadataChanged = true; - _value = new DataValue(session.Codec.Decode(Template.Value, BuiltInType.Variant)); - Valid = true; - return true; - } - - /// - public override bool MergeWith(OpcUaMonitoredItem item, IOpcUaSession session, - out bool metadataChanged) - { - metadataChanged = false; - return false; - } - - /// - public override bool RemoveFrom(Subscription subscription, out bool metadataChanged) - { - metadataChanged = true; - _value = new DataValue(); - Valid = false; - return true; - } - - /// - public override bool TryCompleteChanges(Subscription subscription, - ref bool applyChanges) - { - return true; - } - - /// - public override bool TryGetLastMonitoredItemNotifications( - MonitoredItemNotifications notifications) - { - if (!Valid) - { - return false; - } - notifications.Add(Owner, ToMonitoredItemNotification()); - return true; - } - - /// - public override bool TryGetMonitoredItemNotifications(DateTimeOffset publishTime, - IEncodeable evt, MonitoredItemNotifications notifications) - { - Debug.Fail("Unexpected notification on extension field"); - return false; - } - - /// - protected override IEnumerable CreateTriggeredItems( - ILoggerFactory factory, OpcUaClient client) - { - return Enumerable.Empty(); - } - - /// - protected override bool TryGetErrorMonitoredItemNotifications( - StatusCode statusCode, MonitoredItemNotifications notifications) - { - Debug.Fail("Unexpected notification on extension field"); - return false; - } - - /// - /// Convert to monitored item notifications - /// - /// - protected MonitoredItemNotificationModel ToMonitoredItemNotification() - { - Debug.Assert(Valid); - Debug.Assert(Template != null); - - return new MonitoredItemNotificationModel - { - Id = Template.Id, - DataSetFieldName = Template.DisplayName, - DataSetName = Template.DisplayName, - NodeId = NodeId, - PathFromRoot = null, - Value = _value, - Flags = 0, - SequenceNumber = GetNextSequenceNumber() - }; - } - - private DataValue _value = new(); - private readonly Guid _fieldId = Guid.NewGuid(); - } - } -} diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Heartbeat.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Heartbeat.cs index 7414134306..2f94d5adad 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Heartbeat.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.Heartbeat.cs @@ -308,7 +308,7 @@ private void SendHeartbeatNotifications(object? sender, ElapsedEventArgs e) } var lastValue = lastNotification?.Value; - if (lastValue == null && Status?.Error?.StatusCode != null) + if (lastValue == null && ServiceResult.IsNotGood(Status.Error)) { lastValue = new DataValue(Status.Error.StatusCode); } diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.ModelChange.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.ModelChange.cs index 6482ff01f1..268ec81eb1 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.ModelChange.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.ModelChange.cs @@ -304,8 +304,8 @@ protected override bool OnSamplingIntervalOrQueueSizeRevised( private void OnNodeChange(object? sender, Change e) { Publish(Owner, MessageType.Event, - CreateEvent(_nodeChangeType, e).ToList(), sender as ISession, - EventTypeName); + CreateEvent(_nodeChangeType, e).ToList(), + eventTypeName: EventTypeName); } /// @@ -316,8 +316,8 @@ private void OnNodeChange(object? sender, Change e) private void OnReferenceChange(object? sender, Change e) { Publish(Owner, MessageType.Event, - CreateEvent(_refChangeType, e).ToList(), sender as ISession, - EventTypeName); + CreateEvent(_refChangeType, e).ToList(), + eventTypeName: EventTypeName); } /// @@ -406,7 +406,7 @@ private void EnsureBrowserStarted() return; } // Start the browser - if (_browser == null) + if (_browser == null && Subscription != null) { _browser = _client.Browse(Template.RebrowsePeriod ?? TimeSpan.FromHours(12), Subscription.DisplayName); diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.cs index 2acad4ba34..5246b82263 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaMonitoredItem.cs @@ -248,10 +248,6 @@ public static IEnumerable Create(OpcUaClient client, factory.CreateLogger(), timeProvider); } break; - case ExtensionFieldItemModel efm: - yield return new Field(owner, efm, - factory.CreateLogger(), timeProvider); - break; default: Debug.Fail($"Unexpected type of item {item}"); break; @@ -548,11 +544,15 @@ public virtual bool TryGetLastMonitoredItemNotifications( MonitoredItemNotifications notifications) { var lastValue = LastReceivedValue; - if (lastValue == null || Status?.Error != null) + if (lastValue == null) { return TryGetErrorMonitoredItemNotifications( - Status?.Error.StatusCode ?? StatusCodes.GoodNoData, - notifications); + StatusCodes.BadNoData, notifications); + } + if (Status.Error != null && ServiceResult.IsNotGood(Status.Error)) + { + return TryGetErrorMonitoredItemNotifications( + Status.Error.StatusCode, notifications); } return TryGetMonitoredItemNotifications(TimeProvider.GetUtcNow(), lastValue, notifications); @@ -1018,14 +1018,13 @@ public void Add(ISubscriber callback, /// /// /// - /// /// /// /// protected void Publish(ISubscriber owner, MessageType messageType, IList notifications, - ISession? session = null, string? eventTypeName = null, - bool diagnosticsOnly = false, DateTimeOffset? timestamp = null) + string? eventTypeName = null, bool diagnosticsOnly = false, + DateTimeOffset? timestamp = null) { if (Subscription is not OpcUaSubscription subscription) { @@ -1034,9 +1033,8 @@ protected void Publish(ISubscriber owner, MessageType messageType, this); return; } - subscription.SendNotification( - owner, messageType, notifications, session, eventTypeName, - diagnosticsOnly, timestamp); + subscription.SendNotification(owner, messageType, notifications, + eventTypeName, diagnosticsOnly, timestamp); } /// diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSession.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSession.cs index 03c908026c..2c9c6c074b 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSession.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSession.cs @@ -1078,7 +1078,7 @@ private void Initialize() c => c.NodeId.AsString(MessageContext, namespaceFormat) ?? string.Empty); var conformanceUnits = config.ConformanceUnits.GetValueOrDefault( v => v == null || v.Length == 0 ? null : - v.Select(q => q.AsString(this.MessageContext, namespaceFormat)).ToList()); + v.Select(q => q.AsString(MessageContext, namespaceFormat)).ToList()); return new ServerCapabilitiesModel { OperationLimits = _limits ?? new OperationLimitsModel(), diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSubscription.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSubscription.cs index 99a3e44e93..dbdae3ecc7 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSubscription.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Stack/Services/OpcUaSubscription.cs @@ -17,7 +17,6 @@ namespace Azure.IIoT.OpcUa.Publisher.Stack.Services using Opc.Ua.Client.ComplexTypes; using Opc.Ua.Extensions; using System; - using System.Collections.Frozen; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -39,7 +38,6 @@ namespace Azure.IIoT.OpcUa.Publisher.Stack.Services [KnownType(typeof(OpcUaMonitoredItem.ModelChangeEventItem))] [KnownType(typeof(OpcUaMonitoredItem.Event))] [KnownType(typeof(OpcUaMonitoredItem.Condition))] - [KnownType(typeof(OpcUaMonitoredItem.Field))] internal sealed class OpcUaSubscription : Subscription, IAsyncDisposable, IEquatable { @@ -79,8 +77,7 @@ internal bool IsClosed /// Currently monitored but unordered /// private IEnumerable CurrentlyMonitored - => _additionallyMonitored.Values - .Concat(MonitoredItems.OfType()); + => MonitoredItems.OfType(); public byte DesiredPriority => Template.Priority @@ -154,22 +151,19 @@ public bool ResolveBrowsePathFromRoot /// /// /// - /// /// /// /// /// internal OpcUaSubscription(OpcUaClient client, SubscriptionModel template, - IOptions options, TimeSpan? createSessionTimeout, - ILoggerFactory loggerFactory, IMetricsContext metrics, uint? parentId = null, - TimeProvider? timeProvider = null) + IOptions options, ILoggerFactory loggerFactory, + IMetricsContext metrics, uint? parentId = null, TimeProvider? timeProvider = null) { _client = client; _options = options; _loggerFactory = loggerFactory; _metrics = metrics; _parentId = parentId; - _createSessionTimeout = createSessionTimeout; _timeProvider = timeProvider ?? TimeProvider.System; Template = template; @@ -177,11 +171,8 @@ internal OpcUaSubscription(OpcUaClient client, SubscriptionModel template, SubscriptionId = Opc.Ua.SequenceNumber.Increment32(ref _lastIndex); _logger = _loggerFactory.CreateLogger(); - _additionallyMonitored = FrozenDictionary.Empty; Initialize(); - _timer = _timeProvider.CreateTimer(_ => TriggerManageSubscription(), null, - Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _keepAliveWatcher = _timeProvider.CreateTimer(OnKeepAliveMissing, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _monitoredItemWatcher = _timeProvider.CreateTimer(OnMonitoredItemWatchdog, null, @@ -227,12 +218,9 @@ private OpcUaSubscription(OpcUaSubscription subscription, bool copyEventHandlers _missingKeepAlives = subscription._missingKeepAlives; _unassignedNotifications = subscription._unassignedNotifications; - _additionallyMonitored = subscription._additionallyMonitored; _continuouslyMissingKeepAlives = subscription._continuouslyMissingKeepAlives; Initialize(); - _timer = _timeProvider.CreateTimer(_ => TriggerManageSubscription(), null, - Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _keepAliveWatcher = _timeProvider.CreateTimer(OnKeepAliveMissing, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _monitoredItemWatcher = _timeProvider.CreateTimer(OnMonitoredItemWatchdog, null, @@ -261,12 +249,9 @@ private OpcUaSubscription(OpcUaSubscription subscription, uint parentId) SubscriptionId = Opc.Ua.SequenceNumber.Increment32(ref _lastIndex); _logger = _loggerFactory.CreateLogger(); - _additionallyMonitored = FrozenDictionary.Empty; Initialize(); - _timer = _timeProvider.CreateTimer(_ => TriggerManageSubscription(), null, - Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _keepAliveWatcher = _timeProvider.CreateTimer(OnKeepAliveMissing, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); _monitoredItemWatcher = _timeProvider.CreateTimer(OnMonitoredItemWatchdog, null, @@ -344,60 +329,59 @@ public override int GetHashCode() /// protected override void Dispose(bool disposing) { - base.Dispose(disposing); - if (!disposing || _disposed) - { - return; - } - try + if (disposing && !_disposed) { - ResetMonitoredItemWatchdogTimer(false); - _keepAliveWatcher.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + try + { + ResetMonitoredItemWatchdogTimer(false); + _keepAliveWatcher.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); - FastDataChangeCallback = null; - FastEventCallback = null; - FastKeepAliveCallback = null; + FastDataChangeCallback = null; + FastEventCallback = null; + FastKeepAliveCallback = null; - PublishStatusChanged -= OnPublishStatusChange; - StateChanged -= OnStateChange; + PublishStatusChanged -= OnPublishStatusChange; + StateChanged -= OnStateChange; - var items = CurrentlyMonitored.ToList(); - if (items.Count == 0) - { - _logger.LogInformation("Disposed Subscription {Subscription}.", this); - return; - } - - // - // When the entire session is disposed and recreated we must - // still dispose all monitored items that are remaining - // - items.ForEach(item => item.Dispose()); - RemoveItems(MonitoredItems); - _additionallyMonitored = FrozenDictionary.Empty; - Debug.Assert(!CurrentlyMonitored.Any()); + var items = CurrentlyMonitored.ToList(); + if (items.Count != 0) + { + // + // When the entire session is disposed and recreated we must + // still dispose all monitored items that are remaining + // + items.ForEach(item => item.Dispose()); + RemoveItems(MonitoredItems); + Debug.Assert(!CurrentlyMonitored.Any()); - _logger.LogInformation( - "Disposed Subscription {Subscription} with {Count)} items.", - this, items.Count); - } - catch (Exception ex) - { - _logger.LogError(ex, - "Disposing Subscription {Subscription} encountered error.", this); + _logger.LogInformation( + "Disposed Subscription {Subscription} with {Count)} items.", + this, items.Count); + } + else + { + _logger.LogInformation("Disposed Subscription {Subscription}.", + this); + } + } + catch (Exception ex) + { + _logger.LogError(ex, + "Disposing Subscription {Subscription} encountered error.", this); - // Eat the error - } - finally - { - _disposed = true; - _keepAliveWatcher.Dispose(); - _monitoredItemWatcher.Dispose(); - _timer.Dispose(); - _meter.Dispose(); + // Eat the error + } + finally + { + _disposed = true; + _keepAliveWatcher.Dispose(); + _monitoredItemWatcher.Dispose(); + _meter.Dispose(); - Handle = null; + Handle = null; + } } + base.Dispose(disposing); } /// @@ -600,21 +584,6 @@ internal async ValueTask CollectMetaDataAsync( await CollectMetaDataAsync(owner, session, typeSystem, dataTypes, fields, ct).ConfigureAwait(false); - // - // For full featured messages there are additional fields that are required - // see data set json dataset message encoder for more information. This will - // not apply to other encodings yet, since they do not support full featured - // message modes. - // - if ((dataSetFieldContentMask & DataSetFieldContentFlags.EndpointUrl) != 0) - { - AddExtraField(fields, nameof(DataSetFieldContentFlags.EndpointUrl)); - } - if ((dataSetFieldContentMask & DataSetFieldContentFlags.ApplicationUri) != 0) - { - AddExtraField(fields, nameof(DataSetFieldContentFlags.ApplicationUri)); - } - return new PublishedDataSetMetaDataModel { DataSetMetaData = @@ -630,22 +599,6 @@ await CollectMetaDataAsync(owner, session, typeSystem, dataTypes, fields, MinorVersion = minorVersion }; - - static void AddExtraField(List fields, - string name) - { - if (fields.Any(f => f.Name == name)) - { - return; - } - fields.Add(new PublishedFieldMetaDataModel - { - Name = name, - DataType = "String", - ValueRank = ValueRanks.Scalar, - BuiltInType = (byte)BuiltInType.String - }); - } } /// @@ -673,13 +626,14 @@ internal void Update(SubscriptionModel template) /// /// /// - internal async ValueTask SyncAsync(uint? maxMonitoredItemsPerSubscription, + internal async ValueTask SyncAsync(uint? maxMonitoredItemsPerSubscription, OperationLimitsModel limits, CancellationToken ct) { Debug.Assert(IsRoot); if (_disposed) { - return; + throw ServiceResultException.Create(StatusCodes.BadUnexpectedError, + "Subscription was disposed."); } var maxMonitoredItems = maxMonitoredItemsPerSubscription ?? 0u; @@ -690,87 +644,71 @@ internal async ValueTask SyncAsync(uint? maxMonitoredItemsPerSubscription, } Debug.Assert(Session != null); - if (Session is not OpcUaSession session || !session.Connected) + if (Session is not OpcUaSession session) { - _logger.LogError( - "Session {Session} for {Subscription} not connected.", - Session, this); - _timer.Change(Delay(_createSessionTimeout, TimeSpan.FromSeconds(10)), - Timeout.InfiniteTimeSpan); - return; + throw ServiceResultException.Create(StatusCodes.BadUnexpectedError, + "Session not of expected type."); } - try - { - var retryDelay = Timeout.InfiniteTimeSpan; - _timer.Change(retryDelay, Timeout.InfiniteTimeSpan); - // Force recreate all subscriptions in the chain if needed - await ForceRecreateIfNeededAsync(session).ConfigureAwait(false); + var retryDelay = Timeout.InfiniteTimeSpan; - // Parition the monitored items across subscriptions - var partitions = Partition.Create(_client.GetSubscribers(Template), - maxMonitoredItems, _options.Value); + // Force recreate all subscriptions in the chain if needed + await ForceRecreateIfNeededAsync(session).ConfigureAwait(false); - var subscriptionPartition = this; // The root is the default - for (var partitionIdx = 0; partitionIdx < partitions.Count; partitionIdx++) - { - // Synchronize the subscription of this partition - await subscriptionPartition.SynchronizeSubscriptionAsync( - ct).ConfigureAwait(false); - - // Add partitioned items - var partition = partitions[partitionIdx]; - var delay = await subscriptionPartition.SynchronizeMonitoredItemsAsync( - partition, limits, ct).ConfigureAwait(false); - if (retryDelay > delay) - { - retryDelay = delay; - } + // Parition the monitored items across subscriptions + var partitions = Partition.Create(_client.GetSubscribers(Template), + maxMonitoredItems, _options.Value); - if (partitionIdx == partitions.Count - 1) - { - break; - } + var subscriptionPartition = this; // The root is the default + for (var partitionIdx = 0; partitionIdx < partitions.Count; partitionIdx++) + { + // Synchronize the subscription of this partition + await subscriptionPartition.SynchronizeSubscriptionAsync( + ct).ConfigureAwait(false); - // Get or create a child subscription - subscriptionPartition = subscriptionPartition.GetChildSubscription(true); - if (subscriptionPartition == null) - { - throw ServiceResultException.Create(StatusCodes.BadUnexpectedError, - "Failed to create child subscription."); - } + // Add partitioned items + var partition = partitions[partitionIdx]; + var delay = await subscriptionPartition.SynchronizeMonitoredItemsAsync( + partition, limits, ct).ConfigureAwait(false); + if (retryDelay > delay) + { + retryDelay = delay; } - // - // subscription now is the tail or head subscription. We remove - // all child subscriptions below it as they are not needed anymore. - // - var tail = subscriptionPartition; - while (tail != null) + if (partitionIdx == partitions.Count - 1) { - tail = tail.GetChildSubscription(); - if (tail != null) - { - await tail.DisposeAsync().ConfigureAwait(false); - } + break; } - // Snip off here - subscriptionPartition._childId = null; - - // Force finalize all subscriptions in the (new) chain if needed - await FinalizeSyncAsync(ct).ConfigureAwait(false); - _timer.Change(retryDelay, Timeout.InfiniteTimeSpan); + // Get or create a child subscription + subscriptionPartition = subscriptionPartition.GetChildSubscription(true); + if (subscriptionPartition == null) + { + throw ServiceResultException.Create(StatusCodes.BadUnexpectedError, + "Failed to create child subscription."); + } } - catch (Exception e) + + // + // subscription now is the tail or head subscription. We remove + // all child subscriptions below it as they are not needed anymore. + // + var tail = subscriptionPartition; + while (tail != null) { - _logger.LogError(e, - "Failed to apply state to Subscription {Subscription} in session {Session}...", - this, Session); - // Retry in 1 minute if not automatically retried - _timer.Change(Delay(_options.Value.SubscriptionErrorRetryDelay, - kDefaultErrorRetryDelay), Timeout.InfiniteTimeSpan); + tail = tail.GetChildSubscription(); + if (tail != null) + { + await tail.DisposeAsync().ConfigureAwait(false); + } } + // Snip off here + subscriptionPartition._childId = null; + + // Force finalize all subscriptions in the (new) chain if needed + await FinalizeSyncAsync(ct).ConfigureAwait(false); + + return retryDelay; } /// @@ -846,8 +784,6 @@ private async Task FinalizeSyncAsync(CancellationToken ct) /// private async ValueTask SynchronizeSubscriptionAsync(CancellationToken ct) { - Debug.Assert(Session.DefaultSubscription != null, "No default subscription template."); - if (Handle == null) { Handle = SubscriptionId; // Initialized for the first time @@ -1409,11 +1345,6 @@ private async ValueTask SynchronizeMonitoredItemsAsync( .ToList(); dispose.ForEach(m => m.Dispose()); - // Update subscription state - _additionallyMonitored = set - .Where(m => !m.AttachedToSubscription) - .ToFrozenDictionary(m => m.ClientHandle, m => m); - // Notify semantic change now that we have update the monitored items foreach (var owner in metadataChanged) { @@ -1605,7 +1536,6 @@ private async Task CloseCurrentSubscriptionAsync() // Dispose all monitored items var items = CurrentlyMonitored.ToList(); - _additionallyMonitored = FrozenDictionary.Empty; RemoveItems(MonitoredItems); _currentSequenceNumber = 0; _goodMonitoredItems = 0; @@ -1618,15 +1548,15 @@ private async Task CloseCurrentSubscriptionAsync() ResetMonitoredItemWatchdogTimer(false); - await Try.Async(() => SetPublishingModeAsync(false)).ConfigureAwait(false); + await Try.Async(() => SetPublishingModeAsync(false, default)).ConfigureAwait(false); await Try.Async(() => DeleteItemsAsync(default)).ConfigureAwait(false); - await Try.Async(() => ApplyChangesAsync()).ConfigureAwait(false); + await Try.Async(() => ApplyChangesAsync(default)).ConfigureAwait(false); items.ForEach(item => item.Dispose()); _logger.LogDebug("Deleted {Count} monitored items for '{Subscription}'.", items.Count, this); - await Try.Async(() => DeleteAsync(true)).ConfigureAwait(false); + await Try.Async(() => DeleteAsync(true, default)).ConfigureAwait(false); if (Session != null) { @@ -1771,60 +1701,24 @@ private static TimeSpan Delay(TimeSpan? delay, TimeSpan defaultDelay) return delay.Value; } - /// - /// Trigger managing of this subscription, ensure client exists if it is null - /// - private void TriggerManageSubscription() - { - Debug.Assert(IsRoot); - - if (IsClosed) - { - return; - } - - // Execute creation/update on the session management thread inside the client - - if (IsOnline) - { - _logger.LogInformation("Trigger management of subscription {Subscription}...", - this); - } - else - { - _logger.LogDebug("Trigger management of offline subscription {Subscription}...", - this); - } - - _client.TriggerSubscriptionSynchronization(this); - } - /// /// Send notification /// /// /// /// - /// /// /// /// internal void SendNotification(ISubscriber callback, MessageType messageType, - IList notifications, ISession? session, + IList notifications, string? eventTypeName, bool diagnosticsOnly, DateTimeOffset? timestamp) { - var curSession = session ?? Session; + var curSession = Session; var messageContext = curSession?.MessageContext; if (messageContext == null) { - if (session == null) - { - // Can only send with context - _logger.LogWarning("Failed to send notification since no session exists " + - "to use as context. Notification was dropped."); - return; - } _logger.LogWarning("A session was passed to send notification with but without " + "message context. Using thread context."); messageContext = ServiceMessageContext.ThreadContext; @@ -2225,7 +2119,6 @@ private void OnSubscriptionDataChangeNotification(Subscription subscription, firstDataChangeReceived ? MessageType.DeltaFrame : MessageType.KeyFrame }; #pragma warning restore CA2000 // Dispose objects before losing scope - Debug.Assert(notification.MonitoredItems != null); callback.OnSubscriptionDataChangeReceived(message); @@ -2261,7 +2154,7 @@ private bool TryGetMonitoredItemForNotification(uint clientHandle, [NotNullWhen(true)] out OpcUaMonitoredItem? monitoredItem) { monitoredItem = FindItemByClientHandle(clientHandle) as OpcUaMonitoredItem; - if (monitoredItem != null || _additionallyMonitored.TryGetValue(clientHandle, out monitoredItem)) + if (monitoredItem != null) { return true; } @@ -2470,7 +2363,6 @@ private void RunWatchdogAction(SubscriptionWatchdogBehavior action, string msg) case SubscriptionWatchdogBehavior.Reset: ResetMonitoredItemWatchdogTimer(false); _forceRecreate = true; - TriggerManageSubscription(); break; case SubscriptionWatchdogBehavior.FailFast: Publisher.Runtime.FailFast(msg, null); @@ -2658,7 +2550,7 @@ public static List Create(IEnumerable subscribers, .ToList()) .OrderByDescending(tl => tl.Count)) { - bool placed = false; + var placed = false; foreach (var partition in partitions) { if (partition.Items.Count + @@ -2726,8 +2618,6 @@ public void Dispose() private int _count; } - private long TotalMonitoredItems - => _additionallyMonitored.Count + MonitoredItemCount; private int HeartbeatsEnabled => MonitoredItems.Count(r => r is OpcUaMonitoredItem.Heartbeat h && h.TimerEnabled); private int ConditionsEnabled @@ -2744,7 +2634,7 @@ public void InitializeMetrics() () => new Measurement(_missingKeepAlives, _metrics.TagList), description: "Number of missing keep alives in subscription."); _meter.CreateObservableUpDownCounter("iiot_edge_publisher_monitored_items", - () => new Measurement(TotalMonitoredItems, _metrics.TagList), + () => new Measurement(MonitoredItemCount, _metrics.TagList), description: "Total monitored item count."); _meter.CreateObservableUpDownCounter("iiot_edge_publisher_disabled_nodes", () => new Measurement(_disabledItems, _metrics.TagList), @@ -2811,8 +2701,6 @@ public void InitializeMetrics() } private const int kMaxMonitoredItemPerSubscriptionDefault = 64 * 1024; - private static readonly TimeSpan kDefaultErrorRetryDelay = TimeSpan.FromMinutes(1); - private FrozenDictionary _additionallyMonitored; private uint _previousSequenceNumber; private uint _sequenceNumber; private uint _currentSequenceNumber; @@ -2822,11 +2710,9 @@ public void InitializeMetrics() private readonly uint? _parentId; private readonly OpcUaClient _client; private readonly IOptions _options; - private readonly TimeSpan? _createSessionTimeout; private readonly ILoggerFactory _loggerFactory; private readonly ILogger _logger; private readonly IMetricsContext _metrics; - private readonly ITimer _timer; private readonly ITimer _keepAliveWatcher; private readonly ITimer _monitoredItemWatcher; private readonly TimeProvider _timeProvider; diff --git a/src/Azure.IIoT.OpcUa.Publisher/src/Storage/PublishedNodesConverter.cs b/src/Azure.IIoT.OpcUa.Publisher/src/Storage/PublishedNodesConverter.cs index ba3db97002..03cad145bf 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/src/Storage/PublishedNodesConverter.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/src/Storage/PublishedNodesConverter.cs @@ -122,7 +122,8 @@ public IEnumerable ToPublishedNodes(uint version, Date WriterGroupPartitions = item.WriterGroup.PublishQueuePartitions, WriterGroupQueueName = item.WriterGroup.Publishing?.QueueName, SendKeepAliveDataSetMessages = item.Writer.DataSet?.SendKeepAlive ?? false, - DataSetExtensionFields = item.Writer.DataSet?.ExtensionFields, + DataSetExtensionFields = item.Writer.DataSet?.ExtensionFields?.ToDictionary( + e => e.DataSetFieldName, e => e.Value), MetaDataUpdateTimeTimespan = item.Writer.MetaDataUpdateTime, QueueName = item.Writer.Publishing?.QueueName, QualityOfService = item.Writer.Publishing?.RequestedDeliveryGuarantee, @@ -441,7 +442,13 @@ public IEnumerable ToWriterGroups(IEnumerable new ExtensionFieldModel + { + DataSetFieldName = ef.Key, + Value = ef.Value + }) + .ToList(), SendKeepAlive = b.Header.SendKeepAliveDataSetMessages, Routing = b.Header.DataSetRouting, DataSetSource = new PublishedDataSetSourceModel diff --git a/src/Azure.IIoT.OpcUa.Publisher/tests/Azure.IIoT.OpcUa.Publisher.Tests.csproj b/src/Azure.IIoT.OpcUa.Publisher/tests/Azure.IIoT.OpcUa.Publisher.Tests.csproj index c0e4ac7344..bb306bebad 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/tests/Azure.IIoT.OpcUa.Publisher.Tests.csproj +++ b/src/Azure.IIoT.OpcUa.Publisher/tests/Azure.IIoT.OpcUa.Publisher.Tests.csproj @@ -5,9 +5,9 @@ - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Azure.IIoT.OpcUa.Publisher/tests/Services/Encoder/NetworkMessage.cs b/src/Azure.IIoT.OpcUa.Publisher/tests/Services/Encoder/NetworkMessage.cs index 8c80e250b1..f77bbd22d2 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/tests/Services/Encoder/NetworkMessage.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/tests/Services/Encoder/NetworkMessage.cs @@ -251,6 +251,7 @@ public static IList GenerateSampleSubscriptionNot Retain = false, Ttl = randomTopic ? TimeSpan.FromSeconds(Random.Shared.Next(60)) : null, PublisherId = publisherId, + ExtensionFields = Array.Empty<(string, DataValue)>(), Schema = null, Writer = writer, WriterName = writer.DataSetWriterName ?? Constants.DefaultDataSetWriterName, diff --git a/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/GetSimpleEventFilterTests.cs b/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/GetSimpleEventFilterTests.cs index 22fcf37dc4..de71c8fe5a 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/GetSimpleEventFilterTests.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/GetSimpleEventFilterTests.cs @@ -173,7 +173,6 @@ public async Task SetupSimpleFilterForConditionTypeWithConditionHandlingEnabled( protected override Mock SetupMockedNodeCache(NamespaceTable namespaceTable = null) { - var nodeCache = base.SetupMockedNodeCache(namespaceTable); AddNode(_baseObjectTypeNode); AddNode(_baseEventTypeNode); AddNode(_messageNode); @@ -182,12 +181,6 @@ protected override Mock SetupMockedNodeCache(NamespaceTable namespac AddNode(_commentNode); AddNode(_enabledStateNode); AddNode(_idNode); - var typeTable = nodeCache.Object.TypeTree as TypeTable; - typeTable.Add(_baseObjectTypeNode); - typeTable.Add(_baseEventTypeNode); - typeTable.Add(_conditionTypeNode); - typeTable.AddSubtype(ObjectTypeIds.BaseEventType, ObjectTypeIds.BaseObjectType); - typeTable.AddSubtype(ObjectTypeIds.ConditionType, ObjectTypeIds.BaseEventType); _baseObjectTypeNode.ReferenceTable.Add(ReferenceTypeIds.HasSubtype, false, ObjectTypeIds.BaseEventType); _baseEventTypeNode.ReferenceTable.Add(ReferenceTypeIds.HasSubtype, true, ObjectTypeIds.BaseObjectType); _baseEventTypeNode.ReferenceTable.Add(ReferenceTypeIds.HasProperty, false, _messageNode.NodeId); @@ -202,14 +195,24 @@ protected override Mock SetupMockedNodeCache(NamespaceTable namespac _enabledStateNode.ReferenceTable.Add(ReferenceTypeIds.HasProperty, false, _idNode.NodeId); _idNode.ReferenceTable.Add(ReferenceTypeIds.HasProperty, true, _enabledStateNode.NodeId); _commentNode.ReferenceTable.Add(ReferenceTypeIds.HasProperty, true, ObjectTypeIds.ConditionType); - nodeCache.Setup(x => x.FetchNodeAsync(It.IsAny(), It.IsAny())).Returns((ExpandedNodeId x, CancellationToken _) => - { - if (x.IdType == IdType.Numeric && x.Identifier is uint id) + + var nodeCache = base.SetupMockedNodeCache(namespaceTable); + var typeTable = nodeCache.Object.TypeTree as TypeTable; + typeTable.Add(_baseObjectTypeNode); + typeTable.Add(_baseEventTypeNode); + typeTable.Add(_conditionTypeNode); + typeTable.AddSubtype(ObjectTypeIds.BaseEventType, ObjectTypeIds.BaseObjectType); + typeTable.AddSubtype(ObjectTypeIds.ConditionType, ObjectTypeIds.BaseEventType); + nodeCache.Setup(x => x.FetchNodeAsync(It.IsAny(), It.IsAny())) + .Returns((ExpandedNodeId x, CancellationToken _) => { - return Task.FromResult(_nodes[id]); - } - return Task.FromResult(null); - }); + if (x.IdType == IdType.Numeric && x.Identifier is uint id) + { + return Task.FromResult(_nodes[id]); + } + return Task.FromResult(null); + }); + return nodeCache; } diff --git a/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTests.cs b/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTests.cs index ceee0f5349..49a96d63c9 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTests.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTests.cs @@ -112,7 +112,6 @@ public async Task SetBaseValuesWhenPropertiesAreSetInBaseTemplate() Assert.Equal("DisplayName", monitoredItem.DisplayName); Assert.Equal((uint)NodeAttribute.Value, monitoredItem.AttributeId); Assert.Equal("5:20", monitoredItem.IndexRange); - Assert.Null(monitoredItem.RelativePath); Assert.Equal(Opc.Ua.MonitoringMode.Sampling, monitoredItem.MonitoringMode); Assert.Equal("i=2258", monitoredItem.StartNodeId); Assert.Equal(10u, monitoredItem.QueueSize); diff --git a/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTestsBase.cs b/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTestsBase.cs index aed34b859e..90b10575f4 100644 --- a/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTestsBase.cs +++ b/src/Azure.IIoT.OpcUa.Publisher/tests/Stack/OpcUaMonitoredItemTestsBase.cs @@ -60,7 +60,7 @@ internal async Task GetMonitoredItem(BaseMonitoredItemModel var monitoredItemWrapper = OpcUaMonitoredItem.Create(null!, (subscriber.Object, template).YieldReturn(), Log.ConsoleFactory(), TimeProvider.System).Single(); - using var subscription = new Subscription(); + using var subscription = new SimpleSubscription(); monitoredItemWrapper.AddTo(subscription, session, out _); if (monitoredItemWrapper.FinalizeAddTo != null) { @@ -68,5 +68,22 @@ internal async Task GetMonitoredItem(BaseMonitoredItemModel } return monitoredItemWrapper; } + + internal sealed class SimpleSubscription : Subscription + { + public SimpleSubscription() + { + } + + public SimpleSubscription(Subscription template, bool copyEventHandlers) + : base(template, copyEventHandlers) + { + } + + public override Subscription CloneSubscription(bool copyEventHandlers) + { + throw new NotImplementedException(); + } + } } } diff --git a/src/Azure.IIoT.OpcUa/src/Azure.IIoT.OpcUa.csproj b/src/Azure.IIoT.OpcUa/src/Azure.IIoT.OpcUa.csproj index 45300a856c..7fbb47c5f8 100644 --- a/src/Azure.IIoT.OpcUa/src/Azure.IIoT.OpcUa.csproj +++ b/src/Azure.IIoT.OpcUa/src/Azure.IIoT.OpcUa.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/AvroDecoder.cs b/src/Azure.IIoT.OpcUa/src/Encoders/AvroDecoder.cs index fe2239ad73..725a193b57 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/AvroDecoder.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/AvroDecoder.cs @@ -15,6 +15,7 @@ namespace Azure.IIoT.OpcUa.Encoders using System.IO; using System.Linq; using System.Xml; + using System.Collections.Generic; /// /// Decodes objects from underlying decoder using a provided @@ -286,7 +287,7 @@ DataSet ReadDataSet(Schema currentSchema) $"Invalid schema {currentSchema.ToJson()}. " + "Data sets must be records or maps.", Schema.ToJson()); } - var dataSet = new DataSet(); + var dataSet = new List<(string, DataValue?)>(); var isRaw = false; // Run through the fields and read either using variant or data values @@ -298,13 +299,10 @@ DataSet ReadDataSet(Schema currentSchema) isRaw = true; dataValue.StatusCode = StatusCodes.Good; } - dataSet.Add(SchemaUtils.Unescape(field.Name), dataValue); + dataSet.Add((SchemaUtils.Unescape(field.Name), dataValue)); } - if (isRaw) - { - dataSet.DataSetFieldContentMask = DataSetFieldContentFlags.RawData; - } - return dataSet; + var dataSetFieldContentFlags = isRaw ? DataSetFieldContentFlags.RawData : 0; + return new DataSet(dataSet, dataSetFieldContentFlags); } } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/AvroEncoder.cs b/src/Azure.IIoT.OpcUa/src/Encoders/AvroEncoder.cs index f90a2c8f18..f5725e8ade 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/AvroEncoder.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/AvroEncoder.cs @@ -685,9 +685,10 @@ void WriteDataSet(DataSet dataSet, Schema currentSchema) } // Serialize the fields in the schema + var lookup = dataSet.DataSetFields.ToDictionary(f => f.Name, f => f.Value); foreach (var field in r.Fields) { - if (!dataSet.TryGetValue(SchemaUtils.Unescape(field.Name), + if (!lookup.TryGetValue(SchemaUtils.Unescape(field.Name), out var dataValue)) { dataValue = null; diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroDecoder.cs b/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroDecoder.cs index 30620ee5b6..7cb840b98e 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroDecoder.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroDecoder.cs @@ -404,9 +404,9 @@ public virtual DataSet ReadDataSet(string? fieldName) return ReadUnion(fieldName, avroFieldContent => { var fieldNames = Array.Empty(); - var dataSet = avroFieldContent == 0 ? - new DataSet() : - new DataSet(DataSetFieldContentFlags.RawData); + var dataSet = new List<(string, DataValue?)>(); + var dataSetFieldContentMask = avroFieldContent == 0 ? + 0 : DataSetFieldContentFlags.RawData; if (avroFieldContent == 1) // Raw mode { @@ -416,7 +416,7 @@ public virtual DataSet ReadDataSet(string? fieldName) var variants = ReadVariantArray(null); // TODO: Read map if (variants == null && fieldNames.Length == 0) { - return dataSet; + return new DataSet(dataSet, dataSetFieldContentMask); } if (variants == null || variants.Count != fieldNames.Length) { @@ -425,7 +425,7 @@ public virtual DataSet ReadDataSet(string? fieldName) } for (var index = 0; index < fieldNames.Length; index++) { - dataSet.Add(fieldNames[index], new DataValue(variants[index])); + dataSet.Add((fieldNames[index], new DataValue(variants[index]))); } } else if (avroFieldContent == 0) @@ -436,7 +436,7 @@ public virtual DataSet ReadDataSet(string? fieldName) var dataValues = ReadDataValueArray(null); // TODO: Read map if (dataValues == null && fieldNames.Length == 0) { - return dataSet; + return new DataSet(dataSet, dataSetFieldContentMask); } if (dataValues == null || dataValues.Count != fieldNames.Length) { @@ -445,10 +445,10 @@ public virtual DataSet ReadDataSet(string? fieldName) } for (var index = 0; index < fieldNames.Length; index++) { - dataSet.Add(fieldNames[index], dataValues[index]); + dataSet.Add((fieldNames[index], dataValues[index])); } } - return dataSet; + return new DataSet(dataSet, dataSetFieldContentMask); }); } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroEncoder.cs b/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroEncoder.cs index 35247e1b21..755b0c903a 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroEncoder.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/BaseAvroEncoder.cs @@ -292,16 +292,16 @@ public virtual void WriteDataSet(string? fieldName, DataSet dataSet) if ((fieldContentMask.HasFlag(DataSetFieldContentFlags.RawData)) || fieldContentMask == 0) { - foreach (var value in dataSet) + foreach (var field in dataSet.DataSetFields) { - WriteVariant(value.Key, value.Value?.WrappedValue ?? default); + WriteVariant(field.Name, field.Value?.WrappedValue ?? default); } } else { - foreach (var value in dataSet) + foreach (var field in dataSet.DataSetFields) { - WriteNullable(value.Key, value.Value, WriteDataValue); + WriteNullable(field.Name, field.Value, WriteDataValue); } } } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/JsonDecoderEx.cs b/src/Azure.IIoT.OpcUa/src/Encoders/JsonDecoderEx.cs index bd87e2db7a..0a3bf3d9a9 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/JsonDecoderEx.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/JsonDecoderEx.cs @@ -884,7 +884,7 @@ public StringCollection ReadStringArray(string? fieldName) } /// - public IDictionary? ReadStringDictionary(string? property) + public IReadOnlyList<(string, string?)>? ReadStringDictionary(string? property) { return ReadDictionary(property, () => ReadString(null)); } @@ -1968,17 +1968,17 @@ private ExtensionObjectEncoding ReadEncoding(string? property) /// /// /// - private Dictionary? ReadDictionary(string? property, + private List<(string, T?)>? ReadDictionary(string? property, Func reader) { if (!TryGetToken(property, out var token) || token is not JObject o) { return null; } - var dictionary = new Dictionary(); + var dictionary = new List<(string, T?)>(); foreach (var p in o.Properties()) { - dictionary[p.Name] = ReadToken(p.Value, reader); + dictionary.Add((p.Name, ReadToken(p.Value, reader))); } return dictionary; } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/JsonEncoderEx.cs b/src/Azure.IIoT.OpcUa/src/Encoders/JsonEncoderEx.cs index efa238cf19..95ae27c1c5 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/JsonEncoderEx.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/JsonEncoderEx.cs @@ -1071,7 +1071,7 @@ public void WriteStringArray(string? fieldName, IList? values) /// public void WriteStringDictionary(string? property, - IEnumerable> values) + IEnumerable<(string, string?)> values) { WriteDictionary(property, values, WriteString); } @@ -1168,8 +1168,7 @@ public void WriteEncodeableArray(string? fieldName, IList? values, } /// - public void WriteDataSet(string? property, DataSet? dataSet, - IEnumerable>? extraFields = null) + public void WriteDataSet(string? property, DataSet? dataSet) { if (dataSet == null) { @@ -1181,7 +1180,7 @@ public void WriteDataSet(string? property, DataSet? dataSet, try { var fieldContentMask = dataSet.DataSetFieldContentMask; - var writeSingleValue = (dataSet.Count == 1) && + var writeSingleValue = (dataSet.DataSetFields.Count == 1) && fieldContentMask.HasFlag(DataSetFieldContentFlags.SingleFieldDegradeToValue); if (fieldContentMask.HasFlag(DataSetFieldContentFlags.RawData)) { @@ -1192,7 +1191,7 @@ public void WriteDataSet(string? property, DataSet? dataSet, // UseUriEncoding = true; UseReversibleEncoding = false; - Write(property, dataSet, extraFields, + Write(property, dataSet.DataSetFields, (k, v) => WriteVariant(k, v?.WrappedValue ?? default), writeSingleValue); } else if (fieldContentMask == 0) @@ -1204,7 +1203,7 @@ public void WriteDataSet(string? property, DataSet? dataSet, // UseUriEncoding = false; UseReversibleEncoding = true; - Write(property, dataSet, extraFields, + Write(property, dataSet.DataSetFields, (k, v) => WriteVariant(k, v?.WrappedValue ?? default), writeSingleValue); } else @@ -1214,7 +1213,7 @@ public void WriteDataSet(string? property, DataSet? dataSet, // the field value is a DataValue encoded using the non-reversible OPC UA // JSON Data Encoding or reversible depending on encoder configuration. // - Write(property, dataSet, extraFields, (k, value) => + Write(property, dataSet.DataSetFields, (k, value) => { PushObject(k); try @@ -1251,20 +1250,15 @@ public void WriteDataSet(string? property, DataSet? dataSet, }, writeSingleValue); } - void Write(string? property, IEnumerable> values, - IEnumerable>? extra, Action writer, + void Write(string? property, IEnumerable<(string, T)> values, Action writer, bool writeSingleValue) { if (writeSingleValue) { - writer(property, values.Single().Value); + writer(property, values.Single().Item2); } else { - if (extra != null) - { - values = values.Concat(extra); - } WriteDictionary(property, values, writer); } } @@ -2034,7 +2028,7 @@ internal void WriteObject(string? property, T value, Action writer) /// /// private void WriteDictionary(string? property, - IEnumerable>? values, Action writer) + IEnumerable<(string Key, T Value)>? values, Action writer) { if (values == null) { diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/Models/DataSet.cs b/src/Azure.IIoT.OpcUa/src/Encoders/Models/DataSet.cs index 43f5681a3f..4c973e3b0d 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/Models/DataSet.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/Models/DataSet.cs @@ -7,33 +7,49 @@ namespace Azure.IIoT.OpcUa.Encoders.Models { using Azure.IIoT.OpcUa.Encoders.PubSub; using Azure.IIoT.OpcUa.Publisher.Models; + using Newtonsoft.Json.Linq; using Opc.Ua; using System; using System.Collections.Generic; + using System.Linq; /// /// Encodable dataset message payload /// - public class DataSet : Dictionary + public class DataSet { /// /// Field mask /// public DataSetFieldContentFlags DataSetFieldContentMask { get; set; } + /// + /// Entries + /// + public IReadOnlyList<(string Name, DataValue? Value)> DataSetFields { get; } + /// /// Create payload /// /// /// public DataSet(IDictionary values, + DataSetFieldContentFlags? fieldContentMask = null) + : this(fieldContentMask) + { + DataSetFields = values.Select(kv => (kv.Key, kv.Value)).ToList(); + } + + /// + /// Create payload + /// + /// + /// + public DataSet(IReadOnlyList<(string, DataValue?)> values, DataSetFieldContentFlags? fieldContentMask) : this(fieldContentMask) { - foreach (var value in values) - { - this[value.Key] = value.Value; - } + DataSetFields = values; } /// @@ -46,7 +62,7 @@ public DataSet(string field, DataValue? value, DataSetFieldContentFlags? fieldContentMask) : this(fieldContentMask) { - this[field] = value; + DataSetFields = new[] { (field, value) }; } /// @@ -57,21 +73,19 @@ public DataSet(DataSetFieldContentFlags? fieldContentMask = null) { DataSetFieldContentMask = fieldContentMask ?? PubSubMessage.DefaultDataSetFieldContentFlags; + DataSetFields = Array.Empty<(string, DataValue?)>(); } /// public override bool Equals(object? obj) { - if (obj is not Dictionary set) - { - return false; - } - if (!Keys.SequenceEqualsSafe(set.Keys)) + if (obj is not DataSet set) { return false; } - if (!Values.SequenceEqualsSafe(set.Values, - (x, y) => Utils.IsEqual(x?.Value, y?.Value))) + if (!DataSetFields.SequenceEqualsSafe(set.DataSetFields, + (x, y) => x.Item1 == y.Item1 && + Utils.IsEqual(x.Item2?.Value, y.Item2?.Value))) { return false; } @@ -81,7 +95,52 @@ public override bool Equals(object? obj) /// public override int GetHashCode() { - return HashCode.Combine(Keys); + return HashCode.Combine(DataSetFields.Select(s => s.Item1)); + } + + /// + /// Remove field from dataset + /// + /// + /// + internal DataSet Remove(string field) + { + return new DataSet(DataSetFields + .Where(b => b.Name != field) + .ToList(), DataSetFieldContentMask); + } + + /// + /// Set field from dataset to different value + /// + /// + /// + /// + internal DataSet Set(string field, DataValue? value) + { + return new DataSet(DataSetFields + .Select(b => (b.Name, b.Name == field ? value : b.Value)) + .ToList(), DataSetFieldContentMask); + } + + /// + /// Set field from dataset to different value + /// + /// + /// + /// + /// + internal DataSet Add(string field, DataValue? value, + DataSetFieldContentFlags? additionalFlags = null) + { + var fieldContentMask = DataSetFieldContentMask; + if (additionalFlags.HasValue) + { + fieldContentMask |= additionalFlags.Value; + } + return new DataSet(DataSetFields + .Append((field, value)) + .ToList(), fieldContentMask); } } } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/Models/EncodeableDictionary.cs b/src/Azure.IIoT.OpcUa/src/Encoders/Models/EncodeableDictionary.cs index 3bae5dc413..e1eb289239 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/Models/EncodeableDictionary.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/Models/EncodeableDictionary.cs @@ -84,12 +84,12 @@ public virtual void Decode(IDecoder decoder) var dataSet = jsonDecoder.ReadDataSet(null); if (dataSet != null) { - foreach (var keyValuePair in dataSet) + foreach (var field in dataSet.DataSetFields) { Add(new KeyDataValuePair { - Key = keyValuePair.Key, - Value = keyValuePair.Value + Key = field.Name, + Value = field.Value }); } } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/AvroDataSetMessage.cs b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/AvroDataSetMessage.cs index 2d38bc5aae..7ec79f09ef 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/AvroDataSetMessage.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/AvroDataSetMessage.cs @@ -9,6 +9,7 @@ namespace Azure.IIoT.OpcUa.Encoders.PubSub using Avro; using System; using System.Linq; + using Azure.IIoT.OpcUa.Publisher.Models; /// /// Avro binary data set message @@ -245,9 +246,10 @@ private void WriteDataSetMessageHeader(BaseAvroEncoder encoder) typeof(Opc.Ua.ConfigurationVersionDataType)); encoder.WriteDateTime(nameof(Timestamp), Timestamp?.UtcDateTime ?? default); - var status = Status ?? Payload.Values + var status = Status ?? Payload.DataSetFields .FirstOrDefault(s => Opc.Ua.StatusCode.IsNotGood( - s?.StatusCode ?? Opc.Ua.StatusCodes.BadNoData))?.StatusCode ?? Opc.Ua.StatusCodes.Good; + s.Value?.StatusCode ?? Opc.Ua.StatusCodes.BadNoData)).Value?.StatusCode + ?? Opc.Ua.StatusCodes.Good; encoder.WriteStatusCode(nameof(Status), status); } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/BaseDataSetMessage.cs b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/BaseDataSetMessage.cs index b4c4cca8ff..312f623b4e 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/BaseDataSetMessage.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/BaseDataSetMessage.cs @@ -8,6 +8,7 @@ namespace Azure.IIoT.OpcUa.Encoders.PubSub using Azure.IIoT.OpcUa.Encoders.Models; using Azure.IIoT.OpcUa.Publisher.Models; using System; + using System.Collections.Generic; /// /// Data set message @@ -57,9 +58,17 @@ public abstract class BaseDataSetMessage /// /// Payload /// -#pragma warning disable CA2227 // Collection properties should be read only public DataSet Payload { get; set; } = new DataSet(); -#pragma warning restore CA2227 // Collection properties should be read only + + /// + /// Endpoint url + /// + public string? EndpointUrl { get; set; } + + /// + /// Application uri + /// + public string? ApplicationUri { get; set; } /// public override bool Equals(object? obj) @@ -77,7 +86,9 @@ public override bool Equals(object? obj) !Opc.Ua.Utils.IsEqual(wrapper.Status ?? Opc.Ua.StatusCodes.Good, Status ?? Opc.Ua.StatusCodes.Good) || !Opc.Ua.Utils.IsEqual(wrapper.Timestamp, Timestamp) || !Opc.Ua.Utils.IsEqual(wrapper.MessageType, MessageType) || - !Opc.Ua.Utils.IsEqual(wrapper.MetaDataVersion, MetaDataVersion)) + !Opc.Ua.Utils.IsEqual(wrapper.MetaDataVersion, MetaDataVersion) || + !Opc.Ua.Utils.IsEqual(wrapper.EndpointUrl, EndpointUrl) || + !Opc.Ua.Utils.IsEqual(wrapper.ApplicationUri, ApplicationUri)) { return false; } @@ -100,6 +111,8 @@ public override int GetHashCode() hash.Add(Picoseconds); hash.Add(Status); hash.Add(Payload); + hash.Add(EndpointUrl); + hash.Add(ApplicationUri); return hash.ToHashCode(); } diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/JsonDataSetMessage.cs b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/JsonDataSetMessage.cs index 0059609410..0fb6169b17 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/JsonDataSetMessage.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/JsonDataSetMessage.cs @@ -21,16 +21,6 @@ public class JsonDataSetMessage : BaseDataSetMessage /// public bool UseCompatibilityMode { get; set; } - /// - /// Endpoint url - /// - public string? EndpointUrl { get; set; } - - /// - /// Application uri - /// - public string? ApplicationUri { get; set; } - /// /// Dataset writer name /// @@ -51,9 +41,7 @@ public override bool Equals(object? obj) { return false; } - if (!Opc.Ua.Utils.IsEqual(wrapper.EndpointUrl, EndpointUrl) || - !Opc.Ua.Utils.IsEqual(wrapper.ApplicationUri, ApplicationUri) || - !Opc.Ua.Utils.IsEqual(wrapper.DataSetWriterName, DataSetWriterName)) + if (!Opc.Ua.Utils.IsEqual(wrapper.DataSetWriterName, DataSetWriterName)) { return false; } @@ -66,8 +54,6 @@ public override int GetHashCode() var hash = new HashCode(); hash.Add(base.GetHashCode()); - hash.Add(EndpointUrl); - hash.Add(ApplicationUri); hash.Add(DataSetWriterName); return hash.ToHashCode(); } @@ -105,9 +91,9 @@ internal virtual void Encode(JsonEncoderEx encoder, string? publisherId, bool wi } if ((DataSetMessageContentMask & DataSetMessageContentFlags.Status) != 0) { - var status = Status ?? Payload.Values - .FirstOrDefault(s => Opc.Ua.StatusCode.IsNotGood(s?.StatusCode ?? - Opc.Ua.StatusCodes.BadNoData))?.StatusCode ?? Opc.Ua.StatusCodes.Good; + var status = Status ?? Payload.DataSetFields + .FirstOrDefault(s => Opc.Ua.StatusCode.IsNotGood(s.Value?.StatusCode ?? + Opc.Ua.StatusCodes.BadNoData)).Value?.StatusCode ?? Opc.Ua.StatusCodes.Good; if (!UseCompatibilityMode) { encoder.WriteUInt32(nameof(Status), status.Code); @@ -160,31 +146,8 @@ void WritePayload(JsonEncoderEx jsonEncoder, string? propertyName = null) { jsonEncoder.UseReversibleEncoding = useReversibleEncoding; - if ((Payload.DataSetFieldContentMask & (DataSetFieldContentFlags.EndpointUrl | - DataSetFieldContentFlags.ApplicationUri)) != 0) - { - var extraFields = Enumerable.Empty>(); - if ((Payload.DataSetFieldContentMask & DataSetFieldContentFlags.EndpointUrl) != 0 && - !Payload.ContainsKey(nameof(EndpointUrl)) && - !string.IsNullOrWhiteSpace(EndpointUrl)) - { - extraFields = extraFields.Append(KeyValuePair.Create( - nameof(EndpointUrl), new Opc.Ua.DataValue(EndpointUrl))); - } - if ((Payload.DataSetFieldContentMask & DataSetFieldContentFlags.ApplicationUri) != 0 && - !Payload.ContainsKey(nameof(ApplicationUri)) && - !string.IsNullOrWhiteSpace(ApplicationUri)) - { - extraFields = extraFields.Append(KeyValuePair.Create( - nameof(ApplicationUri), new Opc.Ua.DataValue(ApplicationUri))); - } - jsonEncoder.WriteDataSet(propertyName, Payload, extraFields); - } - else - { - // if propertyname is null we are already inside the object - jsonEncoder.WriteDataSet(propertyName, Payload); - } + // if propertyname is null we are already inside the object + jsonEncoder.WriteDataSet(propertyName, Payload); } finally { diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/MonitoredItemMessage.cs b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/MonitoredItemMessage.cs index 53478e748e..e90a5cafe2 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/MonitoredItemMessage.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/MonitoredItemMessage.cs @@ -6,6 +6,7 @@ namespace Azure.IIoT.OpcUa.Encoders.PubSub { using Azure.IIoT.OpcUa.Encoders; + using Azure.IIoT.OpcUa.Encoders.Models; using Azure.IIoT.OpcUa.Encoders.Utils; using Azure.IIoT.OpcUa.Publisher.Models; using Furly.Extensions.Serializers; @@ -32,19 +33,17 @@ public class MonitoredItemMessage : JsonDataSetMessage /// /// Display name /// - public string? DisplayName => Payload.Keys.SingleOrDefault(); + public string? DisplayName => Payload.DataSetFields.SingleOrDefault().Name; /// /// Data value for variable change notification /// - public Opc.Ua.DataValue? Value => Payload.Values.SingleOrDefault(); + public Opc.Ua.DataValue? Value => Payload.DataSetFields.SingleOrDefault().Value; /// /// Extension fields /// -#pragma warning disable CA2227 // Collection properties should be read only - public IDictionary? ExtensionFields { get; set; } -#pragma warning restore CA2227 // Collection properties should be read only + public IReadOnlyList? ExtensionFields { get; set; } /// public override bool Equals(object? obj) @@ -65,8 +64,8 @@ public override bool Equals(object? obj) { return false; } - if (!wrapper.ExtensionFields.DictionaryEqualsSafe(ExtensionFields, - (a, b) => a.Equals(b))) + if (!wrapper.ExtensionFields.SetEqualsSafe(ExtensionFields, + (a, b) => a?.Equals(b) ?? b == null)) { return false; } @@ -178,25 +177,31 @@ internal override void Encode(JsonEncoderEx encoder, string? publisherId, bool w if (Payload.DataSetFieldContentMask.HasFlag(DataSetFieldContentFlags.ExtensionFields)) { - var extensionFields = new KeyValuePair(nameof(DataSetWriterId), DataSetWriterName) + var extensionFields = (nameof(DataSetWriterId), DataSetWriterName) .YieldReturn(); if (publisherId != null) { extensionFields = extensionFields - .Append(new KeyValuePair(nameof(JsonNetworkMessage.PublisherId), publisherId)); + .Append((nameof(JsonNetworkMessage.PublisherId), publisherId)); } if (WriterGroupId != null) { extensionFields = extensionFields - .Append(new KeyValuePair(nameof(WriterGroupId), WriterGroupId)); + .Append((nameof(WriterGroupId), WriterGroupId)); } if (ExtensionFields != null) { extensionFields = extensionFields.Concat(ExtensionFields - .Where(e => e.Key is not (nameof(DataSetWriterId)) and - not (nameof(JsonNetworkMessage.PublisherId))) - .Select(e => new KeyValuePair(e.Key, e.Value.Value?.ToString()))); + .Where(e => e.DataSetFieldName is + not (nameof(DataSetWriterId)) and + not (nameof(EndpointUrl)) and + not (nameof(ApplicationUri)) and + not (nameof(WriterGroupId)) and + not (nameof(JsonNetworkMessage.PublisherId))) + .Select(e => (e.DataSetFieldName, e.Value.Value?.ToString()))); } + + // We already wrote application uri and endpoint uri, so do not write again encoder.WriteStringDictionary(nameof(ExtensionFields), extensionFields); } } @@ -282,33 +287,45 @@ internal override bool TryDecode(JsonDecoderEx decoder, string? property, ref bo { DataSetMessageContentMask |= DataSetMessageContentFlags.SequenceNumber; } - var extensionFields = decoder.ReadStringDictionary(nameof(ExtensionFields)); - if (extensionFields != null) + var stringDictionary = decoder.ReadStringDictionary(nameof(ExtensionFields)); + if (stringDictionary?.Count > 0) { dataSetFieldContentMask |= DataSetFieldContentFlags.ExtensionFields; - ExtensionFields = new Dictionary(); - foreach (var item in extensionFields) - { - ExtensionFields.AddOrUpdate(item.Key, item.Value); - } - if (extensionFields.TryGetValue(nameof(DataSetWriterId), out var dataSetWriterName)) - { - DataSetWriterName = dataSetWriterName; - } - if (extensionFields.TryGetValue(nameof(WriterGroupId), out var writerGroupid)) + var extensionFields = new List(); + foreach (var (name, v) in stringDictionary) { - WriterGroupId = writerGroupid; + if (name == nameof(DataSetWriterId)) + { + DataSetWriterName = v; + } + else if (name == nameof(JsonNetworkMessage.PublisherId)) + { + publisherId = v; + } + else if (name == nameof(WriterGroupId)) + { + WriterGroupId = v; + } + else + { + extensionFields.Add(new ExtensionFieldModel + { + DataSetFieldName = name, + Value = v + }); + } } - extensionFields.TryGetValue(nameof(JsonNetworkMessage.PublisherId), out publisherId); + ExtensionFields = extensionFields; + } + else + { + ExtensionFields = null; } withHeader |= DataSetMessageContentMask != 0; if (value != null || dataSetFieldContentMask != 0) { - Payload.Clear(); - Payload.DataSetFieldContentMask = dataSetFieldContentMask; - Payload.Add(displayName ?? string.Empty, value); - + Payload = Payload.Add(displayName ?? string.Empty, value, dataSetFieldContentMask); return true; } // Only return true if we otherwise read a header value diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/PubSubMessage.cs b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/PubSubMessage.cs index ac4ab96399..5fc0815d16 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/PubSubMessage.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/PubSubMessage.cs @@ -277,7 +277,7 @@ public static bool TryCreateMonitoredItemMessage(MessageEncoding encoding, string? writerGroupName, DataSetMessageContentFlags? dataSetMessageContentFlags, MessageType messageType, DateTimeOffset? timestamp, uint sequenceNumber, DataSet payload, string? nodeId, string? endpointUrl, string? applicationUri, - bool standardsCompliant, IDictionary? extensionFields, + bool standardsCompliant, IReadOnlyList? extensionFields, [NotNullWhen(true)] out BaseDataSetMessage? message) { if (encoding.HasFlag(MessageEncoding.Json)) diff --git a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/UadpDataSetMessage.cs b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/UadpDataSetMessage.cs index 4104e17d46..5fd0d22750 100644 --- a/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/UadpDataSetMessage.cs +++ b/src/Azure.IIoT.OpcUa/src/Encoders/PubSub/UadpDataSetMessage.cs @@ -430,8 +430,8 @@ private void WriteDataSetMessageHeader(BinaryEncoder encoder) { // This is the high order 16 bits of the StatusCode DataType representing // the numeric value of the Severity and SubCode of the StatusCode DataType. - var status = Status ?? Payload.Values - .FirstOrDefault(s => StatusCode.IsNotGood(s?.StatusCode ?? StatusCodes.BadNoData))? + var status = Status ?? Payload.DataSetFields + .FirstOrDefault(s => StatusCode.IsNotGood(s.Value?.StatusCode ?? StatusCodes.BadNoData)).Value? .StatusCode ?? StatusCodes.Good; encoder.WriteUInt16(null, (ushort)(status.Code >> 16)); } @@ -474,22 +474,23 @@ private void ReadPayloadKeyFrame(BinaryDecoder binaryDecoder, PublishedDataSetMe } // check configuration version + var fields = Payload.DataSetFields.ToList(); switch (fieldType) { case 0: for (var i = 0; i < dataSetFieldCount; i++) { var fieldMetaData = GetFieldMetadata(metadata, i); - Payload.Add(fieldMetaData?.Name ?? i.ToString(CultureInfo.InvariantCulture), - new DataValue(binaryDecoder.ReadVariant(null))); + fields.Add((fieldMetaData?.Name ?? i.ToString(CultureInfo.InvariantCulture), + new DataValue(binaryDecoder.ReadVariant(null)))); } break; case DataSetFlags1EncodingMask.FieldTypeDataValue: for (var i = 0; i < dataSetFieldCount; i++) { var fieldMetaData = GetFieldMetadata(metadata, i); - Payload.Add(fieldMetaData?.Name ?? i.ToString(CultureInfo.InvariantCulture), - binaryDecoder.ReadDataValue(null)); + fields.Add((fieldMetaData?.Name ?? i.ToString(CultureInfo.InvariantCulture), + binaryDecoder.ReadDataValue(null))); } break; case DataSetFlags1EncodingMask.FieldTypeRawData: @@ -499,13 +500,14 @@ private void ReadPayloadKeyFrame(BinaryDecoder binaryDecoder, PublishedDataSetMe if (fieldMetaData != null) { var decodedValue = ReadRawData(binaryDecoder, fieldMetaData); - Payload.Add(fieldMetaData.Name, new DataValue(new Variant(decodedValue))); + fields.Add((fieldMetaData.Name, new DataValue(new Variant(decodedValue)))); } } break; default: throw new DecodingException($"Reserved field type {fieldType} not allowed."); } + Payload = new Models.DataSet(fields, Payload.DataSetFieldContentMask); } /// @@ -520,24 +522,24 @@ private void WritePayloadKeyFrame(BinaryEncoder binaryEncoder, PublishedDataSetM switch (fieldType) { case 0: - Debug.Assert(Payload.Count <= ushort.MaxValue); - binaryEncoder.WriteUInt16(null, (ushort)Payload.Count); - foreach (var value in Payload) + Debug.Assert(Payload.DataSetFields.Count <= ushort.MaxValue); + binaryEncoder.WriteUInt16(null, (ushort)Payload.DataSetFields.Count); + foreach (var field in Payload.DataSetFields) { - binaryEncoder.WriteVariant(null, value.Value?.WrappedValue ?? default); + binaryEncoder.WriteVariant(null, field.Value?.WrappedValue ?? default); } break; case DataSetFlags1EncodingMask.FieldTypeDataValue: - Debug.Assert(Payload.Count <= ushort.MaxValue); - binaryEncoder.WriteUInt16(null, (ushort)Payload.Count); - foreach (var value in Payload) + Debug.Assert(Payload.DataSetFields.Count <= ushort.MaxValue); + binaryEncoder.WriteUInt16(null, (ushort)Payload.DataSetFields.Count); + foreach (var field in Payload.DataSetFields) { - binaryEncoder.WriteDataValue(null, value.Value); + binaryEncoder.WriteDataValue(null, field.Value); } break; case DataSetFlags1EncodingMask.FieldTypeRawData: // DataSetFieldCount is not written for RawData - var values = Payload.ToList(); + var values = Payload.DataSetFields.ToList(); for (var i = 0; i < values.Count; i++) { var fieldMetaData = GetFieldMetadata(metadata, i); @@ -563,6 +565,7 @@ private void ReadPayloadDeltaFrame(BinaryDecoder binaryDecoder, PublishedDataSet var fieldType = DataSetFlags1 & DataSetFlags1EncodingMask.FieldTypeUsedBits; var fieldCount = binaryDecoder.ReadUInt16(null); + var fields = Payload.DataSetFields.ToList(); for (var i = 0; i < fieldCount; i++) { var fieldIndex = binaryDecoder.ReadUInt16(null); @@ -570,25 +573,26 @@ private void ReadPayloadDeltaFrame(BinaryDecoder binaryDecoder, PublishedDataSet switch (fieldType) { case 0: - Payload.Add(fieldMetaData?.Name ?? fieldIndex.ToString(CultureInfo.InvariantCulture), - new DataValue(binaryDecoder.ReadVariant(null))); + fields.Add((fieldMetaData?.Name ?? fieldIndex.ToString(CultureInfo.InvariantCulture), + new DataValue(binaryDecoder.ReadVariant(null)))); break; case DataSetFlags1EncodingMask.FieldTypeDataValue: - Payload.Add(fieldMetaData?.Name ?? fieldIndex.ToString(CultureInfo.InvariantCulture), - binaryDecoder.ReadDataValue(null)); + fields.Add((fieldMetaData?.Name ?? fieldIndex.ToString(CultureInfo.InvariantCulture), + binaryDecoder.ReadDataValue(null))); break; case DataSetFlags1EncodingMask.FieldTypeRawData: if (fieldMetaData != null) { var decodedValue = ReadRawData(binaryDecoder, fieldMetaData); - Payload.Add(fieldMetaData.Name, - new DataValue(new Variant(decodedValue))); + fields.Add((fieldMetaData.Name, + new DataValue(new Variant(decodedValue)))); } break; default: throw new DecodingException($"Reserved field type {fieldType} not allowed."); } } + Payload = new Models.DataSet(fields, Payload.DataSetFieldContentMask); } /// @@ -600,12 +604,12 @@ private void ReadPayloadDeltaFrame(BinaryDecoder binaryDecoder, PublishedDataSet private void WritePayloadDeltaFrame(BinaryEncoder binaryEncoder, PublishedDataSetMetaDataModel? metadata) { // ignore null fields - var fieldCount = Payload.Count(value => value.Value?.Value != null); + var fieldCount = Payload.DataSetFields.Count(value => value.Value?.Value != null); Debug.Assert(fieldCount <= ushort.MaxValue); binaryEncoder.WriteUInt16(null, (ushort)fieldCount); var fieldType = DataSetFlags1 & DataSetFlags1EncodingMask.FieldTypeUsedBits; - var values = Payload.ToList(); + var values = Payload.DataSetFields.ToList(); for (var i = 0; i < values.Count; i++) { var value = values[i]; @@ -615,7 +619,7 @@ private void WritePayloadDeltaFrame(BinaryEncoder binaryEncoder, PublishedDataSe } // write field index corresponding to metadata - var fieldIndex = GetFieldIndex(metadata, value.Key, i); + var fieldIndex = GetFieldIndex(metadata, value.Name, i); binaryEncoder.WriteUInt16(null, fieldIndex); switch (fieldType) { diff --git a/src/Azure.IIoT.OpcUa/src/Publisher/Extensions/PublishedDataSetModelEx.cs b/src/Azure.IIoT.OpcUa/src/Publisher/Extensions/PublishedDataSetModelEx.cs index 3949791e07..b761331ebc 100644 --- a/src/Azure.IIoT.OpcUa/src/Publisher/Extensions/PublishedDataSetModelEx.cs +++ b/src/Azure.IIoT.OpcUa/src/Publisher/Extensions/PublishedDataSetModelEx.cs @@ -25,8 +25,6 @@ public static class PublishedDataSetModelEx { DataSetMetaData = model.DataSetMetaData.Clone(), DataSetSource = model.DataSetSource.Clone(), - ExtensionFields = model.ExtensionFields? - .ToDictionary(k => k.Key, v => v.Value) }); } } diff --git a/src/Azure.IIoT.OpcUa/tests/Azure.IIoT.OpcUa.Tests.csproj b/src/Azure.IIoT.OpcUa/tests/Azure.IIoT.OpcUa.Tests.csproj index 128f2144a0..deb7c90eeb 100644 --- a/src/Azure.IIoT.OpcUa/tests/Azure.IIoT.OpcUa.Tests.csproj +++ b/src/Azure.IIoT.OpcUa/tests/Azure.IIoT.OpcUa.Tests.csproj @@ -8,13 +8,13 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers - + diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/AvroDataSetTests.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/AvroDataSetTests.cs index 5ceadc5a52..e928de9435 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/AvroDataSetTests.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/AvroDataSetTests.cs @@ -15,6 +15,7 @@ namespace Azure.IIoT.OpcUa.Encoders using System.IO; using System.Linq; using Xunit; + using System.Collections.Generic; public class AvroDataSetTests { @@ -226,7 +227,7 @@ public void ReadWriteDataValueArrayWithStringAndSchema(bool concise) public void ReadWriteDataSetTest1(bool concise) { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow), ["http://microsoft.com"] = new DataValue(new Variant(-222222222), StatusCodes.Bad, DateTime.MinValue, DateTime.UtcNow), @@ -234,7 +235,7 @@ public void ReadWriteDataSetTest1(bool concise) ["@#$%^&*()_+~!@#$%^*(){}"] = new DataValue(new Variant(new byte[] { 0, 2, 4, 6 }), StatusCodes.Good), ["1245"] = new DataValue(new Variant("hello"), StatusCodes.Bad, DateTime.UtcNow, DateTime.MinValue), ["..."] = new DataValue(new Variant("imbricated")) - }; + }); byte[] buffer; var context = new ServiceMessageContext(); @@ -264,7 +265,7 @@ public void ReadWriteDataSetTest1(bool concise) public void ReadWriteDataSetTest2(bool concise) { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow), ["http://microsoft.com"] = null, @@ -272,7 +273,7 @@ public void ReadWriteDataSetTest2(bool concise) ["@#$%^&*()_+~!@#$%^*(){}"] = new DataValue(new Variant(new byte[] { 0, 2, 4, 6 }), StatusCodes.Good), ["1245"] = null, ["..."] = new DataValue(new Variant("imbricated")) - }; + }); byte[] buffer; var context = new ServiceMessageContext(); @@ -302,7 +303,7 @@ public void ReadWriteDataSetTest2(bool concise) public void ReadWriteDataSetArrayRawTest1(bool concise) { // Create dummy - var expected = new DataSet(DataSetFieldContentFlags.RawData) + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow), ["http://microsoft.com"] = new DataValue(new Variant(-222222222), StatusCodes.Bad, DateTime.MinValue, DateTime.UtcNow), @@ -310,7 +311,7 @@ public void ReadWriteDataSetArrayRawTest1(bool concise) ["@#$%^&*()_+~!@#$%^*(){}"] = new DataValue(new Variant(new byte[] { 0, 2, 4, 6 }), StatusCodes.Good), ["1245"] = new DataValue(new Variant("hello"), StatusCodes.Bad, DateTime.UtcNow, DateTime.MinValue), ["..."] = new DataValue(new Variant("imbricated")) - }; + }, DataSetFieldContentFlags.RawData); byte[] buffer; var context = new ServiceMessageContext(); @@ -338,7 +339,7 @@ public void ReadWriteDataSetArrayRawTest1(bool concise) public void ReadWriteDataSetArrayRawTest2(bool concise) { // Create dummy - var expected = new DataSet(DataSetFieldContentFlags.RawData) + var expected = new DataSet(new Dictionary { ["abcd"] = null, ["http://microsoft.com"] = new DataValue(new Variant(-222222222), StatusCodes.Bad, DateTime.MinValue, DateTime.UtcNow), @@ -346,7 +347,7 @@ public void ReadWriteDataSetArrayRawTest2(bool concise) ["@#$%^&*()_+~!@#$%^*(){}"] = new DataValue(new Variant(new byte[] { 0, 2, 4, 6 }), StatusCodes.Good), ["1245"] = new DataValue(new Variant("hello"), StatusCodes.Bad, DateTime.UtcNow, DateTime.MinValue), ["..."] = null - }; + }, DataSetFieldContentFlags.RawData); byte[] buffer; var context = new ServiceMessageContext(); @@ -374,11 +375,11 @@ public void ReadWriteDataSetArrayRawTest2(bool concise) public void ReadWriteDataSetWithSingleEntryTest(bool concise) { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) - }; + }); byte[] buffer; var context = new ServiceMessageContext(); @@ -396,7 +397,8 @@ public void ReadWriteDataSetWithSingleEntryTest(bool concise) using (var decoder = new AvroDecoder(stream, schema, context)) { var result = decoder.ReadDataSet(null); - Assert.Equal(expected["abcd"], result["abcd"]); + Assert.Equal(expected.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value, + result.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value); } } @@ -406,11 +408,11 @@ public void ReadWriteDataSetWithSingleEntryTest(bool concise) public void ReadWriteDataSetWithSingleComplexEntryTest(bool concise) { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(VariantVariants.Complex), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) - }; + }); byte[] buffer; var context = new ServiceMessageContext(); @@ -428,8 +430,8 @@ public void ReadWriteDataSetWithSingleComplexEntryTest(bool concise) using (var decoder = new AvroDecoder(stream, schema, context)) { var result = decoder.ReadDataSet(null); - Assert.True(result["abcd"].Value is ExtensionObject); - var eo = (ExtensionObject)result["abcd"].Value; + Assert.True(result.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value is ExtensionObject); + var eo = (ExtensionObject)result.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value; Assert.True(eo.Body is IEncodeable); var e = (IEncodeable)eo.Body; Assert.Equal(VariantVariants.Complex.AsJson(context), e.AsJson(context)); @@ -442,11 +444,11 @@ public void ReadWriteDataSetWithSingleComplexEntryTest(bool concise) public void ReadWriteDataSetWithSingleValueRawTest(bool concise) { // Create dummy - var expected = new DataSet(DataSetFieldContentFlags.RawData) + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) - }; + }, DataSetFieldContentFlags.RawData); byte[] buffer; var context = new ServiceMessageContext(); @@ -464,7 +466,8 @@ public void ReadWriteDataSetWithSingleValueRawTest(bool concise) using (var decoder = new AvroDecoder(stream, schema, context)) { var result = decoder.ReadDataSet(null); - Assert.Equal(expected["abcd"].Value, result["abcd"].Value); + Assert.Equal(expected.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value, + result.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value); } } @@ -474,11 +477,11 @@ public void ReadWriteDataSetWithSingleValueRawTest(bool concise) public void ReadWriteDataSetWithSingleComplexValueRawTest(bool concise) { // Create dummy - var expected = new DataSet(DataSetFieldContentFlags.RawData) + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(VariantVariants.Complex), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) - }; + }, DataSetFieldContentFlags.RawData); byte[] buffer; var context = new ServiceMessageContext(); @@ -496,8 +499,8 @@ public void ReadWriteDataSetWithSingleComplexValueRawTest(bool concise) using (var decoder = new AvroDecoder(stream, schema, context)) { var result = decoder.ReadDataSet(null); - Assert.True(result["abcd"].Value is ExtensionObject); - var eo = (ExtensionObject)result["abcd"].Value; + Assert.True(result.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value is ExtensionObject); + var eo = (ExtensionObject)result.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value; Assert.True(eo.Body is IEncodeable); var e = (IEncodeable)eo.Body; Assert.Equal(VariantVariants.Complex.AsJson(context), e.AsJson(context)); @@ -510,7 +513,7 @@ public void ReadWriteDataSetWithSingleComplexValueRawTest(bool concise) public void ReadWriteDataSetArrayRawStreamTest(bool concise) { // Create dummy - var expected = new DataSet(DataSetFieldContentFlags.RawData) + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow), ["http://microsoft.com"] = new DataValue(new Variant(-222222222), StatusCodes.Bad, DateTime.MinValue, DateTime.UtcNow), @@ -518,7 +521,7 @@ public void ReadWriteDataSetArrayRawStreamTest(bool concise) ["@#$%^&*()_+~!@#$%^*(){}"] = new DataValue(new Variant(new byte[] { 0, 2, 4, 6 }), StatusCodes.Good), ["1245"] = new DataValue(new Variant("hello"), StatusCodes.Bad, DateTime.UtcNow, DateTime.MinValue), ["..."] = new DataValue(new Variant("imbricated")) - }; + }, DataSetFieldContentFlags.RawData); const int count = 10000; byte[] buffer; diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/JsonDataSetTests.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/JsonDataSetTests.cs index 4f5edf637c..57fa9a0254 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/JsonDataSetTests.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/JsonDataSetTests.cs @@ -9,7 +9,9 @@ namespace Azure.IIoT.OpcUa.Encoders using Azure.IIoT.OpcUa.Publisher.Models; using Opc.Ua; using System; + using System.Collections.Generic; using System.IO; + using System.Linq; using Xunit; public class JsonDataSetTests @@ -119,7 +121,7 @@ public void ReadWriteDataValueWithStringStream() public void ReadWriteDataSetArrayTest() { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow), ["http://microsoft.com"] = new DataValue(new Variant(-222222222), StatusCodes.Bad, DateTime.MinValue, DateTime.UtcNow), @@ -127,7 +129,7 @@ public void ReadWriteDataSetArrayTest() ["@#$%^&*()_+~!@#$%^*(){}"] = new DataValue(new Variant(new byte[] { 0, 2, 4, 6 }), StatusCodes.Good), ["1245"] = new DataValue(new Variant("hello"), StatusCodes.Bad, DateTime.UtcNow, DateTime.MinValue), ["..."] = new DataValue(new Variant("imbricated")) - }; + }); const int count = 10000; byte[] buffer; @@ -161,10 +163,10 @@ public void ReadWriteDataSetArrayTest() public void ReadWriteDataSetWithSingleEntryTest() { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) - }; + }); expected.DataSetFieldContentMask |= DataSetFieldContentFlags.SingleFieldDegradeToValue; @@ -183,7 +185,7 @@ public void ReadWriteDataSetWithSingleEntryTest() using (var decoder = new JsonDecoderEx(stream, context)) { var result = decoder.ReadDataValue(null); - Assert.Equal(expected["abcd"], result); + Assert.Equal(expected.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value, result); } } @@ -191,10 +193,10 @@ public void ReadWriteDataSetWithSingleEntryTest() public void ReadWriteDataSetWithSingleValueRawTest() { // Create dummy - var expected = new DataSet + var expected = new DataSet(new Dictionary { ["abcd"] = new DataValue(new Variant(1234), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) - }; + }); expected.DataSetFieldContentMask |= DataSetFieldContentFlags.SingleFieldDegradeToValue; expected.DataSetFieldContentMask |= DataSetFieldContentFlags.RawData; @@ -214,7 +216,7 @@ public void ReadWriteDataSetWithSingleValueRawTest() using (var decoder = new JsonDecoderEx(stream, context)) { var result = decoder.ReadInt32(null); - Assert.Equal(expected["abcd"].Value, result); + Assert.Equal(expected.DataSetFields.FirstOrDefault(f => f.Name == "abcd").Value.Value, result); } } } diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests1.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests1.cs index 8964bcf59c..57e3710836 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests1.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests1.cs @@ -92,7 +92,7 @@ public void EncodeDecodeNetworkMessageWithNullableDataValue(bool compress, var json = schema.ToJson(); // Set null data value - messages.ForEach(messages => messages.Payload["6"] = null); + messages.ForEach(messages => messages.Payload = messages.Payload.Set("6", null)); // Reencode with the schema context = new ServiceMessageContext(); buffer = Assert.Single(networkMessage.Encode(context, 256 * 1000)); @@ -129,7 +129,7 @@ public void EncodeDecodeNetworkMessageWithMissingDataValue(bool compress, var json = schema.ToJson(); // Set null data value - messages.ForEach(messages => messages.Payload.Remove("6")); + messages.ForEach(messages => messages.Payload = messages.Payload.Remove("6")); // Reencode with the schema context = new ServiceMessageContext(); buffer = Assert.Single(networkMessage.Encode(context, 256 * 1000)); @@ -141,8 +141,8 @@ public void EncodeDecodeNetworkMessageWithMissingDataValue(bool compress, // Result will contain the removed field in the data set as it was serialized as null ((BaseNetworkMessage)result).Messages.ToList().ForEach(m => { - Assert.Null(m.Payload["6"]); - m.Payload.Remove("6"); + Assert.Null(m.Payload.DataSetFields.FirstOrDefault(f => f.Name == "6").Value); + m.Payload = m.Payload.Remove("6"); }); Assert.Equal(networkMessage, result); } @@ -217,7 +217,7 @@ public void EncodeDecodeNetworkMessagesWithNullableDataValue(bool compress, var json = schema.ToJson(); // Set null data value - messages.ForEach(messages => messages.Payload["6"] = null); + messages.ForEach(messages => messages.Payload = messages.Payload.Set("6", null)); // Reencode with the schema context = new ServiceMessageContext(); buffers = networkMessage.Encode(context, maxMessageSize); @@ -293,13 +293,13 @@ public void EncodeDecodeNetworkMessagesNoNetworkMessageHeaderRaw( var result = buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context, messageSchema: json)).Messages) - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList(); var serializer = new NewtonsoftJsonSerializer(); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); } @@ -361,7 +361,7 @@ public void EncodeDecodeNetworkMessagesNoDataSetMessageHeaderWithNullableDataVal var json = schema.ToJson(); // Set null data value - messages.ForEach(messages => messages.Payload["6"] = null); + messages.ForEach(messages => messages.Payload = messages.Payload.Set("6", null)); // Reencode with the schema context = new ServiceMessageContext(); buffers = networkMessage.Encode(context, maxMessageSize); @@ -405,14 +405,14 @@ public void EncodeDecodeNetworkMessagesNoDataSetMessageHeaderRaw( var result = buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context, messageSchema: json)).Messages) - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList(); var serializer = new NewtonsoftJsonSerializer(); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); } @@ -519,12 +519,12 @@ public void EncodeDecodeNetworkMessagesNoHeaderRaw( var result = serializer.Parse(serializer.SerializeToString(buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context, messageSchema: json)).Messages) - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); Assert.Equal(expected, result); @@ -556,7 +556,7 @@ public void EncodeDecodeNetworkMessagesNoHeaderRawAndNullableDataValue( var json = schema.ToJson(); // Set null data value - messages.ForEach(messages => messages.Payload["6"] = null); + messages.ForEach(messages => messages.Payload = messages.Payload.Set("6", null)); // Reencode with the schema context = new ServiceMessageContext(); buffers = networkMessage.Encode(context, maxMessageSize); @@ -569,12 +569,12 @@ public void EncodeDecodeNetworkMessagesNoHeaderRawAndNullableDataValue( var result = serializer.Parse(serializer.SerializeToString(buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context, messageSchema: json)).Messages) - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); Assert.Equal(expected, result); @@ -606,7 +606,7 @@ public void EncodeDecodeNetworkMessagesNoHeaderRawAndMissingDataValue( var json = schema.ToJson(); // Set null data value - messages.ForEach(messages => messages.Payload.Remove("6")); + messages.ForEach(messages => messages.Payload = messages.Payload.Remove("6")); // Reencode with the schema context = new ServiceMessageContext(); buffers = networkMessage.Encode(context, maxMessageSize); @@ -619,13 +619,13 @@ public void EncodeDecodeNetworkMessagesNoHeaderRawAndMissingDataValue( var result = serializer.Parse(serializer.SerializeToString(buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context, messageSchema: json)).Messages) - .SelectMany(m => m.Payload) - .Where(m => m.Key != "6") - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Where(m => m.Name != "6") + .Select(v => (v.Name, v.Value?.Value)) .ToList())); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value?.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value?.Value)) .ToList())); Assert.Equal(expected, result); @@ -641,9 +641,9 @@ private static void ConvertToOpcUaUniversalTime(BaseNetworkMessage networkMessag foreach (var dataSetMessage in networkMessage.Messages) { var expectedPayload = new Dictionary(); - foreach (var entry in dataSetMessage.Payload) + foreach (var entry in dataSetMessage.Payload.DataSetFields) { - expectedPayload[entry.Key] = entry.Value == null ? null : new DataValue(entry.Value).ToOpcUaUniversalTime(); + expectedPayload[entry.Name] = entry.Value == null ? null : new DataValue(entry.Value).ToOpcUaUniversalTime(); } dataSetMessage.Payload = new DataSet(expectedPayload, DataSetFieldContentFlags.StatusCode | diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests2.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests2.cs index 9a90cf03fe..240272da39 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests2.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageEncoderDecoderTests2.cs @@ -232,12 +232,12 @@ public void EncodeDecodeNetworkMessagesNoHeaderRaw( var result = serializer.Parse(serializer.SerializeToString(buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context, messageSchema: json)).Messages) - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value.Value)) .ToList())); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value.Value)) .ToList())); Assert.Equal(expected, result); @@ -253,9 +253,9 @@ private static void ConvertToOpcUaUniversalTime(BaseNetworkMessage networkMessag foreach (var dataSetMessage in networkMessage.Messages) { var expectedPayload = new Dictionary(); - foreach (var entry in dataSetMessage.Payload) + foreach (var entry in dataSetMessage.Payload.DataSetFields) { - expectedPayload[entry.Key] = new DataValue(entry.Value).ToOpcUaUniversalTime(); + expectedPayload[entry.Name] = new DataValue(entry.Value).ToOpcUaUniversalTime(); } dataSetMessage.Payload = new DataSet(expectedPayload, DataSetFieldContentFlags.StatusCode | diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageFileWriterReaderTests.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageFileWriterReaderTests.cs index e9ad92652c..036c26ed26 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageFileWriterReaderTests.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/AvroNetworkMessageFileWriterReaderTests.cs @@ -216,9 +216,9 @@ private static void ConvertToOpcUaUniversalTime(BaseNetworkMessage networkMessag foreach (var dataSetMessage in networkMessage.Messages) { var expectedPayload = new Dictionary(); - foreach (var entry in dataSetMessage.Payload) + foreach (var entry in dataSetMessage.Payload.DataSetFields) { - expectedPayload[entry.Key] = new DataValue(entry.Value).ToOpcUaUniversalTime(); + expectedPayload[entry.Name] = new DataValue(entry.Value).ToOpcUaUniversalTime(); } dataSetMessage.Payload = new DataSet(expectedPayload, DataSetFieldContentFlags.StatusCode | diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderDecoderTests.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderDecoderTests.cs index 3d52705e48..f47cf05ffa 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderDecoderTests.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderDecoderTests.cs @@ -358,12 +358,12 @@ public void EncodeDecodeNetworkMessagesNoHeaderRaw(bool useArrayEnvelope, var result = serializer.Parse(serializer.SerializeToString(buffers .SelectMany(buffer => ((BaseNetworkMessage)PubSubMessage .Decode(buffer, networkMessage.ContentType, context)).Messages) - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value.Value)) .ToList())); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value.Value)) .ToList())); Assert.Equal(expected, result); @@ -379,9 +379,9 @@ private static void ConvertToOpcUaUniversalTime(BaseNetworkMessage networkMessag foreach (var dataSetMessage in networkMessage.Messages) { var expectedPayload = new Dictionary(); - foreach (var entry in dataSetMessage.Payload) + foreach (var entry in dataSetMessage.Payload.DataSetFields) { - expectedPayload[entry.Key] = new DataValue(entry.Value).ToOpcUaUniversalTime(); + expectedPayload[entry.Name] = new DataValue(entry.Value).ToOpcUaUniversalTime(); } dataSetMessage.Payload = new DataSet(expectedPayload, DataSetFieldContentFlags.StatusCode | diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests1.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests1.cs index 5472fc3b11..63d3728ee1 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests1.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests1.cs @@ -463,7 +463,8 @@ private static JsonNetworkMessage CreateMessage(uint messageMask, uint datasetMa Status = 1073741824, MessageType = MessageType.KeyFrame, DataSetWriterName = "Writer100", - Payload = new DataSet(PubSubMessageContentFlagHelper.StackToDataSetFieldContentFlags(fieldMask)) { + Payload = new DataSet(new Dictionary + { ["Temperature"] = new DataValue(25, StatusCodes.Good, DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture), DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture)), @@ -473,7 +474,7 @@ private static JsonNetworkMessage CreateMessage(uint messageMask, uint datasetMa ["Humidiy"] = new DataValue(42, StatusCodes.Uncertain, DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture), DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture)) - } + }, PubSubMessageContentFlagHelper.StackToDataSetFieldContentFlags(fieldMask)) } } }; diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests2.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests2.cs index bc41104b98..63f12f12ab 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests2.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/JsonNetworkMessageEncoderTests2.cs @@ -733,7 +733,8 @@ private static JsonNetworkMessage CreateMessage(uint messageMask, uint datasetMa Status = 1073741824, MessageType = MessageType.KeyFrame, DataSetWriterName = "Writer100", - Payload = new DataSet((DataSetFieldContentFlags)fieldMask) { + Payload = new DataSet(new Dictionary + { ["Temperature"] = new DataValue(25, StatusCodes.Good, DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture), DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture)), @@ -743,7 +744,7 @@ private static JsonNetworkMessage CreateMessage(uint messageMask, uint datasetMa ["Humidiy"] = new DataValue(42, StatusCodes.Uncertain, DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture), DateTime.Parse("2021-09-27T18:45:19.555Z", CultureInfo.InvariantCulture)) - } + }, (DataSetFieldContentFlags)fieldMask) }, new JsonDataSetMessage { DataSetMessageContentMask = PubSubMessageContentFlagHelper.StackToDataSetMessageContentFlags(datasetMask), @@ -758,7 +759,8 @@ private static JsonNetworkMessage CreateMessage(uint messageMask, uint datasetMa Status = 1073741824, MessageType = MessageType.DeltaFrame, DataSetWriterName = "Writer100", - Payload = new DataSet(PubSubMessageContentFlagHelper.StackToDataSetFieldContentFlags(fieldMask)) { + Payload = new DataSet(new Dictionary + { ["Temperature"] = new DataValue(26, StatusCodes.Good, DateTime.Parse("2021-09-27T18:45:19.556Z", CultureInfo.InvariantCulture), DateTime.Parse("2021-09-27T18:45:19.556Z", CultureInfo.InvariantCulture)), @@ -768,7 +770,7 @@ private static JsonNetworkMessage CreateMessage(uint messageMask, uint datasetMa ["Humidiy"] = new DataValue(43, StatusCodes.Uncertain, DateTime.Parse("2021-09-27T18:45:19.556Z", CultureInfo.InvariantCulture), DateTime.Parse("2021-09-27T18:45:19.556Z", CultureInfo.InvariantCulture)) - } + }, PubSubMessageContentFlagHelper.StackToDataSetFieldContentFlags(fieldMask)) } } }; diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/MonitoredItemMessageEncoderDecoderTests.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/MonitoredItemMessageEncoderDecoderTests.cs index 4bd02ad65f..613d57d5ce 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/MonitoredItemMessageEncoderDecoderTests.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/MonitoredItemMessageEncoderDecoderTests.cs @@ -223,9 +223,9 @@ private static void ConvertToOpcUaUniversalTime(BaseNetworkMessage networkMessag foreach (var dataSetMessage in networkMessage.Messages) { var expectedPayload = new Dictionary(); - foreach (var entry in dataSetMessage.Payload) + foreach (var entry in dataSetMessage.Payload.DataSetFields) { - expectedPayload[entry.Key] = new DataValue(entry.Value).ToOpcUaUniversalTime(); + expectedPayload[entry.Name] = new DataValue(entry.Value).ToOpcUaUniversalTime(); } dataSetMessage.Payload = new DataSet(expectedPayload, DataSetFieldContentFlags.ApplicationUri | // Important diff --git a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/UadpNetworkMessageEncoderDecoderTests.cs b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/UadpNetworkMessageEncoderDecoderTests.cs index 611734d131..fb89fc5ac4 100644 --- a/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/UadpNetworkMessageEncoderDecoderTests.cs +++ b/src/Azure.IIoT.OpcUa/tests/Encoders/PubSub/UadpNetworkMessageEncoderDecoderTests.cs @@ -322,12 +322,12 @@ public void EncodeDecodeNetworkMessagesNoHeaderRaw(MessageType type, int numberO .ToList(); var result = serializer.Parse(serializer.SerializeToString(decodedMessages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value.Value)) .ToList())); var expected = serializer.Parse(serializer.SerializeToString(messages - .SelectMany(m => m.Payload) - .Select(v => (v.Key, v.Value.Value)) + .SelectMany(m => m.Payload.DataSetFields) + .Select(v => (v.Name, v.Value.Value)) .ToList())); Assert.Equal(expected, result); @@ -343,9 +343,9 @@ private static void ConvertToOpcUaUniversalTime(BaseNetworkMessage networkMessag foreach (var dataSetMessage in networkMessage.Messages) { var expectedPayload = new Dictionary(); - foreach (var entry in dataSetMessage.Payload) + foreach (var entry in dataSetMessage.Payload.DataSetFields) { - expectedPayload[entry.Key] = new DataValue(entry.Value).ToOpcUaUniversalTime(); + expectedPayload[entry.Name] = new DataValue(entry.Value).ToOpcUaUniversalTime(); } dataSetMessage.Payload = new DataSet(expectedPayload, DataSetFieldContentFlags.StatusCode | diff --git a/tools/e2etesting/DeployAKS.ps1 b/tools/e2etesting/DeployAKS.ps1 index db2192d778..9c03c9d4f2 100644 --- a/tools/e2etesting/DeployAKS.ps1 +++ b/tools/e2etesting/DeployAKS.ps1 @@ -5,11 +5,11 @@ Param( $TenantId, [String] $Region = "northeurope", - [string] + [string] $PublisherDeploymentFile = "./K8s-Standalone/publisher/deployment.yaml", - [string] + [string] $ContainerRegistryServer = "mcr.microsoft.com", - [string] + [string] $ContainerRegistryUsername, [string] $ContainerRegistryPassword, diff --git a/tools/e2etesting/steps/cleanup.yml b/tools/e2etesting/steps/cleanup.yml index 6a36b6fc79..3c9abc05c5 100644 --- a/tools/e2etesting/steps/cleanup.yml +++ b/tools/e2etesting/steps/cleanup.yml @@ -14,18 +14,3 @@ steps: $resourceGroup = Get-AzResourceGroup -Name "$(ResourceGroupName)" -ErrorAction SilentlyContinue if ($resourceGroup) { $resourceGroup | Remove-AzResourceGroup -Force } else { Write-Host "Resource group '$(ResourceGroupName)' not found." } - -- task: AzurePowerShell@5 - displayName: "Delete AD App Registrations" - condition: eq( '${{ parameters.CleanupAppRegistrations }}', true) - inputs: - azureSubscription: '$(AzureSubscription)' - azurePowerShellVersion: 'latestVersion' - scriptType: 'InlineScript' - inline: | - Write-Host "Deleting AD App Registration: '$(ApplicationName)-aks' ..." - Remove-AzADApplication -DisplayName "$(ApplicationName)-aks" -ErrorAction SilentlyContinue - Write-Host "Deleting AD App Registration: '$(ApplicationName)-client' ..." - Remove-AzADApplication -DisplayName "$(ApplicationName)-client" -ErrorAction SilentlyContinue - Write-Host "Deleting AD App Registration: '$(ApplicationName)-service' ..." - Remove-AzADApplication -DisplayName "$(ApplicationName)-service" -ErrorAction SilentlyContinue \ No newline at end of file diff --git a/tools/e2etesting/steps/deployplatform.yml b/tools/e2etesting/steps/deployplatform.yml index 4927b64515..134956c0a3 100644 --- a/tools/e2etesting/steps/deployplatform.yml +++ b/tools/e2etesting/steps/deployplatform.yml @@ -53,6 +53,7 @@ steps: | ConvertTo-SecureString -AsPlainText -Force Write-Host "Deploying to '$(ResourceGroupName)'..." . $(BasePath)/deploy/scripts/deploy.ps1 ` + -noAadAppRegistration ` -type services ` -BranchName "$(BranchName)" ` -ImageNamespace "$(ImageNamespace)" ` @@ -95,20 +96,3 @@ steps: azureSubscription: '$(AzureSubscription)' KeyVaultName: '$(KeyVaultName)' SecretsFilter: 'PCS-AUTH-CLIENT-APPID,PCS-AUTH-TENANT' -- task: AzureKeyVault@1 - displayName: 'Retrieve Admin credentials' - inputs: - azureSubscription: '$(AzureSubscription)' - KeyVaultName: 'e2etestingsecrets' - SecretsFilter: 'User,Password' -- task: AzureCLI@2 - displayName: 'Grant admin access to App' - inputs: - azureSubscription: '$(AzureSubscription)' - scriptLocation: 'InlineScript' - scriptType: 'bash' - addSpnToEnvironment: true - inlineScript: | - az login -t $(pcs-auth-tenant) --allow-no-subscriptions -u $(User) -p "$(Password)" - az ad app permission admin-consent --id $(pcs-auth-client-appid) -