Fix diagnostics information presented and update dependencies (#2157)

* Fix endpoint count diagnostics
* Fix that sessions are not closed correctly.
* Support syslog and systemd log formatters

common.props

@@ -42,8 +42,8 @@
     <!--<TreatWarningsAsErrors>true</TreatWarningsAsErrors>-->
   </PropertyGroup>
   <ItemGroup Condition="$(NO_RCS) == ''">
-    <PackageReference Include="Roslynator.Analyzers" Version="4.8.0" PrivateAssets="All"/>
-    <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.8.0" PrivateAssets="All"/>
+    <PackageReference Include="Roslynator.Analyzers" Version="4.9.0" PrivateAssets="All"/>
+    <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.9.0" PrivateAssets="All"/>
   </ItemGroup>
   <!-- only create the SARIF files for the SDL build step in cloud builds -->
   <PropertyGroup Condition="'$(NBGV_NugetPackageVersion)' != ''">

deploy/docker/docker-compose.yaml

@@ -5,20 +5,20 @@ services:
   ############################
   opcplc:
     container_name: opcplc
-    image: mcr.microsoft.com/iotedge/opc-plc:${OPC_PLC_TAG:-2.9.10}
+    image: mcr.microsoft.com/iotedge/opc-plc:${OPC_PLC_TAG:-latest}
     ports:
       - "50000:50000"
     command: [
-      "--sph",
+      "--sph=True",
       "--spf=/shared/pn.json",
       "--pn=50000",
-      "--alm",
-      "--ses",
+      "--alm=True",
+      "--ses=True",
       "--ei=${EVENT_NODES:-100}",
       "--gn=${GUID_NODES:-100}",
       "--fn=${FAST_NODES:-99900}",
       "--sn=${SLOW_NODES:-99900}",
-      "--aa"
+      "--aa=True"
       ]
     volumes:
       - shared:/shared:rw
@@ -41,6 +41,7 @@ services:
       "--cl=5",
       "--rs",
       "--dm=True",
+      "--lfm=syslog",
       "--pki=/shared/pki",
       "--pf=/shared/pn.json",
       "--npd=${NODES_PER_DATASET:-10000}"

deploy/docker/with-monitor.yaml

@@ -13,7 +13,7 @@ services:
     environment:
       DOTNET_DiagnosticPorts: /shared/diag/dotnet-monitor.sock
   ############################
-  # Optional dotnet-monitor
+  # dotnet-monitor
   ############################
   monitor:
     container_name: monitor

deploy/docker/with-pcap-capture.yaml

@@ -0,0 +1,34 @@
+version: "3.9"
+services:
+  ############################
+  # OPC PLC Simulation
+  ############################
+  opcplc:
+    command: [
+      "--sph=True",
+      "--spf=/shared/pn.json",
+      "--pn=50000",
+      "--fn=8000",
+      "--aa=True",
+      "--ut=True"
+      ]
+  ############################
+  # Network capture
+  ############################
+  pcap:
+    container_name: pcap
+    image: travelping/pcap
+    cap_add:
+      - NET_ADMIN
+    network_mode: host
+    volumes:
+      - shared:/data:rw
+    environment:
+      IFACE: any
+      FORMAT: pcapng
+      MAXFILENUM: 10
+      MAXFILESIZE: 200
+      FILENAME: dump.pcap
+      FILTER: "src or dst host 192.168.80.2"
+volumes:
+  shared:

docs/opc-publisher/commandline.md

@@ -18,8 +18,8 @@ When both environment variable and CLI argument are provided, the command line o
 ██║   ██║██╔═══╝ ██║         ██╔═══╝ ██║   ██║██╔══██╗██║     ██║╚════██║██╔══██║██╔══╝  ██╔══██╗
 ╚██████╔╝██║     ╚██████╗    ██║     ╚██████╔╝██████╔╝███████╗██║███████║██║  ██║███████╗██║  ██║
  ╚═════╝ ╚═╝      ╚═════╝    ╚═╝      ╚═════╝ ╚═════╝ ╚══════╝╚═╝╚══════╝╚═╝  ╚═╝╚══════╝╚═╝  ╚═╝
+                                                 2.9.4 (.NET 8.0.1/win-x64/OPC Stack 1.4.372.116)
 
-                                                 2.9.4 (.NET 8.0.0/win-x64/OPC Stack 1.4.372.106)
 General
 -------
 
@@ -806,6 +806,14 @@ Diagnostic options
                                    `Critical`
                                    `None`
                                Default: `Information`.
+      --lfm, --logformat, --LogFormat=VALUE
+                             The logging format to use when writing to the 
+                               console.
+                               Allowed values:
+                                   `simple`
+                                   `syslog`
+                                   `systemd`
+                               Default: `simple`.
       --di, --diagnosticsinterval, --DiagnosticsInterval=VALUE
                              Produce publisher diagnostic information at this
                                specified interval in seconds.
@@ -856,15 +864,6 @@ Diagnostic options
                                metrics directly on the standard path.
                                Default: `disabled` if Otlp collector is
                                configured, otherwise `enabled`.
-      --cap, --capturedevice, --CaptureDevice=VALUE
-                             The capture device to use to capture network
-                               traffic.
-                               Network capture is not supported on this system.
-      --cpf, --capturefile, --CaptureFileName=VALUE
-                             The file name to capture traffic to.
-                               A device must be selected using `--cd` if
-                               capture capability is supported on this system.
-                               Default: `opcua.pcap`.
 ```
 
 Currently supported combinations of `--mm` snd `--me` can be found [here](./messageformats.md).

docs/release-announcement.md

@@ -65,6 +65,7 @@ We are pleased to announce the release of version 2.9.4 of OPC Publisher and the
 - Recreate session if it expires on server (#2138)
 - Log subscription keep alive error only when session is connected (#2137)
 - Update OPC UA .net stack to latest version (1.4.372.116-preview) to enable fully async reconnect and fix several issues in previous versions.
+- Added the ability to switch publisher to emit logs in syslog or systemd format using --lfm command line option.
 - Fix issue where certain publish errors cause reconnect state machine to fail (#2104, #2136)
 
 ## Azure Industrial IoT OPC Publisher 2.9.3

src/Azure.IIoT.OpcUa.Publisher.Models/tests/Azure.IIoT.OpcUa.Publisher.Models.Tests.csproj

@@ -10,7 +10,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="xunit" Version="2.6.5" />
+    <PackageReference Include="xunit" Version="2.6.6" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

src/Azure.IIoT.OpcUa.Publisher.Module/cli/Program.cs

@@ -381,7 +381,7 @@ private static async Task<string> LoadPnJson(string publishProfile,
             }
             if (publishedNodesFile != null && File.Exists(publishedNodesFile))
             {
-                var publishedNodesFilePath = Path.GetTempFileName();
+                const string publishedNodesFilePath = "profile.json";
 
                 await File.WriteAllTextAsync(publishedNodesFilePath,
                     (await File.ReadAllTextAsync(publishedNodesFile, ct).ConfigureAwait(false))

src/Azure.IIoT.OpcUa.Publisher.Module/src/Runtime/CommandLine.cs

@@ -448,9 +448,12 @@ public CommandLine(string[] args, CommandLineLogger? logger = null)
                 "------------------",
                 "",
 
-                { $"ll|loglevel=|{Configuration.Logging.LogLevelKey}=",
+                { $"ll|loglevel=|{Configuration.LoggingLevel.LogLevelKey}=",
                     $"The loglevel to use.\nAllowed values:\n    `{string.Join("`\n    `", Enum.GetNames(typeof(LogLevel)))}`\nDefault: `{LogLevel.Information}`.\n",
-                    (LogLevel l) => this[Configuration.Logging.LogLevelKey] = l.ToString() },
+                    (LogLevel l) => this[Configuration.LoggingLevel.LogLevelKey] = l.ToString() },
+               { $"lfm|logformat=|{Configuration.LoggingFormat.LogFormatKey}=",
+                    $"The log format to use when writing to the console.\nAllowed values:\n    `{string.Join("`\n    `", Configuration.LoggingFormat.LogFormatsSupported)}`\nDefault: `{Configuration.LoggingFormat.LogFormatDefault}`.\n",
+                    (string s) => this[Configuration.LoggingFormat.LogFormatKey] = s },
                 { $"di|diagnosticsinterval=|{PublisherConfig.DiagnosticsIntervalKey}=",
                     "Produce publisher diagnostic information at this specified interval in seconds.\nBy default diagnostics are written to the OPC Publisher logger (which requires at least --loglevel `information`) unless configured differently using `--pd`.\n`0` disables diagnostic output.\nDefault:60000 (60 seconds).\nAlso can be set using `DiagnosticsInterval` environment variable in the form of a duration string in the form `[d.]hh:mm:ss[.fffffff]`\".\n",
                     (int i) => this[PublisherConfig.DiagnosticsIntervalKey] = TimeSpan.FromSeconds(i).ToString() },
@@ -475,12 +478,6 @@ public CommandLine(string[] args, CommandLineLogger? logger = null)
                 { $"em|enableprometheusendpoint=|{Configuration.Otlp.EnableMetricsKey}=",
                     "Explicitly enable or disable exporting prometheus metrics directly on the standard path.\nDefault: `disabled` if Otlp collector is configured, otherwise `enabled`.\n",
                     (bool? b) => this[Configuration.Otlp.EnableMetricsKey] = b?.ToString() ?? "True" },
-                { $"cap|capturedevice=|{OpcUaClientConfig.CaptureDeviceKey}=",
-                    $"The capture device to use to capture network traffic.\n{SupportsCapture(OpcUaClientCapture.AvailableDevices)}\n",
-                    (string s) => this[OpcUaClientConfig.CaptureDeviceKey] = s },
-                { $"cpf|capturefile=|{OpcUaClientConfig.CaptureFileNameKey}=",
-                    $"The file name to capture traffic to.\nA device must be selected using `--cd` if capture capability is supported on this system.\nDefault: `{OpcUaClientConfig.CaptureFileNameDefault}`.\n",
-                    (string s) => this[OpcUaClientConfig.CaptureFileNameKey] = s },
 
                 // testing purposes
 
@@ -618,15 +615,6 @@ void SetStoreType(string s, string storeTypeKey, string optionName)
             }
         }
 
-        private static string SupportsCapture(IReadOnlyList<string> devices)
-        {
-            if (devices.Count == 0)
-            {
-                return "Network capture is not supported on this system.";
-            }
-            return $"Available devices on your system:\n    `{string.Join("`\n    `", devices)}`\nDefault: `null` (disabled).";
-        }
-
         private readonly CommandLineLogger _logger;
     }
 

src/Azure.IIoT.OpcUa.Publisher.Module/src/Runtime/Configuration.cs

@@ -33,6 +33,8 @@ namespace Azure.IIoT.OpcUa.Publisher.Module.Runtime
     using System.Linq;
     using System.Net;
     using System.Text.RegularExpressions;
+    using Microsoft.Extensions.Logging.Console;
+    using static Azure.IIoT.OpcUa.Publisher.Module.Runtime.Configuration;
 
     /// <summary>
     /// Configuration extensions
@@ -54,8 +56,16 @@ public static void AddPublisherServices(this ContainerBuilder builder)
                 .AsImplementedInterfaces().SingleInstance();
             builder.RegisterType<CommandLine>()
                 .AsImplementedInterfaces().AsSelf().SingleInstance();
-            builder.RegisterType<Logging>()
+            builder.RegisterType<LoggingLevel>()
                 .AsImplementedInterfaces();
+            builder.RegisterType<ConsoleLogging<ConsoleFormatterOptions>>()
+                .AsImplementedInterfaces();
+            builder.RegisterType<ConsoleLogging<SimpleConsoleFormatterOptions>>()
+                .AsImplementedInterfaces();
+            builder.RegisterType<ConsoleLogging<JsonConsoleFormatterOptions>>()
+                .AsImplementedInterfaces();
+            builder.RegisterType<Syslog>()
+                .AsImplementedInterfaces().AsSelf().SingleInstance();
             builder.RegisterType<Kestrel>()
                 .AsImplementedInterfaces();
 
@@ -445,9 +455,9 @@ public override void Configure(string? name, KestrelServerOptions options)
         }
 
         /// <summary>
-        /// Configure logger factory
+        /// Configure logger filter
         /// </summary>
-        internal sealed class Logging : ConfigureOptionBase<LoggerFilterOptions>
+        internal sealed class LoggingLevel : ConfigureOptionBase<LoggerFilterOptions>
         {
             /// <summary>
             /// Configuration
@@ -457,17 +467,118 @@ internal sealed class Logging : ConfigureOptionBase<LoggerFilterOptions>
             /// <inheritdoc/>
             public override void Configure(string? name, LoggerFilterOptions options)
             {
-                if (Enum.TryParse<LogLevel>(GetStringOrDefault(LogLevelKey), out var logLevel))
+                var levelString = GetStringOrDefault(LogLevelKey);
+                if (!string.IsNullOrEmpty(levelString))
+                {
+                    if (Enum.TryParse<LogLevel>(levelString, out var logLevel))
+                    {
+                        options.MinLevel = logLevel;
+                    }
+                    else
+                    {
+                        // Compatibilty with serilog
+                        switch (levelString)
+                        {
+                            case "Verbose":
+                                options.MinLevel = LogLevel.Trace;
+                                break;
+                            case "Fatal":
+                                options.MinLevel = LogLevel.Critical;
+                                break;
+                        }
+                    }
+                }
+            }
+
+            /// <summary>
+            /// Create logging configurator
+            /// </summary>
+            /// <param name="configuration"></param>
+            public LoggingLevel(IConfiguration configuration) : base(configuration)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Logging format
+        /// </summary>
+        internal class LoggingFormat : PostConfigureOptionBase<ConsoleLoggerOptions>
+        {
+            /// <summary>
+            /// Supported formats
+            /// </summary>
+            public static readonly string[] LogFormatsSupported = new[]
+            {
+                ConsoleFormatterNames.Simple,
+                Syslog.FormatterName,
+                ConsoleFormatterNames.Systemd
+            };
+
+            /// <summary>
+            /// Configuration
+            /// </summary>
+            public const string LogFormatKey = "LogFormat";
+
+            /// <summary>
+            /// Default format
+            /// </summary>
+            public const string LogFormatDefault = ConsoleFormatterNames.Simple;
+
+            /// <inheritdoc/>
+            public override void PostConfigure(string? name, ConsoleLoggerOptions options)
+            {
+                switch (GetStringOrDefault(LogFormatKey))
                 {
-                    options.MinLevel = logLevel;
+                    case Syslog.FormatterName:
+                        options.FormatterName = Syslog.FormatterName;
+                        break;
+                    case ConsoleFormatterNames.Systemd:
+                        options.FormatterName = ConsoleFormatterNames.Systemd;
+                        break;
+                    case ConsoleFormatterNames.Simple:
+                        options.FormatterName = ConsoleFormatterNames.Simple;
+                        break;
+                    default:
+                        options.FormatterName = LogFormatDefault;
+                        break;
                 }
             }
 
             /// <summary>
             /// Create logging configurator
             /// </summary>
             /// <param name="configuration"></param>
-            public Logging(IConfiguration configuration) : base(configuration)
+            public LoggingFormat(IConfiguration configuration) : base(configuration)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Logging format
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        internal sealed class ConsoleLogging<T> : LoggingFormat,
+            IConfigureOptions<T>, IConfigureNamedOptions<T> where T : ConsoleFormatterOptions
+        {
+            /// <inheritdoc/>
+            public void Configure(string? name, T options)
+            {
+                options.TimestampFormat = "[yy-MM-dd HH:mm:ss.ffff] ";
+                options.IncludeScopes = true;
+                options.UseUtcTimestamp = true;
+            }
+
+            /// <inheritdoc/>
+            public void Configure(T options)
+            {
+                Configure(null, options);
+            }
+
+            /// <summary>
+            /// Create logging configurator
+            /// </summary>
+            /// <param name="configuration"></param>
+            public ConsoleLogging(IConfiguration configuration) : base(configuration)
             {
             }
         }