Merge pull request #118 from microsoft/develop: v1.0.12

FabricHealer 1.0.12

Build-SFPKGs.ps1

@@ -23,11 +23,11 @@ function Build-SFPkg {
 try {
     Push-Location $scriptPath
 
-    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Linux.SelfContained.1.0.11" "$scriptPath\bin\release\FabricHealer\linux-x64\self-contained\FabricHealerType"
-    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Linux.FrameworkDependent.1.0.11" "$scriptPath\bin\release\FabricHealer\linux-x64\framework-dependent\FabricHealerType"
+    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Linux.SelfContained.1.0.12" "$scriptPath\bin\release\FabricHealer\linux-x64\self-contained\FabricHealerType"
+    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Linux.FrameworkDependent.1.0.12" "$scriptPath\bin\release\FabricHealer\linux-x64\framework-dependent\FabricHealerType"
 
-    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Windows.SelfContained.1.0.11" "$scriptPath\bin\release\FabricHealer\win-x64\self-contained\FabricHealerType"
-    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Windows.FrameworkDependent.1.0.11" "$scriptPath\bin\release\FabricHealer\win-x64\framework-dependent\FabricHealerType"
+    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Windows.SelfContained.1.0.12" "$scriptPath\bin\release\FabricHealer\win-x64\self-contained\FabricHealerType"
+    Build-SFPkg "Microsoft.ServiceFabricApps.FabricHealer.Windows.FrameworkDependent.1.0.12" "$scriptPath\bin\release\FabricHealer\win-x64\framework-dependent\FabricHealerType"
 }
 finally {
     Pop-Location

FabricHealer.nuspec.template

@@ -2,12 +2,10 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
   <metadata minClientVersion="3.3.0">
     <id>%PACKAGE_ID%</id>
-    <version>1.0.11</version>
+    <version>1.0.12</version>
     <releaseNotes>
-- Code enhancements.
-- Test improvements.
-- Tested with latest version of Guan (1.0.3).
-- Removed unnecessary package reference (System.Text.Json).
+- Bug fix in Disk Repair feature.
+- Added new named argument to DeleteFiles external predicate: SearchPattern, for use in searching specified folders for files with names matching specified pattern.
 	</releaseNotes>
     <authors>Microsoft</authors>
     <license type="expression">MIT</license>

FabricHealer/FabricHealer.csproj

@@ -11,8 +11,8 @@
     <RuntimeIdentifiers>linux-x64;win-x64</RuntimeIdentifiers>
     <RootNamespace>FabricHealer</RootNamespace>
     <AssemblyName>FabricHealer</AssemblyName>
-    <AssemblyVersion>1.0.11</AssemblyVersion>
-    <FileVersion>1.0.11</FileVersion>
+    <AssemblyVersion>1.0.12</AssemblyVersion>
+    <FileVersion>1.0.12</FileVersion>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <IsServiceFabricServiceProject>true</IsServiceFabricServiceProject>
     <StartupObject>FabricHealer.Program</StartupObject>

FabricHealer/FabricHealerManager.cs

@@ -28,7 +28,7 @@ public sealed class FabricHealerManager : IDisposable
         internal static RepairData RepairHistory;
 
         // Folks often use their own version numbers. This is for internal diagnostic telemetry.
-        private const string InternalVersionNumber = "1.0.11";
+        private const string InternalVersionNumber = "1.0.12";
         private static FabricHealerManager singleton;
         private bool disposedValue;
         private readonly StatelessServiceContext serviceContext;

FabricHealer/PackageRoot/Config/LogicRules/DiskRules.config.txt

@@ -5,30 +5,38 @@
 
 ## Mitigate() :- CheckInsideRunInterval(02:00:00), !.
 
-## The sample rule below checks the folder size of SF query trace logs, fabricobserver logs, E:\temp. If the computed size exceeds a supplied threshold, then try and delete the files in the directory. 
-## The CheckFolderSize external predicate takes a folder path string argument and either a MaxFolderSizeGB or MaxFolderSizeMB argument.
-## Either size arg must be supplied as a positive whole number. If the specified folder is larger than the supplied unsigned integer value for size, then the DeleteFiles external predicate will run.
-## NOTE: You can supply optional arguments (named, non-positional) to the DeleteFiles external predicate. 
-## Optional (named, non-positional) arguments for DeleteFiles: 
+## DeleteFiles external predicate takes 1 required positional argument (position 0), which is the full path to the directory to be cleaned, and 4 optional named arguments.
+## Optional (named, position 1 to n in argument list, first arg (0) is reserved for folder path) arguments for DeleteFiles: 
 ##		SortOrder - File sort order, Ascending or Descending. Defaults to Ascending (oldest to newest)).
 ##		MaxFilesToDelete - The maximum number of files to delete. If not specified (or 0) it will be interpreted to mean delete all files.
 ##		RecurseSubdirectories - Delete files in child folders of specified directory. Defaults to false if not specified.
+##		SearchPattern - Delete only files with file names that match the specified search pattern. Default is "*" (all files).
 
-## Required (positional) arguments for CheckFolderSize external predicate:
-##		0 (first argument): The full path to the folder to be cleaned.
+## The CheckFolderSize external predicate takes a folder path string argument and either a MaxFolderSizeGB or MaxFolderSizeMB argument.
+## Required positional arguments for CheckFolderSize external predicate:
+##		First argument (0 position in argument list): The full path to the folder to be cleaned.
 
-## Optional (named, non-positional) arguments for CheckFolderSize external predicate:
-##		MaxFolderSizezGB (default if not supplied): max size of folder in gigabytes.
+## Required named arguments (1 through n position in argument list) for CheckFolderSize external predicate:
+##		MaxFolderSizezGB: max size of folder in gigabytes.
 ##      MaxFolderSizezMB: max size of folder in megabytes.
 
+## If CheckFolderSize is true, then the next sub-goal will run, which in the rules below is the DeleteFiles external predicate.
+
+## Single rule for different directories and size constraints, but where DeleteFiles takes the same optional arguments (and values). 
 ## Iterate over a list of folders with a system predicate, member (defined and implemented in Guan) and an internal predicate, 
-## config (an internal predicate needs no backing impl, it only exists in this logic).
-## You can just write a rule for each folder should you need to have MB max size values for some folders and GB max size values for others. 
-## In the case below, all values are GB so it makes sense to use enumeration for convenience.
-## The only supported disk repair is file deletion and this is related explicitly to DiskSpace* metrics. There are no repairs for queue length issues, for example.
-## Note the use of the match system predicate to ensure the metric name supplied by FabricObserver contains "DiskSpace".
+## config (an internal predicate needs no backing impl, it only exists in this logic). This is useful if all folder targets share enough
+## of the same argument values (less rules to write..).
 Mitigate(MetricName=?MetricName) :- match(?MetricName, "DiskSpace"), GetRepairHistory(?repairCount, 08:00:00), 
 	?repairCount < 4,
-	member(config(?X,?Y), [config("C:\SFDevCluster\Log\QueryTraces", 50), config("C:\observer_logs", 1), config("E:\temp", 40)]), 
-	CheckFolderSize(?X, MaxFolderSizezGB=?Y),
-	DeleteFiles(?X, SortOrder=Ascending, MaxFilesToDelete=10, RecurseSubdirectories=true).
+	member(config(?X,?Y), [config("C:\SFDevCluster\Log\QueryTraces", 50), config("C:\fabric_observer_logs", 1), config("E:\temp", 10)]), 
+	CheckFolderSize(?X, MaxFolderSizeGB=?Y),
+	DeleteFiles(?X, SortOrder=Ascending, MaxFilesToDelete=10, RecurseSubdirectories=true).
+
+## Single rule for one directory with all optional args supplied, including a search pattern for target file name matching. 
+## Please note: when specifying search pattern string, you must enclose it in quotes since the special characters used can also be interpreted as mathematical operators (like / or *) by Guan
+## if they are not quoted strings. This is true for Guan across the board. Keep this in mind when you pass arguments in rules that contain special characters
+## that are also mathematical operators (*,+,/,-,%, etc...).
+Mitigate(MetricName=?MetricName) :- match(?MetricName, "DiskSpace"), GetRepairHistory(?repairCount, 08:00:00),
+	?repairCount < 4,
+	CheckFolderSize("C:\SFDevCluster\Log\Traces", MaxFolderSizeGB=20),
+	DeleteFiles("C:\SFDevCluster\Log\Traces", SortOrder=Ascending, MaxFilesToDelete=10, RecurseSubdirectories=true, SearchPattern="lease_traces*").

FabricHealer/PackageRoot/Config/LogicRules/SystemAppRules.config.txt

@@ -49,7 +49,7 @@ TimeScopedRestartFabricSystemProcess(?count, ?time) :- GetRepairHistory(?repairC
 	RestartFabricSystemProcess(MaxWaitTimeForHealthStateOk=00:00:30).
 
 
-## Mitigation rules for multiple metrics and targets. NOTE: Do not restart Fabric or FabricHost processes unless you want to take the node down. For the latter (restart node),
+## Mitigation rules for multiple metrics and targets. NOTE: Do not restart Fabric or FabricHost processes unless you want to take the Fabric node down. For the latter (restart node),
 ## use TimeScopedRestartFabricNode (or RestartFabricNode predicate directly), which employs a safe Fabric node restart workflow (with deactivation step), not just a process kill.
 
 

FabricHealer/PackageRoot/ServiceManifest.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <ServiceManifest Name="FabricHealerPkg"
-                 Version="1.0.11"
+                 Version="1.0.12"
                  xmlns="http://schemas.microsoft.com/2011/01/fabric"
                  xmlns:xsd="http://www.w3.org/2001/XMLSchema"
                  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
@@ -11,7 +11,7 @@
   </ServiceTypes>
 
   <!-- Code package is your service executable. -->
-  <CodePackage Name="Code" Version="1.0.11">
+  <CodePackage Name="Code" Version="1.0.12">
     <EntryPoint>
       <ExeHost>
         <Program>FabricHealer</Program>
@@ -21,5 +21,5 @@
 
   <!-- Config package is the contents of the Config directory under PackageRoot that contains an 
        independently-updateable and versioned set of custom configuration settings for your service. -->
-  <ConfigPackage Name="Config" Version="1.0.11" />
+  <ConfigPackage Name="Config" Version="1.0.12" />
 </ServiceManifest>

FabricHealer/Repair/DiskRepairPolicy.cs

@@ -26,6 +26,11 @@ public FileSortOrder FileAgeSortOrder
         {
             get; set;
         }
+
+        public string FileSearchPattern
+        {
+            get; set;
+        }
     }
 
     public enum FileSortOrder

FabricHealer/Repair/Guan/CheckFolderSizePredicateType.cs

@@ -9,6 +9,7 @@
 using FabricHealer.Utilities;
 using FabricHealer.Utilities.Telemetry;
 using System.Threading.Tasks;
+using System;
 
 namespace FabricHealer.Repair.Guan
 {
@@ -36,6 +37,7 @@ protected override async Task<bool> CheckAsync()
                 }
 
                 string folderPath = Input.Arguments[0].Value.GetEffectiveTerm().GetStringValue();
+
                 long maxFolderSizeGB = 0;
                 long maxFolderSizeMB = 0;
 
@@ -52,32 +54,31 @@ protected override async Task<bool> CheckAsync()
                             break;
 
                         default:
-                            maxFolderSizeGB = (long)Input.Arguments[i].Value.GetEffectiveTerm().GetObjectValue();
-                            break;
+                            throw new GuanException($"Unrecognized argument supplied: {Input.Arguments[i].Name}");
                     }
                 }
 
                 if (!Directory.Exists(folderPath))
                 {
                     await RepairTaskManager.TelemetryUtilities.EmitTelemetryEtwHealthEventAsync(
-                                                            LogLevel.Info,
-                                                            "CheckFolderSizePredicate::DirectoryNotFound",
-                                                            $"Directory {folderPath} does not exist.",
-                                                            RepairTaskManager.Token).ConfigureAwait(false);
+                                                                LogLevel.Info,
+                                                                "CheckFolderSizePredicate::DirectoryNotFound",
+                                                                $"Directory {folderPath} does not exist.",
+                                                                RepairTaskManager.Token).ConfigureAwait(false);
                     return false;
                 }
 
                 if (Directory.GetFiles(folderPath, "*", new EnumerationOptions { RecurseSubdirectories = true }).Length == 0)
                 {
                     await RepairTaskManager.TelemetryUtilities.EmitTelemetryEtwHealthEventAsync(
-                                                            LogLevel.Info,
-                                                            "CheckFolderSizePredicate::NoFilesFound",
-                                                            $"Directory {folderPath} does not contain any files.",
-                                                            RepairTaskManager.Token).ConfigureAwait(false);
-                    return false;
+                                                                LogLevel.Info,
+                                                                "CheckFolderSizePredicate::NoFilesFound",
+                                                                $"Directory {folderPath} does not contain any files.",
+                                                                RepairTaskManager.Token).ConfigureAwait(false);
+                        return false;
                 }
 
-                long size = 0;
+                double size = 0.0;
 
                 if (maxFolderSizeGB > 0)
                 {
@@ -99,28 +100,28 @@ await RepairTaskManager.TelemetryUtilities.EmitTelemetryEtwHealthEventAsync(
                 }
 
                 string message =
-                $"Repair {FOHealthData.RepairId}: Supplied Maximum folder size value ({(maxFolderSizeGB > 0 ? maxFolderSizeGB + "GB" : maxFolderSizeMB + "MB")}) " +
-                $"for path {folderPath} is less than computed folder size ({size}{(maxFolderSizeGB > 0 ? "GB" : "MB")}). " +
-                "Will not attempt repair.";
+                        $"Repair {FOHealthData.RepairId}: Supplied Maximum folder size value ({(maxFolderSizeGB > 0 ? maxFolderSizeGB + "GB" : maxFolderSizeMB + "MB")}) " +
+                        $"for path {folderPath} is less than computed folder size ({size}{(maxFolderSizeGB > 0 ? "GB" : "MB")}). " +
+                        "Will not attempt repair.";
 
                 await RepairTaskManager.TelemetryUtilities.EmitTelemetryEtwHealthEventAsync(
-                                                        LogLevel.Info,
-                                                        "CheckFolderSizePredicate",
-                                                        message,
-                                                        RepairTaskManager.Token).ConfigureAwait(false);
+                                                            LogLevel.Info,
+                                                            "CheckFolderSizePredicate",
+                                                            message,
+                                                            RepairTaskManager.Token).ConfigureAwait(false);
                 return false;
             }
 
-            private static async Task<long> GetFolderSizeAsync(string path, SizeUnit unit)
+            private static async Task<double> GetFolderSizeAsync(string path, SizeUnit unit)
             {
                 var dir = new DirectoryInfo(path);
-                var folderSizeInBytes = dir.EnumerateFiles("*", SearchOption.AllDirectories).Sum(fi => fi.Length);
+                double folderSizeInBytes = Convert.ToDouble(dir.EnumerateFiles("*", SearchOption.AllDirectories).Sum(fi => fi.Length));
 
                 await RepairTaskManager.TelemetryUtilities.EmitTelemetryEtwHealthEventAsync(
-                                                        LogLevel.Info,
-                                                        "CheckFolderSizePredicate::Size",
-                                                        $"Directory {path} size: {folderSizeInBytes} bytes.",
-                                                        RepairTaskManager.Token).ConfigureAwait(false);
+                                                            LogLevel.Info,
+                                                            "CheckFolderSizePredicate::Size",
+                                                            $"Directory {path} size: {folderSizeInBytes} bytes.",
+                                                            RepairTaskManager.Token).ConfigureAwait(false);
                 if (unit == SizeUnit.GB)
                 {
                     return folderSizeInBytes / 1024 / 1024 / 1024;