more robust test for idle GPUs

icl-utk-edu · Oct 29, 2024 · 07ab223 · 07ab223
1 parent 47376ed
commit 07ab223
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/test/idle_gpus.py b/test/idle_gpus.py
@@ -48,13 +48,15 @@
             if (s):
                 gpu = s.group( 1 )
 
-            # If using > half the memory, assume it is not idle.
+            # If using >= 10 MiB or 5% utilization, assume it is not idle.
+            # Typically idle is 1 MiB and 0% utilization.
             # Docker can't see processes in section 2.
-            s = re.search( '^\| +N/A +\d+C +\w+ +\d+W +/ +\d+W *\| +(\d+)MiB +/ +(\d+)MiB', line )
+            s = re.search( '^\| +N/A +\d+C +\w+ +\d+W +/ +\d+W *\| +(\d+)MiB +/ +(\d+)MiB *\| +(\d+)%', line )
             if (s):
                 used_mem  = int( s.group( 1 ) )
                 total_mem = int( s.group( 2 ) )
-                if (used_mem > 0.5*total_mem):
+                percent   = int( s.group( 3 ) )
+                if (used_mem >= 10 or percent >= 5):
                     gpus[ gpu ] = 0
         else:
             # Match process lines: