Fix histogram bug in libpymo

Signed-off-by: Kyunggeun Lee <[email protected]>
quic · Jan 15, 2025 · 48cb4df · 48cb4df
1 parent ec09a37
commit 48cb4df
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/ModelOptimizations/DlQuantization/src/math_functions.cpp b/ModelOptimizations/DlQuantization/src/math_functions.cpp
@@ -391,7 +391,7 @@ void GetHistogram_cpu(const DTYPE* data, int cnt, uint32_t histogram[PDF_SIZE],
     {
         // Map a floating point number to the appropriate bucket.
         int index =
-            is_signed ? round(data[i] / bucket_size - pdf_offset) : round(std::abs(data[i]) / bucket_size - pdf_offset);
+            is_signed ? floor(data[i] / bucket_size - pdf_offset) : floor(std::abs(data[i]) / bucket_size - pdf_offset);
 
         // Add to histogram, if inside the histogram range.
         if (index >= 0 && index < PDF_SIZE)

diff --git a/ModelOptimizations/DlQuantization/src/math_functions.cu b/ModelOptimizations/DlQuantization/src/math_functions.cu
@@ -174,8 +174,8 @@ __global__ static void histogramCountKernel(const DTYPE* data,
     {
         // Map a floating point number to the appropriate bucket.
         int index = is_signed ?
-                    round(data[i] / bucket_size - histogram_offset) :
-                    round(abs(data[i]) / bucket_size - histogram_offset);
+                    floor(data[i] / bucket_size - histogram_offset) :
+                    floor(abs(data[i]) / bucket_size - histogram_offset);
 
         // Add to histogram, if inside the histogram range.
         if (index >= 0 && index < PDF_SIZE)