From 48cb4df5dde286577f2537c707a388137bfc1f0b Mon Sep 17 00:00:00 2001 From: Kyunggeun Lee Date: Wed, 8 Jan 2025 21:08:48 -0800 Subject: [PATCH] Fix histogram bug in libpymo Signed-off-by: Kyunggeun Lee --- ModelOptimizations/DlQuantization/src/math_functions.cpp | 2 +- ModelOptimizations/DlQuantization/src/math_functions.cu | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ModelOptimizations/DlQuantization/src/math_functions.cpp b/ModelOptimizations/DlQuantization/src/math_functions.cpp index 7c9972cd103..3eebe0ef066 100644 --- a/ModelOptimizations/DlQuantization/src/math_functions.cpp +++ b/ModelOptimizations/DlQuantization/src/math_functions.cpp @@ -391,7 +391,7 @@ void GetHistogram_cpu(const DTYPE* data, int cnt, uint32_t histogram[PDF_SIZE], { // Map a floating point number to the appropriate bucket. int index = - is_signed ? round(data[i] / bucket_size - pdf_offset) : round(std::abs(data[i]) / bucket_size - pdf_offset); + is_signed ? floor(data[i] / bucket_size - pdf_offset) : floor(std::abs(data[i]) / bucket_size - pdf_offset); // Add to histogram, if inside the histogram range. if (index >= 0 && index < PDF_SIZE) diff --git a/ModelOptimizations/DlQuantization/src/math_functions.cu b/ModelOptimizations/DlQuantization/src/math_functions.cu index 80340cbcf87..23b0ed90f64 100644 --- a/ModelOptimizations/DlQuantization/src/math_functions.cu +++ b/ModelOptimizations/DlQuantization/src/math_functions.cu @@ -174,8 +174,8 @@ __global__ static void histogramCountKernel(const DTYPE* data, { // Map a floating point number to the appropriate bucket. int index = is_signed ? - round(data[i] / bucket_size - histogram_offset) : - round(abs(data[i]) / bucket_size - histogram_offset); + floor(data[i] / bucket_size - histogram_offset) : + floor(abs(data[i]) / bucket_size - histogram_offset); // Add to histogram, if inside the histogram range. if (index >= 0 && index < PDF_SIZE)