From 48af5279b9fa099ce384f8417dd26396edfc7649 Mon Sep 17 00:00:00 2001 From: ilyaShein Date: Thu, 16 Jul 2020 14:27:01 +0300 Subject: [PATCH 1/3] segmentation --- .travis.yml | 2 +- .../6_segmentation/include/segmentation.hpp | 5 +++ modules/6_segmentation/src/segmentation.cpp | 38 ++++++++++++++++- modules/6_segmentation/src/unet_histology.cpp | 42 ++++++++++++++++--- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7851dea..4fd45d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ before_install: # origin weights are from https://github.com/NifTK/NiftyNetModelZoo/tree/5-reorganising-with-lfs/unet_histology - megadl 'https://mega.nz/#!RAFU1I4T!xjuol1lZ9t_hA67AxgAxp8w1JIb5FXm53617UXR0NO8' --path data --no-progress # frozen_unet_histology.pb - # TODO: Implement frozen_unet_histology.pb model conversion with OpenVINO + - /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model data/frozen_unet_histology.pb --input_shape [1,952,1144,1,1,3] --output_dir data # (note that TensorFlow accepts input tensor of shape 1,952,1144,1,1,3) - megadl 'https://mega.nz/#!AR1HwIIA!U3p46tM9SFMhKzbSwJIEiugdJDQ5mUSOC3twZL6UYTM' --path data --no-progress # celeba_generator.xml - megadl 'https://mega.nz/#!hAslCCaL!2vWXjiPq3EbuLZbOvEDOI0y9VorGn5enUm7UMNWGrWo' --path data --no-progress # celeba_generator.bin diff --git a/modules/6_segmentation/include/segmentation.hpp b/modules/6_segmentation/include/segmentation.hpp index 7c2dcb1..aef78e9 100644 --- a/modules/6_segmentation/include/segmentation.hpp +++ b/modules/6_segmentation/include/segmentation.hpp @@ -23,6 +23,10 @@ class ADAS : public Segmenter { ADAS(); virtual void segment(const cv::Mat& image, cv::Mat& mask); + +private: + InferenceEngine::InferRequest req; + std::string outputName; }; // Glands segmentation in colon histology images @@ -53,4 +57,5 @@ class UNetHistology : public Segmenter { private: InferenceEngine::InferRequest req; + std::string outputName; }; diff --git a/modules/6_segmentation/src/segmentation.cpp b/modules/6_segmentation/src/segmentation.cpp index 74b7ed1..01f0359 100644 --- a/modules/6_segmentation/src/segmentation.cpp +++ b/modules/6_segmentation/src/segmentation.cpp @@ -7,7 +7,7 @@ using namespace cv; using namespace cv::utils::fs; float Dice(const Mat& a, const Mat& b) { - CV_Error(Error::StsNotImplemented, "Dice score computation"); + return (float)countNonZero(a&b) * 2.0f / (float)(countNonZero(a) + countNonZero(b)); } ADAS::ADAS() { @@ -16,8 +16,42 @@ ADAS::ADAS() { // Load deep learning network into memory auto net = ie.ReadNetwork(join(DATA_FOLDER, "semantic-segmentation-adas-0001.xml"), join(DATA_FOLDER, "semantic-segmentation-adas-0001.bin")); + + InputInfo::Ptr inputInfo = net.getInputsInfo()["data"]; + inputInfo->getPreProcess().setResizeAlgorithm(ResizeAlgorithm::RESIZE_BILINEAR); + inputInfo->setLayout(Layout::NHWC); + inputInfo->setPrecision(Precision::U8); + outputName = net.getOutputsInfo().begin()->first; + + // Initialize runnable object on CPU device + ExecutableNetwork execNet = ie.LoadNetwork(net, "CPU"); + + // Create a single processing thread + req = execNet.CreateInferRequest(); } +Blob::Ptr wrapMatToBlob2(const Mat& m) { + CV_Assert(m.depth() == CV_8U); + std::vector dims = { 1, (size_t)m.channels(), (size_t)m.rows, (size_t)m.cols }; + return make_shared_blob(TensorDesc(Precision::U8, dims, Layout::NHWC), + m.data); +} + + void ADAS::segment(const Mat& image, Mat& mask) { - CV_Error(Error::StsNotImplemented, "ADAS semantic segmentation"); + Blob::Ptr input = wrapMatToBlob2(image); + + req.SetBlob("data", input); + + req.Infer(); + int* output = req.GetBlob(outputName)->buffer(); + int Height = 1024; + int Width = 2048; + Mat res = Mat::zeros(Height, Width, CV_8UC1); + for (int i = 0; i < Height; i++) { + for (int j = 0; j < Width; j++) { + res.at(i, j) = output[i * Width + j]; + } + } + resize(res, mask, image.size()); } diff --git a/modules/6_segmentation/src/unet_histology.cpp b/modules/6_segmentation/src/unet_histology.cpp index d497904..429ee4f 100644 --- a/modules/6_segmentation/src/unet_histology.cpp +++ b/modules/6_segmentation/src/unet_histology.cpp @@ -25,15 +25,21 @@ UNetHistology::UNetHistology() { // Create a single processing thread req = execNet.CreateInferRequest(); + outputName = net.getOutputsInfo().begin()->first; } void UNetHistology::bgr2rgb(const Mat& src, Mat& dst) { - CV_Error(Error::StsNotImplemented, "bgr2rgb"); + cvtColor(src, dst, COLOR_BGR2RGB); } void UNetHistology::normalize(const Mat& src, Mat& dst) { - CV_Error(Error::StsNotImplemented, "normalize"); + Scalar mean, std; + meanStdDev(src, mean, std); + dst = Mat::zeros(src.size(), CV_32FC3); + dst += src; + dst -= mean; + dst /= std; } void UNetHistology::segment(const Mat& image, Mat& mask) { @@ -64,12 +70,38 @@ void UNetHistology::segment(const Mat& image, Mat& mask) { Blob::Ptr inputBlob = wrapMatToBlob(inp); // TODO: Put inputBlob to the network, perform inference and return mask - - CV_Error(Error::StsNotImplemented, "UNetHistology semantic segmentation"); + req.SetBlob("worker_0/validation/IteratorGetNext", inputBlob); + + // Launch network + req.Infer(); + + // Copy output. "prob" is a name of output from .xml file + int* output = req.GetBlob(outputName)->buffer(); + int Height = 772; + int Width = 964; + Mat res = Mat::zeros(Height, Width, CV_8UC1); + for (int i = 0; i < Height; i++) { + for (int j = 0; j < Width; j++) { + res.at(i, j) = output[i * Width + j]; + } + } + resize(res, mask, image.size()); + } int UNetHistology::countGlands(const cv::Mat& segm) { - CV_Error(Error::StsNotImplemented, "countGlands"); + Mat thresh; + threshold(segm, thresh, 0, 255, THRESH_BINARY_INV + THRESH_OTSU); + morphologyEx(thresh, thresh, MORPH_CLOSE, Mat::ones(3, 3, CV_8U), Point(-1, -1), 3); + dilate(thresh, thresh, 3); + distanceTransform(thresh, thresh, DIST_L2, 5); + double maxValue; + minMaxLoc(thresh, 0 , &maxValue); + threshold(thresh, thresh, 0.45*maxValue, 255, 0); + thresh.convertTo(thresh, CV_8U); + std::vector > contours; + findContours(thresh, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + return contours.size(); } void UNetHistology::padMinimum(const Mat& src, int width, int height, Mat& dst) { From 2274136db8a217a5f20e76b6560f0a4f609db97c Mon Sep 17 00:00:00 2001 From: ilyaShein Date: Thu, 16 Jul 2020 14:36:10 +0300 Subject: [PATCH 2/3] fix --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4fd45d0..e4062d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,10 +42,12 @@ before_install: # origin weights are from https://github.com/NifTK/NiftyNetModelZoo/tree/5-reorganising-with-lfs/unet_histology - megadl 'https://mega.nz/#!RAFU1I4T!xjuol1lZ9t_hA67AxgAxp8w1JIb5FXm53617UXR0NO8' --path data --no-progress # frozen_unet_histology.pb - - /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model data/frozen_unet_histology.pb --input_shape [1,952,1144,1,1,3] --output_dir data # (note that TensorFlow accepts input tensor of shape 1,952,1144,1,1,3) - megadl 'https://mega.nz/#!AR1HwIIA!U3p46tM9SFMhKzbSwJIEiugdJDQ5mUSOC3twZL6UYTM' --path data --no-progress # celeba_generator.xml - megadl 'https://mega.nz/#!hAslCCaL!2vWXjiPq3EbuLZbOvEDOI0y9VorGn5enUm7UMNWGrWo' --path data --no-progress # celeba_generator.bin + - /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model data/frozen_unet_histology.pb --input_shape [1,952,1144,1,1,3] --output_dir data + - megadl 'https://mega.nz/#!AR1HwIIA!U3p46tM9SFMhKzbSwJIEiugdJDQ5mUSOC3twZL6UYTM' --path data --no-progress # celeba_generator.xml + - megadl 'https://mega.nz/#!hAslCCaL!2vWXjiPq3EbuLZbOvEDOI0y9VorGn5enUm7UMNWGrWo' --path data --no-progress # celeba_generator.bin script: - source /opt/intel/openvino/bin/setupvars.sh -pyver 3.5 From 9455e7c50e5f848af88b387005b4cf0dbcbddf0b Mon Sep 17 00:00:00 2001 From: ilyaShein Date: Thu, 16 Jul 2020 14:45:49 +0300 Subject: [PATCH 3/3] fix v2 --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4062d3..418dfda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,11 +43,7 @@ before_install: # origin weights are from https://github.com/NifTK/NiftyNetModelZoo/tree/5-reorganising-with-lfs/unet_histology - megadl 'https://mega.nz/#!RAFU1I4T!xjuol1lZ9t_hA67AxgAxp8w1JIb5FXm53617UXR0NO8' --path data --no-progress # frozen_unet_histology.pb # (note that TensorFlow accepts input tensor of shape 1,952,1144,1,1,3) - - megadl 'https://mega.nz/#!AR1HwIIA!U3p46tM9SFMhKzbSwJIEiugdJDQ5mUSOC3twZL6UYTM' --path data --no-progress # celeba_generator.xml - - megadl 'https://mega.nz/#!hAslCCaL!2vWXjiPq3EbuLZbOvEDOI0y9VorGn5enUm7UMNWGrWo' --path data --no-progress # celeba_generator.bin - /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model data/frozen_unet_histology.pb --input_shape [1,952,1144,1,1,3] --output_dir data - - megadl 'https://mega.nz/#!AR1HwIIA!U3p46tM9SFMhKzbSwJIEiugdJDQ5mUSOC3twZL6UYTM' --path data --no-progress # celeba_generator.xml - - megadl 'https://mega.nz/#!hAslCCaL!2vWXjiPq3EbuLZbOvEDOI0y9VorGn5enUm7UMNWGrWo' --path data --no-progress # celeba_generator.bin script: - source /opt/intel/openvino/bin/setupvars.sh -pyver 3.5