From ad797b1864c7bd2256a531af982fe037604fb3af Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 9 Aug 2024 14:13:40 +0000 Subject: [PATCH] Update documentation --- .../openvino_xai/explainer/explanation.html | 61 ++++++++-- .../openvino_xai/methods/black_box/aise.html | 7 +- .../openvino_xai/methods/black_box/rise.html | 2 +- develop/_sources/user-guide.md.txt | 106 +++++++++++++++++- develop/api-manual.html | 51 ++++++++- develop/index.html | 1 + develop/objects.inv | Bin 1947 -> 1962 bytes develop/searchindex.js | 2 +- develop/user-guide.html | 103 ++++++++++++++++- 9 files changed, 298 insertions(+), 35 deletions(-) diff --git a/develop/_modules/openvino_xai/explainer/explanation.html b/develop/_modules/openvino_xai/explainer/explanation.html index 278d1f4e..3e9f5781 100644 --- a/develop/_modules/openvino_xai/explainer/explanation.html +++ b/develop/_modules/openvino_xai/explainer/explanation.html @@ -535,21 +535,58 @@

Source code for openvino_xai.explainer.explanation

[docs] - def save(self, dir_path: Path | str, name: str | None = None) -> None: - """Dumps saliency map.""" + def save( + self, + dir_path: Path | str, + prefix: str = "", + postfix: str = "", + confidence_scores: Dict[int, float] | None = None, + ) -> None: + """ + Dumps saliency map images to the specified directory. + + Allows flexibly name the files with the prefix and postfix. + {prefix} + target_id + {postfix}.jpg + + Also allows to add confidence scores to the file names. + {prefix} + target_id + {postfix} + confidence.jpg + + save(output_dir) -> aeroplane.jpg + save(output_dir, prefix="image_name_target_") -> image_name_target_aeroplane.jpg + save(output_dir, postfix="_class_map") -> aeroplane_class_map.jpg + save( + output_dir, prefix="image_name_", postfix="_conf_", confidence_scores=scores + ) -> image_name_aeroplane_conf_0.85.jpg + + Parameters: + :param dir_path: The directory path where the saliency maps will be saved. + :type dir_path: Path | str + :param prefix: Optional prefix for the saliency map names. Default is an empty string. + :type prefix: str + :param postfix: Optional postfix for the saliency map names. Default is an empty string. + :type postfix: str + :param confidence_scores: Dict with confidence scores for each class index. Default is None. + :type confidence_scores: Dict[int, float] | None + + """ + os.makedirs(dir_path, exist_ok=True) - save_name = name if name else "" - for cls_idx, map_to_save in self._saliency_map.items(): + + template = f"{prefix}{{target_name}}{postfix}{{conf_score}}.jpg" + for target_idx, map_to_save in self._saliency_map.items(): + conf_score = "" map_to_save = cv2.cvtColor(map_to_save, code=cv2.COLOR_RGB2BGR) - if isinstance(cls_idx, str): - cv2.imwrite(os.path.join(dir_path, f"{save_name}.jpg"), img=map_to_save) - return + if isinstance(target_idx, str): + target_name = "activation_map" + elif self.label_names and isinstance(target_idx, np.int64): + target_name = self.label_names[target_idx] else: - if self.label_names: - target_name = self.label_names[cls_idx] - else: - target_name = str(cls_idx) - image_name = f"{save_name}_target_{target_name}.jpg" if save_name else f"target_{target_name}.jpg" + target_name = str(target_idx) + + if confidence_scores and target_idx in confidence_scores: + conf_score = f"{confidence_scores[int(target_idx)]:.2f}" + + image_name = template.format(target_name=target_name, conf_score=conf_score) cv2.imwrite(os.path.join(dir_path, image_name), img=map_to_save)
diff --git a/develop/_modules/openvino_xai/methods/black_box/aise.html b/develop/_modules/openvino_xai/methods/black_box/aise.html index abafbb80..49888653 100644 --- a/develop/_modules/openvino_xai/methods/black_box/aise.html +++ b/develop/_modules/openvino_xai/methods/black_box/aise.html @@ -400,11 +400,10 @@

Source code for openvino_xai.methods.black_box.aise

import collections import math -from typing import Callable, Dict, List, Tuple +from typing import Callable, Dict, List, Mapping, Tuple import numpy as np import openvino.runtime as ov -from openvino.runtime.utils.data_helpers.wrappers import OVDict from scipy.optimize import Bounds, direct from openvino_xai.common.utils import ( @@ -427,7 +426,7 @@

Source code for openvino_xai.methods.black_box.aise

:param model: OpenVINO model. :type model: ov.Model :param postprocess_fn: Post-processing function that extract scores from IR model output. - :type postprocess_fn: Callable[[OVDict], np.ndarray] + :type postprocess_fn: Callable[[Mapping], np.ndarray] :param preprocess_fn: Pre-processing function, identity function by default (assume input images are already preprocessed by user). :type preprocess_fn: Callable[[np.ndarray], np.ndarray] @@ -440,7 +439,7 @@

Source code for openvino_xai.methods.black_box.aise

def __init__( self, model: ov.Model, - postprocess_fn: Callable[[OVDict], np.ndarray], + postprocess_fn: Callable[[Mapping], np.ndarray], preprocess_fn: Callable[[np.ndarray], np.ndarray] = IdentityPreprocessFN(), device_name: str = "CPU", prepare_model: bool = True, diff --git a/develop/_modules/openvino_xai/methods/black_box/rise.html b/develop/_modules/openvino_xai/methods/black_box/rise.html index b0062297..87f256cc 100644 --- a/develop/_modules/openvino_xai/methods/black_box/rise.html +++ b/develop/_modules/openvino_xai/methods/black_box/rise.html @@ -419,7 +419,7 @@

Source code for openvino_xai.methods.black_box.rise

:param model: OpenVINO model. :type model: ov.Model :param postprocess_fn: Post-processing function that extract scores from IR model output. - :type postprocess_fn: Callable[[OVDict], np.ndarray] + :type postprocess_fn: Callable[[Mapping], np.ndarray] :param preprocess_fn: Pre-processing function, identity function by default (assume input images are already preprocessed by user). :type preprocess_fn: Callable[[np.ndarray], np.ndarray] diff --git a/develop/_sources/user-guide.md.txt b/develop/_sources/user-guide.md.txt index 23db8cf4..8916e046 100644 --- a/develop/_sources/user-guide.md.txt +++ b/develop/_sources/user-guide.md.txt @@ -23,6 +23,7 @@ Content: - [Black-Box mode](#black-box-mode) - [XAI insertion (white-box usage)](#xai-insertion-white-box-usage) - [Plot saliency maps](#plot-saliency-maps) + - [Saving saliency maps](#saving-saliency-maps) - [Example scripts](#example-scripts) @@ -100,8 +101,8 @@ Here's the example how we can avoid passing `preprocess_fn` by preprocessing dat ```python import cv2 import numpy as np +from typing import Mapping import openvino.runtime as ov -from from typing import Mapping import openvino_xai as xai @@ -137,7 +138,7 @@ explanation = explainer( ) # Save saliency maps -explanation.save("output_path", "name") +explanation.save("output_path", "name_") ``` ### Specifying `preprocess_fn` @@ -146,8 +147,8 @@ explanation.save("output_path", "name") ```python import cv2 import numpy as np -import openvino.runtime as ov from typing import Mapping +import openvino.runtime as ov import openvino_xai as xai @@ -184,7 +185,7 @@ explanation = explainer( ) # Save saliency maps -explanation.save("output_path", "name") +explanation.save("output_path", "name_") ``` @@ -242,7 +243,7 @@ explanation = explainer( ) # Save saliency maps -explanation.save("output_path", "name") +explanation.save("output_path", "name_") ``` @@ -298,7 +299,7 @@ explanation = explainer( ) # Save saliency maps -explanation.save("output_path", "name") +explanation.save("output_path", "name_") ``` @@ -343,7 +344,9 @@ The `cv` backend is better for visualization in Python scripts, as it opens extr import cv2 import numpy as np import openvino.runtime as ov + import openvino_xai as xai +from openvino_xai.explainer import ExplainMode def preprocess_fn(image: np.ndarray) -> np.ndarray: """Preprocess the input image.""" @@ -391,6 +394,97 @@ explanation.plot(targets=[7], backend="cv") explanation.plot(targets=["cat"], backend="cv") ``` +## Saving saliency maps + +You can easily save saliency maps with flexible naming options by using a `prefix` and `postfix`. The `prefix` allows saliency maps from the same image to have consistent naming. + +The format for naming is: + +`{prefix} + target_id + {postfix}.jpg` + +Additionally, you can include the confidence score for each class in the saved saliency map's name. + +`{prefix} + target_id + {postfix} + confidence.jpg` + +```python +import cv2 +import numpy as np +import openvino.runtime as ov +from typing import Mapping + +import openvino_xai as xai +from openvino_xai.explainer import ExplainMode + +def preprocess_fn(image: np.ndarray) -> np.ndarray: + """Preprocess the input image.""" + x = cv2.resize(src=image, dsize=(224, 224)) + x = x.transpose((2, 0, 1)) + processed_image = np.expand_dims(x, 0) + return processed_image + +def postprocess_fn(output: Mapping): + """Postprocess the model output.""" + output = softmax(output) + return output[0] + +def softmax(x: np.ndarray) -> np.ndarray: + """Compute softmax values of x.""" + e_x = np.exp(x - np.max(x)) + return e_x / e_x.sum() + +# Generate and process saliency maps (as many as required, sequentially) +image = cv2.imread("path/to/image.jpg") + +# Create ov.Model +MODEL_PATH = "path/to/model.xml" +model = ov.Core().read_model(MODEL_PATH) # type: ov.Model + +# The Explainer object will prepare and load the model once in the beginning +explainer = xai.Explainer( + model, + task=xai.Task.CLASSIFICATION, + preprocess_fn=preprocess_fn, +) + +voc_labels = [ + 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', + 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor' +] + +# Get predicted confidences for the image +compiled_model = core.compile_model(model=model, device_name="AUTO") +logits = compiled_model(preprocess_fn(image))[0] +result_infer = postprocess_fn(logits) + +# Generate list of predicted class indices and scores +result_idxs = np.argwhere(result_infer > 0.4).flatten() +result_scores = result_infer[result_idxs] + +# Generate dict {class_index: confidence} to save saliency maps +scores_dict = {i: score for i, score in zip(result_idxs, result_scores)} + +# Run explanation +explanation = explainer( + image, + explain_mode=ExplainMode.WHITEBOX, + label_names=voc_labels, + target_explain_labels=result_idxs, # target classes to explain +) + +# Save saliency maps flexibly +OUTPUT_PATH = "output_path" +explanation.save(OUTPUT_PATH) # aeroplane.jpg +explanation.save(OUTPUT_PATH, "image_name_target_") # image_name_target_aeroplane.jpg +explanation.save(OUTPUT_PATH, prefix="image_name_target_") # image_name_target_aeroplane.jpg +explanation.save(OUTPUT_PATH, postfix="_class_map") # aeroplane_class_map.jpg +explanation.save(OUTPUT_PATH, prefix="image_name_", postfix="_class_map") # image_name_aeroplane_class_map.jpg + +# Save saliency maps with confidence scores +explanation.save( + OUTPUT_PATH, prefix="image_name_", postfix="_conf_", confidence_scores=scores_dict +) # image_name_aeroplane_conf_0.85.jpg +``` + ## Example scripts More usage scenarios that can be used with your own models and images as arguments are available in [examples](../../examples). diff --git a/develop/api-manual.html b/develop/api-manual.html index 7add6137..86c70c5b 100644 --- a/develop/api-manual.html +++ b/develop/api-manual.html @@ -617,8 +617,49 @@

Explanation
-save(dir_path: Path | str, name: str | None = None) → None[source]#
-

Dumps saliency map.

+save(dir_path: Path | str, prefix: str = '', postfix: str = '', confidence_scores: Dict[int, float] | None = None) → None[source]# +

Dumps saliency map images to the specified directory.

+

Allows flexibly name the files with the prefix and postfix. +{prefix} + target_id + {postfix}.jpg

+

Also allows to add confidence scores to the file names. +{prefix} + target_id + {postfix} + confidence.jpg

+

save(output_dir) -> aeroplane.jpg +save(output_dir, prefix=”image_name_target_”) -> image_name_target_aeroplane.jpg +save(output_dir, postfix=”_class_map”) -> aeroplane_class_map.jpg +save(

+
+

output_dir, prefix=”image_name_”, postfix=”_conf_”, confidence_scores=scores

+
+

) -> image_name_aeroplane_conf_0.85.jpg

+
+
Parameters:
+
param dir_path:
+

The directory path where the saliency maps will be saved.

+
+
type dir_path:
+

Path | str

+
+
param prefix:
+

Optional prefix for the saliency map names. Default is an empty string.

+
+
type prefix:
+

str

+
+
param postfix:
+

Optional postfix for the saliency map names. Default is an empty string.

+
+
type postfix:
+

str

+
+
param confidence_scores:
+

Dict with confidence scores for each class index. Default is None.

+
+
type confidence_scores:
+

Dict[int, float] | None

+
+
+
+
@@ -717,7 +758,7 @@

Methods#<

XAI algorithms.

-class openvino_xai.methods.AISE(model: ~openvino.runtime.ie_api.Model, postprocess_fn: ~typing.Callable[[~openvino.runtime.utils.data_helpers.wrappers.OVDict], ~numpy.ndarray], preprocess_fn: ~typing.Callable[[~numpy.ndarray], ~numpy.ndarray] = <openvino_xai.common.utils.IdentityPreprocessFN object>, device_name: str = 'CPU', prepare_model: bool = True)[source]#
+class openvino_xai.methods.AISE(model: ~openvino.runtime.ie_api.Model, postprocess_fn: ~typing.Callable[[~typing.Mapping], ~numpy.ndarray], preprocess_fn: ~typing.Callable[[~numpy.ndarray], ~numpy.ndarray] = <openvino_xai.common.utils.IdentityPreprocessFN object>, device_name: str = 'CPU', prepare_model: bool = True)[source]#

Bases: BlackBoxXAIMethod

AISE explains classification models in black-box mode using AISE: Adaptive Input Sampling for Explanation of Black-box Models @@ -726,7 +767,7 @@

Methods#<
Parameters:
@@ -434,6 +435,7 @@

OpenVINO™ Explainable AI Toolkit User GuideBlack-Box mode

  • XAI insertion (white-box usage)

  • Plot saliency maps

    • +

  • Saving saliency maps

  • Example scripts

    • @@ -502,8 +504,8 @@

    Running without Here’s the example how we can avoid passing preprocess_fn by preprocessing data beforehand (like resizing and adding a batch dimension).

    import cv2
 import numpy as np
+from typing import Mapping
 import openvino.runtime as ov
-from from typing import Mapping
 
 import openvino_xai as xai
 
@@ -539,7 +541,7 @@ 
    Running without )
 
 # Save saliency maps
-explanation.save("output_path", "name")
+explanation.save("output_path", "name_")
    @@ -547,8 +549,8 @@

    Running without Specifying preprocess_fn#

    import cv2
 import numpy as np
-import openvino.runtime as ov
 from typing import Mapping
+import openvino.runtime as ov
 
 import openvino_xai as xai
 
@@ -585,7 +587,7 @@ 
    Specifying prep
 )
 
 # Save saliency maps
-explanation.save("output_path", "name")
+explanation.save("output_path", "name_")
    @@ -640,7 +642,7 @@

    White-Box mode) # Save saliency maps -explanation.save("output_path", "name") +explanation.save("output_path", "name_")

    @@ -693,7 +695,7 @@

    Black-Box mode) # Save saliency maps -explanation.save("output_path", "name") +explanation.save("output_path", "name_")

    @@ -731,7 +733,9 @@

    Plot saliency maps
    import cv2
 import numpy as np
 import openvino.runtime as ov
+
 import openvino_xai as xai
+from openvino_xai.explainer import ExplainMode
 
 def preprocess_fn(image: np.ndarray) -> np.ndarray:
     """Preprocess the input image."""
@@ -780,6 +784,92 @@ 
    Plot saliency maps
+
    Saving saliency maps#
    
+
    You can easily save saliency maps with flexible naming options by using a prefix and postfix. The prefix allows saliency maps from the same image to have consistent naming.
    
+
    The format for naming is:
    
+
    {prefix} + target_id + {postfix}.jpg
    
+
    Additionally, you can include the confidence score for each class in the saved saliency map’s name.
    
+
    {prefix} + target_id + {postfix} + confidence.jpg
    
+
    import cv2
+import numpy as np
+import openvino.runtime as ov
+from typing import Mapping
+
+import openvino_xai as xai
+from openvino_xai.explainer import ExplainMode
+
+def preprocess_fn(image: np.ndarray) -> np.ndarray:
+    """Preprocess the input image."""
+    x = cv2.resize(src=image, dsize=(224, 224))
+    x = x.transpose((2, 0, 1))
+    processed_image = np.expand_dims(x, 0)
+    return processed_image
+
+def postprocess_fn(output: Mapping):
+    """Postprocess the model output."""
+    output = softmax(output)
+    return output[0]
+
+def softmax(x: np.ndarray) -> np.ndarray:
+    """Compute softmax values of x."""
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum()
+
+# Generate and process saliency maps (as many as required, sequentially)
+image = cv2.imread("path/to/image.jpg")
+
+# Create ov.Model
+MODEL_PATH = "path/to/model.xml"
+model = ov.Core().read_model(MODEL_PATH)  # type: ov.Model
+
+# The Explainer object will prepare and load the model once in the beginning
+explainer = xai.Explainer(
+    model,
+    task=xai.Task.CLASSIFICATION,
+    preprocess_fn=preprocess_fn,
+)
+
+voc_labels = [
+    'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable',
+    'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'
+]
+
+# Get predicted confidences for the image
+compiled_model = core.compile_model(model=model, device_name="AUTO")
+logits = compiled_model(preprocess_fn(image))[0]
+result_infer = postprocess_fn(logits)
+
+# Generate list of predicted class indices and scores
+result_idxs = np.argwhere(result_infer > 0.4).flatten()
+result_scores = result_infer[result_idxs]
+
+# Generate dict {class_index: confidence} to save saliency maps
+scores_dict = {i: score for i, score in zip(result_idxs, result_scores)}
+
+# Run explanation
+explanation = explainer(
+    image,
+    explain_mode=ExplainMode.WHITEBOX,
+    label_names=voc_labels,
+    target_explain_labels=result_idxs,  # target classes to explain
+)
+
+# Save saliency maps flexibly
+OUTPUT_PATH = "output_path"
+explanation.save(OUTPUT_PATH)  # aeroplane.jpg
+explanation.save(OUTPUT_PATH, "image_name_target_")  # image_name_target_aeroplane.jpg
+explanation.save(OUTPUT_PATH, prefix="image_name_target_")  # image_name_target_aeroplane.jpg
+explanation.save(OUTPUT_PATH, postfix="_class_map")  # aeroplane_class_map.jpg
+explanation.save(OUTPUT_PATH, prefix="image_name_", postfix="_class_map")  # image_name_aeroplane_class_map.jpg
+
+# Save saliency maps with confidence scores
+explanation.save(
+    OUTPUT_PATH, prefix="image_name_", postfix="_conf_", confidence_scores=scores_dict
+)  # image_name_aeroplane_conf_0.85.jpg
+
    
+
    
+
 
    
 
    Example scripts#
    
 
    More usage scenarios that can be used with your own models and images as arguments are available in examples.
    
@@ -858,6 +948,7 @@ 
    Example scriptsBlack-Box mode
 
  • XAI insertion (white-box usage)
    • 
 
  • Plot saliency maps
    • 
+
  • Saving saliency maps
    • 
 
  • Example scripts
    •