diff --git a/docs/source/user-guide.md b/docs/source/user-guide.md index 3f46ff0f..d96bfea2 100644 --- a/docs/source/user-guide.md +++ b/docs/source/user-guide.md @@ -394,7 +394,7 @@ Target layer is the part of the model graph where XAI branch will be inserted (a All supported methods are gradient-free, which suits deployment framework settings (e.g. OpenVINO™), where the model is in optimized or compiled representation. -## Methods performance-accuracy comparison +### Methods performance-accuracy comparison The table below compares accuracy and performace of different models and explain methods (learn more about [Quality Metrics](#measure-quality-metrics-of-saliency-maps)). @@ -404,16 +404,18 @@ Metrics were measured on a 10% random subset of the [ILSVRC 2012](https://www.im |:---------------------------:|:------------:|:--------------:|:---------------------------------:|---|:-------------:|---|:---------:|:--------:|---|:--------:|:---------:|:----------:|:------------:| | deit - tiny (transformer) | White box | VIT ReciproCAM | 1* | | **89.9** | | 22.4 | **4.5** | | 70.4 | 88.9 | **38.1** | 34.3 | | | | Activation map | 1 | | 56.6 | | 7.8 | 7.0 | | 46.9 | 74.0 | 53.7 | 65.4 | -| | Black Box | AISE | 60 | | 73.9 | | 15.9 | 8.9 | | 66.6 | 73.9 | 44.3 | 26.0 | +| | Black Box** | AISE | 60 | | 73.9 | | 15.9 | 8.9 | | 66.6 | 73.9 | 44.3 | 26.0 | | | | RISE | 2000 | | 85.5 | | **23.2** | 5.8 | | **74.8** | **92.5** | 42.3 | **16.6** | | | | | | | | | | | | | | | | | resnet18 | White box | ReciproCAM | 1* | | **89.5** | | 33.9 | **5.9** | | **77.3** | 91.1 | 30.2 | 25.9 | | | | Activation map | 1 | | 87.0 | | **36.3** | 10.5 | | 74.4 | **97.9** | **25.2** | 40.2 | -| | Black Box | AISE | 60 | | 72.0 | | 22.5 | 12.4 | | 67.4 | 69.3 | 44.5 | 16.9 | +| | Black Box** | AISE | 60 | | 72.0 | | 22.5 | 12.4 | | 67.4 | 69.3 | 44.5 | 16.9 | | | | RISE | 2000 | | 87.0 | | 34.6 | 7.1 | | 77.1 | 93.0 | 42.0 | **8.3** | \* Recipro-CAM re-infers part of the graph (usually neck + head or last transformer block) H*W times, where HxW is the feature map size of the target layer. +\*\* For Black Box Methods preset = `SPEED` + ### White-Box methods diff --git a/tests/perf/test_accuracy.py b/tests/perf/test_accuracy.py index 70bbefff..769af1e7 100644 --- a/tests/perf/test_accuracy.py +++ b/tests/perf/test_accuracy.py @@ -62,7 +62,6 @@ ] TEST_MODELS = IMAGENET_MODELS + VOC_MODELS + TRANSFORMER_MODELS -IMAGENET_LABELS = get_imagenet_labels() EXPLAIN_METHODS = [Method.RECIPROCAM, Method.AISE, Method.RISE, Method.ACTIVATIONMAP] @@ -100,8 +99,9 @@ def setup_model(self, data_dir, model_name): return model, None elif model_name in IMAGENET_MODELS + TRANSFORMER_MODELS: - self.dataset_label_list = IMAGENET_LABELS _, model_cfg = convert_timm_to_ir(model_name, data_dir, self.supported_num_classes) + version = "1k" if model_cfg["num_classes"] == 1000 else "21k" + self.dataset_label_list = get_imagenet_labels(version) ir_path = data_dir / "timm_models" / "converted_models" / model_name / "model_fp32.xml" model = ov.Core().read_model(ir_path) return model, model_cfg @@ -157,7 +157,7 @@ def setup_explainer(self, model, explain_method): def setup(self, fxt_data_root, fxt_output_root, fxt_dataset_parameters): self.data_dir = fxt_data_root self.output_dir = fxt_output_root - self.supported_num_classes = {1000: 1000} + self.supported_num_classes = {1000: 1000, 21841: 21841, 21843: 21843} self.setup_dataset(fxt_dataset_parameters) self.dataset_name = self.dataset_type.value diff --git a/tests/unit/explainer/test_explanation_utils.py b/tests/unit/explainer/test_explanation_utils.py index cc1f827c..8ad594e3 100644 --- a/tests/unit/explainer/test_explanation_utils.py +++ b/tests/unit/explainer/test_explanation_utils.py @@ -84,8 +84,13 @@ def test_is_bhwc_layout(): assert is_bhwc_layout(np.empty((1, 3, 224, 224))) == False -def get_imagenet_labels(file_path="imagenet_2012.txt"): - url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt" +def get_imagenet_labels(version="1k"): + if version == "1k": + url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt" + file_path = "imagenet_2012.txt" + elif version == "21k": + url = "https://storage.googleapis.com/bit_models/imagenet21k_wordnet_ids.txt" + file_path = "imagenet21k_wordnet_ids.txt" if not os.path.exists(file_path): response = requests.get(url)