diff --git a/pyproject.toml b/pyproject.toml index 060a4835..b8f818aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,9 @@ dev = [ val = [ "timm==0.9.5", "onnx==1.14.1", + "pandas", + "py-cpuinfo", + "openpyxl", ] [project.urls] diff --git a/tests/conftest.py b/tests/conftest.py index 77b21511..373ce54e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import logging import os +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest @@ -17,13 +18,12 @@ def pytest_addoption(parser: pytest.Parser): "--data-root", action="store", default=".data", - help="Data root directory.", + help="Data root directory. Defaults to '.data'", ) parser.addoption( "--output-root", action="store", - default=".data", - help="Output root directory.", + help="Output root directory. Defaults to temp dir.", ) parser.addoption( "--clear-cache", @@ -44,9 +44,15 @@ def fxt_data_root(request: pytest.FixtureRequest) -> Path: @pytest.fixture(scope="session") -def fxt_output_root(request: pytest.FixtureRequest) -> Path: - """Output root directory path.""" - output_root = Path(request.config.getoption("--output-root")) +def fxt_output_root( + request: pytest.FixtureRequest, + tmp_path_factory: pytest.TempPathFactory, +) -> Path: + """Output root.""" + output_root = request.config.getoption("--output-root") + if output_root is None: + output_root = tmp_path_factory.mktemp("openvino_xai") + output_root = Path(output_root) output_root.mkdir(parents=True, exist_ok=True) msg = f"{output_root = }" log.info(msg) diff --git a/tests/intg/test_classification_timm.py b/tests/intg/test_classification_timm.py index 4e075982..42f458be 100644 --- a/tests/intg/test_classification_timm.py +++ b/tests/intg/test_classification_timm.py @@ -155,7 +155,7 @@ def test_classification_white_box(self, model_id, dump_maps=False): # self.check_for_saved_map(model_id, "timm_models/maps_wb/") if model_id in NON_SUPPORTED_BY_WB_MODELS: - pytest.xfail(reason="Not supported yet") + pytest.skip(reason="Not supported yet") timm_model, model_cfg = self.get_timm_model(model_id) self.update_report("report_wb.csv", model_id) @@ -251,13 +251,6 @@ def test_classification_white_box(self, model_id, dump_maps=False): self.update_report("report_wb.csv", model_id, "True", "True", "True", shape_str, str(map_saved)) self.clear_cache() - # sudo ln -s /usr/local/cuda-11.8/ cuda - # pip uninstall torch torchvision - # pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu118 - # - # ulimit -a - # ulimit -Sn 10000 - # ulimit -a @pytest.mark.parametrize("model_id", TEST_MODELS) def test_classification_black_box(self, model_id, dump_maps=False): # self.check_for_saved_map(model_id, "timm_models/maps_bb/") diff --git a/tests/perf/__init__.py b/tests/perf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/perf/conftest.py b/tests/perf/conftest.py new file mode 100644 index 00000000..138846cb --- /dev/null +++ b/tests/perf/conftest.py @@ -0,0 +1,175 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import logging +import os +import platform +import subprocess +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pandas as pd +import pytest +from cpuinfo import get_cpu_info + +log = logging.getLogger(__name__) + + +def pytest_addoption(parser: pytest.Parser): + """Add custom options for OpenVINO XAI perf tests.""" + parser.addoption( + "--num-repeat", + action="store", + default=5, + help="Number of trials for each model explain. " + "Random seeds are set to 0 ~ num_repeat-1 for the trials. " + "Defaults to 10.", + ) + parser.addoption( + "--num-masks", + action="store", + default=5000, + help="Number of masks for black box methods." "Defaults to 5000.", + ) + + +@pytest.fixture(scope="session") +def fxt_num_repeat(request: pytest.FixtureRequest) -> int: + """Number of repeated trials.""" + num_repeat = int(request.config.getoption("--num-repeat")) + msg = f"{num_repeat = }" + log.info(msg) + print(msg) + return num_repeat + + +@pytest.fixture(scope="session") +def fxt_num_masks(request: pytest.FixtureRequest) -> int: + """Number of masks for black box methods.""" + num_masks = int(request.config.getoption("--num-masks")) + msg = f"{num_masks = }" + log.info(msg) + print(msg) + return num_masks + + +@pytest.fixture(scope="session") +def fxt_current_date() -> str: + tz = timezone(offset=timedelta(hours=9), name="Seoul") + return datetime.now(tz=tz).strftime("%Y%m%d-%H%M%S") + + +@pytest.fixture(scope="session") +def fxt_output_root( + request: pytest.FixtureRequest, + tmp_path_factory: pytest.TempPathFactory, + fxt_current_date: str, +) -> Path: + """Output root + dateh.""" + output_root = request.config.getoption("--output-root") + if output_root is None: + output_root = tmp_path_factory.mktemp("openvino_xai") + output_root = Path(output_root) / "perf" / fxt_current_date + output_root.mkdir(parents=True, exist_ok=True) + msg = f"{output_root = }" + log.info(msg) + print(msg) + return output_root + + +@pytest.fixture(scope="session") +def fxt_tags(fxt_current_date: str) -> dict[str, str]: + """Tag fields to record various metadata.""" + try: + from importlib.metadata import version + + version_str = version("openvino_xai") + except Exception: + version_str = "unknown" + try: + branch_str = ( + subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("ascii").strip() + ) # noqa: S603, S607 + except Exception: + branch_str = os.environ.get("GH_CTX_REF_NAME", "unknown") + try: + commit_str = ( + subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("ascii").strip() + ) # noqa: S603, S607 + except Exception: + commit_str = os.environ.get("GH_CTX_SHA", "unknown") + tags = { + "version": version_str, + "branch": branch_str, + "commit": commit_str, + "date": fxt_current_date, + "machine_name": platform.node(), + "cpu_info": get_cpu_info()["brand_raw"], + } + msg = f"{tags = }" + log.info(msg) + return tags + + +@pytest.fixture(scope="session", autouse=True) +def fxt_perf_summary( + fxt_output_root: Path, + fxt_tags: dict[str, str], +): + """Summarize all results at the end of test session.""" + yield + + # Merge all raw data + raw_data = [] + csv_files = fxt_output_root.rglob("perf-raw-*-*.csv") + for csv_file in csv_files: + data = pd.read_csv(csv_file) + raw_data.append(data) + if len(raw_data) == 0: + print("No raw data to summarize") + return + raw_data = pd.concat(raw_data, ignore_index=True) + raw_data = raw_data.drop(["Unnamed: 0"], axis=1) + raw_data = raw_data.replace( + { + "Method.RECIPROCAM": "RECIPROCAM", + "Method.VITRECIPROCAM": "RECIPROCAM", + "Method.RISE": "RISE", + } + ) + raw_data.to_csv(fxt_output_root / "perf-raw-all.csv", index=False) + + # Summarize + data = raw_data.pivot_table( + index=["model", "version"], + columns=["method"], + values=["time"], + aggfunc=["mean", "std"], + ) + data.columns = data.columns.rename(["stat", "metric", "method"]) + data = data.reorder_levels(["method", "metric", "stat"], axis=1) + data0 = data + + data = raw_data.pivot_table( + index=["version"], + columns=["method"], + values=["time"], + aggfunc=["mean", "std"], + ) + indices = data.index.to_frame() + indices["model"] = "all" + data.index = pd.MultiIndex.from_frame(indices) + data = data.reorder_levels(["model", "version"], axis=0) + data.columns = data.columns.rename(["stat", "metric", "method"]) + data = data.reorder_levels(["method", "metric", "stat"], axis=1) + data1 = data + + data = pd.concat([data0, data1], axis=0) + data = data.sort_index(axis=0).sort_index(axis=1) + + print("=" * 20, "[Perf summary]") + print(data) + data.to_csv(fxt_output_root / "perf-summary.csv") + data.to_excel(fxt_output_root / "perf-summary.xlsx") + print(f" -> Saved to {fxt_output_root}") diff --git a/tests/perf/test_performance.py b/tests/perf/test_performance.py new file mode 100644 index 00000000..91bf167f --- /dev/null +++ b/tests/perf/test_performance.py @@ -0,0 +1,244 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import csv +import os +import shutil +from pathlib import Path +from time import time + +import cv2 +import numpy as np +import openvino.runtime as ov +import pandas as pd +import pytest + +from openvino_xai.common.parameters import Method, Task +from openvino_xai.explainer.explainer import Explainer, ExplainMode +from openvino_xai.explainer.utils import ( + ActivationType, + get_postprocess_fn, + get_preprocess_fn, + get_score, +) +from openvino_xai.explainer.visualizer import Visualizer +from openvino_xai.utils.model_export import export_to_ir, export_to_onnx + +timm = pytest.importorskip("timm") +torch = pytest.importorskip("torch") +pytest.importorskip("onnx") + + +from tests.intg.test_classification_timm import ( + LIMITED_DIVERSE_SET_OF_CNN_MODELS, + LIMITED_DIVERSE_SET_OF_VISION_TRANSFORMER_MODELS, + NON_SUPPORTED_BY_WB_MODELS, +) + +TEST_MODELS = ( + LIMITED_DIVERSE_SET_OF_CNN_MODELS + LIMITED_DIVERSE_SET_OF_VISION_TRANSFORMER_MODELS + NON_SUPPORTED_BY_WB_MODELS +) + + +def seed_everything(seed: int): + """Set random seed.""" + import os + import random + + import numpy as np + + random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + np.random.seed(seed) + + +class TestPerfClassificationTimm: + clear_cache_converted_models = False + clear_cache_hf_models = False + supported_num_classes = { + 1000: 293, # 293 is a cheetah class_id in the ImageNet-1k dataset + 21841: 2441, # 2441 is a cheetah class_id in the ImageNet-21k dataset + 21843: 2441, # 2441 is a cheetah class_id in the ImageNet-21k dataset + 11821: 1652, # 1652 is a cheetah class_id in the ImageNet-12k dataset + } + + @pytest.fixture(autouse=True) + def setup(self, fxt_data_root, fxt_output_root, fxt_clear_cache): + self.data_dir = fxt_data_root + self.output_dir = fxt_output_root + self.cache_dir = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")).expanduser() + self.clear_cache_hf_models = fxt_clear_cache + self.clear_cache_converted_models = fxt_clear_cache + + @pytest.mark.parametrize("model_id", TEST_MODELS) + def test_classification_white_box(self, model_id: str, fxt_num_repeat: int, fxt_tags: dict): + if model_id in NON_SUPPORTED_BY_WB_MODELS: + pytest.skip(reason="Not supported yet") + + timm_model, model_cfg = self.get_timm_model(model_id) + + ir_path = self.data_dir / "timm_models" / "converted_models" / model_id / "model_fp32.xml" + if not ir_path.is_file(): + output_model_dir = self.output_dir / "timm_models" / "converted_models" / model_id + output_model_dir.mkdir(parents=True, exist_ok=True) + ir_path = output_model_dir / "model_fp32.xml" + input_size = [1] + list(timm_model.default_cfg["input_size"]) + dummy_tensor = torch.rand(input_size) + onnx_path = output_model_dir / "model_fp32.onnx" + set_dynamic_batch = model_id in LIMITED_DIVERSE_SET_OF_VISION_TRANSFORMER_MODELS + export_to_onnx(timm_model, onnx_path, dummy_tensor, set_dynamic_batch) + export_to_ir(onnx_path, output_model_dir / "model_fp32.xml") + + if model_id in LIMITED_DIVERSE_SET_OF_CNN_MODELS: + explain_method = Method.RECIPROCAM + elif model_id in LIMITED_DIVERSE_SET_OF_VISION_TRANSFORMER_MODELS: + explain_method = Method.VITRECIPROCAM + else: + raise ValueError + + mean_values = [(item * 255) for item in model_cfg["mean"]] + scale_values = [(item * 255) for item in model_cfg["std"]] + preprocess_fn = get_preprocess_fn( + change_channel_order=True, + input_size=model_cfg["input_size"][1:], + mean=mean_values, + std=scale_values, + hwc_to_chw=True, + ) + + target_class = self.supported_num_classes[model_cfg["num_classes"]] + image = cv2.imread("tests/assets/cheetah_person.jpg") + + records = [] + for seed in range(fxt_num_repeat): + seed_everything(seed) + + record = fxt_tags.copy() + record["model"] = model_id + record["method"] = explain_method + record["seed"] = seed + + model = ov.Core().read_model(ir_path) + + start_time = time() + + explainer = Explainer( + model=model, + task=Task.CLASSIFICATION, + preprocess_fn=preprocess_fn, + explain_mode=ExplainMode.WHITEBOX, # defaults to AUTO + explain_method=explain_method, + embed_scaling=False, + ) + explanation = explainer( + image, + targets=[target_class], + resize=True, + colormap=True, + overlay=True, + ) + + explain_time = time() - start_time + record["time"] = explain_time + + assert explanation is not None + assert explanation.shape[-1] > 1 and explanation.shape[-2] > 1 + print(record) + records.append(record) + + df = pd.DataFrame(records) + df.to_csv(self.output_dir / f"perf-raw-wb-{model_id}.csv") + + self.clear_cache() + + @pytest.mark.parametrize("model_id", TEST_MODELS) + def test_classification_black_box(self, model_id, fxt_num_repeat: int, fxt_num_masks: int, fxt_tags: dict): + timm_model, model_cfg = self.get_timm_model(model_id) + + onnx_path = self.data_dir / "timm_models" / "converted_models" / model_id / "model_fp32.onnx" + if not onnx_path.is_file(): + output_model_dir = self.output_dir / "timm_models" / "converted_models" / model_id + output_model_dir.mkdir(parents=True, exist_ok=True) + onnx_path = output_model_dir / "model_fp32.onnx" + input_size = [1] + list(timm_model.default_cfg["input_size"]) + dummy_tensor = torch.rand(input_size) + onnx_path = output_model_dir / "model_fp32.onnx" + export_to_onnx(timm_model, onnx_path, dummy_tensor, False) + + model = ov.Core().read_model(onnx_path) + + mean_values = [(item * 255) for item in model_cfg["mean"]] + scale_values = [(item * 255) for item in model_cfg["std"]] + preprocess_fn = get_preprocess_fn( + change_channel_order=True, + input_size=model_cfg["input_size"][1:], + mean=mean_values, + std=scale_values, + hwc_to_chw=True, + ) + + postprocess_fn = get_postprocess_fn() + + image = cv2.imread("tests/assets/cheetah_person.jpg") + target_class = self.supported_num_classes[model_cfg["num_classes"]] + + records = [] + for seed in range(fxt_num_repeat): + seed_everything(seed) + + record = fxt_tags.copy() + record["model"] = model_id + record["method"] = Method.RISE + record["seed"] = seed + record["num_masks"] = fxt_num_masks + + start_time = time() + + explainer = Explainer( + model=model, + task=Task.CLASSIFICATION, + preprocess_fn=preprocess_fn, + postprocess_fn=postprocess_fn, + explain_mode=ExplainMode.BLACKBOX, # defaults to AUTO + ) + explanation = explainer( + image, + targets=[target_class], + resize=True, + colormap=True, + overlay=True, + num_masks=fxt_num_masks, # kwargs of the RISE algo + ) + + explain_time = time() - start_time + record["time"] = explain_time + + assert explanation is not None + assert explanation.shape[-1] > 1 and explanation.shape[-2] > 1 + print(record) + records.append(record) + + df = pd.DataFrame(records) + df.to_csv(self.output_dir / f"perf-raw-bb-{model_id}.csv", index=False) + + self.clear_cache() + + def get_timm_model(self, model_id): + timm_model = timm.create_model(model_id, in_chans=3, pretrained=True, checkpoint_path="") + timm_model.eval() + model_cfg = timm_model.default_cfg + num_classes = model_cfg["num_classes"] + if num_classes not in self.supported_num_classes: + self.clear_cache() + pytest.skip(f"Number of model classes {num_classes} unknown") + return timm_model, model_cfg + + def clear_cache(self): + if self.clear_cache_converted_models: + ir_model_dir = self.data_dir / "timm_models" / "converted_models" + if ir_model_dir.is_dir(): + shutil.rmtree(ir_model_dir) + if self.clear_cache_hf_models: + huggingface_hub_dir = self.cache_dir / "huggingface" / "hub" + if huggingface_hub_dir.is_dir(): + shutil.rmtree(huggingface_hub_dir)