Skip to content

Commit

Permalink
Move accuracy tests from regression to integration (#59)
Browse files Browse the repository at this point in the history
Regression tests -> integration tests
  • Loading branch information
GalyaZalesskaya authored Aug 30, 2024
1 parent 634caed commit 656345c
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Dict, List, Tuple

import cv2
import numpy as np
import openvino as ov
import pytest

Expand Down Expand Up @@ -53,35 +54,34 @@ def load_gt_bboxes(json_coco_path: str) -> List[Dict[str, List[Tuple[int, int, i
return list(result.values())


class TestDummyRegression:
class TestAccuracyMetrics:
image = cv2.imread(IMAGE_PATH)
gt_bboxes = load_gt_bboxes(COCO_ANN_PATH)

preprocess_fn = get_preprocess_fn(
change_channel_order=True,
input_size=(224, 224),
hwc_to_chw=True,
)

postprocess_fn = get_postprocess_fn(activation=ActivationType.SIGMOID)

@pytest.fixture(autouse=True)
def setup(self, fxt_data_root):
data_dir = fxt_data_root
retrieve_otx_model(data_dir, MODEL_NAME)
model_path = data_dir / "otx_models" / (MODEL_NAME + ".xml")
model = ov.Core().read_model(model_path)

preprocess_fn = get_preprocess_fn(
change_channel_order=True,
input_size=(224, 224),
hwc_to_chw=True,
)
postprocess_fn = get_postprocess_fn(activation=ActivationType.SIGMOID)

self.explainer = Explainer(
model=model,
task=Task.CLASSIFICATION,
preprocess_fn=self.preprocess_fn,
preprocess_fn=preprocess_fn,
explain_mode=ExplainMode.WHITEBOX,
)

self.pointing_game = PointingGame()
self.auc = InsertionDeletionAUC(model, self.preprocess_fn, self.postprocess_fn)
self.adcc = ADCC(model, self.preprocess_fn, self.postprocess_fn, self.explainer)
self.auc = InsertionDeletionAUC(model, preprocess_fn, postprocess_fn)
self.adcc = ADCC(model, preprocess_fn, postprocess_fn, self.explainer)

def test_explainer_image(self):
explanation = self.explainer(self.image, targets=["person"], label_names=VOC_NAMES, colormap=False)
Expand All @@ -92,12 +92,12 @@ def test_explainer_image(self):

auc_score = self.auc.evaluate([explanation], [self.image], steps=10).values()
insertion_auc_score, deletion_auc_score, delta_auc_score = auc_score
assert insertion_auc_score >= 0.9
assert deletion_auc_score >= 0.2
assert delta_auc_score >= 0.7
assert np.abs(insertion_auc_score - 0.94) <= 0.01
assert np.abs(deletion_auc_score - 0.23) <= 0.01
assert np.abs(delta_auc_score - 0.71) <= 0.01

adcc_score = self.adcc.evaluate([explanation], [self.image])["adcc"]
assert adcc_score > 0.9
assert np.abs(adcc_score - 0.96) <= 0.01

def test_explainer_image_2_classes(self):
explanation = self.explainer(self.image, targets=["person", "cat"], label_names=VOC_NAMES, colormap=False)
Expand All @@ -108,12 +108,12 @@ def test_explainer_image_2_classes(self):

auc_score = self.auc.evaluate([explanation], [self.image], steps=10).values()
insertion_auc_score, deletion_auc_score, delta_auc_score = auc_score
assert insertion_auc_score >= 0.5
assert deletion_auc_score >= 0.1
assert delta_auc_score >= 0.35
assert np.abs(insertion_auc_score - 0.53) <= 0.01
assert np.abs(deletion_auc_score - 0.13) <= 0.01
assert np.abs(delta_auc_score - 0.39) <= 0.01

adcc_score = self.adcc.evaluate([explanation], [self.image])["adcc"]
assert adcc_score > 0.5
assert np.abs(adcc_score - 0.55) <= 0.01

def test_explainer_images(self):
images = [self.image, self.image]
Expand All @@ -128,9 +128,9 @@ def test_explainer_images(self):

auc_score = self.auc.evaluate(explanations, images, steps=10).values()
insertion_auc_score, deletion_auc_score, delta_auc_score = auc_score
assert insertion_auc_score >= 0.9
assert deletion_auc_score >= 0.2
assert delta_auc_score >= 0.7
assert np.abs(insertion_auc_score - 0.94) <= 0.01
assert np.abs(deletion_auc_score - 0.23) <= 0.01
assert np.abs(delta_auc_score - 0.71) <= 0.01

adcc_score = self.adcc.evaluate(explanations, images)["adcc"]
assert adcc_score > 0.9
assert np.abs(adcc_score - 0.96) <= 0.01
Empty file removed tests/regression/__init__.py
Empty file.

0 comments on commit 656345c

Please sign in to comment.