From 2afa66a355b8b3c496bfd028a8979ce4b45697b4 Mon Sep 17 00:00:00 2001 From: Beat Buesser Date: Thu, 16 Jan 2025 10:45:18 +0100 Subject: [PATCH 1/5] Update typing and documentation of AdversarialPatchPyTorch for object detection Signed-off-by: Beat Buesser --- .../adversarial_patch_pytorch.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py index cc7d578f19..a43ce14687 100644 --- a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py +++ b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py @@ -183,7 +183,10 @@ def __init__( self._optimizer = torch.optim.Adam([self._patch], lr=self.learning_rate) def _train_step( - self, images: "torch.Tensor", target: "torch.Tensor", mask: "torch.Tensor" | None = None + self, + images: "torch.Tensor", + target: "torch.Tensor" | list[dict[str, "torch.Tensor"]], + mask: "torch.Tensor" | None = None, ) -> "torch.Tensor": import torch @@ -227,7 +230,12 @@ def _predictions( return predictions, target - def _loss(self, images: "torch.Tensor", target: "torch.Tensor", mask: "torch.Tensor" | None) -> "torch.Tensor": + def _loss( + self, + images: "torch.Tensor", + target: "torch.Tensor" | list[dict[str, "torch.Tensor"]], + mask: "torch.Tensor" | None, + ) -> "torch.Tensor": import torch if isinstance(target, torch.Tensor): @@ -475,13 +483,13 @@ def _random_overlay( return patched_images def generate( # type: ignore - self, x: np.ndarray, y: np.ndarray | None = None, **kwargs + self, x: np.ndarray, y: np.ndarray | list[dict[str, np.ndarray | "torch.Tensor"]] | None = None, **kwargs ) -> tuple[np.ndarray, np.ndarray]: """ Generate an adversarial patch and return the patch and its mask in arrays. :param x: An array with the original input images of shape NCHW or input videos of shape NFCHW. - :param y: An array with the original true labels. + :param y: The true or target labels. :param mask: A boolean array of shape equal to the shape of a single samples (1, H, W) or the shape of `x` (N, H, W) without their channel dimensions. Any features for which the mask is True can be the center location of the patch during sampling. @@ -499,11 +507,12 @@ def generate( # type: ignore if self.patch_location is not None and mask is not None: raise ValueError("Masks can only be used if the `patch_location` is `None`.") - if y is None: # pragma: no cover - logger.info("Setting labels to estimator predictions and running untargeted attack because `y=None`.") - y = to_categorical(np.argmax(self.estimator.predict(x=x), axis=1), nb_classes=self.estimator.nb_classes) - if hasattr(self.estimator, "nb_classes"): + + if y is None: # pragma: no cover + logger.info("Setting labels to estimator classification predictions.") + y = to_categorical(np.argmax(self.estimator.predict(x=x), axis=1), nb_classes=self.estimator.nb_classes) + y = check_and_transform_label_format(labels=y, nb_classes=self.estimator.nb_classes) # check if logits or probabilities @@ -513,6 +522,10 @@ def generate( # type: ignore self.use_logits = False else: self.use_logits = True + else: + if y is None: # pragma: no cover + logger.info("Setting labels to estimator object detection predictions.") + y = self.estimator.predict(x=x) if isinstance(y, np.ndarray): x_tensor = torch.Tensor(x) From 0aa91b5d2c95bf0178dc04d34089e20ed27ff4d8 Mon Sep 17 00:00:00 2001 From: Beat Buesser Date: Thu, 16 Jan 2025 10:49:46 +0100 Subject: [PATCH 2/5] Update estimator typing in AdversarialPatchPyTorch Signed-off-by: Beat Buesser --- .../evasion/adversarial_patch/adversarial_patch_pytorch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py index a43ce14687..1a88f4a875 100644 --- a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py +++ b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py @@ -41,7 +41,7 @@ import torch - from art.utils import CLASSIFIER_NEURALNETWORK_TYPE + from art.utils import CLASSIFIER_NEURALNETWORK_TYPE, PYTORCH_OBJECT_DETECTOR_TYPE logger = logging.getLogger(__name__) @@ -72,7 +72,7 @@ class AdversarialPatchPyTorch(EvasionAttack): def __init__( self, - estimator: "CLASSIFIER_NEURALNETWORK_TYPE", + estimator: "CLASSIFIER_NEURALNETWORK_TYPE | PYTORCH_OBJECT_DETECTOR_TYPE", rotation_max: float = 22.5, scale_min: float = 0.1, scale_max: float = 1.0, @@ -91,7 +91,7 @@ def __init__( """ Create an instance of the :class:`.AdversarialPatchPyTorch`. - :param estimator: A trained estimator. + :param estimator: A trained PyTorch estimator for classification or object detection. :param rotation_max: The maximum rotation applied to random patches. The value is expected to be in the range `[0, 180]`. :param scale_min: The minimum scaling applied to random patches. The value should be in the range `[0, 1]`, From 911884bf6e909106cb9b22e8ba03220fa3d6f4dd Mon Sep 17 00:00:00 2001 From: Beat Buesser Date: Thu, 16 Jan 2025 14:06:26 +0100 Subject: [PATCH 3/5] Fix mypy warning on typing Signed-off-by: Beat Buesser --- .../evasion/adversarial_patch/adversarial_patch_pytorch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py index 1a88f4a875..55d45aa508 100644 --- a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py +++ b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py @@ -513,7 +513,9 @@ def generate( # type: ignore logger.info("Setting labels to estimator classification predictions.") y = to_categorical(np.argmax(self.estimator.predict(x=x), axis=1), nb_classes=self.estimator.nb_classes) - y = check_and_transform_label_format(labels=y, nb_classes=self.estimator.nb_classes) + y_array: np.ndarray = y + + y = check_and_transform_label_format(labels=y_array, nb_classes=self.estimator.nb_classes) # check if logits or probabilities y_pred = self.estimator.predict(x=x[[0]]) From 33782201eea6ed7cd2a89048a9d5e99f092a73da Mon Sep 17 00:00:00 2001 From: Beat Buesser Date: Mon, 20 Jan 2025 11:19:26 +0100 Subject: [PATCH 4/5] Update docstring and fix order of function calls Signed-off-by: Beat Buesser --- .../adversarial_patch/adversarial_patch_pytorch.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py index 55d45aa508..78b54edf22 100644 --- a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py +++ b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py @@ -489,7 +489,11 @@ def generate( # type: ignore Generate an adversarial patch and return the patch and its mask in arrays. :param x: An array with the original input images of shape NCHW or input videos of shape NFCHW. - :param y: The true or target labels. + :param y: True or target labels of format `list[dict[str, Union[np.ndarray, torch.Tensor]]]`, one for each + input image. The fields of the dict are as follows: + + - boxes [N, 4]: the boxes in [x1, y1, x2, y2] format, with 0 <= x1 < x2 <= W and 0 <= y1 < y2 <= H. + - labels [N]: the labels for each image. :param mask: A boolean array of shape equal to the shape of a single samples (1, H, W) or the shape of `x` (N, H, W) without their channel dimensions. Any features for which the mask is True can be the center location of the patch during sampling. @@ -511,11 +515,13 @@ def generate( # type: ignore if y is None: # pragma: no cover logger.info("Setting labels to estimator classification predictions.") - y = to_categorical(np.argmax(self.estimator.predict(x=x), axis=1), nb_classes=self.estimator.nb_classes) - + y_array: np.ndarray = to_categorical( + np.argmax(self.estimator.predict(x=x), axis=1), nb_classes=self.estimator.nb_classes + ) + else: y_array: np.ndarray = y - y = check_and_transform_label_format(labels=y_array, nb_classes=self.estimator.nb_classes) + y = check_and_transform_label_format(labels=y_array, nb_classes=self.estimator.nb_classes) # check if logits or probabilities y_pred = self.estimator.predict(x=x[[0]]) From d7e9f2b8dbbd58ae125b414e8891a0bbd69a3cde Mon Sep 17 00:00:00 2001 From: Beat Buesser Date: Mon, 20 Jan 2025 15:06:46 +0100 Subject: [PATCH 5/5] Fix typing Signed-off-by: Beat Buesser --- .../adversarial_patch/adversarial_patch_pytorch.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py index 78b54edf22..737d065feb 100644 --- a/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py +++ b/art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py @@ -26,7 +26,7 @@ import logging import math from packaging.version import parse -from typing import Any, TYPE_CHECKING +from typing import Any, cast, TYPE_CHECKING import numpy as np from tqdm.auto import trange @@ -513,13 +513,15 @@ def generate( # type: ignore if hasattr(self.estimator, "nb_classes"): + y_array: np.ndarray + if y is None: # pragma: no cover logger.info("Setting labels to estimator classification predictions.") - y_array: np.ndarray = to_categorical( + y_array = to_categorical( np.argmax(self.estimator.predict(x=x), axis=1), nb_classes=self.estimator.nb_classes ) else: - y_array: np.ndarray = y + y_array = cast(np.ndarray, y) y = check_and_transform_label_format(labels=y_array, nb_classes=self.estimator.nb_classes)