Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Aggregate Metric Results #1731

Open
wants to merge 7 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions supervision/metrics/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,36 @@ def compute(self, *args, **kwargs) -> Any:
raise NotImplementedError


class MetricResult(ABC):
"""
Base class for all metric results.
"""

@abstractmethod
def to_pandas():
"""
Convert the result to a pandas DataFrame.

Returns:
(pd.DataFrame): The result as a DataFrame.
"""
raise NotImplementedError()

@abstractmethod
def plot():
"""
Plot the results.
"""
raise NotImplementedError()

@abstractmethod
def _get_plot_details():
"""
Get the metric details to be plotted.
"""
raise NotImplementedError()


class MetricTarget(Enum):
"""
Specifies what type of detection is used to compute the metric.
Expand Down
38 changes: 25 additions & 13 deletions supervision/metrics/f1_score.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
oriented_box_iou_batch,
)
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import AveragingMethod, Metric, MetricTarget
from supervision.metrics.core import AveragingMethod, Metric, MetricResult, MetricTarget
from supervision.metrics.utils.object_size import (
ObjectSizeCategory,
get_detection_size_category,
Expand Down Expand Up @@ -455,7 +455,7 @@ def _filter_predictions_and_targets_by_size(


@dataclass
class F1ScoreResult:
class F1ScoreResult(MetricResult):
"""
The results of the F1 score metric calculation.

Expand Down Expand Up @@ -583,15 +583,15 @@ def to_pandas(self) -> "pd.DataFrame":

return pd.DataFrame(pandas_data, index=[0])

def plot(self):
def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]:
"""
Plot the F1 results.
Obtain the metric details for plotting them.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/f1_plot_example.png\
){ align=center width="800" }
Returns:
Tuple[List[str], List[float], str, List[str]]: The details for plotting the
metric. It is a tuple of four elements: a list of labels, a list of
values, the title of the plot and the bar colors.
"""

labels = ["F1@50", "F1@75"]
values = [self.f1_50, self.f1_75]
colors = [LEGACY_COLOR_PALETTE[0]] * 2
Expand All @@ -614,16 +614,28 @@ def plot(self):
values += [large_objects.f1_50, large_objects.f1_75]
colors += [LEGACY_COLOR_PALETTE[4]] * 2

plt.rcParams["font.family"] = "monospace"

_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
title = (
f"F1 Score, by Object Size"
f"\n(target: {self.metric_target.value},"
f" averaging: {self.averaging_method.value})"
)
return labels, values, title, colors

def plot(self):
"""
Plot the F1 results.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/f1_plot_example.png\
){ align=center width="800" }
"""
labels, values, title, colors = self._get_plot_details()

plt.rcParams["font.family"] = "monospace"

_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
ax.set_title(title, fontweight="bold")

x_positions = range(len(labels))
Expand Down
32 changes: 23 additions & 9 deletions supervision/metrics/mean_average_precision.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
oriented_box_iou_batch,
)
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import Metric, MetricTarget
from supervision.metrics.core import Metric, MetricResult, MetricTarget
from supervision.metrics.utils.object_size import (
ObjectSizeCategory,
get_detection_size_category,
Expand Down Expand Up @@ -418,7 +418,7 @@ def _filter_detections_by_size(


@dataclass
class MeanAveragePrecisionResult:
class MeanAveragePrecisionResult(MetricResult):
"""
The result of the Mean Average Precision calculation.

Expand Down Expand Up @@ -559,18 +559,19 @@ def to_pandas(self) -> "pd.DataFrame":
index=[0],
)

def plot(self):
def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]:
"""
Plot the mAP results.
Obtain the metric details for plotting them.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/mAP_plot_example.png\
){ align=center width="800" }
Returns:
Tuple[List[str], List[float], str, List[str]]: The details for plotting the
metric. It is a tuple of four elements: a list of labels, a list of
values, the title of the plot and the bar colors.
"""

labels = ["mAP@50:95", "mAP@50", "mAP@75"]
values = [self.map50_95, self.map50, self.map75]
colors = [LEGACY_COLOR_PALETTE[0]] * 3
title = "Mean Average Precision"

if self.small_objects is not None:
labels += ["Small: mAP@50:95", "Small: mAP@50", "Small: mAP@75"]
Expand Down Expand Up @@ -599,12 +600,25 @@ def plot(self):
]
colors += [LEGACY_COLOR_PALETTE[4]] * 3

return labels, values, title, colors

def plot(self):
"""
Plot the mAP results.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/mAP_plot_example.png\
){ align=center width="800" }
"""

labels, values, title, colors = self._get_plot_details()

plt.rcParams["font.family"] = "monospace"

_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
ax.set_title("Mean Average Precision", fontweight="bold")
ax.set_title(title, fontweight="bold")

x_positions = range(len(labels))
bars = ax.bar(x_positions, values, color=colors, align="center")
Expand Down
36 changes: 25 additions & 11 deletions supervision/metrics/mean_average_recall.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
oriented_box_iou_batch,
)
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import Metric, MetricTarget
from supervision.metrics.core import Metric, MetricResult, MetricTarget
from supervision.metrics.utils.object_size import (
ObjectSizeCategory,
get_detection_size_category,
Expand Down Expand Up @@ -460,7 +460,7 @@ def _filter_predictions_and_targets_by_size(


@dataclass
class MeanAverageRecallResult:
class MeanAverageRecallResult(MetricResult):
# """
# The results of the recall metric calculation.

Expand Down Expand Up @@ -622,13 +622,14 @@ def to_pandas(self) -> "pd.DataFrame":

return pd.DataFrame(pandas_data, index=[0])

def plot(self):
def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]:
"""
Plot the Mean Average Recall results.
Obtain the metric details for plotting them.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png\
){ align=center width="800" }
Returns:
Tuple[List[str], List[float], str, List[str]]: The details for plotting the
metric. It is a tuple of four elements: a list of labels, a list of
values, the title of the plot and the bar colors.
"""
labels = ["mAR @ 1", "mAR @ 10", "mAR @ 100"]
values = [self.mAR_at_1, self.mAR_at_10, self.mAR_at_100]
Expand Down Expand Up @@ -664,15 +665,28 @@ def plot(self):
]
colors += [LEGACY_COLOR_PALETTE[4]] * 3

title = (
f"Mean Average Recall, by Object Size"
f"\n(target: {self.metric_target.value})"
)
return labels, values, title, colors

def plot(self):
"""
Plot the Mean Average Recall results.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png\
){ align=center width="800" }
"""

labels, values, title, colors = self._get_plot_details()

plt.rcParams["font.family"] = "monospace"

_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
title = (
f"Mean Average Recall, by Object Size"
f"\n(target: {self.metric_target.value})"
)
ax.set_title(title, fontweight="bold")

x_positions = range(len(labels))
Expand Down
39 changes: 26 additions & 13 deletions supervision/metrics/precision.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
oriented_box_iou_batch,
)
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import AveragingMethod, Metric, MetricTarget
from supervision.metrics.core import AveragingMethod, Metric, MetricResult, MetricTarget
from supervision.metrics.utils.object_size import (
ObjectSizeCategory,
get_detection_size_category,
Expand Down Expand Up @@ -458,7 +458,7 @@ def _filter_predictions_and_targets_by_size(


@dataclass
class PrecisionResult:
class PrecisionResult(MetricResult):
"""
The results of the precision metric calculation.

Expand Down Expand Up @@ -588,15 +588,15 @@ def to_pandas(self) -> "pd.DataFrame":

return pd.DataFrame(pandas_data, index=[0])

def plot(self):
def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]:
"""
Plot the precision results.
Obtain the metric details for plotting them.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/precision_plot_example.png\
){ align=center width="800" }
Returns:
Tuple[List[str], List[float], str, List[str]]: The details for plotting the
metric. It is a tuple of four elements: a list of labels, a list of
values, the title of the plot and the bar colors.
"""

labels = ["Precision@50", "Precision@75"]
values = [self.precision_at_50, self.precision_at_75]
colors = [LEGACY_COLOR_PALETTE[0]] * 2
Expand All @@ -619,16 +619,29 @@ def plot(self):
values += [large_objects.precision_at_50, large_objects.precision_at_75]
colors += [LEGACY_COLOR_PALETTE[4]] * 2

plt.rcParams["font.family"] = "monospace"

_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
title = (
f"Precision, by Object Size"
f"\n(target: {self.metric_target.value},"
f" averaging: {self.averaging_method.value})"
)
return labels, values, title, colors

def plot(self):
"""
Plot the precision results.

![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/precision_plot_example.png\
){ align=center width="800" }
"""

labels, values, title, colors = self._get_plot_details()

plt.rcParams["font.family"] = "monospace"

_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
ax.set_title(title, fontweight="bold")

x_positions = range(len(labels))
Expand Down
Loading
Loading