from abc import ABC, abstractmethod
from typing import List, Tuple, Union
import numpy as np
from kitcar_ml.utils.bounding_box import BoundingBox
[docs]class Evaluator(ABC):
@abstractmethod
def __str__(self):
pass
@abstractmethod
def __call__(
self, groundtruth: List[List[BoundingBox]], detections: List[List[BoundingBox]]
):
pass
[docs] @classmethod
def split_bbs_per_class(
cls, groundtruth: List[List[BoundingBox]], detections: List[List[BoundingBox]]
):
"""Split the bounding boxes into lists for each class. The images are all separated
into their own list.
Args:
groundtruth: The groundtruth bounding boxes.
detections: The detection bounding boxes.
Returns:
The dictionary with the bounding boxes per class and a list of classes.
"""
all_classes, gt_classes = cls.find_all_classes(groundtruth, detections)
classes_bbs = {c: {"gt": [], "det": []} for c in all_classes}
for gt_boxes, det_boxes in zip(groundtruth, detections):
for key in classes_bbs:
classes_bbs[key]["gt"].append([])
classes_bbs[key]["det"].append([])
for bb in gt_boxes:
classes_bbs[bb.class_label]["gt"][-1].append(bb)
for bb in det_boxes:
classes_bbs[bb.class_label]["det"][-1].append(bb)
return classes_bbs, gt_classes
[docs] @classmethod
def calculate_tp(
cls,
detections: List[BoundingBox],
groundtruths: List[BoundingBox],
iou_threshold: float,
) -> Tuple[Union[np.ndarray, np.ndarray], Union[np.ndarray, np.ndarray]]:
"""Iterates over all detections and create the accumulated true positive and false
positive arrays.
Args:
detections: The detections for this image.
groundtruths: The groundtruths for this image.
iou_threshold: The intersection over union threshold.
Returns:
The true positive array.
"""
found_gts = set()
max_gts = [cls.find_max_iou(groundtruths, det) for det in detections]
true_positives = []
for iou, id_match_gt in max_gts:
# The intersection over union is high enough to count as true positive
accepted = iou >= iou_threshold
true_positives.append(accepted and id_match_gt not in found_gts)
if accepted:
found_gts.add(id_match_gt)
return np.array(true_positives, dtype=bool)
[docs] @classmethod
def calculate_all_tp(
cls,
groundtruth: List[List[BoundingBox]],
detections: List[List[BoundingBox]],
iou_threshold: float,
) -> Tuple[np.ndarray, np.ndarray]:
"""Calculate the True and False positive array for a list of images.
Args:
groundtruth: A list of bounding box for each image.
detections: A list of bounding box for each image.
iou_threshold: The threshold that is needed for a true positive.
Returns:
The true_positives for all images
"""
true_positives = [
cls.calculate_tp(dets_per_image, gts_per_image, iou_threshold)
for dets_per_image, gts_per_image in zip(detections, groundtruth)
]
return np.concatenate(true_positives)
[docs] @staticmethod
def find_max_iou(
groundtruth: List[BoundingBox], detection: BoundingBox
) -> Tuple[float, int]:
"""Find the groundtruth with the maximal IOU.
Returns:
iou_max: The maximal IoU.
id_match_gt: The index of the groundtruth with the maximal IOU.
"""
if len(groundtruth) == 0:
# No groundtruth available, there should be no detection.
return -1, -1
ious = [BoundingBox.iou(gt, detection) for gt in groundtruth]
max_iou = max(ious)
index = ious.index(max_iou)
return max_iou, index
[docs] @staticmethod
def find_all_classes(
groundtruth: List[List[BoundingBox]], detections: List[List[BoundingBox]]
):
"""Calculate the set of all classes and the set of all classes contained in the
groundtruth.
Args:
groundtruth: List of bounding box per image.
detections: List of bounding box per image.
Returns:
Set for all classes and classes represented in the groundtruth.
"""
gt_classes = {gt.class_label for img_gts in groundtruth for gt in img_gts}
det_classes = {det.class_label for im_dets in detections for det in im_dets}
return det_classes.union(gt_classes), gt_classes