import math
import hypothesis
import numpy as np
from hypothesis import strategies as st
from kitcar_ml.utils.bounding_box import BoundingBox
from kitcar_ml.utils.evaluation.interpolation_evaluator import InterpolationEvaluator
from kitcar_ml.utils.evaluation.test.basic_utility import create_bounding_boxes
[docs]@hypothesis.settings(deadline=1500)
@hypothesis.given(
accuracy=st.floats(min_value=0.1, max_value=1),
iou_threshold=st.floats(min_value=0.1, max_value=0.9),
num_classes=st.integers(min_value=1, max_value=5),
)
def test_m_ap(accuracy, iou_threshold, num_classes):
print(f"Test m_ap {locals()}")
groundtruth, detections, accuracy = create_bounding_boxes(
accuracy=accuracy, num_images=10, num_classes=num_classes, max_boxes=300
)
positive_vector = [det.confidence == 1 for dets in detections for det in dets]
groundtruth_m_ap = (
0 if len(positive_vector) == 0 else sum(positive_vector) / len(positive_vector)
)
interpolation_evaluator = InterpolationEvaluator((iou_threshold,))
interpolation_evaluator(groundtruth, detections, [str(x) for x in range(num_classes)])
m_ap = interpolation_evaluator.m_ap[iou_threshold]
assert math.isclose(
m_ap, groundtruth_m_ap
), f"mAP should be calculated to {accuracy} but was {m_ap}"
iou = 0.5
interpolation_evaluator = InterpolationEvaluator((iou,))
interpolation_evaluator(groundtruth, detections, "all")
m_ap = interpolation_evaluator.m_ap[iou]
assert math.isclose(
m_ap, accuracy
), f"mAP should be calculated to {accuracy} but was {m_ap}"
precision = interpolation_evaluator.results[iou]["all"].precision
recall = interpolation_evaluator.results[iou]["all"].recall
assert not [r for r in recall if r > accuracy]
accuracy_recall = np.where(recall == accuracy)[0]
assert not len(accuracy_recall) == 0
assert np.logical_and.reduce(precision[: accuracy_recall[0]] == 1)
[docs]def test_edge_cases(evaluator: InterpolationEvaluator):
print("Test Edge Cases")
b1 = BoundingBox(0, 0, 1, 1, "test class 1", 1)
# No Detections
gts = [[b1], []]
dets = [[], []]
evaluator(gts, dets, "all")
assert evaluator.m_ap[0.5] == 0
gts = [[], []]
dets = [[], []]
evaluator(gts, dets, "all")
assert evaluator.m_ap[0.5] == 1
# Wrong detections
gts = [[b1], []]
dets = [[], [b1]]
evaluator(gts, dets, "all")
assert evaluator.m_ap[0.5] == 0
gts = [[]]
dets = [[b1]]
evaluator(gts, dets, "all")
assert evaluator.m_ap[0.5] == 0
[docs]def main():
print("Test Interpolation Evaluator")
test_m_ap()
test_edge_cases(InterpolationEvaluator(iou_thresholds=(0.5,)))
test_edge_cases(
InterpolationEvaluator(iou_thresholds=(0.5,), use_every_point_interpolation=False)
)
if __name__ == "__main__":
main()