research/object_detection/utils/vrd_evaluation.py
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Evaluator class for Visual Relations Detection.
VRDDetectionEvaluator is a class which manages ground truth information of a
visual relations detection (vrd) dataset, and computes frequently used detection
metrics such as Precision, Recall, Recall@k, of the provided vrd detection
results.
It supports the following operations:
1) Adding ground truth information of images sequentially.
2) Adding detection results of images sequentially.
3) Evaluating detection metrics on already inserted detection results.
Note1: groundtruth should be inserted before evaluation.
Note2: This module operates on numpy boxes and box lists.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from abc import abstractmethod
import collections
import logging
import numpy as np
import six
from six.moves import range
from object_detection.core import standard_fields
from object_detection.utils import metrics
from object_detection.utils import object_detection_evaluation
from object_detection.utils import per_image_vrd_evaluation
# Below standard input numpy datatypes are defined:
# box_data_type - datatype of the groundtruth visual relations box annotations;
# this datatype consists of two named boxes: subject bounding box and object
# bounding box. Each box is of the format [y_min, x_min, y_max, x_max], each
# coordinate being of type float32.
# label_data_type - corresponding datatype of the visual relations label
# annotaions; it consists of three numerical class labels: subject class label,
# object class label and relation class label, each class label being of type
# int32.
vrd_box_data_type = np.dtype([('subject', 'f4', (4,)), ('object', 'f4', (4,))])
single_box_data_type = np.dtype([('box', 'f4', (4,))])
label_data_type = np.dtype([('subject', 'i4'), ('object', 'i4'), ('relation',
'i4')])
class VRDDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
"""A class to evaluate VRD detections.
This class serves as a base class for VRD evaluation in two settings:
- phrase detection
- relation detection.
"""
def __init__(self, matching_iou_threshold=0.5, metric_prefix=None):
"""Constructor.
Args:
matching_iou_threshold: IOU threshold to use for matching groundtruth
boxes to detection boxes.
metric_prefix: (optional) string prefix for metric name; if None, no
prefix is used.
"""
super(VRDDetectionEvaluator, self).__init__([])
self._matching_iou_threshold = matching_iou_threshold
self._evaluation = _VRDDetectionEvaluation(
matching_iou_threshold=self._matching_iou_threshold)
self._image_ids = set([])
self._metric_prefix = (metric_prefix + '_') if metric_prefix else ''
self._evaluatable_labels = {}
self._negative_labels = {}
@abstractmethod
def _process_groundtruth_boxes(self, groundtruth_box_tuples):
"""Pre-processes boxes before adding them to the VRDDetectionEvaluation.
Phrase detection and Relation detection subclasses re-implement this method
depending on the task.
Args:
groundtruth_box_tuples: A numpy array of structures with the shape
[M, 1], each structure containing the same number of named bounding
boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see
datatype vrd_box_data_type, single_box_data_type above).
"""
raise NotImplementedError(
'_process_groundtruth_boxes method should be implemented in subclasses'
'of VRDDetectionEvaluator.')
@abstractmethod
def _process_detection_boxes(self, detections_box_tuples):
"""Pre-processes boxes before adding them to the VRDDetectionEvaluation.
Phrase detection and Relation detection subclasses re-implement this method
depending on the task.
Args:
detections_box_tuples: A numpy array of structures with the shape
[M, 1], each structure containing the same number of named bounding
boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see
datatype vrd_box_data_type, single_box_data_type above).
"""
raise NotImplementedError(
'_process_detection_boxes method should be implemented in subclasses'
'of VRDDetectionEvaluator.')
def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
"""Adds groundtruth for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
groundtruth_dict: A dictionary containing -
standard_fields.InputDataFields.groundtruth_boxes: A numpy array
of structures with the shape [M, 1], representing M tuples, each tuple
containing the same number of named bounding boxes.
Each box is of the format [y_min, x_min, y_max, x_max] (see
datatype vrd_box_data_type, single_box_data_type above).
standard_fields.InputDataFields.groundtruth_classes: A numpy array of
structures shape [M, 1], representing the class labels of the
corresponding bounding boxes and possibly additional classes (see
datatype label_data_type above).
standard_fields.InputDataFields.groundtruth_image_classes: numpy array
of shape [K] containing verified labels.
Raises:
ValueError: On adding groundtruth for an image more than once.
"""
if image_id in self._image_ids:
raise ValueError('Image with id {} already added.'.format(image_id))
groundtruth_class_tuples = (
groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes])
groundtruth_box_tuples = (
groundtruth_dict[standard_fields.InputDataFields.groundtruth_boxes])
self._evaluation.add_single_ground_truth_image_info(
image_key=image_id,
groundtruth_box_tuples=self._process_groundtruth_boxes(
groundtruth_box_tuples),
groundtruth_class_tuples=groundtruth_class_tuples)
self._image_ids.update([image_id])
all_classes = []
for field in groundtruth_box_tuples.dtype.fields:
all_classes.append(groundtruth_class_tuples[field])
groudtruth_positive_classes = np.unique(np.concatenate(all_classes))
verified_labels = groundtruth_dict.get(
standard_fields.InputDataFields.groundtruth_image_classes,
np.array([], dtype=int))
self._evaluatable_labels[image_id] = np.unique(
np.concatenate((verified_labels, groudtruth_positive_classes)))
self._negative_labels[image_id] = np.setdiff1d(verified_labels,
groudtruth_positive_classes)
def add_single_detected_image_info(self, image_id, detections_dict):
"""Adds detections for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
detections_dict: A dictionary containing -
standard_fields.DetectionResultFields.detection_boxes: A numpy array of
structures with shape [N, 1], representing N tuples, each tuple
containing the same number of named bounding boxes.
Each box is of the format [y_min, x_min, y_max, x_max] (as an example
see datatype vrd_box_data_type, single_box_data_type above).
standard_fields.DetectionResultFields.detection_scores: float32 numpy
array of shape [N] containing detection scores for the boxes.
standard_fields.DetectionResultFields.detection_classes: A numpy array
of structures shape [N, 1], representing the class labels of the
corresponding bounding boxes and possibly additional classes (see
datatype label_data_type above).
"""
if image_id not in self._image_ids:
logging.warning('No groundtruth for the image with id %s.', image_id)
# Since for the correct work of evaluator it is assumed that groundtruth
# is inserted first we make sure to break the code if is it not the case.
self._image_ids.update([image_id])
self._negative_labels[image_id] = np.array([])
self._evaluatable_labels[image_id] = np.array([])
num_detections = detections_dict[
standard_fields.DetectionResultFields.detection_boxes].shape[0]
detection_class_tuples = detections_dict[
standard_fields.DetectionResultFields.detection_classes]
detection_box_tuples = detections_dict[
standard_fields.DetectionResultFields.detection_boxes]
negative_selector = np.zeros(num_detections, dtype=bool)
selector = np.ones(num_detections, dtype=bool)
# Only check boxable labels
for field in detection_box_tuples.dtype.fields:
# Verify if one of the labels is negative (this is sure FP)
negative_selector |= np.isin(detection_class_tuples[field],
self._negative_labels[image_id])
# Verify if all labels are verified
selector &= np.isin(detection_class_tuples[field],
self._evaluatable_labels[image_id])
selector |= negative_selector
self._evaluation.add_single_detected_image_info(
image_key=image_id,
detected_box_tuples=self._process_detection_boxes(
detection_box_tuples[selector]),
detected_scores=detections_dict[
standard_fields.DetectionResultFields.detection_scores][selector],
detected_class_tuples=detection_class_tuples[selector])
def evaluate(self, relationships=None):
"""Compute evaluation result.
Args:
relationships: A dictionary of numerical label-text label mapping; if
specified, returns per-relationship AP.
Returns:
A dictionary of metrics with the following fields -
summary_metrics:
'weightedAP@<matching_iou_threshold>IOU' : weighted average precision
at the specified IOU threshold.
'AP@<matching_iou_threshold>IOU/<relationship>' : AP per relationship.
'mAP@<matching_iou_threshold>IOU': mean average precision at the
specified IOU threshold.
'Recall@50@<matching_iou_threshold>IOU': recall@50 at the specified IOU
threshold.
'Recall@100@<matching_iou_threshold>IOU': recall@100 at the specified
IOU threshold.
if relationships is specified, returns <relationship> in AP metrics as
readable names, otherwise the names correspond to class numbers.
"""
(weighted_average_precision, mean_average_precision, average_precisions, _,
_, recall_50, recall_100, _, _) = (
self._evaluation.evaluate())
vrd_metrics = {
(self._metric_prefix + 'weightedAP@{}IOU'.format(
self._matching_iou_threshold)):
weighted_average_precision,
self._metric_prefix + 'mAP@{}IOU'.format(self._matching_iou_threshold):
mean_average_precision,
self._metric_prefix + 'Recall@50@{}IOU'.format(
self._matching_iou_threshold):
recall_50,
self._metric_prefix + 'Recall@100@{}IOU'.format(
self._matching_iou_threshold):
recall_100,
}
if relationships:
for key, average_precision in six.iteritems(average_precisions):
vrd_metrics[self._metric_prefix + 'AP@{}IOU/{}'.format(
self._matching_iou_threshold,
relationships[key])] = average_precision
else:
for key, average_precision in six.iteritems(average_precisions):
vrd_metrics[self._metric_prefix + 'AP@{}IOU/{}'.format(
self._matching_iou_threshold, key)] = average_precision
return vrd_metrics
def clear(self):
"""Clears the state to prepare for a fresh evaluation."""
self._evaluation = _VRDDetectionEvaluation(
matching_iou_threshold=self._matching_iou_threshold)
self._image_ids.clear()
self._negative_labels.clear()
self._evaluatable_labels.clear()
class VRDRelationDetectionEvaluator(VRDDetectionEvaluator):
"""A class to evaluate VRD detections in relations setting.
Expected groundtruth box datatype is vrd_box_data_type, expected groudtruth
labels datatype is label_data_type.
Expected detection box datatype is vrd_box_data_type, expected detection
labels
datatype is label_data_type.
"""
def __init__(self, matching_iou_threshold=0.5):
super(VRDRelationDetectionEvaluator, self).__init__(
matching_iou_threshold=matching_iou_threshold,
metric_prefix='VRDMetric_Relationships')
def _process_groundtruth_boxes(self, groundtruth_box_tuples):
"""Pre-processes boxes before adding them to the VRDDetectionEvaluation.
Args:
groundtruth_box_tuples: A numpy array of structures with the shape
[M, 1], each structure containing the same number of named bounding
boxes. Each box is of the format [y_min, x_min, y_max, x_max].
Returns:
Unchanged input.
"""
return groundtruth_box_tuples
def _process_detection_boxes(self, detections_box_tuples):
"""Pre-processes boxes before adding them to the VRDDetectionEvaluation.
Phrase detection and Relation detection subclasses re-implement this method
depending on the task.
Args:
detections_box_tuples: A numpy array of structures with the shape
[M, 1], each structure containing the same number of named bounding
boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see
datatype vrd_box_data_type, single_box_data_type above).
Returns:
Unchanged input.
"""
return detections_box_tuples
class VRDPhraseDetectionEvaluator(VRDDetectionEvaluator):
"""A class to evaluate VRD detections in phrase setting.
Expected groundtruth box datatype is vrd_box_data_type, expected groudtruth
labels datatype is label_data_type.
Expected detection box datatype is single_box_data_type, expected detection
labels datatype is label_data_type.
"""
def __init__(self, matching_iou_threshold=0.5):
super(VRDPhraseDetectionEvaluator, self).__init__(
matching_iou_threshold=matching_iou_threshold,
metric_prefix='VRDMetric_Phrases')
def _process_groundtruth_boxes(self, groundtruth_box_tuples):
"""Pre-processes boxes before adding them to the VRDDetectionEvaluation.
In case of phrase evaluation task, evaluation expects exactly one bounding
box containing all objects in the phrase. This bounding box is computed
as an enclosing box of all groundtruth boxes of a phrase.
Args:
groundtruth_box_tuples: A numpy array of structures with the shape
[M, 1], each structure containing the same number of named bounding
boxes. Each box is of the format [y_min, x_min, y_max, x_max]. See
vrd_box_data_type for an example of structure.
Returns:
result: A numpy array of structures with the shape [M, 1], each
structure containing exactly one named bounding box. i-th output
structure corresponds to the result of processing i-th input structure,
where the named bounding box is computed as an enclosing bounding box
of all bounding boxes of the i-th input structure.
"""
first_box_key = next(six.iterkeys(groundtruth_box_tuples.dtype.fields))
miny = groundtruth_box_tuples[first_box_key][:, 0]
minx = groundtruth_box_tuples[first_box_key][:, 1]
maxy = groundtruth_box_tuples[first_box_key][:, 2]
maxx = groundtruth_box_tuples[first_box_key][:, 3]
for fields in groundtruth_box_tuples.dtype.fields:
miny = np.minimum(groundtruth_box_tuples[fields][:, 0], miny)
minx = np.minimum(groundtruth_box_tuples[fields][:, 1], minx)
maxy = np.maximum(groundtruth_box_tuples[fields][:, 2], maxy)
maxx = np.maximum(groundtruth_box_tuples[fields][:, 3], maxx)
data_result = []
for i in range(groundtruth_box_tuples.shape[0]):
data_result.append(([miny[i], minx[i], maxy[i], maxx[i]],))
result = np.array(data_result, dtype=[('box', 'f4', (4,))])
return result
def _process_detection_boxes(self, detections_box_tuples):
"""Pre-processes boxes before adding them to the VRDDetectionEvaluation.
In case of phrase evaluation task, evaluation expects exactly one bounding
box containing all objects in the phrase. This bounding box is computed
as an enclosing box of all groundtruth boxes of a phrase.
Args:
detections_box_tuples: A numpy array of structures with the shape
[M, 1], each structure containing the same number of named bounding
boxes. Each box is of the format [y_min, x_min, y_max, x_max]. See
vrd_box_data_type for an example of this structure.
Returns:
result: A numpy array of structures with the shape [M, 1], each
structure containing exactly one named bounding box. i-th output
structure corresponds to the result of processing i-th input structure,
where the named bounding box is computed as an enclosing bounding box
of all bounding boxes of the i-th input structure.
"""
first_box_key = next(six.iterkeys(detections_box_tuples.dtype.fields))
miny = detections_box_tuples[first_box_key][:, 0]
minx = detections_box_tuples[first_box_key][:, 1]
maxy = detections_box_tuples[first_box_key][:, 2]
maxx = detections_box_tuples[first_box_key][:, 3]
for fields in detections_box_tuples.dtype.fields:
miny = np.minimum(detections_box_tuples[fields][:, 0], miny)
minx = np.minimum(detections_box_tuples[fields][:, 1], minx)
maxy = np.maximum(detections_box_tuples[fields][:, 2], maxy)
maxx = np.maximum(detections_box_tuples[fields][:, 3], maxx)
data_result = []
for i in range(detections_box_tuples.shape[0]):
data_result.append(([miny[i], minx[i], maxy[i], maxx[i]],))
result = np.array(data_result, dtype=[('box', 'f4', (4,))])
return result
VRDDetectionEvalMetrics = collections.namedtuple('VRDDetectionEvalMetrics', [
'weighted_average_precision', 'mean_average_precision',
'average_precisions', 'precisions', 'recalls', 'recall_50', 'recall_100',
'median_rank_50', 'median_rank_100'
])
class _VRDDetectionEvaluation(object):
"""Performs metric computation for the VRD task. This class is internal.
"""
def __init__(self, matching_iou_threshold=0.5):
"""Constructor.
Args:
matching_iou_threshold: IOU threshold to use for matching groundtruth
boxes to detection boxes.
"""
self._per_image_eval = per_image_vrd_evaluation.PerImageVRDEvaluation(
matching_iou_threshold=matching_iou_threshold)
self._groundtruth_box_tuples = {}
self._groundtruth_class_tuples = {}
self._num_gt_instances = 0
self._num_gt_imgs = 0
self._num_gt_instances_per_relationship = {}
self.clear_detections()
def clear_detections(self):
"""Clears detections."""
self._detection_keys = set()
self._scores = []
self._relation_field_values = []
self._tp_fp_labels = []
self._average_precisions = {}
self._precisions = []
self._recalls = []
def add_single_ground_truth_image_info(
self, image_key, groundtruth_box_tuples, groundtruth_class_tuples):
"""Adds groundtruth for a single image to be used for evaluation.
Args:
image_key: A unique string/integer identifier for the image.
groundtruth_box_tuples: A numpy array of structures with the shape
[M, 1], representing M tuples, each tuple containing the same number
of named bounding boxes.
Each box is of the format [y_min, x_min, y_max, x_max].
groundtruth_class_tuples: A numpy array of structures shape [M, 1],
representing the class labels of the corresponding bounding boxes and
possibly additional classes.
"""
if image_key in self._groundtruth_box_tuples:
logging.warning(
'image %s has already been added to the ground truth database.',
image_key)
return
self._groundtruth_box_tuples[image_key] = groundtruth_box_tuples
self._groundtruth_class_tuples[image_key] = groundtruth_class_tuples
self._update_groundtruth_statistics(groundtruth_class_tuples)
def add_single_detected_image_info(self, image_key, detected_box_tuples,
detected_scores, detected_class_tuples):
"""Adds detections for a single image to be used for evaluation.
Args:
image_key: A unique string/integer identifier for the image.
detected_box_tuples: A numpy array of structures with shape [N, 1],
representing N tuples, each tuple containing the same number of named
bounding boxes.
Each box is of the format [y_min, x_min, y_max, x_max].
detected_scores: A float numpy array of shape [N, 1], representing
the confidence scores of the detected N object instances.
detected_class_tuples: A numpy array of structures shape [N, 1],
representing the class labels of the corresponding bounding boxes and
possibly additional classes.
"""
self._detection_keys.add(image_key)
if image_key in self._groundtruth_box_tuples:
groundtruth_box_tuples = self._groundtruth_box_tuples[image_key]
groundtruth_class_tuples = self._groundtruth_class_tuples[image_key]
else:
groundtruth_box_tuples = np.empty(
shape=[0, 4], dtype=detected_box_tuples.dtype)
groundtruth_class_tuples = np.array([], dtype=detected_class_tuples.dtype)
scores, tp_fp_labels, mapping = (
self._per_image_eval.compute_detection_tp_fp(
detected_box_tuples=detected_box_tuples,
detected_scores=detected_scores,
detected_class_tuples=detected_class_tuples,
groundtruth_box_tuples=groundtruth_box_tuples,
groundtruth_class_tuples=groundtruth_class_tuples))
self._scores += [scores]
self._tp_fp_labels += [tp_fp_labels]
self._relation_field_values += [detected_class_tuples[mapping]['relation']]
def _update_groundtruth_statistics(self, groundtruth_class_tuples):
"""Updates grouth truth statistics.
Args:
groundtruth_class_tuples: A numpy array of structures shape [M, 1],
representing the class labels of the corresponding bounding boxes and
possibly additional classes.
"""
self._num_gt_instances += groundtruth_class_tuples.shape[0]
self._num_gt_imgs += 1
for relation_field_value in np.unique(groundtruth_class_tuples['relation']):
if relation_field_value not in self._num_gt_instances_per_relationship:
self._num_gt_instances_per_relationship[relation_field_value] = 0
self._num_gt_instances_per_relationship[relation_field_value] += np.sum(
groundtruth_class_tuples['relation'] == relation_field_value)
def evaluate(self):
"""Computes evaluation result.
Returns:
A named tuple with the following fields -
average_precision: a float number corresponding to average precision.
precisions: an array of precisions.
recalls: an array of recalls.
recall@50: recall computed on 50 top-scoring samples.
recall@100: recall computed on 100 top-scoring samples.
median_rank@50: median rank computed on 50 top-scoring samples.
median_rank@100: median rank computed on 100 top-scoring samples.
"""
if self._num_gt_instances == 0:
logging.warning('No ground truth instances')
if not self._scores:
scores = np.array([], dtype=float)
tp_fp_labels = np.array([], dtype=bool)
else:
scores = np.concatenate(self._scores)
tp_fp_labels = np.concatenate(self._tp_fp_labels)
relation_field_values = np.concatenate(self._relation_field_values)
for relation_field_value, _ in (six.iteritems(
self._num_gt_instances_per_relationship)):
precisions, recalls = metrics.compute_precision_recall(
scores[relation_field_values == relation_field_value],
tp_fp_labels[relation_field_values == relation_field_value],
self._num_gt_instances_per_relationship[relation_field_value])
self._average_precisions[
relation_field_value] = metrics.compute_average_precision(
precisions, recalls)
self._mean_average_precision = np.mean(
list(self._average_precisions.values()))
self._precisions, self._recalls = metrics.compute_precision_recall(
scores, tp_fp_labels, self._num_gt_instances)
self._weighted_average_precision = metrics.compute_average_precision(
self._precisions, self._recalls)
self._recall_50 = (
metrics.compute_recall_at_k(self._tp_fp_labels, self._num_gt_instances,
50))
self._median_rank_50 = (
metrics.compute_median_rank_at_k(self._tp_fp_labels, 50))
self._recall_100 = (
metrics.compute_recall_at_k(self._tp_fp_labels, self._num_gt_instances,
100))
self._median_rank_100 = (
metrics.compute_median_rank_at_k(self._tp_fp_labels, 100))
return VRDDetectionEvalMetrics(
self._weighted_average_precision, self._mean_average_precision,
self._average_precisions, self._precisions, self._recalls,
self._recall_50, self._recall_100, self._median_rank_50,
self._median_rank_100)