coco
627 lines
19 KiB
C++
627 lines
19 KiB
C++
// Copyright (C) 2011 Davis E. King (davis@dlib.net)
|
|
// License: Boost Software License See LICENSE.txt for the full license.
|
|
#ifndef DLIB_OBJECT_DeTECTOR_Hh_
|
|
#define DLIB_OBJECT_DeTECTOR_Hh_
|
|
|
|
#include "object_detector_abstract.h"
|
|
#include "../geometry.h"
|
|
#include <vector>
|
|
#include "box_overlap_testing.h"
|
|
#include "full_object_detection.h"
|
|
|
|
namespace dlib
|
|
{
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
struct rect_detection
|
|
{
|
|
double detection_confidence;
|
|
unsigned long weight_index;
|
|
rectangle rect;
|
|
|
|
bool operator<(const rect_detection& item) const { return detection_confidence < item.detection_confidence; }
|
|
};
|
|
|
|
struct full_detection
|
|
{
|
|
double detection_confidence;
|
|
unsigned long weight_index;
|
|
full_object_detection rect;
|
|
|
|
bool operator<(const full_detection& item) const { return detection_confidence < item.detection_confidence; }
|
|
};
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <typename image_scanner_type>
|
|
struct processed_weight_vector
|
|
{
|
|
processed_weight_vector(){}
|
|
|
|
typedef typename image_scanner_type::feature_vector_type feature_vector_type;
|
|
|
|
void init (
|
|
const image_scanner_type&
|
|
)
|
|
/*!
|
|
requires
|
|
- w has already been assigned its value. Note that the point of this
|
|
function is to allow an image scanner to overload the
|
|
processed_weight_vector template and provide some different kind of
|
|
object as the output of get_detect_argument(). For example, the
|
|
scan_fhog_pyramid object uses an overload that causes
|
|
get_detect_argument() to return the special fhog_filterbank object
|
|
instead of a feature_vector_type. This avoids needing to construct the
|
|
fhog_filterbank during each call to detect and therefore speeds up
|
|
detection.
|
|
!*/
|
|
{}
|
|
|
|
// return the first argument to image_scanner_type::detect()
|
|
const feature_vector_type& get_detect_argument() const { return w; }
|
|
|
|
feature_vector_type w;
|
|
};
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type_
|
|
>
|
|
class object_detector
|
|
{
|
|
public:
|
|
typedef image_scanner_type_ image_scanner_type;
|
|
typedef typename image_scanner_type::feature_vector_type feature_vector_type;
|
|
|
|
object_detector (
|
|
);
|
|
|
|
object_detector (
|
|
const object_detector& item
|
|
);
|
|
|
|
object_detector (
|
|
const image_scanner_type& scanner_,
|
|
const test_box_overlap& overlap_tester_,
|
|
const feature_vector_type& w_
|
|
);
|
|
|
|
object_detector (
|
|
const image_scanner_type& scanner_,
|
|
const test_box_overlap& overlap_tester_,
|
|
const std::vector<feature_vector_type>& w_
|
|
);
|
|
|
|
explicit object_detector (
|
|
const std::vector<object_detector>& detectors
|
|
);
|
|
|
|
unsigned long num_detectors (
|
|
) const { return w.size(); }
|
|
|
|
const feature_vector_type& get_w (
|
|
unsigned long idx = 0
|
|
) const { return w[idx].w; }
|
|
|
|
const processed_weight_vector<image_scanner_type>& get_processed_w (
|
|
unsigned long idx = 0
|
|
) const { return w[idx]; }
|
|
|
|
const test_box_overlap& get_overlap_tester (
|
|
) const;
|
|
|
|
const image_scanner_type& get_scanner (
|
|
) const;
|
|
|
|
object_detector& operator= (
|
|
const object_detector& item
|
|
);
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
std::vector<rectangle> operator() (
|
|
const image_type& img,
|
|
double adjust_threshold = 0
|
|
);
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void operator() (
|
|
const image_type& img,
|
|
std::vector<std::pair<double, rectangle> >& final_dets,
|
|
double adjust_threshold = 0
|
|
);
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void operator() (
|
|
const image_type& img,
|
|
std::vector<std::pair<double, full_object_detection> >& final_dets,
|
|
double adjust_threshold = 0
|
|
);
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void operator() (
|
|
const image_type& img,
|
|
std::vector<full_object_detection>& final_dets,
|
|
double adjust_threshold = 0
|
|
);
|
|
|
|
// These typedefs are here for backwards compatibility with previous versions of
|
|
// dlib.
|
|
typedef ::dlib::rect_detection rect_detection;
|
|
typedef ::dlib::full_detection full_detection;
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void operator() (
|
|
const image_type& img,
|
|
std::vector<rect_detection>& final_dets,
|
|
double adjust_threshold = 0
|
|
);
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void operator() (
|
|
const image_type& img,
|
|
std::vector<full_detection>& final_dets,
|
|
double adjust_threshold = 0
|
|
);
|
|
|
|
template <typename T>
|
|
friend void serialize (
|
|
const object_detector<T>& item,
|
|
std::ostream& out
|
|
);
|
|
|
|
template <typename T>
|
|
friend void deserialize (
|
|
object_detector<T>& item,
|
|
std::istream& in
|
|
);
|
|
|
|
private:
|
|
|
|
bool overlaps_any_box (
|
|
const std::vector<rect_detection>& rects,
|
|
const dlib::rectangle& rect
|
|
) const
|
|
{
|
|
for (unsigned long i = 0; i < rects.size(); ++i)
|
|
{
|
|
if (boxes_overlap(rects[i].rect, rect))
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
test_box_overlap boxes_overlap;
|
|
std::vector<processed_weight_vector<image_scanner_type> > w;
|
|
image_scanner_type scanner;
|
|
};
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <typename T>
|
|
void serialize (
|
|
const object_detector<T>& item,
|
|
std::ostream& out
|
|
)
|
|
{
|
|
int version = 2;
|
|
serialize(version, out);
|
|
|
|
T scanner;
|
|
scanner.copy_configuration(item.scanner);
|
|
serialize(scanner, out);
|
|
serialize(item.boxes_overlap, out);
|
|
// serialize all the weight vectors
|
|
serialize(item.w.size(), out);
|
|
for (unsigned long i = 0; i < item.w.size(); ++i)
|
|
serialize(item.w[i].w, out);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <typename T>
|
|
void deserialize (
|
|
object_detector<T>& item,
|
|
std::istream& in
|
|
)
|
|
{
|
|
int version = 0;
|
|
deserialize(version, in);
|
|
if (version == 1)
|
|
{
|
|
deserialize(item.scanner, in);
|
|
item.w.resize(1);
|
|
deserialize(item.w[0].w, in);
|
|
item.w[0].init(item.scanner);
|
|
deserialize(item.boxes_overlap, in);
|
|
}
|
|
else if (version == 2)
|
|
{
|
|
deserialize(item.scanner, in);
|
|
deserialize(item.boxes_overlap, in);
|
|
unsigned long num_detectors = 0;
|
|
deserialize(num_detectors, in);
|
|
item.w.resize(num_detectors);
|
|
for (unsigned long i = 0; i < item.w.size(); ++i)
|
|
{
|
|
deserialize(item.w[i].w, in);
|
|
item.w[i].init(item.scanner);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
throw serialization_error("Unexpected version encountered while deserializing a dlib::object_detector object.");
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------------------
|
|
// object_detector member functions
|
|
// ----------------------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
object_detector<image_scanner_type>::
|
|
object_detector (
|
|
)
|
|
{
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
object_detector<image_scanner_type>::
|
|
object_detector (
|
|
const object_detector& item
|
|
)
|
|
{
|
|
boxes_overlap = item.boxes_overlap;
|
|
w = item.w;
|
|
scanner.copy_configuration(item.scanner);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
object_detector<image_scanner_type>::
|
|
object_detector (
|
|
const image_scanner_type& scanner_,
|
|
const test_box_overlap& overlap_tester,
|
|
const feature_vector_type& w_
|
|
) :
|
|
boxes_overlap(overlap_tester)
|
|
{
|
|
// make sure requires clause is not broken
|
|
DLIB_ASSERT(scanner_.get_num_detection_templates() > 0 &&
|
|
w_.size() == scanner_.get_num_dimensions() + 1,
|
|
"\t object_detector::object_detector(scanner_,overlap_tester,w_)"
|
|
<< "\n\t Invalid inputs were given to this function "
|
|
<< "\n\t scanner_.get_num_detection_templates(): " << scanner_.get_num_detection_templates()
|
|
<< "\n\t w_.size(): " << w_.size()
|
|
<< "\n\t scanner_.get_num_dimensions(): " << scanner_.get_num_dimensions()
|
|
<< "\n\t this: " << this
|
|
);
|
|
|
|
scanner.copy_configuration(scanner_);
|
|
w.resize(1);
|
|
w[0].w = w_;
|
|
w[0].init(scanner);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
object_detector<image_scanner_type>::
|
|
object_detector (
|
|
const image_scanner_type& scanner_,
|
|
const test_box_overlap& overlap_tester,
|
|
const std::vector<feature_vector_type>& w_
|
|
) :
|
|
boxes_overlap(overlap_tester)
|
|
{
|
|
// make sure requires clause is not broken
|
|
DLIB_CASSERT(scanner_.get_num_detection_templates() > 0 && w_.size() > 0,
|
|
"\t object_detector::object_detector(scanner_,overlap_tester,w_)"
|
|
<< "\n\t Invalid inputs were given to this function "
|
|
<< "\n\t scanner_.get_num_detection_templates(): " << scanner_.get_num_detection_templates()
|
|
<< "\n\t w_.size(): " << w_.size()
|
|
<< "\n\t this: " << this
|
|
);
|
|
|
|
for (unsigned long i = 0; i < w_.size(); ++i)
|
|
{
|
|
DLIB_CASSERT(w_[i].size() == scanner_.get_num_dimensions() + 1,
|
|
"\t object_detector::object_detector(scanner_,overlap_tester,w_)"
|
|
<< "\n\t Invalid inputs were given to this function "
|
|
<< "\n\t scanner_.get_num_detection_templates(): " << scanner_.get_num_detection_templates()
|
|
<< "\n\t w_["<<i<<"].size(): " << w_[i].size()
|
|
<< "\n\t scanner_.get_num_dimensions(): " << scanner_.get_num_dimensions()
|
|
<< "\n\t this: " << this
|
|
);
|
|
}
|
|
|
|
scanner.copy_configuration(scanner_);
|
|
w.resize(w_.size());
|
|
for (unsigned long i = 0; i < w.size(); ++i)
|
|
{
|
|
w[i].w = w_[i];
|
|
w[i].init(scanner);
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
object_detector<image_scanner_type>::
|
|
object_detector (
|
|
const std::vector<object_detector>& detectors
|
|
)
|
|
{
|
|
DLIB_CASSERT(detectors.size() != 0,
|
|
"\t object_detector::object_detector(detectors)"
|
|
<< "\n\t Invalid inputs were given to this function "
|
|
<< "\n\t this: " << this
|
|
);
|
|
std::vector<feature_vector_type> weights;
|
|
weights.reserve(detectors.size());
|
|
for (unsigned long i = 0; i < detectors.size(); ++i)
|
|
{
|
|
for (unsigned long j = 0; j < detectors[i].num_detectors(); ++j)
|
|
weights.push_back(detectors[i].get_w(j));
|
|
}
|
|
|
|
*this = object_detector(detectors[0].get_scanner(), detectors[0].get_overlap_tester(), weights);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
object_detector<image_scanner_type>& object_detector<image_scanner_type>::
|
|
operator= (
|
|
const object_detector& item
|
|
)
|
|
{
|
|
if (this == &item)
|
|
return *this;
|
|
|
|
boxes_overlap = item.boxes_overlap;
|
|
w = item.w;
|
|
scanner.copy_configuration(item.scanner);
|
|
return *this;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
template <
|
|
typename image_type
|
|
>
|
|
void object_detector<image_scanner_type>::
|
|
operator() (
|
|
const image_type& img,
|
|
std::vector<rect_detection>& final_dets,
|
|
double adjust_threshold
|
|
)
|
|
{
|
|
scanner.load(img);
|
|
std::vector<std::pair<double, rectangle> > dets;
|
|
std::vector<rect_detection> dets_accum;
|
|
for (unsigned long i = 0; i < w.size(); ++i)
|
|
{
|
|
const double thresh = w[i].w(scanner.get_num_dimensions());
|
|
scanner.detect(w[i].get_detect_argument(), dets, thresh + adjust_threshold);
|
|
for (unsigned long j = 0; j < dets.size(); ++j)
|
|
{
|
|
rect_detection temp;
|
|
temp.detection_confidence = dets[j].first-thresh;
|
|
temp.weight_index = i;
|
|
temp.rect = dets[j].second;
|
|
dets_accum.push_back(temp);
|
|
}
|
|
}
|
|
|
|
// Do non-max suppression
|
|
final_dets.clear();
|
|
if (w.size() > 1)
|
|
std::sort(dets_accum.rbegin(), dets_accum.rend());
|
|
for (unsigned long i = 0; i < dets_accum.size(); ++i)
|
|
{
|
|
if (overlaps_any_box(final_dets, dets_accum[i].rect))
|
|
continue;
|
|
|
|
final_dets.push_back(dets_accum[i]);
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
template <
|
|
typename image_type
|
|
>
|
|
void object_detector<image_scanner_type>::
|
|
operator() (
|
|
const image_type& img,
|
|
std::vector<full_detection>& final_dets,
|
|
double adjust_threshold
|
|
)
|
|
{
|
|
std::vector<rect_detection> dets;
|
|
(*this)(img,dets,adjust_threshold);
|
|
|
|
final_dets.resize(dets.size());
|
|
|
|
// convert all the rectangle detections into full_object_detections.
|
|
for (unsigned long i = 0; i < dets.size(); ++i)
|
|
{
|
|
final_dets[i].detection_confidence = dets[i].detection_confidence;
|
|
final_dets[i].weight_index = dets[i].weight_index;
|
|
final_dets[i].rect = scanner.get_full_object_detection(dets[i].rect, w[dets[i].weight_index].w);
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
template <
|
|
typename image_type
|
|
>
|
|
std::vector<rectangle> object_detector<image_scanner_type>::
|
|
operator() (
|
|
const image_type& img,
|
|
double adjust_threshold
|
|
)
|
|
{
|
|
std::vector<rect_detection> dets;
|
|
(*this)(img,dets,adjust_threshold);
|
|
|
|
std::vector<rectangle> final_dets(dets.size());
|
|
for (unsigned long i = 0; i < dets.size(); ++i)
|
|
final_dets[i] = dets[i].rect;
|
|
|
|
return final_dets;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
template <
|
|
typename image_type
|
|
>
|
|
void object_detector<image_scanner_type>::
|
|
operator() (
|
|
const image_type& img,
|
|
std::vector<std::pair<double, rectangle> >& final_dets,
|
|
double adjust_threshold
|
|
)
|
|
{
|
|
std::vector<rect_detection> dets;
|
|
(*this)(img,dets,adjust_threshold);
|
|
|
|
final_dets.resize(dets.size());
|
|
for (unsigned long i = 0; i < dets.size(); ++i)
|
|
final_dets[i] = std::make_pair(dets[i].detection_confidence,dets[i].rect);
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
template <
|
|
typename image_type
|
|
>
|
|
void object_detector<image_scanner_type>::
|
|
operator() (
|
|
const image_type& img,
|
|
std::vector<std::pair<double, full_object_detection> >& final_dets,
|
|
double adjust_threshold
|
|
)
|
|
{
|
|
std::vector<rect_detection> dets;
|
|
(*this)(img,dets,adjust_threshold);
|
|
|
|
final_dets.clear();
|
|
final_dets.reserve(dets.size());
|
|
|
|
// convert all the rectangle detections into full_object_detections.
|
|
for (unsigned long i = 0; i < dets.size(); ++i)
|
|
{
|
|
final_dets.push_back(std::make_pair(dets[i].detection_confidence,
|
|
scanner.get_full_object_detection(dets[i].rect, w[dets[i].weight_index].w)));
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
template <
|
|
typename image_type
|
|
>
|
|
void object_detector<image_scanner_type>::
|
|
operator() (
|
|
const image_type& img,
|
|
std::vector<full_object_detection>& final_dets,
|
|
double adjust_threshold
|
|
)
|
|
{
|
|
std::vector<rect_detection> dets;
|
|
(*this)(img,dets,adjust_threshold);
|
|
|
|
final_dets.clear();
|
|
final_dets.reserve(dets.size());
|
|
|
|
// convert all the rectangle detections into full_object_detections.
|
|
for (unsigned long i = 0; i < dets.size(); ++i)
|
|
{
|
|
final_dets.push_back(scanner.get_full_object_detection(dets[i].rect, w[dets[i].weight_index].w));
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
const test_box_overlap& object_detector<image_scanner_type>::
|
|
get_overlap_tester (
|
|
) const
|
|
{
|
|
return boxes_overlap;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename image_scanner_type
|
|
>
|
|
const image_scanner_type& object_detector<image_scanner_type>::
|
|
get_scanner (
|
|
) const
|
|
{
|
|
return scanner;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
}
|
|
|
|
#endif // DLIB_OBJECT_DeTECTOR_Hh_
|
|
|
|
|