coco
395 lines
17 KiB
C++
395 lines
17 KiB
C++
// Copyright (C) 2013 Davis E. King (davis@dlib.net)
|
|
// License: Boost Software License See LICENSE.txt for the full license.
|
|
#undef DLIB_SCAN_IMAGE_bOXES_ABSTRACT_Hh_
|
|
#ifdef DLIB_SCAN_IMAGE_bOXES_ABSTRACT_Hh_
|
|
|
|
#include "../matrix.h"
|
|
#include "../geometry.h"
|
|
#include "../image_processing.h"
|
|
#include "../array2d.h"
|
|
#include "full_object_detection_abstract.h"
|
|
#include "../image_transforms/segment_image_abstract.h"
|
|
#include <vector>
|
|
|
|
namespace dlib
|
|
{
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
class default_box_generator
|
|
{
|
|
/*!
|
|
WHAT THIS OBJECT REPRESENTS
|
|
This is a function object that takes in an image and outputs a set of
|
|
candidate object locations. It is also the default box generator used by
|
|
the scan_image_boxes object defined below.
|
|
!*/
|
|
|
|
public:
|
|
|
|
template <typename image_type>
|
|
void operator() (
|
|
const image_type& img,
|
|
std::vector<rectangle>& rects
|
|
) const
|
|
/*!
|
|
ensures
|
|
- #rects == the set of candidate object locations which should be searched
|
|
inside img. That is, these are the rectangles which might contain
|
|
objects of interest within the given image.
|
|
!*/
|
|
{
|
|
rects.clear();
|
|
find_candidate_object_locations(img, rects);
|
|
}
|
|
};
|
|
|
|
inline void serialize (const default_box_generator&, std::ostream& ) {}
|
|
inline void deserialize( default_box_generator&, std::istream& ) {}
|
|
/*!
|
|
ensures
|
|
- provides serialization support.
|
|
!*/
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename Feature_extractor_type,
|
|
typename Box_generator = default_box_generator
|
|
>
|
|
class scan_image_boxes : noncopyable
|
|
{
|
|
/*!
|
|
REQUIREMENTS ON Feature_extractor_type
|
|
- must be an object with an interface compatible with the hashed_feature_image
|
|
object defined in dlib/image_keypoint/hashed_feature_image_abstract.h or
|
|
with the nearest_neighbor_feature_image object defined in
|
|
dlib/image_keypoint/nearest_neighbor_feature_image_abstract.h
|
|
|
|
REQUIREMENTS ON Box_generator
|
|
- must be an object with an interface compatible with the
|
|
default_box_generator object defined at the top of this file.
|
|
|
|
INITIAL VALUE
|
|
- get_num_spatial_pyramid_levels() == 3
|
|
- is_loaded_with_image() == false
|
|
|
|
WHAT THIS OBJECT REPRESENTS
|
|
This object is a tool for running a classifier over an image with the goal
|
|
of localizing each object present. The localization is in the form of the
|
|
bounding box around each object of interest.
|
|
|
|
Unlike the scan_image_pyramid object which scans a fixed sized window over
|
|
an image pyramid, the scan_image_boxes tool allows you to define your own
|
|
list of "candidate object locations" which should be evaluated. This is
|
|
simply a list of rectangle objects which might contain objects of interest.
|
|
The scan_image_boxes object will then evaluate the classifier at each of
|
|
these locations and return the subset of rectangles which appear to have
|
|
objects in them. The candidate object location generation is provided by
|
|
the Box_generator that is passed in as a template argument.
|
|
|
|
This object can also be understood as a general tool for implementing the
|
|
spatial pyramid models described in the paper:
|
|
Beyond Bags of Features: Spatial Pyramid Matching for Recognizing
|
|
Natural Scene Categories by Svetlana Lazebnik, Cordelia Schmid,
|
|
and Jean Ponce
|
|
|
|
|
|
The classifiers used by this object have three parts:
|
|
1. The underlying feature extraction provided by Feature_extractor_type
|
|
objects, which associate a vector with each location in an image.
|
|
|
|
2. A rule for extracting a feature vector from a candidate object
|
|
location. In this object we use the spatial pyramid matching method.
|
|
This means we cut an object's detection window into a set of "feature
|
|
extraction regions" and extract a bag-of-words vector from each
|
|
before finally concatenating them to form the final feature vector
|
|
representing the entire object window. The set of feature extraction
|
|
regions can be configured by the user by calling
|
|
set_num_spatial_pyramid_levels(). To be a little more precise, the
|
|
feature vector for a candidate object window is defined as follows:
|
|
- Let N denote the number of feature extraction zones.
|
|
- Let M denote the dimensionality of the vectors output by
|
|
Feature_extractor_type objects.
|
|
- Let F(i) == the M dimensional vector which is the sum of all
|
|
vectors given by our Feature_extractor_type object inside the
|
|
i-th feature extraction zone. So this is notionally a
|
|
bag-of-words vector from the i-th zone.
|
|
- Then the feature vector for an object window is an M*N
|
|
dimensional vector [F(1) F(2) F(3) ... F(N)] (i.e. it is a
|
|
concatenation of the N vectors). This feature vector can be
|
|
thought of as a collection of N bags-of-words, each bag coming
|
|
from a spatial location determined by one of the feature
|
|
extraction zones.
|
|
|
|
3. A weight vector and a threshold value. The dot product between the
|
|
weight vector and the feature vector for a candidate object location
|
|
gives the score of the location. If this score is greater than the
|
|
threshold value then the candidate object location is output as a
|
|
detection.
|
|
|
|
THREAD SAFETY
|
|
Concurrent access to an instance of this object is not safe and should be
|
|
protected by a mutex lock except for the case where you are copying the
|
|
configuration (via copy_configuration()) of a scan_image_boxes object to
|
|
many other threads. In this case, it is safe to copy the configuration of
|
|
a shared object so long as no other operations are performed on it.
|
|
!*/
|
|
|
|
public:
|
|
|
|
typedef matrix<double,0,1> feature_vector_type;
|
|
|
|
typedef Feature_extractor_type feature_extractor_type;
|
|
typedef Box_generator box_generator;
|
|
|
|
scan_image_boxes (
|
|
);
|
|
/*!
|
|
ensures
|
|
- this object is properly initialized
|
|
!*/
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void load (
|
|
const image_type& img
|
|
);
|
|
/*!
|
|
requires
|
|
- image_type must be a type with the following properties:
|
|
- image_type objects can be loaded into Feature_extractor_type
|
|
objects via Feature_extractor_type::load().
|
|
- image_type objects can be passed to the first argument of
|
|
Box_generator::operator()
|
|
ensures
|
|
- #is_loaded_with_image() == true
|
|
- This object is ready to run a classifier over img to detect object
|
|
locations. Call detect() to do this.
|
|
!*/
|
|
|
|
bool is_loaded_with_image (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns true if this object has been loaded with an image to process and
|
|
false otherwise.
|
|
!*/
|
|
|
|
const feature_extractor_type& get_feature_extractor (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns a const reference to the feature_extractor_type object used
|
|
internally for local feature extraction.
|
|
!*/
|
|
|
|
void copy_configuration(
|
|
const feature_extractor_type& fe
|
|
);
|
|
/*!
|
|
ensures
|
|
- This function performs the equivalent of
|
|
get_feature_extractor().copy_configuration(fe) (i.e. this function allows
|
|
you to configure the parameters of the underlying feature extractor used
|
|
by a scan_image_boxes object)
|
|
!*/
|
|
|
|
void copy_configuration(
|
|
const box_generator& bg
|
|
);
|
|
/*!
|
|
ensures
|
|
- #get_box_generator() == bg
|
|
(i.e. this function allows you to configure the parameters of the
|
|
underlying box generator used by a scan_image_boxes object)
|
|
!*/
|
|
|
|
const box_generator& get_box_generator (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns the box_generator used by this object to generate candidate
|
|
object locations.
|
|
!*/
|
|
|
|
void copy_configuration (
|
|
const scan_image_boxes& item
|
|
);
|
|
/*!
|
|
ensures
|
|
- Copies all the state information of item into *this, except for state
|
|
information populated by load(). More precisely, given two scan_image_boxes
|
|
objects S1 and S2, the following sequence of instructions should always
|
|
result in both of them having the exact same state:
|
|
S2.copy_configuration(S1);
|
|
S1.load(img);
|
|
S2.load(img);
|
|
!*/
|
|
|
|
long get_num_dimensions (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns the number of dimensions in the feature vector for a candidate
|
|
object location. This value is the dimensionality of the underlying
|
|
feature vectors produced by Feature_extractor_type times the number of
|
|
feature extraction regions used. Note that the number of feature
|
|
extraction regions used is a function of
|
|
get_num_spatial_pyramid_levels().
|
|
!*/
|
|
|
|
unsigned long get_num_spatial_pyramid_levels (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns the number of layers in the spatial pyramid. For example, if
|
|
this function returns 1 then it means we use a simple bag-of-words
|
|
representation over the whole object window. If it returns 2 then it
|
|
means the feature representation is the concatenation of 5 bag-of-words
|
|
vectors, one from the entire object window and 4 others from 4 different
|
|
parts of the object window. If it returns 3 then there are 1+4+16
|
|
bag-of-words vectors concatenated together in the feature representation,
|
|
and so on.
|
|
!*/
|
|
|
|
void set_num_spatial_pyramid_levels (
|
|
unsigned long levels
|
|
);
|
|
/*!
|
|
requires
|
|
- levels > 0
|
|
ensures
|
|
- #get_num_spatial_pyramid_levels() == levels
|
|
!*/
|
|
|
|
void detect (
|
|
const feature_vector_type& w,
|
|
std::vector<std::pair<double, rectangle> >& dets,
|
|
const double thresh
|
|
) const;
|
|
/*!
|
|
requires
|
|
- w.size() >= get_num_dimensions()
|
|
- is_loaded_with_image() == true
|
|
ensures
|
|
- Scans over all the candidate object locations as discussed in the WHAT
|
|
THIS OBJECT REPRESENTS section and stores all detections into #dets.
|
|
- for all valid i:
|
|
- #dets[i].second == The candidate object location which produced this
|
|
detection. This rectangle gives the location of the detection.
|
|
- #dets[i].first == The score for this detection. This value is equal
|
|
to dot(w, feature vector for this candidate object location).
|
|
- #dets[i].first >= thresh
|
|
- #dets will be sorted in descending order.
|
|
(i.e. #dets[i].first >= #dets[j].first for all i, and j>i)
|
|
- Elements of w beyond index get_num_dimensions()-1 are ignored. I.e. only
|
|
the first get_num_dimensions() are used.
|
|
- Note that no form of non-max suppression is performed. If a locations
|
|
has a score >= thresh then it is reported in #dets.
|
|
!*/
|
|
|
|
void get_feature_vector (
|
|
const full_object_detection& obj,
|
|
feature_vector_type& psi
|
|
) const;
|
|
/*!
|
|
requires
|
|
- obj.num_parts() == 0
|
|
- is_loaded_with_image() == true
|
|
- psi.size() >= get_num_dimensions()
|
|
(i.e. psi must have preallocated its memory before this function is called)
|
|
ensures
|
|
- This function allows you to determine the feature vector used for a
|
|
candidate object location output from detect(). Note that this vector is
|
|
added to psi. Note also that you must use get_full_object_detection() to
|
|
convert a rectangle from detect() into the needed full_object_detection.
|
|
- The dimensionality of the vector added to psi is get_num_dimensions(). This
|
|
means that elements of psi after psi(get_num_dimensions()-1) are not modified.
|
|
- Since scan_image_boxes only searches a limited set of object locations,
|
|
not all possible rectangles can be output by detect(). So in the case
|
|
where obj.get_rect() could not arise from a call to detect(), this
|
|
function will map obj.get_rect() to the nearest possible rectangle and
|
|
then add the feature vector for the mapped rectangle into #psi.
|
|
- get_best_matching_rect(obj.get_rect()) == the rectangle obj.get_rect()
|
|
gets mapped to for feature extraction.
|
|
!*/
|
|
|
|
full_object_detection get_full_object_detection (
|
|
const rectangle& rect,
|
|
const feature_vector_type& w
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns full_object_detection(rect)
|
|
(This function is here only for compatibility with the scan_image_pyramid
|
|
object)
|
|
!*/
|
|
|
|
const rectangle get_best_matching_rect (
|
|
const rectangle& rect
|
|
) const;
|
|
/*!
|
|
requires
|
|
- is_loaded_with_image() == true
|
|
ensures
|
|
- Since scan_image_boxes only searches a limited set of object locations,
|
|
not all possible rectangles can be represented. Therefore, this function
|
|
allows you to supply a rectangle and obtain the nearest possible
|
|
candidate object location rectangle.
|
|
!*/
|
|
|
|
unsigned long get_num_detection_templates (
|
|
) const { return 1; }
|
|
/*!
|
|
ensures
|
|
- returns 1. Note that this function is here only for compatibility with
|
|
the scan_image_pyramid object. Notionally, its return value indicates
|
|
that a scan_image_boxes object is always ready to detect objects once
|
|
an image has been loaded.
|
|
!*/
|
|
|
|
unsigned long get_num_movable_components_per_detection_template (
|
|
) const { return 0; }
|
|
/*!
|
|
ensures
|
|
- returns 0. Note that this function is here only for compatibility with
|
|
the scan_image_pyramid object. Its return value means that this object
|
|
does not support using movable part models.
|
|
!*/
|
|
};
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename Feature_extractor_type,
|
|
typename Box_generator
|
|
>
|
|
void serialize (
|
|
const scan_image_boxes<Feature_extractor_type,Box_generator>& item,
|
|
std::ostream& out
|
|
);
|
|
/*!
|
|
provides serialization support
|
|
!*/
|
|
|
|
template <
|
|
typename Feature_extractor_type,
|
|
typename Box_generator
|
|
>
|
|
void deserialize (
|
|
scan_image_boxes<Feature_extractor_type,Box_generator>& item,
|
|
std::istream& in
|
|
);
|
|
/*!
|
|
provides deserialization support
|
|
!*/
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
}
|
|
|
|
#endif // DLIB_SCAN_IMAGE_bOXES_ABSTRACT_Hh_
|
|
|