coco
391 lines
16 KiB
C++
391 lines
16 KiB
C++
// Copyright (C) 2013 Davis E. King (davis@dlib.net)
|
|
// License: Boost Software License See LICENSE.txt for the full license.
|
|
#undef DLIB_SCAN_IMAGE_CuSTOM_ABSTRACT_Hh_
|
|
#ifdef DLIB_SCAN_IMAGE_CuSTOM_ABSTRACT_Hh_
|
|
|
|
#include <vector>
|
|
#include "../matrix.h"
|
|
#include "../geometry.h"
|
|
#include "../image_processing/full_object_detection_abstract.h"
|
|
|
|
namespace dlib
|
|
{
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
class example_feature_extractor
|
|
{
|
|
/*!
|
|
WHAT THIS OBJECT REPRESENTS
|
|
This object defines the interface a feature extractor must implement if it
|
|
is to be used with the scan_image_custom object defined at the bottom of
|
|
this file.
|
|
|
|
In this case, the purpose of a feature extractor is to associated a
|
|
complete feature vector with each rectangle in an image. In particular,
|
|
each rectangle is scored by taking the dot product between this feature
|
|
vector and a weight vector. If this score is greater than a threshold then
|
|
the rectangle is output as a detection.
|
|
!*/
|
|
|
|
public:
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void load (
|
|
const image_type& image,
|
|
std::vector<rectangle>& candidate_objects
|
|
);
|
|
/*!
|
|
ensures
|
|
- Loads the given image into this feature extractor. This means that
|
|
subsequent calls to get_feature_vector() will return the feature vector
|
|
corresponding to locations in the image given to load().
|
|
- #candidate_objects == a set of bounding boxes in the given image that
|
|
might contain objects of interest. These are the locations that will be
|
|
checked for the presents of objects when this feature extractor is used
|
|
with the scan_image_custom object.
|
|
|
|
!*/
|
|
|
|
void copy_configuration (
|
|
const feature_extractor& item
|
|
);
|
|
/*!
|
|
ensures
|
|
- Copies all the state information of item into *this, except for state
|
|
information populated by load(). More precisely, given two
|
|
feature extractor objects S1 and S2, the following sequence of
|
|
instructions should always result in both of them having the exact same
|
|
state:
|
|
S2.copy_configuration(S1);
|
|
S1.load(img, temp);
|
|
S2.load(img, temp);
|
|
!*/
|
|
|
|
unsigned long get_num_dimensions (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns the dimensionality of the feature vectors output by this object.
|
|
!*/
|
|
|
|
void get_feature_vector (
|
|
const rectangle& obj,
|
|
matrix<double,0,1>& psi
|
|
) const;
|
|
/*!
|
|
requires
|
|
- psi.size() >= get_num_dimensions()
|
|
(i.e. psi must have preallocated its memory before this function is called)
|
|
ensures
|
|
- This function computes the feature vector associated with the given rectangle
|
|
in obj. This rectangle is interpreted as a bounding box within the last image
|
|
given to this->load() and a feature vector describing that bounding box is
|
|
output into psi.
|
|
- The feature vector is added into psi. That is, it does not overwrite the
|
|
previous contents of psi, but instead, it adds the vector to psi.
|
|
- The dimensionality of the vector added to psi is get_num_dimensions(). This
|
|
means that elements of psi after psi(get_num_dimensions()-1) are not modified.
|
|
- #psi.size() == psi.size()
|
|
(i.e. this function does not change the size of the psi vector)
|
|
!*/
|
|
|
|
double compute_object_score (
|
|
const matrix<double,0,1>& w,
|
|
const rectangle& obj
|
|
) const;
|
|
/*!
|
|
requires
|
|
- w.size() >= get_num_dimensions()
|
|
ensures
|
|
- This function returns the dot product between the feature vector for
|
|
object box obj and the given w vector. That is, this function computes
|
|
the same number as the following code snippet:
|
|
matrix<double,0,1> psi(w.size());
|
|
psi = 0;
|
|
get_feature_vector(obj, psi);
|
|
return dot(psi, w);
|
|
The point of the compute_object_score() routine is to compute this dot
|
|
product in a much more efficient way than directly calling
|
|
get_feature_vector() and dot(). Therefore, compute_object_score() is an
|
|
optional function. If you can't think of a faster way to compute these
|
|
scores then do not implement compute_object_score() and the
|
|
scan_image_custom object will simply compute these scores for you.
|
|
However, it is often the case that there is something clever you can do
|
|
to make this computation faster. If that is the case, then you can
|
|
provide an implementation of this function with your feature extractor
|
|
and then scan_image_custom will use it instead of using the default
|
|
calculation method shown in the above code snippet.
|
|
!*/
|
|
|
|
};
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
void serialize(
|
|
const feature_extractor& item,
|
|
std::ostream& out
|
|
);
|
|
/*!
|
|
provides serialization support
|
|
!*/
|
|
|
|
void deserialize(
|
|
feature_extractor& item,
|
|
std::istream& in
|
|
);
|
|
/*!
|
|
provides deserialization support
|
|
!*/
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <
|
|
typename Feature_extractor_type
|
|
>
|
|
class scan_image_custom : noncopyable
|
|
{
|
|
/*!
|
|
REQUIREMENTS ON Feature_extractor_type
|
|
- must be an object with an interface compatible with the
|
|
example_feature_extractor defined at the top of this file.
|
|
|
|
INITIAL VALUE
|
|
- is_loaded_with_image() == false
|
|
|
|
WHAT THIS OBJECT REPRESENTS
|
|
This object is a tool for running a classifier over an image with the goal
|
|
of localizing each object present. The localization is in the form of the
|
|
bounding box around each object of interest.
|
|
|
|
Unlike the scan_image_pyramid and scan_image_boxes objects, this image
|
|
scanner delegates all the work of constructing the object feature vector to
|
|
its Feature_extractor_type template argument. That is, scan_image_custom
|
|
simply asks the supplied feature extractor what boxes in the image we
|
|
should investigate and then asks the feature extractor for the complete
|
|
feature vector for each box. That is, scan_image_custom does not apply any
|
|
kind of pyramiding or other higher level processing to the features coming
|
|
out of the feature extractor. That means that when you use
|
|
scan_image_custom it is completely up to you to define the feature vector
|
|
used with each image box.
|
|
|
|
THREAD SAFETY
|
|
Concurrent access to an instance of this object is not safe and should be
|
|
protected by a mutex lock except for the case where you are copying the
|
|
configuration (via copy_configuration()) of a scan_image_custom object to
|
|
many other threads. In this case, it is safe to copy the configuration of
|
|
a shared object so long as no other operations are performed on it.
|
|
!*/
|
|
|
|
public:
|
|
|
|
typedef matrix<double,0,1> feature_vector_type;
|
|
typedef Feature_extractor_type feature_extractor_type;
|
|
|
|
scan_image_custom (
|
|
);
|
|
/*!
|
|
ensures
|
|
- this object is properly initialized
|
|
!*/
|
|
|
|
template <
|
|
typename image_type
|
|
>
|
|
void load (
|
|
const image_type& img
|
|
);
|
|
/*!
|
|
requires
|
|
- image_type must be a type with the following properties:
|
|
- image_type objects can be loaded into Feature_extractor_type
|
|
objects via Feature_extractor_type::load().
|
|
ensures
|
|
- #is_loaded_with_image() == true
|
|
- Calls get_feature_extractor().load() on the given image. That is, we
|
|
will have loaded the image into the feature extractor in this
|
|
scan_image_custom object. We will also have stored the candidate
|
|
object locations generated by the feature extractor and will scan
|
|
over them when this->detect() is called.
|
|
- This object is ready to run a classifier over img to detect object
|
|
locations. Call detect() to do this.
|
|
!*/
|
|
|
|
bool is_loaded_with_image (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns true if this object has been loaded with an image to process and
|
|
false otherwise.
|
|
!*/
|
|
|
|
const feature_extractor_type& get_feature_extractor (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns a const reference to the feature_extractor_type object used
|
|
internally for local feature extraction.
|
|
!*/
|
|
|
|
void copy_configuration(
|
|
const feature_extractor_type& fe
|
|
);
|
|
/*!
|
|
ensures
|
|
- This function performs the equivalent of
|
|
get_feature_extractor().copy_configuration(fe) (i.e. this function allows
|
|
you to configure the parameters of the underlying feature extractor used
|
|
by a scan_image_custom object)
|
|
!*/
|
|
|
|
void copy_configuration (
|
|
const scan_image_custom& item
|
|
);
|
|
/*!
|
|
ensures
|
|
- Copies all the state information of item into *this, except for state
|
|
information populated by load(). More precisely, given two
|
|
scan_image_custom objects S1 and S2, the following sequence of
|
|
instructions should always result in both of them having the exact same
|
|
state:
|
|
S2.copy_configuration(S1);
|
|
S1.load(img);
|
|
S2.load(img);
|
|
!*/
|
|
|
|
long get_num_dimensions (
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns the number of dimensions in the feature vector for a candidate
|
|
object location. That is, this function returns get_feature_extractor().get_num_dimensions().
|
|
!*/
|
|
|
|
void detect (
|
|
const feature_vector_type& w,
|
|
std::vector<std::pair<double, rectangle> >& dets,
|
|
const double thresh
|
|
) const;
|
|
/*!
|
|
requires
|
|
- w.size() >= get_num_dimensions()
|
|
- is_loaded_with_image() == true
|
|
ensures
|
|
- Scans over all the candidate object locations produced by the feature
|
|
extractor during image loading and stores all detections into #dets.
|
|
- for all valid i:
|
|
- #dets[i].second == The candidate object location which produced this
|
|
detection. This rectangle gives the location of the detection.
|
|
- #dets[i].first == The score for this detection. This value is equal
|
|
to dot(w, feature vector for this candidate object location).
|
|
- #dets[i].first >= thresh
|
|
- #dets will be sorted in descending order.
|
|
(i.e. #dets[i].first >= #dets[j].first for all i, and j>i)
|
|
- Elements of w beyond index get_num_dimensions()-1 are ignored. I.e. only
|
|
the first get_num_dimensions() are used.
|
|
- Note that no form of non-max suppression is performed. If a locations
|
|
has a score >= thresh then it is reported in #dets.
|
|
!*/
|
|
|
|
void get_feature_vector (
|
|
const full_object_detection& obj,
|
|
feature_vector_type& psi
|
|
) const;
|
|
/*!
|
|
requires
|
|
- obj.num_parts() == 0
|
|
- is_loaded_with_image() == true
|
|
- psi.size() >= get_num_dimensions()
|
|
(i.e. psi must have preallocated its memory before this function is called)
|
|
ensures
|
|
- This function allows you to determine the feature vector used for a
|
|
candidate object location output from detect(). Note that this vector is
|
|
added to psi. Note also that you must use get_full_object_detection() to
|
|
convert a rectangle from detect() into the needed full_object_detection.
|
|
- The dimensionality of the vector added to psi is get_num_dimensions(). This
|
|
means that elements of psi after psi(get_num_dimensions()-1) are not modified.
|
|
- Since scan_image_custom only searches a limited set of object locations,
|
|
not all possible rectangles can be output by detect(). So in the case
|
|
where obj.get_rect() could not arise from a call to detect(), this
|
|
function will map obj.get_rect() to the nearest possible rectangle and
|
|
then add the feature vector for the mapped rectangle into #psi.
|
|
- get_best_matching_rect(obj.get_rect()) == the rectangle obj.get_rect()
|
|
gets mapped to for feature extraction.
|
|
!*/
|
|
|
|
full_object_detection get_full_object_detection (
|
|
const rectangle& rect,
|
|
const feature_vector_type& w
|
|
) const;
|
|
/*!
|
|
ensures
|
|
- returns full_object_detection(rect)
|
|
(This function is here only for compatibility with the scan_image_pyramid
|
|
object)
|
|
!*/
|
|
|
|
const rectangle get_best_matching_rect (
|
|
const rectangle& rect
|
|
) const;
|
|
/*!
|
|
requires
|
|
- is_loaded_with_image() == true
|
|
ensures
|
|
- Since scan_image_custom only searches a limited set of object locations,
|
|
not all possible rectangles can be represented. Therefore, this function
|
|
allows you to supply a rectangle and obtain the nearest possible
|
|
candidate object location rectangle.
|
|
!*/
|
|
|
|
unsigned long get_num_detection_templates (
|
|
) const { return 1; }
|
|
/*!
|
|
ensures
|
|
- returns 1. Note that this function is here only for compatibility with
|
|
the scan_image_pyramid object. Notionally, its return value indicates
|
|
that a scan_image_custom object is always ready to detect objects once an
|
|
image has been loaded.
|
|
!*/
|
|
|
|
unsigned long get_num_movable_components_per_detection_template (
|
|
) const { return 0; }
|
|
/*!
|
|
ensures
|
|
- returns 0. Note that this function is here only for compatibility with
|
|
the scan_image_pyramid object. Its return value means that this object
|
|
does not support using movable part models.
|
|
!*/
|
|
|
|
};
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
template <typename T>
|
|
void serialize (
|
|
const scan_image_custom<T>& item,
|
|
std::ostream& out
|
|
);
|
|
/*!
|
|
provides serialization support
|
|
!*/
|
|
|
|
template <typename T>
|
|
void deserialize (
|
|
scan_image_custom<T>& item,
|
|
std::istream& in
|
|
);
|
|
/*!
|
|
provides deserialization support
|
|
!*/
|
|
|
|
// ----------------------------------------------------------------------------------------
|
|
|
|
}
|
|
|
|
#endif // DLIB_SCAN_IMAGE_CuSTOM_ABSTRACT_Hh_
|
|
|