Fawkes API  Fawkes Development Version
sift.cpp
1 
2 /***************************************************************************
3  * sift.cpp - Feature-based classifier using OpenCV structures
4  *
5  * Created: Mon Mar 15 15:47:11 2008
6  * Copyright 2008 Stefan Schiffer [stefanschiffer.de]
7  *
8  ****************************************************************************/
9 
10 /* This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version. A runtime exception applies to
14  * this software (see LICENSE.GPL_WRE file mentioned below for details).
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * GNU Library General Public License for more details.
20  *
21  * Read the full text in the LICENSE.GPL_WRE file in the doc directory.
22  */
23 
24 #include <fvclassifiers/sift.h>
25 
26 #include <iostream>
27 #include <vector>
28 //#ifdef SIFT_TIMETRACKER
29 #include <utils/time/clock.h>
30 #include <utils/time/tracker.h>
31 //#endif
32 
33 extern "C" {
34 #include <sift/imgfeatures.h>
35 #include <sift/kdtree.h>
36 #include <sift/sift.h>
37 #include <sift/utils.h>
38 #include <sift/xform.h>
39 }
40 
41 #include <core/exception.h>
42 #include <core/exceptions/software.h>
43 #include <fvutils/color/colorspaces.h>
44 #include <fvutils/color/conversions.h>
45 #include <opencv/cv.h>
46 #include <opencv/cxcore.h>
47 #include <opencv/highgui.h>
48 
49 using namespace fawkes;
50 
51 namespace firevision {
52 
53 /** @class SiftClassifier <fvclassifiers/sift.h>
54  * SIFT classifier.
55  *
56  * This class provides a classifier that uses OpenCV to detect objects in a given
57  * image by matching features using SIFT. The objects are reported back as regions
58  * of interest. Each ROI contains an object.
59  *
60  * This code is based on the sift package provided by Rob Hess.
61  * at http://web.engr.oregonstate.edu/~hess/
62  *
63  * @author Stefan Schiffer
64  */
65 
66 /** Constructor.
67  * @param object_file file that contains the object to detect
68  * @param pixel_width width of images that will be processed
69  * @param pixel_height height of images that will be processed
70  * @param kdtree_bbf_max_nn_chks maximum number of keypoint NN candidates to check during BBF search
71  * @param nn_sq_dist_ratio_thr threshold on squared ratio of distances between NN and 2nd NN
72  * @param flags flags, not used yet.
73  */
74 SiftClassifier::SiftClassifier(const char * object_file,
75  unsigned int pixel_width,
76  unsigned int pixel_height,
77  int kdtree_bbf_max_nn_chks,
78  float nn_sq_dist_ratio_thr,
79  int flags)
80 : Classifier("SiftClassifier")
81 {
82  kdtree_bbf_max_nn_chks_ = kdtree_bbf_max_nn_chks;
83  nn_sq_dist_ratio_thr_ = nn_sq_dist_ratio_thr;
84  flags_ = flags;
85 
86  //#ifdef SIFT_TIMETRACKER
87  tt_ = new TimeTracker();
88  loop_count_ = 0;
89  ttc_objconv_ = tt_->add_class("ObjectConvert");
90  ttc_objfeat_ = tt_->add_class("ObjectFeatures");
91  ttc_imgconv_ = tt_->add_class("ImageConvert");
92  ttc_imgfeat_ = tt_->add_class("ImageFeatures");
93  ttc_matchin_ = tt_->add_class("Matching");
94  ttc_roimerg_ = tt_->add_class("MergeROIs");
95  //#endif
96 
97  //#ifdef SIFT_TIMETRACKER
98  tt_->ping_start(ttc_objconv_);
99  //#endif
100  obj_img_ = cvLoadImage(object_file, 1);
101  if (!obj_img_) {
102  throw Exception("Could not load object file");
103  }
104  //#ifdef SIFT_TIMETRACKER
105  tt_->ping_end(ttc_objconv_);
106  //#endif
107 
108  //#ifdef SIFT_TIMETRACKER
109  tt_->ping_start(ttc_objfeat_);
110  //#endif
111  obj_num_features_ = 0;
112  obj_num_features_ = sift_features(obj_img_, &obj_features_);
113  if (!obj_num_features_ > 0) {
114  throw Exception("Could not compute object features");
115  }
116  std::cout << "SiftClassifier(classify): computed '" << obj_num_features_
117  << "' features from object" << std::endl;
118  //cvReleaseImage(&obj_img_);
119  //#ifdef SIFT_TIMETRACKER
120  tt_->ping_end(ttc_objfeat_);
121  //#endif
122 
123  // create space for OpenCV image
124  image_ = cvCreateImage(cvSize(pixel_width, pixel_height), IPL_DEPTH_8U, 3);
125 }
126 
127 /** Destructor. */
129 {
130  //
131  cvReleaseImage(&obj_img_);
132  cvReleaseImage(&image_);
133 }
134 
135 std::list<ROI> *
137 {
138  //#ifdef SIFT_TIMETRACKER
139  tt_->ping_start(0);
140  //#endif
141 
142  // list of ROIs to return
143  std::list<ROI> *rv = new std::list<ROI>();
144 
145  struct feature * feat;
146  struct feature **nbrs;
147  struct kd_node * kd_root;
148  CvPoint pt1, pt2;
149 
150  // for ROI calculation
151  CvPoint ftpt;
152  std::vector<CvPoint> ftlist;
153  //= new std::vector< CvPoint >();
154  int x_min = _width;
155  int y_min = _height;
156  int x_max = 0;
157  int y_max = 0;
158 
159  double d0, d1; // = 0.0;
160  int k, m = 0;
161 
162  //#ifdef SIFT_TIMETRACKER
163  tt_->ping_start(ttc_imgconv_);
164  //#endif
165  //std::cout << "SiftClassifier(classify): convert frame to IplImage" << std::endl;
166  convert(YUV422_PLANAR, BGR, _src, (unsigned char *)image_->imageData, _width, _height);
167  //#ifdef SIFT_TIMETRACKER
168  tt_->ping_end(ttc_imgconv_);
169  //#endif
170 
171  //#ifdef SIFT_TIMETRACKER
172  tt_->ping_start(ttc_imgfeat_);
173  //#endif
174  //std::cout << "SiftClassifier(classify): compute features on current frame " << std::endl;
175  int num_img_ft = sift_features(image_, &img_features_);
176  kd_root = kdtree_build(img_features_, num_img_ft);
177  //#ifdef SIFT_TIMETRACKER
178  tt_->ping_end(ttc_imgfeat_);
179  //#endif
180 
181  if (!kd_root) {
182  std::cerr << "SiftClassifier(classify): KD-Root NULL!" << std::endl;
183  }
184 
185  //#ifdef SIFT_TIMETRACKER
186  tt_->ping_start(ttc_matchin_);
187  //#endif
188  std::cout << "SiftClassifier(classify): matching ..." << std::endl;
189  for (int i = 0; i < obj_num_features_; ++i) {
190  //std::cout << "SiftClassifier(classify): ... feature '" << i << "'" << std::endl;
191  feat = obj_features_ + i;
192  k = kdtree_bbf_knn(kd_root, feat, 2, &nbrs, kdtree_bbf_max_nn_chks_);
193  if (k == 2) {
194  d0 = descr_dist_sq(feat, nbrs[0]);
195  d1 = descr_dist_sq(feat, nbrs[1]);
196  if (d0 < d1 * nn_sq_dist_ratio_thr_) {
197  pt1 = cvPoint(cvRound(feat->x), cvRound(feat->y));
198  pt2 = cvPoint(cvRound(nbrs[0]->x), cvRound(nbrs[0]->y));
199  m++;
200  obj_features_[i].fwd_match = nbrs[0];
201  // save matched feature points
202  ftpt = cvPoint(cvRound(nbrs[0]->x), cvRound(nbrs[0]->y));
203  ftlist.push_back(ftpt);
204  // save matched features as ROIs
205  ROI r(pt2.x - 5, pt2.y - 5, 11, 11, _width, _height);
206  rv->push_back(r);
207  }
208  }
209  free(nbrs);
210  }
211  std::cout << "SiftClassifier(classify): found '" << m << "' matches" << std::endl;
212  kdtree_release(kd_root);
213  //#ifdef SIFT_TIMETRACKER
214  tt_->ping_end(ttc_matchin_);
215  //#endif
216 
217  //#ifdef SIFT_TIMETRACKER
218  tt_->ping_start(ttc_roimerg_);
219  //#endif
220  std::cout << "SiftClassifier(classify): computing ROI" << std::endl;
221  //for ( int i = 0; i < m; ++i) {
222  for (std::vector<CvPoint>::size_type i = 0; i < ftlist.size(); ++i) {
223  if (ftlist[i].x < x_min)
224  x_min = ftlist[i].x;
225  if (ftlist[i].y < y_min)
226  y_min = ftlist[i].y;
227  if (ftlist[i].x > x_max)
228  x_max = ftlist[i].x;
229  if (ftlist[i].y > y_max)
230  y_max = ftlist[i].y;
231  }
232  if (m != 0) {
233  ROI r(x_min, y_min, x_max - x_min, y_max - y_min, _width, _height);
234  rv->push_back(r);
235  }
236  //#ifdef SIFT_TIMETRACKER
237  tt_->ping_end(ttc_roimerg_);
238  //#endif
239 
240  //#ifdef SIFT_TIMETRACKER
241  tt_->ping_end(0);
242  //#endif
243 
244  //#ifdef SIFT_TIMETRACKER
245  tt_->print_to_stdout();
246  //#endif
247 
248  std::cout << "SiftClassifier(classify): done ... returning '" << rv->size() << "' ROIs."
249  << std::endl;
250  return rv;
251 }
252 
253 } // end namespace firevision
void ping_start(unsigned int cls)
Start of given class task.
Definition: tracker.cpp:218
Fawkes library namespace.
unsigned int _width
Width in pixels of _src buffer.
Definition: classifier.h:51
Region of interest.
Definition: roi.h:54
unsigned int _height
Height in pixels of _src buffer.
Definition: classifier.h:53
Base class for exceptions in Fawkes.
Definition: exception.h:35
unsigned int add_class(std::string name)
Add a new class.
Definition: tracker.cpp:149
Time tracking utility.
Definition: tracker.h:36
virtual ~SiftClassifier()
Destructor.
Definition: sift.cpp:128
void ping_end(unsigned int cls)
End of given class task.
Definition: tracker.cpp:243
void print_to_stdout()
Print results to stdout.
Definition: tracker.cpp:307
virtual std::list< ROI > * classify()
Classify image.
Definition: sift.cpp:136
Classifier to extract regions of interest.
Definition: classifier.h:35
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
Definition: classifier.h:49