MTF
AppearanceModel.h
1 #ifndef MTF_APPEARANCE_MODEL_H
2 #define MTF_APPEARANCE_MODEL_H
3 
4 #define am_func_not_implemeted(func_name) \
5  throw mtf::utils::FunctonNotImplemented(cv::format("%s :: %s :: Not implemented Yet", name.c_str(), #func_name))
6 
7 #include "ImageBase.h"
8 #include "AMParams.h"
9 #include "mtf/Utilities/excpUtils.h"
10 #include <memory>
11 
12 _MTF_BEGIN_NAMESPACE
13 
20  bool similarity, grad, hess;
21  bool sampler;
22 
23  AMStatus() : ImgStatus(){ clear(); }
24 
25  void set(){
26  setPixState();
27  similarity = grad = hess = true;
28  sampler = true;
29  }
30  void clear(){
31  clearPixState();
32  similarity = grad = hess = false;
33  sampler = false;
34  }
35 };
36 
40 struct AMDist{
41  typedef double ElementType;
42  typedef double ResultType;
43  const string name;
44  AMDist(const string &_name) : name(_name){}
45  virtual ~AMDist() {}
50  virtual double operator()(const double* a, const double* b,
51  size_t dist_size, double worst_dist = -1) const {
52  am_func_not_implemeted(distance_functor);
53  }
54 };
55 
63 class AppearanceModel : public ImageBase{
64 public:
66  string name;
67 
69  explicit AppearanceModel(const AMParams *params = nullptr,
70  const int _n_channels = 1) :
71  ImageBase(params, _n_channels), f(0), state_size(0),
72  first_iter(false), spi_mask(nullptr){}
74  virtual ~AppearanceModel(){}
75 
77  virtual int getStateSize() const{ return 0; }
78  virtual double getSimilarity() const{ return f; }
84  virtual double getLikelihood() const{ am_func_not_implemeted(getLikelihood); }
85  virtual const VectorXd& getState(){ return p_am; }
86  virtual const RowVectorXd& getInitGrad() const{ return df_dI0; }
87  virtual const RowVectorXd& getCurrGrad() const{ return df_dIt; }
88 
90  virtual void setSimilarity(double _similarity){ f = _similarity; }
91  virtual void setInitGrad(const RowVectorXd &df_dI){ df_dI0 = df_dI; }
92  virtual void setCurrGrad(const RowVectorXd &df_dI){ df_dIt = df_dI; }
93 
94 
110  virtual void initializeSimilarity(){ am_func_not_implemeted(initializeSimilarity); }
111  virtual void initializeGrad() { am_func_not_implemeted(initializeGrad); }
117  virtual void initializeHess(){ am_func_not_implemeted(initializeHess); }
118 
119  virtual void reinitialize(){
120  if(is_initialized.similarity){ initializeSimilarity(); }
121  if(is_initialized.grad){ initializeGrad(); }
122  if(is_initialized.hess){ initializeHess(); }
123  }
124 
132  virtual void updateSimilarity(bool prereq_only = true){ am_func_not_implemeted(updateSimilarity); }
133  virtual void updateState(const VectorXd& state_update){}
134  virtual void invertState(VectorXd& inv_p, const VectorXd& p){}
135  virtual void updateInitGrad(){ am_func_not_implemeted(updateInitGrad); }
136  virtual void updateCurrGrad() { am_func_not_implemeted(updateCurrGrad); }
137  virtual void updateParamGrad() { am_func_not_implemeted(updateParamGrad); }
138 
139  //virtual void updateInitHess() { am_func_not_implemeted(updateInitHess); }
140  //virtual void updateCurrHess() { am_func_not_implemeted(updateCurrHess); }
141 
146  virtual void cmptInitJacobian(RowVectorXd &df_dp, const MatrixXd &dI0_dpssm){
147  assert(dI0_dpssm.rows() == n_pix && dI0_dpssm.cols() == df_dp.cols());
148  df_dp.noalias() = df_dI0 * dI0_dpssm;
149  }
150  virtual void cmptCurrJacobian(RowVectorXd &df_dp, const MatrixXd &dIt_dpssm){
151  assert(dIt_dpssm.rows() == n_pix && dIt_dpssm.cols() == df_dp.cols());
152  df_dp.noalias() = df_dIt * dIt_dpssm;
153  }
161  virtual void cmptDifferenceOfJacobians(RowVectorXd &df_dp_diff,
162  const MatrixXd &dI0_dpssm, const MatrixXd &dIt_dpssm){
163  df_dp_diff.noalias() = (df_dIt * dIt_dpssm) - (df_dI0 * dI0_dpssm);
164  }
165 
170  virtual void cmptInitHessian(MatrixXd &d2f_dp2, const MatrixXd &dI0_dpssm){
171  am_func_not_implemeted(cmptInitHessian(first order));
172  }
173  virtual void cmptCurrHessian(MatrixXd &d2f_dp2, const MatrixXd &dIt_dpssm){
174  am_func_not_implemeted(cmptCurrHessian(first order));
175  }
176  virtual void cmptSelfHessian(MatrixXd &d2f_dp2, const MatrixXd &dIt_dpssm) {
177  am_func_not_implemeted(cmptSelfHessian(first order));
178  }
179 
181  virtual void cmptInitHessian(MatrixXd &d2f_dp2, const MatrixXd &dI0_dpssm, const MatrixXd &d2I0_dpssm2){
182  am_func_not_implemeted(cmptInitHessian(second order));
183  }
184  virtual void cmptCurrHessian(MatrixXd &d2f_dp2, const MatrixXd &dIt_dpssm,
185  const MatrixXd &d2It_dpssm2){
186  am_func_not_implemeted(cmptCurrHessian(second order));
187  }
188  virtual void cmptSelfHessian(MatrixXd &d2f_dp2, const MatrixXd &dIt_dpssm,
189  const MatrixXd &d2I_dpssm2) {
190  am_func_not_implemeted(cmptSelfHessian(second order));
191  }
192 
194  virtual void cmptSumOfHessians(MatrixXd &d2f_dp2_sum,
195  const MatrixXd &dI0_dpssm, const MatrixXd &dIt_dpssm){
196  int pssm_size = dIt_dpssm.cols();
197  assert(d2f_dp2_sum.rows() == pssm_size && d2f_dp2_sum.cols() == pssm_size);
198  MatrixXd d2f_dp2_0(pssm_size, pssm_size);
199  cmptInitHessian(d2f_dp2_0, dI0_dpssm);
200 
201  MatrixXd d2f_dp2_t(pssm_size, pssm_size);
202  cmptCurrHessian(d2f_dp2_t, dIt_dpssm);
203 
204  d2f_dp2_sum = d2f_dp2_0 + d2f_dp2_t;
205  }
206  virtual void cmptSumOfHessians(MatrixXd &d2f_dp2_sum,
207  const MatrixXd &dI0_dpssm, const MatrixXd &dIt_dpssm,
208  const MatrixXd &d2I0_dpssm2, const MatrixXd &d2It_dpssm2){
209  int pssm_size = dIt_dpssm.cols();
210  assert(d2f_dp2_sum.rows() == pssm_size && d2f_dp2_sum.cols() == pssm_size);
211  MatrixXd d2f_dp2_0(pssm_size, pssm_size);
212  cmptInitHessian(d2f_dp2_0, dI0_dpssm, d2I0_dpssm2);
213 
214  MatrixXd d2f_dp2_t(pssm_size, pssm_size);
215  cmptCurrHessian(d2f_dp2_t, dIt_dpssm, d2It_dpssm2);
216 
217  d2f_dp2_sum = d2f_dp2_0 + d2f_dp2_t;
218  }
219 
220  virtual void estimateOpticalFlow(std::vector<cv::Point2f> &curr_pts,
221  const cv::Mat &prev_img, const std::vector<cv::Point2f> &prev_pts,
222  const cv::Size &win_size, unsigned int n_pts, int max_iters,
223  double term_eps, bool const_grad = true) const{
224  am_func_not_implemeted(estimateOpticalFlow);
225  }
226 
227  virtual void setSPIMask(const bool *_spi_mask){ spi_mask = _spi_mask; }
228  virtual const bool* getSPIMask() const{ return spi_mask; }
229  virtual void clearSPIMask(){ spi_mask = nullptr; }
230 
235  virtual bool supportsSPI() const { return false; }
236 
241  virtual void setFirstIter(){ first_iter = true; }
243  virtual void clearFirstIter(){ first_iter = false; }
244 
245  ImgStatus* isInitialized() override { return &is_initialized; }
246  virtual void setInitStatus(){ is_initialized.set(); }
247  virtual void clearInitStatus(){ is_initialized.clear(); }
248 
253  virtual bool isSymmetrical() const{ return true; }
254 
260  virtual void updateModel(const PtsT& curr_pts){ am_func_not_implemeted(updateModel); }
261 
262  // ------------------------ Distance Feature ------------------------ //
263 
265  virtual void initializeDistFeat() {
266  am_func_not_implemeted(initializeDistFeat);
267  }
277  virtual void updateDistFeat() {
278  am_func_not_implemeted(updateDistFeat);
279  }
280  virtual const double* getDistFeat(){
281  am_func_not_implemeted(getDistFeat);
282  }
287  virtual void updateDistFeat(double* feat_addr) {
288  am_func_not_implemeted(updateDistFeat);
289  }
291  virtual unsigned int getDistFeatSize() {
292  am_func_not_implemeted(getDistFeatSize);
293  }
294  virtual const AMDist* getDistFunc() {
295  am_func_not_implemeted(getDistFunc);
296  }
297 
298  // -------------------------------------------------------------------------- //
299  // --------------------------- Stochastic Sampler --------------------------- //
300  // -------------------------------------------------------------------------- //
301 
302  virtual void initializeSampler(const VectorXd &state_sigma,
303  const VectorXd &state_mean){
304  am_func_not_implemeted(initializeSampler(VectorXd, VectorXd));
305  }
306  virtual void setSampler(const VectorXd &state_sigma,
307  const VectorXd &state_mean){
308  am_func_not_implemeted(setSampler);
309  }
310  virtual void setSamplerMean(const VectorXd &mean){
311  am_func_not_implemeted(setSamplerMean(VectorXd));
312  }
313  virtual void setSamplerSigma(const VectorXd &std){
314  am_func_not_implemeted(setSamplerSigma(VectorXd));
315  }
316  virtual void getSamplerMean(VectorXd &mean){
317  am_func_not_implemeted(getSamplerMean);
318  }
319  virtual void getSamplerSigma(VectorXd &std){
320  am_func_not_implemeted(getSamplerMean);
321  }
322  virtual void generatePerturbation(VectorXd &perturbation){
323  am_func_not_implemeted(generatePerturbation);
324  }
325 
326  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
327 
328 protected:
335  double f;
336 
341  RowVectorXd df_dI0, df_dIt;
342 
344  VectorXd p_am;
345 
348 
350  RowVectorXd df_dpam;
351 
353  MatrixXd d2f_dpam2;
354 
359  RowVectorXd df_dg0, df_dgt;
360 
366  MatrixXd d2f_dpam_dIt;
367 
374  MatrixXd d2f_dI02, d2f_dIt2;
375 
381 
389  const bool *spi_mask;
390 
395 };
396 
397 _MTF_END_NAMESPACE
398 
399 #endif
400 
401 
402 
bool first_iter
indicator variable that can be set by iterative search methods to indicate if the initial or first it...
Definition: AppearanceModel.h:380
virtual void updateModel(const PtsT &curr_pts)
optional function to incorporate online learning or adaptation of the model used to represent the app...
Definition: AppearanceModel.h:260
string name
name of the appearance model
Definition: AppearanceModel.h:66
virtual void cmptSumOfHessians(MatrixXd &d2f_dp2_sum, const MatrixXd &dI0_dpssm, const MatrixXd &dIt_dpssm)
analogous to cmptDifferenceOfJacobians except for computing the mean of the current and initial Hessi...
Definition: AppearanceModel.h:194
MatrixXd d2f_dpam2
K x K Hessian of the similarity w.r.t.
Definition: AppearanceModel.h:353
virtual double getLikelihood() const
returns a normalized version of the similarity that lies between 0 and 1 and can be interpreted as th...
Definition: AppearanceModel.h:84
AMStatus is_initialized
indicator variables used to keep track of which state variables have been initialized; ...
Definition: AppearanceModel.h:394
virtual void cmptInitHessian(MatrixXd &d2f_dp2, const MatrixXd &dI0_dpssm, const MatrixXd &d2I0_dpssm2)
compute the exact Hessian by considering the second order terms too
Definition: AppearanceModel.h:181
virtual void updateDistFeat(double *feat_addr)
overloaded version to write the distance feature directly to a row (or column) of a matrix storing th...
Definition: AppearanceModel.h:287
const bool * spi_mask
pixels corresponding to false entries in the mask will be ignored in all respective computations wher...
Definition: AppearanceModel.h:389
MatrixXd d2f_dI02
these NxN Hessians of the similarity wrt pixel values are usually not stored or computed explicitly b...
Definition: AppearanceModel.h:374
virtual int getStateSize() const
accessor methods
Definition: AppearanceModel.h:77
virtual void cmptInitHessian(MatrixXd &d2f_dp2, const MatrixXd &dI0_dpssm)
compute the S x S Hessian of the error norm using the supplied N x S Jacobian of pixel values w...
Definition: AppearanceModel.h:170
virtual void setSimilarity(double _similarity)
modifier methods
Definition: AppearanceModel.h:90
VectorXd p_am
parameters of the photomtric model
Definition: AppearanceModel.h:344
virtual void initializeHess()
even though the Hessian of the error norm w.r.t.
Definition: AppearanceModel.h:117
virtual void updateSimilarity(bool prereq_only=true)
functions for updating state variables when a new image arrives
Definition: AppearanceModel.h:132
MatrixXd d2f_dpam_dIt
K x N cross Hessian of the similarity w.r.t.
Definition: AppearanceModel.h:366
Definition: AMParams.h:12
set of indicator variables to keep track of which dependent state variables need updating and executi...
Definition: AppearanceModel.h:19
RowVectorXd df_dI0
1 x N gradients of the similarity w.r.t.
Definition: AppearanceModel.h:341
virtual void setFirstIter()
should be called before performing the first iteration on a new image to indicate that the image has ...
Definition: AppearanceModel.h:241
RowVectorXd df_dg0
1 x N gradient of the similarity w.r.t.
Definition: AppearanceModel.h:359
virtual bool isSymmetrical() const
return false if the similarity function f is not symmetrical, i.e.
Definition: AppearanceModel.h:253
Distance functor for FLANN.
Definition: AppearanceModel.h:40
virtual void cmptDifferenceOfJacobians(RowVectorXd &df_dp_diff, const MatrixXd &dI0_dpssm, const MatrixXd &dIt_dpssm)
multiplies the gradients of the current error norm w.r.t.
Definition: AppearanceModel.h:161
Definition: ImageBase.h:51
AppearanceModel(const AMParams *params=nullptr, const int _n_channels=1)
default and value constructor
Definition: AppearanceModel.h:69
virtual double operator()(const double *a, const double *b, size_t dist_size, double worst_dist=-1) const
computes the distance / dissimilarity between two patches where each is codified or represented by a ...
Definition: AppearanceModel.h:50
virtual unsigned int getDistFeatSize()
returns the size of the distance vector
Definition: AppearanceModel.h:291
virtual void clearFirstIter()
should be called after the first iteration on a new frame is done
Definition: AppearanceModel.h:243
virtual void updateDistFeat()
computes a "distance" vector using the current image patch such that, when the distance vectors corre...
Definition: AppearanceModel.h:277
RowVectorXd df_dpam
1 x K Jacobian of the similarity function w.r.t.
Definition: AppearanceModel.h:350
double f
f(I_0, I_t, p_am): R(N) x R(N) x R(K) -> R measures the similarity between the current (I_t) and init...
Definition: AppearanceModel.h:335
virtual void initializeSimilarity()
methods to initialize the state variables - to be called once when the tracker is initialized...
Definition: AppearanceModel.h:110
Similarity function that indicates how well a candidate warped patch matches the template.
Definition: AppearanceModel.h:63
Definition: ImageBase.h:38
int state_size
size of p_am, i.e.
Definition: AppearanceModel.h:347
virtual void cmptInitJacobian(RowVectorXd &df_dp, const MatrixXd &dI0_dpssm)
–— interfacing functions that take pixel jacobians/Hessians w.r.t.
Definition: AppearanceModel.h:146
virtual bool supportsSPI() const
should be overridden by an implementing class once it implements SPI functionality for all functions ...
Definition: AppearanceModel.h:235
virtual ~AppearanceModel()
destructor
Definition: AppearanceModel.h:74
virtual void initializeDistFeat()
to be called once during initialization if any of the distance feature functionality is to be used ...
Definition: AppearanceModel.h:265