Merge pull request #379 from sbokov:sparseMatchInterpolationSquashed

modules/optflow/CMakeLists.txt

@@ -1,2 +1,2 @@
 set(the_description "Optical Flow Algorithms")
-ocv_define_module(optflow opencv_core opencv_imgproc opencv_video opencv_highgui WRAP python)
+ocv_define_module(optflow opencv_core opencv_imgproc opencv_video opencv_highgui opencv_ximgproc WRAP python)

modules/optflow/include/opencv2/optflow.hpp

@@ -109,7 +109,30 @@ CV_EXPORTS_W void calcOpticalFlowSF( InputArray from, InputArray to, OutputArray
                                      double sigma_dist, double sigma_color, int postprocess_window,
                                      double sigma_dist_fix, double sigma_color_fix, double occ_thr,
                                      int upscale_averaging_radius, double upscale_sigma_dist,
-                                     double upscale_sigma_color, double speed_up_thr );    
+                                     double upscale_sigma_color, double speed_up_thr );
+
+/** @brief Fast dense optical flow based on PyrLK sparse matches interpolation.
+
+@param from first 8-bit 3-channel or 1-channel image.
+@param to  second 8-bit 3-channel or 1-channel image of the same size as from
+@param flow computed flow image that has the same size as from and CV_32FC2 type
+@param grid_step stride used in sparse match computation. Lower values usually
+       result in higher quality but slow down the algorithm.
+@param k number of nearest-neighbor matches considered, when fitting a locally affine
+       model. Lower values can make the algorithm noticeably faster at the cost of
+       some quality degradation.
+@param sigma parameter defining how fast the weights decrease in the locally-weighted affine
+       fitting. Higher values can help preserve fine details, lower values can help to get rid
+       of the noise in the output flow.
+@param use_post_proc defines whether the ximgproc::fastGlobalSmootherFilter() is used
+       for post-processing after interpolation
+@param fgs_lambda see the respective parameter of the ximgproc::fastGlobalSmootherFilter()
+@param fgs_sigma  see the respective parameter of the ximgproc::fastGlobalSmootherFilter()
+ */
+CV_EXPORTS_W void calcOpticalFlowSparseToDense ( InputArray from, InputArray to, OutputArray flow,
+                                                 int grid_step = 8, int k = 128, float sigma = 0.05f,
+                                                 bool use_post_proc = true, float fgs_lambda = 500.0f,
+                                                 float fgs_sigma = 1.5f );
 
 /** @brief Read a .flo file
 
@@ -165,6 +188,9 @@ CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SimpleFlow();
 //! Additional interface to the Farneback's algorithm - calcOpticalFlowFarneback()
 CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_Farneback();
 
+//! Additional interface to the SparseToDenseFlow algorithm - calcOpticalFlowSparseToDense()
+CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SparseToDense();
+
 //! @}
 
 } //optflow

modules/optflow/samples/optical_flow_evaluation.cpp

@@ -10,7 +10,7 @@ using namespace optflow;
 const String keys = "{help h usage ? |      | print this message   }"
         "{@image1        |      | image1               }"
         "{@image2        |      | image2               }"
-        "{@algorithm     |      | [farneback, simpleflow, tvl1 or deepflow] }"
+        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow or sparsetodenseflow] }"
         "{@groundtruth   |      | path to the .flo file  (optional), Middlebury format }"
         "{m measure      |endpoint| error measure - [endpoint or angular] }"
         "{r region       |all   | region to compute stats about [all, discontinuities, untextured] }"
@@ -249,6 +249,8 @@ int main( int argc, char** argv )
         algorithm = createOptFlow_DualTVL1();
     else if ( method == "deepflow" )
         algorithm = createOptFlow_DeepFlow();
+    else if ( method == "sparsetodenseflow" )
+        algorithm = createOptFlow_SparseToDense();
     else
     {
         printf("Wrong method!\n");

modules/optflow/src/interfaces.cpp

@@ -177,5 +177,42 @@ Ptr<DenseOpticalFlow> createOptFlow_Farneback()
 {
     return makePtr<OpticalFlowFarneback>();
 }
+
+class OpticalFlowSparseToDense : public DenseOpticalFlow
+{
+public:
+    OpticalFlowSparseToDense(int _grid_step, int _k, float _sigma, bool _use_post_proc, float _fgs_lambda, float _fgs_sigma);
+    void calc(InputArray I0, InputArray I1, InputOutputArray flow);
+    void collectGarbage();
+protected:
+    int grid_step;
+    int k;
+    float sigma;
+    bool use_post_proc;
+    float fgs_lambda;
+    float fgs_sigma;
+};
+
+OpticalFlowSparseToDense::OpticalFlowSparseToDense(int _grid_step, int _k, float _sigma, bool _use_post_proc, float _fgs_lambda, float _fgs_sigma)
+{
+    grid_step     = _grid_step;
+    k             = _k;
+    sigma         = _sigma;
+    use_post_proc = _use_post_proc;
+    fgs_lambda    = _fgs_lambda;
+    fgs_sigma     = _fgs_sigma;
+}
+
+void OpticalFlowSparseToDense::calc(InputArray I0, InputArray I1, InputOutputArray flow)
+{
+    calcOpticalFlowSparseToDense(I0,I1,flow,grid_step,k,sigma,use_post_proc,fgs_lambda,fgs_sigma);
+}
+
+void OpticalFlowSparseToDense::collectGarbage() {}
+
+Ptr<DenseOpticalFlow> createOptFlow_SparseToDense()
+{
+    return makePtr<OpticalFlowSparseToDense>(8,128,0.05f,true,500.0f,1.5f);
+}
 }
 }

modules/optflow/src/sparsetodenseflow.cpp

@@ -0,0 +1,112 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "precomp.hpp"
+#include "opencv2/ximgproc/sparse_match_interpolator.hpp"
+using namespace std;
+
+namespace cv {
+namespace optflow {
+
+CV_EXPORTS_W void calcOpticalFlowSparseToDense(InputArray from, InputArray to, OutputArray flow,
+                                               int grid_step, int k,
+                                               float sigma, bool use_post_proc,
+                                               float fgs_lambda, float fgs_sigma)
+{
+    CV_Assert( grid_step>1 && k>3 && sigma>0.0001f && fgs_lambda>1.0f && fgs_sigma>0.01f );
+    CV_Assert( !from.empty() && from.depth() == CV_8U && (from.channels() == 3 || from.channels() == 1) );
+    CV_Assert( !to  .empty() && to  .depth() == CV_8U && (to  .channels() == 3 || to  .channels() == 1) );
+    CV_Assert( from.sameSize(to) );
+
+    Mat prev = from.getMat();
+    Mat cur  = to.getMat();
+    Mat prev_grayscale, cur_grayscale;
+
+    while( (prev.cols/grid_step)*(prev.rows/grid_step) > SHRT_MAX ) //ensure that the number matches is not too big
+        grid_step*=2;
+
+    if(prev.channels()==3)
+    {
+        cvtColor(prev,prev_grayscale,COLOR_BGR2GRAY);
+        cvtColor(cur, cur_grayscale, COLOR_BGR2GRAY);
+    }
+    else
+    {
+        prev.copyTo(prev_grayscale);
+        cur .copyTo(cur_grayscale);
+    }
+
+    vector<Point2f> points;
+    vector<Point2f> dst_points;
+    vector<unsigned char> status;
+    vector<float> err;
+    vector<Point2f> points_filtered, dst_points_filtered;
+
+    for(int i=0;i<prev.rows;i+=grid_step)
+        for(int j=0;j<prev.cols;j+=grid_step)
+            points.push_back(Point2f((float)j,(float)i));
+
+    calcOpticalFlowPyrLK(prev_grayscale,cur_grayscale,points,dst_points,status,err,Size(21,21));
+
+    for(unsigned int i=0;i<points.size();i++)
+    {
+        if(status[i]!=0)
+        {
+            points_filtered.push_back(points[i]);
+            dst_points_filtered.push_back(dst_points[i]);
+        }
+    }
+
+    flow.create(from.size(),CV_32FC2);
+    Mat dense_flow = flow.getMat();
+
+    Ptr<ximgproc::EdgeAwareInterpolator> gd = ximgproc::createEdgeAwareInterpolator();
+    gd->setK(k);
+    gd->setSigma(sigma);
+    gd->setUsePostProcessing(use_post_proc);
+    gd->setFGSLambda(fgs_lambda);
+    gd->setFGSSigma (fgs_sigma);
+    gd->interpolate(prev,points_filtered,cur,dst_points_filtered,dense_flow);
+}
+
+}
+}

modules/optflow/test/test_sparsetodenseflow.cpp

@@ -0,0 +1,146 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                        Intel License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000, Intel Corporation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of Intel Corporation may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "test_precomp.hpp"
+
+#include <string>
+
+using namespace std;
+using namespace cv;
+
+/* ///////////////////// sparsetodenseflow_test ///////////////////////// */
+
+class CV_SparseToDenseFlowTest : public cvtest::BaseTest
+{
+protected:
+    void run(int);
+};
+
+static bool isFlowCorrect(float u) {
+  return !cvIsNaN(u) && (fabs(u) < 1e9);
+}
+
+static float calc_rmse(Mat flow1, Mat flow2) {
+  float sum = 0;
+  int counter = 0;
+  const int rows = flow1.rows;
+  const int cols = flow1.cols;
+
+  for (int y = 0; y < rows; ++y) {
+    for (int x = 0; x < cols; ++x) {
+      Vec2f flow1_at_point = flow1.at<Vec2f>(y, x);
+      Vec2f flow2_at_point = flow2.at<Vec2f>(y, x);
+
+      float u1 = flow1_at_point[0];
+      float v1 = flow1_at_point[1];
+      float u2 = flow2_at_point[0];
+      float v2 = flow2_at_point[1];
+
+      if (isFlowCorrect(u1) && isFlowCorrect(u2) && isFlowCorrect(v1) && isFlowCorrect(v2)) {
+        sum += (u1-u2)*(u1-u2) + (v1-v2)*(v1-v2);
+        counter++;
+      }
+    }
+  }
+  return (float)sqrt(sum / (1e-9 + counter));
+}
+
+void CV_SparseToDenseFlowTest::run(int) {
+    const float MAX_RMSE = 0.6f;
+    const string frame1_path = ts->get_data_path() + "optflow/RubberWhale1.png";
+    const string frame2_path = ts->get_data_path() + "optflow/RubberWhale2.png";
+    const string gt_flow_path = ts->get_data_path() + "optflow/RubberWhale.flo";
+
+    Mat frame1 = imread(frame1_path);
+    Mat frame2 = imread(frame2_path);
+
+    if (frame1.empty()) {
+      ts->printf(cvtest::TS::LOG, "could not read image %s\n", frame2_path.c_str());
+      ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
+      return;
+    }
+
+    if (frame2.empty()) {
+      ts->printf(cvtest::TS::LOG, "could not read image %s\n", frame2_path.c_str());
+      ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
+      return;
+    }
+
+    if (frame1.rows != frame2.rows && frame1.cols != frame2.cols) {
+      ts->printf(cvtest::TS::LOG, "images should be of equal sizes (%s and %s)",
+                 frame1_path.c_str(), frame2_path.c_str());
+      ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
+      return;
+    }
+
+    if (frame1.type() != 16 || frame2.type() != 16) {
+      ts->printf(cvtest::TS::LOG, "images should be of equal type CV_8UC3 (%s and %s)",
+                 frame1_path.c_str(), frame2_path.c_str());
+      ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
+      return;
+    }
+
+    Mat flow_gt = optflow::readOpticalFlow(gt_flow_path);
+    if(flow_gt.empty()) {
+      ts->printf(cvtest::TS::LOG, "error while reading flow data from file %s",
+                 gt_flow_path.c_str());
+      ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
+      return;
+    }
+
+    Mat flow;
+    optflow::calcOpticalFlowSparseToDense(frame1, frame2, flow);
+
+    float rmse = calc_rmse(flow_gt, flow);
+
+    ts->printf(cvtest::TS::LOG, "Optical flow estimation RMSE for SparseToDenseFlow algorithm : %lf\n",
+               rmse);
+
+    if (rmse > MAX_RMSE) {
+      ts->printf( cvtest::TS::LOG,
+                 "Too big rmse error : %lf ( >= %lf )\n", rmse, MAX_RMSE);
+      ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ACCURACY);
+      return;
+    }
+}
+
+
+TEST(Video_OpticalFlowSparseToDenseFlow, accuracy) { CV_SparseToDenseFlowTest test; test.safe_run(); }