From 1565ff1037600aeb63aa8aa2eee6a2ebabaff19f Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 27 Jul 2015 15:41:19 +0900 Subject: [PATCH] Added optimization to Multi-target TLD update --- .../include/opencv2/tracking/tracker.hpp | 2 +- modules/tracking/src/multiTracker.cpp | 150 +++++++++++++++++- modules/tracking/src/tldTracker.hpp | 1 - 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/modules/tracking/include/opencv2/tracking/tracker.hpp b/modules/tracking/include/opencv2/tracking/tracker.hpp index c17f65133..2a3db47f6 100644 --- a/modules/tracking/include/opencv2/tracking/tracker.hpp +++ b/modules/tracking/include/opencv2/tracking/tracker.hpp @@ -565,7 +565,7 @@ class CV_EXPORTS_W Tracker : public virtual Algorithm virtual void read( const FileNode& fn )=0; virtual void write( FileStorage& fs ) const=0; - protected: + public: virtual bool initImpl( const Mat& image, const Rect2d& boundingBox ) = 0; virtual bool updateImpl( const Mat& image, Rect2d& boundingBox ) = 0; diff --git a/modules/tracking/src/multiTracker.cpp b/modules/tracking/src/multiTracker.cpp index d8c53d22f..36f4b70a2 100644 --- a/modules/tracking/src/multiTracker.cpp +++ b/modules/tracking/src/multiTracker.cpp @@ -1,4 +1,4 @@ -#include +#include "tldTracker.hpp" namespace cv { @@ -35,14 +35,156 @@ namespace cv return true; } - + //Multitracker TLD /*Optimized update method for TLD Multitracker */ bool MultiTrackerTLD::update(const Mat& image) { - for (int i = 0; i < trackers.size(); i++) - if (!trackers[i]->update(image, boundingBoxes[i])) + + for (int k = 0; k < trackers.size(); k++) + { + //Set current target(tracker) parameters + Rect2d boundingBox = boundingBoxes[k]; + Ptr tracker = (Ptr)static_cast> (trackers[k]); + tld::TrackerTLDModel* tldModel = ((tld::TrackerTLDModel*)static_cast(tracker->model)); + Ptr data = tracker->data; + double scale = data->getScale(); + + + Mat image_gray, image_blurred, imageForDetector; + cvtColor(image, image_gray, COLOR_BGR2GRAY); + + if (scale > 1.0) + resize(image_gray, imageForDetector, Size(cvRound(image.cols*scale), cvRound(image.rows*scale)), 0, 0, tld::DOWNSCALE_MODE); + else + imageForDetector = image_gray; + GaussianBlur(imageForDetector, image_blurred, tld::GaussBlurKernelSize, 0.0); + + data->frameNum++; + Mat_ standardPatch(tld::STANDARD_PATCH_SIZE, tld::STANDARD_PATCH_SIZE); + std::vector detectorResults; + //best overlap around 92% + + std::vector candidates; + std::vector candidatesRes; + bool trackerNeedsReInit = false; + bool DETECT_FLG = false; + for (int i = 0; i < 2; i++) + { + Rect2d tmpCandid = boundingBox; + + if (i == 1) + { + if (ocl::haveOpenCL()) + DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + else + DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize()); + } + + if (((i == 0) && !data->failedLastTime && tracker->trackerProxy->update(image, tmpCandid)) || (DETECT_FLG)) + { + candidates.push_back(tmpCandid); + if (i == 0) + tld::resample(image_gray, tmpCandid, standardPatch); + else + tld::resample(imageForDetector, tmpCandid, standardPatch); + candidatesRes.push_back(tldModel->detector->Sc(standardPatch)); + } + else + { + if (i == 0) + trackerNeedsReInit = true; + } + } + + std::vector::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end()); + + //dfprintf((stdout, "scale = %f\n", log(1.0 * boundingBox.width / (data->getMinSize()).width) / log(SCALE_STEP))); + //for( int i = 0; i < (int)candidatesRes.size(); i++ ) + //dprintf(("\tcandidatesRes[%d] = %f\n", i, candidatesRes[i])); + //data->printme(); + //tldModel->printme(stdout); + + if (it == candidatesRes.end()) + { + data->confident = false; + data->failedLastTime = true; return false; + } + else + { + boundingBox = candidates[it - candidatesRes.begin()]; + boundingBoxes[k] = boundingBox; + data->failedLastTime = false; + if (trackerNeedsReInit || it != candidatesRes.begin()) + tracker->trackerProxy->init(image, boundingBox); + } + +#if 1 + if (it != candidatesRes.end()) + { + tld::resample(imageForDetector, candidates[it - candidatesRes.begin()], standardPatch); + //dfprintf((stderr, "%d %f %f\n", data->frameNum, tldModel->Sc(standardPatch), tldModel->Sr(standardPatch))); + //if( candidatesRes.size() == 2 && it == (candidatesRes.begin() + 1) ) + //dfprintf((stderr, "detector WON\n")); + } + else + { + //dfprintf((stderr, "%d x x\n", data->frameNum)); + } +#endif + + if (*it > tld::CORE_THRESHOLD) + data->confident = true; + + if (data->confident) + { + tld::TrackerTLDImpl::Pexpert pExpert(imageForDetector, image_blurred, boundingBox, tldModel->detector, tracker->params, data->getMinSize()); + tld::TrackerTLDImpl::Nexpert nExpert(imageForDetector, boundingBox, tldModel->detector, tracker->params); + std::vector > examplesForModel, examplesForEnsemble; + examplesForModel.reserve(100); examplesForEnsemble.reserve(100); + int negRelabeled = 0; + for (int i = 0; i < (int)detectorResults.size(); i++) + { + bool expertResult; + if (detectorResults[i].isObject) + { + expertResult = nExpert(detectorResults[i].rect); + if (expertResult != detectorResults[i].isObject) + negRelabeled++; + } + else + { + expertResult = pExpert(detectorResults[i].rect); + } + + detectorResults[i].shouldBeIntegrated = detectorResults[i].shouldBeIntegrated || (detectorResults[i].isObject != expertResult); + detectorResults[i].isObject = expertResult; + } + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); + //dprintf(("%d relabeled by nExpert\n", negRelabeled)); + pExpert.additionalExamples(examplesForModel, examplesForEnsemble); + if (ocl::haveOpenCL()) + tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true); + else + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, true); + examplesForModel.clear(); examplesForEnsemble.clear(); + nExpert.additionalExamples(examplesForModel, examplesForEnsemble); + + if (ocl::haveOpenCL()) + tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false); + else + tldModel->integrateAdditional(examplesForModel, examplesForEnsemble, false); + } + else + { +#ifdef CLOSED_LOOP + tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults); +#endif + } + + + } return true; } diff --git a/modules/tracking/src/tldTracker.hpp b/modules/tracking/src/tldTracker.hpp index e04926d5d..40de754b7 100644 --- a/modules/tracking/src/tldTracker.hpp +++ b/modules/tracking/src/tldTracker.hpp @@ -128,7 +128,6 @@ public: void read(const FileNode& fn); void write(FileStorage& fs) const; -protected: class Pexpert { public: