diff --git a/modules/saliency/doc/uml b/modules/saliency/doc/uml index 4b13f7c04..aa0a32750 160000 --- a/modules/saliency/doc/uml +++ b/modules/saliency/doc/uml @@ -1 +1 @@ -Subproject commit 4b13f7c043ee897c429d28066c1f4c487d550b0e +Subproject commit aa0a3275061fc1bc8c0d6464ff21b2c7b41c6179 diff --git a/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp b/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp index 551b70ce8..fe5b52141 100644 --- a/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp +++ b/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp @@ -43,20 +43,34 @@ #define __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP_ #include "saliencyBaseClasses.hpp" -#include "kyheader.h" -#include "ValStructVec.h" -#include "FilterTIG.h" +#include "BING/kyheader.h" +#include "BING/ValStructVec.h" +#include "BING/FilterTIG.h" +#include "SuBSENSE/BackgroundSubtractorLBSP.h" #include #include #include +//! defines the default value for BackgroundSubtractorLBSP::m_fRelLBSPThreshold +#define BGSSUBSENSE_DEFAULT_LBSP_REL_SIMILARITY_THRESHOLD (0.333f) +//! defines the default value for BackgroundSubtractorLBSP::m_nDescDistThreshold +#define BGSSUBSENSE_DEFAULT_DESC_DIST_THRESHOLD (3) +//! defines the default value for BackgroundSubtractorSuBSENSE::m_nMinColorDistThreshold +#define BGSSUBSENSE_DEFAULT_COLOR_DIST_THRESHOLD (30) +//! defines the default value for BackgroundSubtractorSuBSENSE::m_nBGSamples +#define BGSSUBSENSE_DEFAULT_NB_BG_SAMPLES (50) +//! defines the default value for BackgroundSubtractorSuBSENSE::m_nRequiredBGSamples +#define BGSSUBSENSE_DEFAULT_REQUIRED_NB_BG_SAMPLES (2) +//! defines the default value for BackgroundSubtractorSuBSENSE::m_nSamplesForMovingAvgs +#define BGSSUBSENSE_DEFAULT_N_SAMPLES_FOR_MV_AVGS (25) + namespace cv { /************************************ Specific Static Saliency Specialized Classes ************************************/ /** - * \brief Saliency based on algorithms based on [1] + * \brief Saliency based on algorithms described in [1] * [1]Hou, Xiaodi, and Liqing Zhang. "Saliency detection: A spectral residual approach." Computer Vision and Pattern Recognition, 2007. CVPR'07. IEEE Conference on. IEEE, 2007. */ class CV_EXPORTS_W StaticSaliencySpectralResidual : public StaticSaliency @@ -67,10 +81,10 @@ class CV_EXPORTS_W StaticSaliencySpectralResidual : public StaticSaliency StaticSaliencySpectralResidual(); ~StaticSaliencySpectralResidual(); - typedef cv::Ptr (cv::Algorithm::*SizeGetter)(); + typedef Ptr (cv::Algorithm::*SizeGetter)(); typedef void (cv::Algorithm::*SizeSetter)( const cv::Ptr & ); - cv::Ptr getWsize(); + Ptr getWsize(); void setWsize( const cv::Ptr &arrPtr ); void read( const FileNode& fn ); @@ -86,7 +100,7 @@ class CV_EXPORTS_W StaticSaliencySpectralResidual : public StaticSaliency /************************************ Specific Motion Saliency Specialized Classes ************************************/ /** - * \brief Saliency based on algorithms based on [2] + * \brief Saliency based on algorithms described in [2] * [2] Hofmann, Martin, Philipp Tiefenbacher, and Gerhard Rigoll. "Background segmentation with feedback: The pixel-based adaptive segmenter." * Computer Vision and Pattern Recognition Workshops (CVPRW), 2012 IEEE Computer Society Conference on. IEEE, 2012. */ @@ -107,11 +121,125 @@ class CV_EXPORTS_W MotionSaliencyPBAS : public MotionSaliency }; +/*! + Self-Balanced Sensitivity segmenTER (SuBSENSE) foreground-background segmentation algorithm. + + Note: both grayscale and RGB/BGR images may be used with this extractor (parameters are adjusted automatically). + For optimal grayscale results, use CV_8UC1 frames instead of CV_8UC3. + + For more details on the different parametersor on the algorithm itself, see P.-L. St-Charles et al., + "Flexible Background Subtraction With Self-Balanced Local Sensitivity", in CVPRW 2014. + + This algorithm is currently NOT thread-safe. + */ + +class CV_EXPORTS_W MotionSaliencySuBSENSE : public BackgroundSubtractorLBSP, public MotionSaliency +{ + public: + + //! full constructor + MotionSaliencySuBSENSE( float fRelLBSPThreshold = BGSSUBSENSE_DEFAULT_LBSP_REL_SIMILARITY_THRESHOLD, size_t nMinDescDistThreshold = + BGSSUBSENSE_DEFAULT_DESC_DIST_THRESHOLD, + size_t nMinColorDistThreshold = BGSSUBSENSE_DEFAULT_COLOR_DIST_THRESHOLD, size_t nBGSamples = + BGSSUBSENSE_DEFAULT_NB_BG_SAMPLES, + size_t nRequiredBGSamples = BGSSUBSENSE_DEFAULT_REQUIRED_NB_BG_SAMPLES, size_t nSamplesForMovingAvgs = + BGSSUBSENSE_DEFAULT_N_SAMPLES_FOR_MV_AVGS ); + + virtual ~MotionSaliencySuBSENSE(); + //! (re)initiaization method; needs to be called before starting background subtraction (note: also reinitializes the keypoints vector) + virtual void initialize( const cv::Mat& oInitImg, const std::vector& voKeyPoints ); + //! refreshes all samples based on the last analyzed frame + virtual void refreshModel( float fSamplesRefreshFrac ); + //! primary model update function; the learning param is used to override the internal learning thresholds (ignored when <= 0) + virtual void operator()( cv::InputArray image, cv::OutputArray fgmask, double learningRateOverride = 0 ); + //! returns a copy of the latest reconstructed background image + void getBackgroundImage( cv::OutputArray backgroundImage ) const; + //! turns automatic model reset on or off + void setAutomaticModelReset( bool ); + + void read( const FileNode& fn ); + void write( FileStorage& fs ) const; + + protected: + bool computeSaliencyImpl( const InputArray src, OutputArray dst ); + AlgorithmInfo* info() const; + + //! indicates whether internal structures have already been initialized (LBSP lookup tables, samples, etc.) + bool m_bInitializedInternalStructs; + //! absolute minimal color distance threshold ('R' or 'radius' in the original ViBe paper, used as the default/initial 'R(x)' value here, paired with BackgroundSubtractorLBSP::m_nDescDistThreshold) + const size_t m_nMinColorDistThreshold; + //! number of different samples per pixel/block to be taken from input frames to build the background model (same as 'N' in ViBe/PBAS) + const size_t m_nBGSamples; + //! number of similar samples needed to consider the current pixel/block as 'background' (same as '#_min' in ViBe/PBAS) + const size_t m_nRequiredBGSamples; + //! number of samples to use to compute the learning rate of moving averages + const size_t m_nSamplesForMovingAvgs; + //! current frame index, frame count since last model reset & model reset cooldown counters + size_t m_nFrameIndex, m_nFramesSinceLastReset, m_nModelResetCooldown; + //! last calculated non-zero desc ratio + float m_fLastNonZeroDescRatio; + //! specifies whether automatic model reset is enabled or not + bool m_bAutoModelResetEnabled; + //! specifies whether Tmin/Tmax scaling is enabled or not + bool m_bLearningRateScalingEnabled; + //! current learning rate caps + float m_fCurrLearningRateLowerCap, m_fCurrLearningRateUpperCap; + //! current kernel size for median blur post-proc filtering + int m_nMedianBlurKernelSize; + //! specifies the px update spread range + bool m_bUse3x3Spread; + //! specifies the downsampled frame size (used for cam motion analysis) + Size m_oDownSampledFrameSize; + + //! background model pixel color intensity samples (equivalent to 'B(x)' in PBAS, but also paired with BackgroundSubtractorLBSP::m_voBGDescSamples to create our complete model) + std::vector m_voBGColorSamples; + + //! per-pixel update rates ('T(x)' in PBAS, which contains pixel-level 'sigmas', as referred to in ViBe) + cv::Mat m_oUpdateRateFrame; + //! per-pixel distance thresholds (equivalent to 'R(x)' in PBAS, but used as a relative value to determine both intensity and descriptor variation thresholds) + cv::Mat m_oDistThresholdFrame; + //! per-pixel distance variation modulators ('v(x)', relative value used to modulate 'R(x)' and 'T(x)' variations) + cv::Mat m_oVariationModulatorFrame; + //! per-pixel mean distances between consecutive frames ('D_last(x)', used to detect ghosts and high variation regions in the sequence) + cv::Mat m_oMeanLastDistFrame; + //! per-pixel mean minimal distances from the model ('D_min(x)' in PBAS, used to control variation magnitude and direction of 'T(x)' and 'R(x)') + cv::Mat m_oMeanMinDistFrame_LT, m_oMeanMinDistFrame_ST; + //! per-pixel mean downsampled distances between consecutive frames (used to analyze camera movement and control max learning rates globally) + cv::Mat m_oMeanDownSampledLastDistFrame_LT, m_oMeanDownSampledLastDistFrame_ST; + //! per-pixel mean raw segmentation results + cv::Mat m_oMeanRawSegmResFrame_LT, m_oMeanRawSegmResFrame_ST; + //! per-pixel mean final segmentation results + cv::Mat m_oMeanFinalSegmResFrame_LT, m_oMeanFinalSegmResFrame_ST; + //! a lookup map used to keep track of unstable regions (based on segm. noise & local dist. thresholds) + cv::Mat m_oUnstableRegionMask; + //! per-pixel blink detection results ('Z(x)') + cv::Mat m_oBlinksFrame; + //! pre-allocated matrix used to downsample (1/8) the input frame when needed + cv::Mat m_oDownSampledColorFrame; + //! copy of previously used pixel intensities used to calculate 'D_last(x)' + cv::Mat m_oLastColorFrame; + //! copy of previously used descriptors used to calculate 'D_last(x)' + cv::Mat m_oLastDescFrame; + //! the foreground mask generated by the method at [t-1] (without post-proc, used for blinking px detection) + cv::Mat m_oRawFGMask_last; + //! the foreground mask generated by the method at [t-1] (with post-proc) + cv::Mat m_oFGMask_last; + + //! pre-allocated CV_8UC1 matrices used to speed up morph ops + cv::Mat m_oFGMask_PreFlood; + cv::Mat m_oFGMask_FloodedHoles; + cv::Mat m_oFGMask_last_dilated; + cv::Mat m_oFGMask_last_dilated_inverted; + cv::Mat m_oRawFGBlinkMask_curr; + cv::Mat m_oRawFGBlinkMask_last; + +}; + /************************************ Specific Objectness Specialized Classes ************************************/ /** - * \brief Objectness algorithms based on [3] - * [3] Cheng, Ming-Ming, et al. "BING: Binarized normed gradients for objectness estimation at 300fps." IEEE CVPR. 2014. + * \brief Objectness algorithms based on [4] + * [4] Cheng, Ming-Ming, et al. "BING: Binarized normed gradients for objectness estimation at 300fps." IEEE CVPR. 2014. */ class CV_EXPORTS_W ObjectnessBING : public Objectness { diff --git a/modules/saliency/src/CmFile.cpp b/modules/saliency/src/BING/CmFile.cpp similarity index 100% rename from modules/saliency/src/CmFile.cpp rename to modules/saliency/src/BING/CmFile.cpp diff --git a/modules/saliency/src/CmFile.h b/modules/saliency/src/BING/CmFile.h similarity index 100% rename from modules/saliency/src/CmFile.h rename to modules/saliency/src/BING/CmFile.h diff --git a/modules/saliency/src/CmShow.cpp b/modules/saliency/src/BING/CmShow.cpp similarity index 100% rename from modules/saliency/src/CmShow.cpp rename to modules/saliency/src/BING/CmShow.cpp diff --git a/modules/saliency/src/CmShow.h b/modules/saliency/src/BING/CmShow.h similarity index 100% rename from modules/saliency/src/CmShow.h rename to modules/saliency/src/BING/CmShow.h diff --git a/modules/saliency/src/CmTimer.h b/modules/saliency/src/BING/CmTimer.h similarity index 100% rename from modules/saliency/src/CmTimer.h rename to modules/saliency/src/BING/CmTimer.h diff --git a/modules/saliency/src/FilterTIG.cpp b/modules/saliency/src/BING/FilterTIG.cpp similarity index 100% rename from modules/saliency/src/FilterTIG.cpp rename to modules/saliency/src/BING/FilterTIG.cpp diff --git a/modules/saliency/src/FilterTIG.h b/modules/saliency/src/BING/FilterTIG.h similarity index 100% rename from modules/saliency/src/FilterTIG.h rename to modules/saliency/src/BING/FilterTIG.h diff --git a/modules/saliency/src/ValStructVec.h b/modules/saliency/src/BING/ValStructVec.h similarity index 100% rename from modules/saliency/src/ValStructVec.h rename to modules/saliency/src/BING/ValStructVec.h diff --git a/modules/saliency/src/kyheader.h b/modules/saliency/src/BING/kyheader.h similarity index 100% rename from modules/saliency/src/kyheader.h rename to modules/saliency/src/BING/kyheader.h diff --git a/modules/saliency/src/objectnessBING.cpp b/modules/saliency/src/BING/objectnessBING.cpp similarity index 100% rename from modules/saliency/src/objectnessBING.cpp rename to modules/saliency/src/BING/objectnessBING.cpp diff --git a/modules/saliency/src/SuBSENSE/BackgroundSubtractorLBSP.cpp b/modules/saliency/src/SuBSENSE/BackgroundSubtractorLBSP.cpp new file mode 100644 index 000000000..4817cda42 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/BackgroundSubtractorLBSP.cpp @@ -0,0 +1,60 @@ +#include "BackgroundSubtractorLBSP.h" +#include "DistanceUtils.h" +#include "RandUtils.h" +#include +#include +#include +#include +#include + +BackgroundSubtractorLBSP::BackgroundSubtractorLBSP(float fRelLBSPThreshold, size_t nDescDistThreshold, size_t nLBSPThresholdOffset) + : nDebugCoordX(0),nDebugCoordY(0) + ,m_nKeyPoints(0) + ,m_nImgChannels(0) + ,m_nImgType(0) + ,m_nDescDistThreshold(nDescDistThreshold) + ,m_nLBSPThresholdOffset(nLBSPThresholdOffset) + ,m_fRelLBSPThreshold(fRelLBSPThreshold) + ,m_bInitialized(false) { + CV_Assert(m_fRelLBSPThreshold>=0); +} + +BackgroundSubtractorLBSP::~BackgroundSubtractorLBSP() {} + +void BackgroundSubtractorLBSP::initialize(const cv::Mat& oInitImg) { + this->initialize(oInitImg,std::vector()); +} + +cv::AlgorithmInfo* BackgroundSubtractorLBSP::info() const { + return nullptr; +} + +void BackgroundSubtractorLBSP::getBackgroundDescriptorsImage(cv::OutputArray backgroundDescImage) const { + CV_Assert(LBSP::DESC_SIZE==2); + CV_Assert(m_bInitialized); + cv::Mat oAvgBGDesc = cv::Mat::zeros(m_oImgSize,CV_32FC((int)m_nImgChannels)); + for(size_t n=0; n BackgroundSubtractorLBSP::getBGKeyPoints() const { + return m_voKeyPoints; +} + +void BackgroundSubtractorLBSP::setBGKeyPoints(std::vector& keypoints) { + LBSP::validateKeyPoints(keypoints,m_oImgSize); + CV_Assert(!keypoints.empty()); + m_voKeyPoints = keypoints; + m_nKeyPoints = keypoints.size(); +} diff --git a/modules/saliency/src/SuBSENSE/BackgroundSubtractorLBSP.h b/modules/saliency/src/SuBSENSE/BackgroundSubtractorLBSP.h new file mode 100644 index 000000000..adf6539f6 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/BackgroundSubtractorLBSP.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include "LBSP.h" + +/*! + Local Binary Similarity Pattern (LBSP) foreground-background segmentation algorithm (abstract version). + + For more details on the different parameters, see P.-L. St-Charles and G.-A. Bilodeau, "Improving Background + Subtraction using Local Binary Similarity Patterns", in WACV 2014, or G.-A. Bilodeau et al, "Change Detection + in Feature Space Using Local Binary Similarity Patterns", in CRV 2013. + + This algorithm is currently NOT thread-safe. + */ +class BackgroundSubtractorLBSP : public cv::BackgroundSubtractor { +public: + //! full constructor + BackgroundSubtractorLBSP(float fRelLBSPThreshold, size_t nDescDistThreshold, size_t nLBSPThresholdOffset=0); + //! default destructor + virtual ~BackgroundSubtractorLBSP(); + //! (re)initiaization method; needs to be called before starting background subtraction + virtual void initialize(const cv::Mat& oInitImg); + //! (re)initiaization method; needs to be called before starting background subtraction (note: also reinitializes the keypoints vector) + virtual void initialize(const cv::Mat& oInitImg, const std::vector& voKeyPoints)=0; + //! primary model update function; the learning param is used to override the internal learning speed (ignored when <= 0) + virtual void operator()(cv::InputArray image, cv::OutputArray fgmask, double learningRate=0)=0; + //! unused, always returns nullptr + virtual cv::AlgorithmInfo* info() const; + //! returns a copy of the latest reconstructed background descriptors image + virtual void getBackgroundDescriptorsImage(cv::OutputArray backgroundDescImage) const; + //! returns the keypoints list used for descriptor extraction (note: by default, these are generated from the DenseFeatureDetector class, and the border points are removed) + virtual std::vector getBGKeyPoints() const; + //! sets the keypoints to be used for descriptor extraction, effectively setting the BGModel ROI (note: this function will remove all border keypoints) + virtual void setBGKeyPoints(std::vector& keypoints); + + // ######## DEBUG PURPOSES ONLY ########## + int nDebugCoordX, nDebugCoordY; + +protected: + //! background model descriptors samples (tied to m_voKeyPoints but shaped like the input frames) + std::vector m_voBGDescSamples; + //! background model keypoints used for LBSP descriptor extraction (specific to the input image size) + std::vector m_voKeyPoints; + //! defines the current number of used keypoints (always tied to m_voKeyPoints) + size_t m_nKeyPoints; + //! input image size + cv::Size m_oImgSize; + //! input image channel size + size_t m_nImgChannels; + //! input image type + int m_nImgType; + //! absolute descriptor distance threshold + const size_t m_nDescDistThreshold; + //! LBSP internal threshold offset value -- used to reduce texture noise in dark regions + const size_t m_nLBSPThresholdOffset; + //! LBSP relative internal threshold (kept here since we don't keep an LBSP object) + const float m_fRelLBSPThreshold; + //! pre-allocated internal LBSP threshold values for all possible 8-bit intensity values + size_t m_anLBSPThreshold_8bitLUT[UCHAR_MAX+1]; + //! defines whether or not the subtractor is fully initialized + bool m_bInitialized; +}; + diff --git a/modules/saliency/src/SuBSENSE/DistanceUtils.h b/modules/saliency/src/SuBSENSE/DistanceUtils.h new file mode 100644 index 000000000..5d58a9186 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/DistanceUtils.h @@ -0,0 +1,106 @@ +#pragma once + +#include + +//! computes the absolute difference of two unsigned char values +static inline size_t absdiff_uchar(uchar a, uchar b) { + return (size_t)abs((int)a-(int)b); // should return the same as (a>8)]; +} + +//! computes the population count of 3x16bit vectors using an 8bit popcount LUT (min=0, max=48) +static inline uchar popcount_ushort_8bitsLUT(const ushort* x) { + return popcount_LUT8[(uchar)x[0]] + popcount_LUT8[(uchar)(x[0]>>8)] + + popcount_LUT8[(uchar)x[1]] + popcount_LUT8[(uchar)(x[1]>>8)] + + popcount_LUT8[(uchar)x[2]] + popcount_LUT8[(uchar)(x[2]>>8)]; +} + +//! computes the hamming distance between two 16bit vectors (min=0, max=16) +static inline size_t hdist_ushort_8bitLUT(ushort a, ushort b) { + return popcount_ushort_8bitsLUT(a^b); +} + +//! computes the sum of hamming distances between two 3x16 bits vectors (min=0, max=48) +static inline size_t hdist_ushort_8bitLUT(const ushort* a, const ushort* b) { + return popcount_ushort_8bitsLUT(a[0]^b[0])+popcount_ushort_8bitsLUT(a[1]^b[1])+popcount_ushort_8bitsLUT(a[2]^b[2]); +} + +//! computes the gradient magnitude distance between two 16 bits vectors (min=0, max=16) +static inline size_t gdist_ushort_8bitLUT(ushort a, ushort b) { + return (size_t)abs((int)popcount_ushort_8bitsLUT(a)-(int)popcount_ushort_8bitsLUT(b)); +} + +//! computes the sum of gradient magnitude distances between two 3x16 bits vectors (min=0, max=48) +static inline size_t gdist_ushort_8bitLUT(const ushort* a, const ushort* b) { + return (size_t)abs((int)popcount_ushort_8bitsLUT(a)-(int)popcount_ushort_8bitsLUT(b)); +} diff --git a/modules/saliency/src/SuBSENSE/LBSP.cpp b/modules/saliency/src/SuBSENSE/LBSP.cpp new file mode 100644 index 000000000..7d733a480 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/LBSP.cpp @@ -0,0 +1,308 @@ +#include "LBSP.h" + +LBSP::LBSP(size_t nThreshold) + : m_bOnlyUsingAbsThreshold(true) + ,m_fRelThreshold(0) // unused + ,m_nThreshold(nThreshold) + ,m_oRefImage() {} + +LBSP::LBSP(float fRelThreshold, size_t nThresholdOffset) + : m_bOnlyUsingAbsThreshold(false) + ,m_fRelThreshold(fRelThreshold) + ,m_nThreshold(nThresholdOffset) + ,m_oRefImage() { + CV_Assert(m_fRelThreshold>=0); +} + +LBSP::~LBSP() {} + +void LBSP::read(const cv::FileNode& /*fn*/) { + // ... = fn["..."]; +} + +void LBSP::write(cv::FileStorage& /*fs*/) const { + //fs << "..." << ...; +} + +void LBSP::setReference(const cv::Mat& img) { + CV_DbgAssert(img.empty() || img.type()==CV_8UC1 || img.type()==CV_8UC3); + m_oRefImage = img; +} + +int LBSP::descriptorSize() const { + return DESC_SIZE; +} + +int LBSP::descriptorType() const { + return CV_16U; +} + +bool LBSP::isUsingRelThreshold() const { + return !m_bOnlyUsingAbsThreshold; +} + +float LBSP::getRelThreshold() const { + return m_fRelThreshold; +} + +size_t LBSP::getAbsThreshold() const { + return m_nThreshold; +} + +static inline void lbsp_computeImpl( const cv::Mat& oInputImg, + const cv::Mat& oRefImg, + const std::vector& voKeyPoints, + cv::Mat& oDesc, + size_t _t) { + CV_DbgAssert(oRefImg.empty() || (oRefImg.size==oInputImg.size && oRefImg.type()==oInputImg.type())); + CV_DbgAssert(oInputImg.type()==CV_8UC1 || oInputImg.type()==CV_8UC3); + CV_DbgAssert(LBSP::DESC_SIZE==2); // @@@ also relies on a constant desc size + const size_t nChannels = (size_t)oInputImg.channels(); + const size_t _step_row = oInputImg.step.p[0]; + const uchar* _data = oInputImg.data; + const uchar* _refdata = oRefImg.empty()?oInputImg.data:oRefImg.data; + const size_t nKeyPoints = voKeyPoints.size(); + if(nChannels==1) { + oDesc.create((int)nKeyPoints,1,CV_16UC1); + for(size_t k=0; k((int)k); + #include "LBSP_16bits_dbcross_1ch.i" + } + } + else { //nChannels==3 + oDesc.create((int)nKeyPoints,1,CV_16UC3); + for(size_t k=0; k& voKeyPoints, + cv::Mat& oDesc, + float fThreshold, + size_t nThresholdOffset) { + CV_DbgAssert(oRefImg.empty() || (oRefImg.size==oInputImg.size && oRefImg.type()==oInputImg.type())); + CV_DbgAssert(oInputImg.type()==CV_8UC1 || oInputImg.type()==CV_8UC3); + CV_DbgAssert(LBSP::DESC_SIZE==2); // @@@ also relies on a constant desc size + CV_DbgAssert(fThreshold>=0); + const size_t nChannels = (size_t)oInputImg.channels(); + const size_t _step_row = oInputImg.step.p[0]; + const uchar* _data = oInputImg.data; + const uchar* _refdata = oRefImg.empty()?oInputImg.data:oRefImg.data; + const size_t nKeyPoints = voKeyPoints.size(); + if(nChannels==1) { + oDesc.create((int)nKeyPoints,1,CV_16UC1); + for(size_t k=0; k((int)k); + const size_t _t = (size_t)(_ref*fThreshold)+nThresholdOffset; + #include "LBSP_16bits_dbcross_1ch.i" + } + } + else { //nChannels==3 + oDesc.create((int)nKeyPoints,1,CV_16UC3); + for(size_t k=0; k& voKeyPoints, + cv::Mat& oDesc, + size_t _t) { + CV_DbgAssert(oRefImg.empty() || (oRefImg.size==oInputImg.size && oRefImg.type()==oInputImg.type())); + CV_DbgAssert(oInputImg.type()==CV_8UC1 || oInputImg.type()==CV_8UC3); + CV_DbgAssert(LBSP::DESC_SIZE==2); // @@@ also relies on a constant desc size + const size_t nChannels = (size_t)oInputImg.channels(); + const size_t _step_row = oInputImg.step.p[0]; + const uchar* _data = oInputImg.data; + const uchar* _refdata = oRefImg.empty()?oInputImg.data:oRefImg.data; + const size_t nKeyPoints = voKeyPoints.size(); + if(nChannels==1) { + oDesc.create(oInputImg.size(),CV_16UC1); + for(size_t k=0; k(_y,_x); + #include "LBSP_16bits_dbcross_1ch.i" + } + } + else { //nChannels==3 + oDesc.create(oInputImg.size(),CV_16UC3); + for(size_t k=0; k& voKeyPoints, + cv::Mat& oDesc, + float fThreshold, + size_t nThresholdOffset) { + CV_DbgAssert(oRefImg.empty() || (oRefImg.size==oInputImg.size && oRefImg.type()==oInputImg.type())); + CV_DbgAssert(oInputImg.type()==CV_8UC1 || oInputImg.type()==CV_8UC3); + CV_DbgAssert(LBSP::DESC_SIZE==2); // @@@ also relies on a constant desc size + CV_DbgAssert(fThreshold>=0); + const size_t nChannels = (size_t)oInputImg.channels(); + const size_t _step_row = oInputImg.step.p[0]; + const uchar* _data = oInputImg.data; + const uchar* _refdata = oRefImg.empty()?oInputImg.data:oRefImg.data; + const size_t nKeyPoints = voKeyPoints.size(); + if(nChannels==1) { + oDesc.create(oInputImg.size(),CV_16UC1); + for(size_t k=0; k(_y,_x); + const size_t _t = (size_t)(_ref*fThreshold)+nThresholdOffset; + #include "LBSP_16bits_dbcross_1ch.i" + } + } + else { //nChannels==3 + oDesc.create(oInputImg.size(),CV_16UC3); + for(size_t k=0; k& voKeypoints, cv::Mat& oDescriptors) const { + CV_Assert(!oImage.empty()); + cv::KeyPointsFilter::runByImageBorder(voKeypoints,oImage.size(),PATCH_SIZE/2); + cv::KeyPointsFilter::runByKeypointSize(voKeypoints,std::numeric_limits::epsilon()); + if(voKeypoints.empty()) { + oDescriptors.release(); + return; + } + if(m_bOnlyUsingAbsThreshold) + lbsp_computeImpl2(oImage,m_oRefImage,voKeypoints,oDescriptors,m_nThreshold); + else + lbsp_computeImpl2(oImage,m_oRefImage,voKeypoints,oDescriptors,m_fRelThreshold,m_nThreshold); +} + +void LBSP::compute2(const std::vector& voImageCollection, std::vector >& vvoPointCollection, std::vector& voDescCollection) const { + CV_Assert(voImageCollection.size() == vvoPointCollection.size()); + voDescCollection.resize(voImageCollection.size()); + for(size_t i=0; i& voKeypoints, cv::Mat& oDescriptors) const { + CV_Assert(!oImage.empty()); + cv::KeyPointsFilter::runByImageBorder(voKeypoints,oImage.size(),PATCH_SIZE/2); + cv::KeyPointsFilter::runByKeypointSize(voKeypoints,std::numeric_limits::epsilon()); + if(voKeypoints.empty()) { + oDescriptors.release(); + return; + } + if(m_bOnlyUsingAbsThreshold) + lbsp_computeImpl(oImage,m_oRefImage,voKeypoints,oDescriptors,m_nThreshold); + else + lbsp_computeImpl(oImage,m_oRefImage,voKeypoints,oDescriptors,m_fRelThreshold,m_nThreshold); +} + +void LBSP::reshapeDesc(cv::Size oSize, const std::vector& voKeypoints, const cv::Mat& oDescriptors, cv::Mat& oOutput) { + CV_DbgAssert(!voKeypoints.empty()); + CV_DbgAssert(!oDescriptors.empty() && oDescriptors.cols==1); + CV_DbgAssert(oSize.width>0 && oSize.height>0); + CV_DbgAssert(DESC_SIZE==2); // @@@ also relies on a constant desc size + CV_DbgAssert(oDescriptors.type()==CV_16UC1 || oDescriptors.type()==CV_16UC3); + const size_t nChannels = (size_t)oDescriptors.channels(); + const size_t nKeyPoints = voKeypoints.size(); + if(nChannels==1) { + oOutput.create(oSize,CV_16UC1); + oOutput = cv::Scalar_(0); + for(size_t k=0; k(voKeypoints[k].pt) = oDescriptors.at((int)k); + } + else { //nChannels==3 + oOutput.create(oSize,CV_16UC3); + oOutput = cv::Scalar_(0,0,0); + for(size_t k=0; k(i,j) = (uchar)(fScaleFactor*hdist_ushort_8bitLUT(desc1_ptr[j],desc2_ptr[j])); + } + } + else { //nChannels==3 + if(bForceMergeChannels) + oOutput.create(oDesc1.size(),CV_8UC1); + else + oOutput.create(oDesc1.size(),CV_8UC3); + for(int i=0; i& voKeypoints, cv::Size oImgSize) { + cv::KeyPointsFilter::runByImageBorder(voKeypoints,oImgSize,PATCH_SIZE/2); +} diff --git a/modules/saliency/src/SuBSENSE/LBSP.h b/modules/saliency/src/SuBSENSE/LBSP.h new file mode 100644 index 000000000..46566d26a --- /dev/null +++ b/modules/saliency/src/SuBSENSE/LBSP.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include +#include "DistanceUtils.h" + +/*! + Local Binary Similarity Pattern (LBSP) feature extractor + + Note 1: both grayscale and RGB/BGR images may be used with this extractor. + Note 2: using LBSP::compute2(...) is logically equivalent to using LBSP::compute(...) followed by LBSP::reshapeDesc(...). + + For more details on the different parameters, see G.-A. Bilodeau et al, "Change Detection in Feature Space Using Local + Binary Similarity Patterns", in CRV 2013. + + This algorithm is currently NOT thread-safe. + */ +class LBSP : public cv::DescriptorExtractor { +public: + //! constructor 1, threshold = absolute intensity 'similarity' threshold used when computing comparisons + LBSP(size_t nThreshold); + //! constructor 2, threshold = relative intensity 'similarity' threshold used when computing comparisons + LBSP(float fRelThreshold, size_t nThresholdOffset=0); + //! default destructor + virtual ~LBSP(); + //! loads extractor params from the specified file node @@@@ not impl + virtual void read(const cv::FileNode&); + //! writes extractor params to the specified file storage @@@@ not impl + virtual void write(cv::FileStorage&) const; + //! sets the 'reference' image to be used for inter-frame comparisons (note: if no image is set or if the image is empty, the algorithm will default back to intra-frame comparisons) + virtual void setReference(const cv::Mat&); + //! returns the current descriptor size, in bytes + virtual int descriptorSize() const; + //! returns the current descriptor data type + virtual int descriptorType() const; + //! returns whether this extractor is using a relative threshold or not + virtual bool isUsingRelThreshold() const; + //! returns the current relative threshold used for comparisons (-1 = invalid/not used) + virtual float getRelThreshold() const; + //! returns the current absolute threshold used for comparisons (-1 = invalid/not used) + virtual size_t getAbsThreshold() const; + + //! similar to DescriptorExtractor::compute(const cv::Mat& image, ...), but in this case, the descriptors matrix has the same shape as the input matrix (possibly slower, but the result can be displayed) + void compute2(const cv::Mat& oImage, std::vector& voKeypoints, cv::Mat& oDescriptors) const; + //! batch version of LBSP::compute2(const cv::Mat& image, ...), also similar to DescriptorExtractor::compute(const std::vector& imageCollection, ...) + void compute2(const std::vector& voImageCollection, std::vector >& vvoPointCollection, std::vector& voDescCollection) const; + + // utility function, shortcut/lightweight/direct single-point LBSP computation function for extra flexibility (1-channel version) + inline static void computeGrayscaleDescriptor(const cv::Mat& oInputImg, const uchar _ref, const int _x, const int _y, const size_t _t, ushort& _res) { + CV_DbgAssert(!oInputImg.empty()); + CV_DbgAssert(oInputImg.type()==CV_8UC1); + CV_DbgAssert(LBSP::DESC_SIZE==2); // @@@ also relies on a constant desc size + CV_DbgAssert(_x>=(int)LBSP::PATCH_SIZE/2 && _y>=(int)LBSP::PATCH_SIZE/2); + CV_DbgAssert(_x=(int)LBSP::PATCH_SIZE/2 && _y>=(int)LBSP::PATCH_SIZE/2); + CV_DbgAssert(_x=(int)LBSP::PATCH_SIZE/2 && _y>=(int)LBSP::PATCH_SIZE/2); + CV_DbgAssert(_x=(int)LBSP::PATCH_SIZE/2 && _y>=(int)LBSP::PATCH_SIZE/2); + CV_DbgAssert(_x& voKeypoints, const cv::Mat& oDescriptors, cv::Mat& oOutput); + //! utility function, used to illustrate the difference between two descriptor images + static void calcDescImgDiff(const cv::Mat& oDesc1, const cv::Mat& oDesc2, cv::Mat& oOutput, bool bForceMergeChannels=false); + //! utility function, used to filter out bad keypoints that would trigger out of bounds error because they're too close to the image border + static void validateKeyPoints(std::vector& voKeypoints, cv::Size oImgSize); + //! utility, specifies the pixel size of the pattern used (width and height) + static const size_t PATCH_SIZE = 5; + //! utility, specifies the number of bytes per descriptor (should be the same as calling 'descriptorSize()') + static const size_t DESC_SIZE = 2; + +protected: + //! classic 'compute' implementation, based on the regular DescriptorExtractor::computeImpl arguments & expected output + virtual void computeImpl(const cv::Mat& oImage, std::vector& voKeypoints, cv::Mat& oDescriptors) const; + + const bool m_bOnlyUsingAbsThreshold; + const float m_fRelThreshold; + const size_t m_nThreshold; + cv::Mat m_oRefImage; +}; diff --git a/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_1ch.i b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_1ch.i new file mode 100644 index 000000000..487153559 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_1ch.i @@ -0,0 +1,44 @@ +// note: this is the LBSP 16 bit double-cross single channel pattern as used in +// the original article by G.-A. Bilodeau et al. +// +// O O O 4 .. 3 .. 6 +// O O O .. 15 8 13 .. +// O O X O O => 0 9 X 11 1 +// O O O .. 12 10 14 .. +// O O O 7 .. 2 .. 5 +// +// must be defined externally: +// _t (size_t, absolute threshold used for comparisons) +// _ref (uchar, 'central' value used for comparisons) +// _data (uchar*, single-channel data to be covered by the pattern) +// _y (int, pattern rows location in the image data) +// _x (int, pattern cols location in the image data) +// _step_row (size_t, step size between rows, including padding) +// _res (ushort, 16 bit result vector) +// absdiff_uchar (function, returns the absolute difference between two uchars) + +#ifdef _val +#error "definitions clash detected" +#else +#define _val(x,y) _data[_step_row*(_y+y)+_x+x] +#endif + +_res= ((absdiff_uchar(_val(-1, 1),_ref) > _t) << 15) + + ((absdiff_uchar(_val( 1,-1),_ref) > _t) << 14) + + ((absdiff_uchar(_val( 1, 1),_ref) > _t) << 13) + + ((absdiff_uchar(_val(-1,-1),_ref) > _t) << 12) + + ((absdiff_uchar(_val( 1, 0),_ref) > _t) << 11) + + ((absdiff_uchar(_val( 0,-1),_ref) > _t) << 10) + + ((absdiff_uchar(_val(-1, 0),_ref) > _t) << 9) + + ((absdiff_uchar(_val( 0, 1),_ref) > _t) << 8) + + ((absdiff_uchar(_val(-2,-2),_ref) > _t) << 7) + + ((absdiff_uchar(_val( 2, 2),_ref) > _t) << 6) + + ((absdiff_uchar(_val( 2,-2),_ref) > _t) << 5) + + ((absdiff_uchar(_val(-2, 2),_ref) > _t) << 4) + + ((absdiff_uchar(_val( 0, 2),_ref) > _t) << 3) + + ((absdiff_uchar(_val( 0,-2),_ref) > _t) << 2) + + ((absdiff_uchar(_val( 2, 0),_ref) > _t) << 1) + + ((absdiff_uchar(_val(-2, 0),_ref) > _t)); + +#undef _val + \ No newline at end of file diff --git a/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_3ch1t.i b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_3ch1t.i new file mode 100644 index 000000000..ce044a052 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_3ch1t.i @@ -0,0 +1,46 @@ +// note: this is the LBSP 16 bit double-cross indiv RGB pattern as used in +// the original article by G.-A. Bilodeau et al. +// +// O O O 4 .. 3 .. 6 +// O O O .. 15 8 13 .. +// O O X O O => 0 9 X 11 1 +// O O O .. 12 10 14 .. +// O O O 7 .. 2 .. 5 +// 3x 3x +// +// must be defined externally: +// _t (size_t, absolute threshold used for comparisons) +// _ref (uchar[3], 'central' values used for comparisons) +// _data (uchar*, triple-channel data to be covered by the pattern) +// _y (int, pattern rows location in the image data) +// _x (int, pattern cols location in the image data) +// _step_row (size_t, step size between rows, including padding) +// _res (ushort[3], 16 bit result vectors vector) +// absdiff_uchar (function, returns the absolute difference between two uchars) + +#ifdef _val +#error "definitions clash detected" +#else +#define _val(x,y,n) _data[_step_row*(_y+y)+3*(_x+x)+n] +#endif + +for(int n=0; n<3; ++n) { + _res[n] = ((absdiff_uchar(_val(-1, 1, n),_ref[n]) > _t) << 15) + + ((absdiff_uchar(_val( 1,-1, n),_ref[n]) > _t) << 14) + + ((absdiff_uchar(_val( 1, 1, n),_ref[n]) > _t) << 13) + + ((absdiff_uchar(_val(-1,-1, n),_ref[n]) > _t) << 12) + + ((absdiff_uchar(_val( 1, 0, n),_ref[n]) > _t) << 11) + + ((absdiff_uchar(_val( 0,-1, n),_ref[n]) > _t) << 10) + + ((absdiff_uchar(_val(-1, 0, n),_ref[n]) > _t) << 9) + + ((absdiff_uchar(_val( 0, 1, n),_ref[n]) > _t) << 8) + + ((absdiff_uchar(_val(-2,-2, n),_ref[n]) > _t) << 7) + + ((absdiff_uchar(_val( 2, 2, n),_ref[n]) > _t) << 6) + + ((absdiff_uchar(_val( 2,-2, n),_ref[n]) > _t) << 5) + + ((absdiff_uchar(_val(-2, 2, n),_ref[n]) > _t) << 4) + + ((absdiff_uchar(_val( 0, 2, n),_ref[n]) > _t) << 3) + + ((absdiff_uchar(_val( 0,-2, n),_ref[n]) > _t) << 2) + + ((absdiff_uchar(_val( 2, 0, n),_ref[n]) > _t) << 1) + + ((absdiff_uchar(_val(-2, 0, n),_ref[n]) > _t)); +} + +#undef _val diff --git a/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_3ch3t.i b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_3ch3t.i new file mode 100644 index 000000000..28ffedd96 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_3ch3t.i @@ -0,0 +1,46 @@ +// note: this is the LBSP 16 bit double-cross indiv RGB pattern as used in +// the original article by G.-A. Bilodeau et al. +// +// O O O 4 .. 3 .. 6 +// O O O .. 15 8 13 .. +// O O X O O => 0 9 X 11 1 +// O O O .. 12 10 14 .. +// O O O 7 .. 2 .. 5 +// 3x 3x +// +// must be defined externally: +// _t (size_t[3], absolute thresholds used for comparisons) +// _ref (uchar[3], 'central' values used for comparisons) +// _data (uchar*, triple-channel data to be covered by the pattern) +// _y (int, pattern rows location in the image data) +// _x (int, pattern cols location in the image data) +// _step_row (size_t, step size between rows, including padding) +// _res (ushort[3], 16 bit result vectors vector) +// absdiff_uchar (function, returns the absolute difference between two uchars) + +#ifdef _val +#error "definitions clash detected" +#else +#define _val(x,y,n) _data[_step_row*(_y+y)+3*(_x+x)+n] +#endif + +for(int n=0; n<3; ++n) { + _res[n] = ((absdiff_uchar(_val(-1, 1, n),_ref[n]) > _t[n]) << 15) + + ((absdiff_uchar(_val( 1,-1, n),_ref[n]) > _t[n]) << 14) + + ((absdiff_uchar(_val( 1, 1, n),_ref[n]) > _t[n]) << 13) + + ((absdiff_uchar(_val(-1,-1, n),_ref[n]) > _t[n]) << 12) + + ((absdiff_uchar(_val( 1, 0, n),_ref[n]) > _t[n]) << 11) + + ((absdiff_uchar(_val( 0,-1, n),_ref[n]) > _t[n]) << 10) + + ((absdiff_uchar(_val(-1, 0, n),_ref[n]) > _t[n]) << 9) + + ((absdiff_uchar(_val( 0, 1, n),_ref[n]) > _t[n]) << 8) + + ((absdiff_uchar(_val(-2,-2, n),_ref[n]) > _t[n]) << 7) + + ((absdiff_uchar(_val( 2, 2, n),_ref[n]) > _t[n]) << 6) + + ((absdiff_uchar(_val( 2,-2, n),_ref[n]) > _t[n]) << 5) + + ((absdiff_uchar(_val(-2, 2, n),_ref[n]) > _t[n]) << 4) + + ((absdiff_uchar(_val( 0, 2, n),_ref[n]) > _t[n]) << 3) + + ((absdiff_uchar(_val( 0,-2, n),_ref[n]) > _t[n]) << 2) + + ((absdiff_uchar(_val( 2, 0, n),_ref[n]) > _t[n]) << 1) + + ((absdiff_uchar(_val(-2, 0, n),_ref[n]) > _t[n])); +} + +#undef _val diff --git a/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_s3ch.i b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_s3ch.i new file mode 100644 index 000000000..03ba2ffa8 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/LBSP_16bits_dbcross_s3ch.i @@ -0,0 +1,46 @@ +// note: this is the LBSP 16 bit double-cross indiv RGB pattern as used in +// the original article by G.-A. Bilodeau et al. +// +// O O O 4 .. 3 .. 6 +// O O O .. 15 8 13 .. +// O O X O O => 0 9 X 11 1 +// O O O .. 12 10 14 .. +// O O O 7 .. 2 .. 5 +// (single/3x) (single/3x) +// +// must be defined externally: +// _t (size_t, absolute threshold used for comparisons) +// _ref (uchar, 'central' value used for comparisons) +// _data (uchar*, triple-channel data to be covered by the pattern) +// _y (int, pattern rows location in the image data) +// _x (int, pattern cols location in the image data) +// _c (size_t, pattern channel location in the image data) +// _step_row (size_t, step size between rows, including padding) +// _res (ushort, 16 bit result vector) +// absdiff_uchar (function, returns the absolute difference between two uchars) + +#ifdef _val +#error "definitions clash detected" +#else +#define _val(x,y,n) _data[_step_row*(_y+y)+3*(_x+x)+n] +#endif + +_res = ((absdiff_uchar(_val(-1, 1, _c),_ref) > _t) << 15) + + ((absdiff_uchar(_val( 1,-1, _c),_ref) > _t) << 14) + + ((absdiff_uchar(_val( 1, 1, _c),_ref) > _t) << 13) + + ((absdiff_uchar(_val(-1,-1, _c),_ref) > _t) << 12) + + ((absdiff_uchar(_val( 1, 0, _c),_ref) > _t) << 11) + + ((absdiff_uchar(_val( 0,-1, _c),_ref) > _t) << 10) + + ((absdiff_uchar(_val(-1, 0, _c),_ref) > _t) << 9) + + ((absdiff_uchar(_val( 0, 1, _c),_ref) > _t) << 8) + + ((absdiff_uchar(_val(-2,-2, _c),_ref) > _t) << 7) + + ((absdiff_uchar(_val( 2, 2, _c),_ref) > _t) << 6) + + ((absdiff_uchar(_val( 2,-2, _c),_ref) > _t) << 5) + + ((absdiff_uchar(_val(-2, 2, _c),_ref) > _t) << 4) + + ((absdiff_uchar(_val( 0, 2, _c),_ref) > _t) << 3) + + ((absdiff_uchar(_val( 0,-2, _c),_ref) > _t) << 2) + + ((absdiff_uchar(_val( 2, 0, _c),_ref) > _t) << 1) + + ((absdiff_uchar(_val(-2, 0, _c),_ref) > _t)); + +#undef _val + \ No newline at end of file diff --git a/modules/saliency/src/SuBSENSE/RandUtils.h b/modules/saliency/src/SuBSENSE/RandUtils.h new file mode 100644 index 000000000..f5ea32363 --- /dev/null +++ b/modules/saliency/src/SuBSENSE/RandUtils.h @@ -0,0 +1,96 @@ +#pragma once + +/*// gaussian 3x3 pattern, based on 'floor(fspecial('gaussian', 3, 1)*256)' +static const int s_nSamplesInitPatternWidth = 3; +static const int s_nSamplesInitPatternHeight = 3; +static const int s_nSamplesInitPatternTot = 256; +static const int s_anSamplesInitPattern[s_nSamplesInitPatternHeight][s_nSamplesInitPatternWidth] = { + {19, 32, 19,}, + {32, 52, 32,}, + {19, 32, 19,}, +};*/ + +// gaussian 7x7 pattern, based on 'floor(fspecial('gaussian',7,1)*4096)' +static const int s_nSamplesInitPatternWidth = 7; +static const int s_nSamplesInitPatternHeight = 7; +static const int s_nSamplesInitPatternTot = 4096; +static const int s_anSamplesInitPattern[s_nSamplesInitPatternHeight][s_nSamplesInitPatternWidth] = { + {0, 0, 4, 7, 4, 0, 0,}, + {0, 11, 53, 88, 53, 11, 0,}, + {4, 53, 240, 399, 240, 53, 4,}, + {7, 88, 399, 660, 399, 88, 7,}, + {4, 53, 240, 399, 240, 53, 4,}, + {0, 11, 53, 88, 53, 11, 0,}, + {0, 0, 4, 7, 4, 0, 0,}, +}; + +//! returns a random init/sampling position for the specified pixel position; also guards against out-of-bounds values via image/border size check. +static inline void getRandSamplePosition(int& x_sample, int& y_sample, const int x_orig, const int y_orig, const int border, const cv::Size& imgsize) { + int r = 1+rand()%s_nSamplesInitPatternTot; + for(x_sample=0; x_sample=imgsize.width-border) + x_sample = imgsize.width-border-1; + if(y_sample=imgsize.height-border) + y_sample = imgsize.height-border-1; +} + +// simple 8-connected (3x3) neighbors pattern +static const int s_anNeighborPatternSize_3x3 = 8; +static const int s_anNeighborPattern_3x3[8][2] = { + {-1, 1}, { 0, 1}, { 1, 1}, + {-1, 0}, { 1, 0}, + {-1,-1}, { 0,-1}, { 1,-1}, +}; + +//! returns a random neighbor position for the specified pixel position; also guards against out-of-bounds values via image/border size check. +static inline void getRandNeighborPosition_3x3(int& x_neighbor, int& y_neighbor, const int x_orig, const int y_orig, const int border, const cv::Size& imgsize) { + int r = rand()%s_anNeighborPatternSize_3x3; + x_neighbor = x_orig+s_anNeighborPattern_3x3[r][0]; + y_neighbor = y_orig+s_anNeighborPattern_3x3[r][1]; + if(x_neighbor=imgsize.width-border) + x_neighbor = imgsize.width-border-1; + if(y_neighbor=imgsize.height-border) + y_neighbor = imgsize.height-border-1; +} + +// 5x5 neighbors pattern +static const int s_anNeighborPatternSize_5x5 = 24; +static const int s_anNeighborPattern_5x5[24][2] = { + {-2, 2}, {-1, 2}, { 0, 2}, { 1, 2}, { 2, 2}, + {-2, 1}, {-1, 1}, { 0, 1}, { 1, 1}, { 2, 1}, + {-2, 0}, {-1, 0}, { 1, 0}, { 2, 0}, + {-2,-1}, {-1,-1}, { 0,-1}, { 1,-1}, { 2,-1}, + {-2,-2}, {-1,-2}, { 0,-2}, { 1,-2}, { 2,-2}, +}; + +//! returns a random neighbor position for the specified pixel position; also guards against out-of-bounds values via image/border size check. +static inline void getRandNeighborPosition_5x5(int& x_neighbor, int& y_neighbor, const int x_orig, const int y_orig, const int border, const cv::Size& imgsize) { + int r = rand()%s_anNeighborPatternSize_5x5; + x_neighbor = x_orig+s_anNeighborPattern_5x5[r][0]; + y_neighbor = y_orig+s_anNeighborPattern_5x5[r][1]; + if(x_neighbor=imgsize.width-border) + x_neighbor = imgsize.width-border-1; + if(y_neighbor=imgsize.height-border) + y_neighbor = imgsize.height-border-1; +} diff --git a/modules/saliency/src/SuBSENSE/motionSaliencySuBSENSE.cpp b/modules/saliency/src/SuBSENSE/motionSaliencySuBSENSE.cpp new file mode 100644 index 000000000..e49a4c26c --- /dev/null +++ b/modules/saliency/src/SuBSENSE/motionSaliencySuBSENSE.cpp @@ -0,0 +1,775 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2013, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" + +#include "DistanceUtils.h" +#include "RandUtils.h" +#include +#include +#include +#include + +/* + * + * Intrinsic parameters for our method are defined here; tuning these for better + * performance should not be required in most cases -- although improvements in + * very specific scenarios are always possible. + * + * Note that the current configuration was used to obtain the results presented + * in our paper, in conjunction with the 2014 CVPRW on Change Detection. + * + */ + +//! defines the threshold value(s) used to detect long-term ghosting and trigger the fast edge-based absorption heuristic +#define GHOSTDET_D_MAX (0.010f) // defines 'negligible' change here +#define GHOSTDET_S_MIN (0.995f) // defines the required minimum local foreground saturation value +//! parameter used to scale dynamic distance threshold adjustments ('R(x)') +#define FEEDBACK_R_VAR (0.01f) +//! parameters used to adjust the variation step size of 'v(x)' +#define FEEDBACK_V_INCR (1.000f) +#define FEEDBACK_V_DECR (0.100f) +//! parameters used to scale dynamic learning rate adjustments ('T(x)') +#define FEEDBACK_T_DECR (0.2500f) +#define FEEDBACK_T_INCR (0.5000f) +#define FEEDBACK_T_LOWER (2.0000f) +#define FEEDBACK_T_UPPER (256.00f) +//! parameters used to define 'unstable' regions, based on segm noise/bg dynamics and local dist threshold values +#define UNSTABLE_REG_RATIO_MIN 0.100f +#define UNSTABLE_REG_RDIST_MIN 3.000f +//! parameters used to scale the relative LBSP intensity threshold used for internal comparisons +#define LBSPDESC_NONZERO_RATIO_MIN 0.100f +#define LBSPDESC_NONZERO_RATIO_MAX 0.500f +//! parameters used to define model reset/learning rate boosts in our frame-level component +#define FRAMELEVEL_COLOR_DIFF_RESET_THRESHOLD 15 +#define FRAMELEVEL_ANALYSIS_DOWNSAMPLE_RATIO 8 + +// local define used for debug purposes only +#define DISPLAY_SUBSENSE_DEBUG_INFO 0 +// local define used to specify the default internal frame size +#define DEFAULT_FRAME_SIZE cv::Size(320,240) +// local define used to specify the color dist threshold offset used for unstable regions +#define STAB_COLOR_DIST_OFFSET m_nMinColorDistThreshold/5 +// local define used to specify the desc dist threshold offset used for unstable regions +#define UNSTAB_DESC_DIST_OFFSET m_nDescDistThreshold +// local define used to determine the median blur kernel size +#define DEFAULT_MEDIAN_BLUR_KERNEL_SIZE (9) + +static const size_t s_nColorMaxDataRange_1ch = UCHAR_MAX; +static const size_t s_nDescMaxDataRange_1ch = LBSP::DESC_SIZE*8; +static const size_t s_nColorMaxDataRange_3ch = s_nColorMaxDataRange_1ch*3; +static const size_t s_nDescMaxDataRange_3ch = s_nDescMaxDataRange_1ch*3; + +namespace cv +{ + + + +MotionSaliencySuBSENSE::MotionSaliencySuBSENSE(float fRelLBSPThreshold + ,size_t nMinDescDistThreshold + ,size_t nMinColorDistThreshold + ,size_t nBGSamples + ,size_t nRequiredBGSamples + ,size_t nSamplesForMovingAvgs) + + : BackgroundSubtractorLBSP(fRelLBSPThreshold,nMinDescDistThreshold) + ,m_bInitializedInternalStructs(false) + ,m_nMinColorDistThreshold(nMinColorDistThreshold) + ,m_nBGSamples(nBGSamples) + ,m_nRequiredBGSamples(nRequiredBGSamples) + ,m_nSamplesForMovingAvgs(nSamplesForMovingAvgs) + ,m_nFrameIndex(SIZE_MAX) + ,m_nFramesSinceLastReset(0) + ,m_nModelResetCooldown(0) + ,m_fLastNonZeroDescRatio(0.0f) + ,m_bAutoModelResetEnabled(true) + ,m_bLearningRateScalingEnabled(true) + ,m_fCurrLearningRateLowerCap(FEEDBACK_T_LOWER) + ,m_fCurrLearningRateUpperCap(FEEDBACK_T_UPPER) + ,m_nMedianBlurKernelSize(DEFAULT_MEDIAN_BLUR_KERNEL_SIZE) + ,m_bUse3x3Spread(true) { + CV_Assert(m_nBGSamples>0 && m_nRequiredBGSamples<=m_nBGSamples); + CV_Assert(m_nMinColorDistThreshold>=STAB_COLOR_DIST_OFFSET); + className = "PBAS"; +} + +MotionSaliencySuBSENSE::~MotionSaliencySuBSENSE() +{ + +} + + +bool MotionSaliencySuBSENSE::computeSaliencyImpl( const InputArray /*src*/, OutputArray /*dst*/) +{ + + return true; +} + +void MotionSaliencySuBSENSE::initialize(const cv::Mat& oInitImg, const std::vector& voKeyPoints) { + // == init + CV_Assert(!oInitImg.empty() && oInitImg.cols>0 && oInitImg.rows>0); + CV_Assert(oInitImg.type()==CV_8UC3 || oInitImg.type()==CV_8UC1); + if(oInitImg.type()==CV_8UC3) { + std::vector voInitImgChannels; + cv::split(oInitImg,voInitImgChannels); + bool eq = std::equal(voInitImgChannels[0].begin(), voInitImgChannels[0].end(), voInitImgChannels[1].begin()) + && std::equal(voInitImgChannels[1].begin(), voInitImgChannels[1].end(), voInitImgChannels[2].begin()); + if(eq) + std::cout << std::endl << "\tMotionSaliencySuBSENSE : Warning, grayscale images should always be passed in CV_8UC1 format for optimal performance." << std::endl; + } + std::vector voNewKeyPoints; + if(voKeyPoints.empty()) { + cv::DenseFeatureDetector oKPDDetector(1.f, 1, 1.f, 1, 0, true, false); + voNewKeyPoints.reserve(oInitImg.rows*oInitImg.cols); + oKPDDetector.detect(cv::Mat(oInitImg.size(),oInitImg.type()),voNewKeyPoints); + } + else + voNewKeyPoints = voKeyPoints; + const size_t nOrigKeyPointsCount = voNewKeyPoints.size(); + CV_Assert(nOrigKeyPointsCount>0); + LBSP::validateKeyPoints(voNewKeyPoints,oInitImg.size()); + CV_Assert(!voNewKeyPoints.empty()); + m_voKeyPoints = voNewKeyPoints; + m_nKeyPoints = m_voKeyPoints.size(); + m_oImgSize = oInitImg.size(); + m_nImgType = oInitImg.type(); + m_nImgChannels = oInitImg.channels(); + m_nFrameIndex = 0; + m_nFramesSinceLastReset = 0; + m_nModelResetCooldown = 0; + m_fLastNonZeroDescRatio = 0.0f; + const int nTotImgPixels = m_oImgSize.height*m_oImgSize.width; + if((int)nOrigKeyPointsCount>=nTotImgPixels/2 && nTotImgPixels>=DEFAULT_FRAME_SIZE.area()) { + m_bLearningRateScalingEnabled = true; + m_bAutoModelResetEnabled = true; + m_bUse3x3Spread = !(nTotImgPixels>DEFAULT_FRAME_SIZE.area()*2); + const int nRawMedianBlurKernelSize = std::min((int)floor((float)nTotImgPixels/DEFAULT_FRAME_SIZE.area()+0.5f)+DEFAULT_MEDIAN_BLUR_KERNEL_SIZE,14); + m_nMedianBlurKernelSize = (nRawMedianBlurKernelSize%2)?nRawMedianBlurKernelSize:nRawMedianBlurKernelSize-1; + m_fCurrLearningRateLowerCap = FEEDBACK_T_LOWER; + m_fCurrLearningRateUpperCap = FEEDBACK_T_UPPER; + } + else { + m_bLearningRateScalingEnabled = false; + m_bAutoModelResetEnabled = false; + m_bUse3x3Spread = true; + m_nMedianBlurKernelSize = DEFAULT_MEDIAN_BLUR_KERNEL_SIZE; + m_fCurrLearningRateLowerCap = FEEDBACK_T_LOWER*2; + m_fCurrLearningRateUpperCap = FEEDBACK_T_UPPER*2; + } + //std::cout << m_oImgSize << " => m_nMedianBlurKernelSize=" << m_nMedianBlurKernelSize << ", with 3x3Spread=" << m_bUse3x3Spread << ", with Tscaling=" << m_bLearningRateScalingEnabled << std::endl; + m_oUpdateRateFrame.create(m_oImgSize,CV_32FC1); + m_oUpdateRateFrame = cv::Scalar(m_fCurrLearningRateLowerCap); + m_oDistThresholdFrame.create(m_oImgSize,CV_32FC1); + m_oDistThresholdFrame = cv::Scalar(1.0f); + m_oVariationModulatorFrame.create(m_oImgSize,CV_32FC1); + m_oVariationModulatorFrame = cv::Scalar(10.0f); // should always be >= FEEDBACK_V_DECR + m_oMeanLastDistFrame.create(m_oImgSize,CV_32FC1); + m_oMeanLastDistFrame = cv::Scalar(0.0f); + m_oMeanMinDistFrame_LT.create(m_oImgSize,CV_32FC1); + m_oMeanMinDistFrame_LT = cv::Scalar(0.0f); + m_oMeanMinDistFrame_ST.create(m_oImgSize,CV_32FC1); + m_oMeanMinDistFrame_ST = cv::Scalar(0.0f); + m_oDownSampledFrameSize = cv::Size(m_oImgSize.width/FRAMELEVEL_ANALYSIS_DOWNSAMPLE_RATIO,m_oImgSize.height/FRAMELEVEL_ANALYSIS_DOWNSAMPLE_RATIO); + m_oMeanDownSampledLastDistFrame_LT.create(m_oDownSampledFrameSize,CV_32FC((int)m_nImgChannels)); + m_oMeanDownSampledLastDistFrame_LT = cv::Scalar(0.0f); + m_oMeanDownSampledLastDistFrame_ST.create(m_oDownSampledFrameSize,CV_32FC((int)m_nImgChannels)); + m_oMeanDownSampledLastDistFrame_ST = cv::Scalar(0.0f); + m_oMeanRawSegmResFrame_LT.create(m_oImgSize,CV_32FC1); + m_oMeanRawSegmResFrame_LT = cv::Scalar(0.0f); + m_oMeanRawSegmResFrame_ST.create(m_oImgSize,CV_32FC1); + m_oMeanRawSegmResFrame_ST = cv::Scalar(0.0f); + m_oMeanFinalSegmResFrame_LT.create(m_oImgSize,CV_32FC1); + m_oMeanFinalSegmResFrame_LT = cv::Scalar(0.0f); + m_oMeanFinalSegmResFrame_ST.create(m_oImgSize,CV_32FC1); + m_oMeanFinalSegmResFrame_ST = cv::Scalar(0.0f); + m_oUnstableRegionMask.create(m_oImgSize,CV_8UC1); + m_oUnstableRegionMask = cv::Scalar_(0); + m_oBlinksFrame.create(m_oImgSize,CV_8UC1); + m_oBlinksFrame = cv::Scalar_(0); + m_oDownSampledColorFrame.create(m_oDownSampledFrameSize,CV_8UC((int)m_nImgChannels)); + m_oDownSampledColorFrame = cv::Scalar_::all(0); + m_oLastColorFrame.create(m_oImgSize,CV_8UC((int)m_nImgChannels)); + m_oLastColorFrame = cv::Scalar_::all(0); + m_oLastDescFrame.create(m_oImgSize,CV_16UC((int)m_nImgChannels)); + m_oLastDescFrame = cv::Scalar_::all(0); + m_oRawFGMask_last.create(m_oImgSize,CV_8UC1); + m_oRawFGMask_last = cv::Scalar_(0); + m_oFGMask_last.create(m_oImgSize,CV_8UC1); + m_oFGMask_last = cv::Scalar_(0); + m_oFGMask_last_dilated.create(m_oImgSize,CV_8UC1); + m_oFGMask_last_dilated = cv::Scalar_(0); + m_oFGMask_last_dilated_inverted.create(m_oImgSize,CV_8UC1); + m_oFGMask_last_dilated_inverted = cv::Scalar_(0); + m_oFGMask_FloodedHoles.create(m_oImgSize,CV_8UC1); + m_oFGMask_FloodedHoles = cv::Scalar_(0); + m_oFGMask_PreFlood.create(m_oImgSize,CV_8UC1); + m_oFGMask_PreFlood = cv::Scalar_(0); + m_oRawFGBlinkMask_curr.create(m_oImgSize,CV_8UC1); + m_oRawFGBlinkMask_curr = cv::Scalar_(0); + m_oRawFGBlinkMask_last.create(m_oImgSize,CV_8UC1); + m_oRawFGBlinkMask_last = cv::Scalar_(0); + m_voBGColorSamples.resize(m_nBGSamples); + m_voBGDescSamples.resize(m_nBGSamples); + for(size_t s=0; s::all(0); + m_voBGDescSamples[s].create(m_oImgSize,CV_16UC((int)m_nImgChannels)); + m_voBGDescSamples[s] = cv::Scalar_::all(0); + } + if(m_nImgChannels==1) { + for(size_t t=0; t<=UCHAR_MAX; ++t) + m_anLBSPThreshold_8bitLUT[t] = cv::saturate_cast((m_nLBSPThresholdOffset+t*m_fRelLBSPThreshold)/3); + for(size_t k=0; k(m_nLBSPThresholdOffset+t*m_fRelLBSPThreshold); + for(size_t k=0; k0.0f && fSamplesRefreshFrac<=1.0f); + const size_t nBGSamplesToRefresh = fSamplesRefreshFrac<1.0f?(size_t)(fSamplesRefreshFrac*m_nBGSamples):m_nBGSamples; + const size_t nRefreshStartPos = fSamplesRefreshFrac<1.0f?rand()%m_nBGSamples:0; + if(m_nImgChannels==1) { + for(size_t k=0; kUNSTABLE_REG_RDIST_MIN || (*pfCurrMeanRawSegmRes_LT-*pfCurrMeanFinalSegmRes_LT)>UNSTABLE_REG_RATIO_MIN || (*pfCurrMeanRawSegmRes_ST-*pfCurrMeanFinalSegmRes_ST)>UNSTABLE_REG_RATIO_MIN)?1:0; + size_t nGoodSamplesCount=0, nSampleIdx=0; + while(nGoodSamplesCountnCurrColorDistThreshold) + goto failedcheck1ch; + const ushort& nBGIntraDesc = *((ushort*)(m_voBGDescSamples[nSampleIdx].data+idx_ushrt)); + const size_t nIntraDescDist = hdist_ushort_8bitLUT(nCurrIntraDesc,nBGIntraDesc); + LBSP::computeGrayscaleDescriptor(oInputImg,nBGColor,x,y,m_anLBSPThreshold_8bitLUT[nBGColor],nCurrInterDesc); + const size_t nInterDescDist = hdist_ushort_8bitLUT(nCurrInterDesc,nBGIntraDesc); + const size_t nDescDist = (nIntraDescDist+nInterDescDist)/2; + if(nDescDist>nCurrDescDistThreshold) + goto failedcheck1ch; + const size_t nSumDist = std::min((nDescDist/4)*(s_nColorMaxDataRange_1ch/s_nDescMaxDataRange_1ch)+nColorDist,s_nColorMaxDataRange_1ch); + if(nSumDist>nCurrColorDistThreshold) + goto failedcheck1ch; + if(nMinDescDist>nDescDist) + nMinDescDist = nDescDist; + if(nMinSumDist>nSumDist) + nMinSumDist = nSumDist; + nGoodSamplesCount++; + } + failedcheck1ch: + nSampleIdx++; + } + const float fNormalizedLastDist = ((float)absdiff_uchar(nLastColor,nCurrColor)/s_nColorMaxDataRange_1ch+(float)hdist_ushort_8bitLUT(nLastIntraDesc,nCurrIntraDesc)/s_nDescMaxDataRange_1ch)/2; + *pfCurrMeanLastDist = (*pfCurrMeanLastDist)*(1.0f-fRollAvgFactor_ST) + fNormalizedLastDist*fRollAvgFactor_ST; + if(nGoodSamplesCount0?(size_t)ceil(learningRateOverride):(size_t)ceil(*pfCurrLearningRate); + if((rand()%nLearningRate)==0) { + const size_t s_rand = rand()%m_nBGSamples; + *((ushort*)(m_voBGDescSamples[s_rand].data+idx_ushrt)) = nCurrIntraDesc; + m_voBGColorSamples[s_rand].data[idx_uchar] = nCurrColor; + } + int x_rand,y_rand; + const bool bCurrUsing3x3Spread = m_bUse3x3Spread && !m_oUnstableRegionMask.data[idx_uchar]; + if(bCurrUsing3x3Spread) + getRandNeighborPosition_3x3(x_rand,y_rand,x,y,LBSP::PATCH_SIZE/2,m_oImgSize); + else + getRandNeighborPosition_5x5(x_rand,y_rand,x,y,LBSP::PATCH_SIZE/2,m_oImgSize); + const size_t n_rand = rand(); + const size_t idx_rand_uchar = m_oImgSize.width*y_rand + x_rand; + const size_t idx_rand_flt32 = idx_rand_uchar*4; + const float fRandMeanLastDist = *((float*)(m_oMeanLastDistFrame.data+idx_rand_flt32)); + const float fRandMeanRawSegmRes = *((float*)(m_oMeanRawSegmResFrame_ST.data+idx_rand_flt32)); + if((n_rand%(bCurrUsing3x3Spread?nLearningRate:(nLearningRate/2+1)))==0 + || (fRandMeanRawSegmRes>GHOSTDET_S_MIN && fRandMeanLastDistm_fCurrLearningRateLowerCap) + *pfCurrLearningRate -= FEEDBACK_T_DECR*(*pfCurrVariationFactor)/std::max(*pfCurrMeanMinDist_LT,*pfCurrMeanMinDist_ST); + if((*pfCurrLearningRate)m_fCurrLearningRateUpperCap) + *pfCurrLearningRate = m_fCurrLearningRateUpperCap; + if(std::max(*pfCurrMeanMinDist_LT,*pfCurrMeanMinDist_ST)>UNSTABLE_REG_RATIO_MIN && m_oBlinksFrame.data[idx_uchar]) + (*pfCurrVariationFactor) += FEEDBACK_V_INCR; + else if((*pfCurrVariationFactor)>FEEDBACK_V_DECR) { + (*pfCurrVariationFactor) -= m_oFGMask_last.data[idx_uchar]?FEEDBACK_V_DECR/4:m_oUnstableRegionMask.data[idx_uchar]?FEEDBACK_V_DECR/2:FEEDBACK_V_DECR; + if((*pfCurrVariationFactor)=2) + ++nNonZeroDescCount; + nLastIntraDesc = nCurrIntraDesc; + nLastColor = nCurrColor; + } + } + else { //m_nImgChannels==3 + for(size_t k=0; kUNSTABLE_REG_RDIST_MIN || (*pfCurrMeanRawSegmRes_LT-*pfCurrMeanFinalSegmRes_LT)>UNSTABLE_REG_RATIO_MIN || (*pfCurrMeanRawSegmRes_ST-*pfCurrMeanFinalSegmRes_ST)>UNSTABLE_REG_RATIO_MIN)?1:0; + size_t nGoodSamplesCount=0, nSampleIdx=0; + while(nGoodSamplesCountnCurrSCColorDistThreshold) + goto failedcheck3ch; + size_t nIntraDescDist = hdist_ushort_8bitLUT(anCurrIntraDesc[c],anBGIntraDesc[c]); + LBSP::computeSingleRGBDescriptor(oInputImg,anBGColor[c],x,y,c,m_anLBSPThreshold_8bitLUT[anBGColor[c]],anCurrInterDesc[c]); + size_t nInterDescDist = hdist_ushort_8bitLUT(anCurrInterDesc[c],anBGIntraDesc[c]); + const size_t nDescDist = (nIntraDescDist+nInterDescDist)/2; + const size_t nSumDist = std::min((nDescDist/2)*(s_nColorMaxDataRange_1ch/s_nDescMaxDataRange_1ch)+nColorDist,s_nColorMaxDataRange_1ch); + if(nSumDist>nCurrSCColorDistThreshold) + goto failedcheck3ch; + nTotDescDist += nDescDist; + nTotSumDist += nSumDist; + } + if(nTotDescDist>nCurrTotDescDistThreshold || nTotSumDist>nCurrTotColorDistThreshold) + goto failedcheck3ch; + if(nMinTotDescDist>nTotDescDist) + nMinTotDescDist = nTotDescDist; + if(nMinTotSumDist>nTotSumDist) + nMinTotSumDist = nTotSumDist; + nGoodSamplesCount++; + failedcheck3ch: + nSampleIdx++; + } + const float fNormalizedLastDist = ((float)L1dist_uchar(anLastColor,anCurrColor)/s_nColorMaxDataRange_3ch+(float)hdist_ushort_8bitLUT(anLastIntraDesc,anCurrIntraDesc)/s_nDescMaxDataRange_3ch)/2; + *pfCurrMeanLastDist = (*pfCurrMeanLastDist)*(1.0f-fRollAvgFactor_ST) + fNormalizedLastDist*fRollAvgFactor_ST; + if(nGoodSamplesCount0?(size_t)ceil(learningRateOverride):(size_t)ceil(*pfCurrLearningRate); + if((rand()%nLearningRate)==0) { + const size_t s_rand = rand()%m_nBGSamples; + for(size_t c=0; c<3; ++c) { + *((ushort*)(m_voBGDescSamples[s_rand].data+idx_ushrt_rgb+2*c)) = anCurrIntraDesc[c]; + *(m_voBGColorSamples[s_rand].data+idx_uchar_rgb+c) = anCurrColor[c]; + } + } + int x_rand,y_rand; + const bool bCurrUsing3x3Spread = m_bUse3x3Spread && !m_oUnstableRegionMask.data[idx_uchar]; + if(bCurrUsing3x3Spread) + getRandNeighborPosition_3x3(x_rand,y_rand,x,y,LBSP::PATCH_SIZE/2,m_oImgSize); + else + getRandNeighborPosition_5x5(x_rand,y_rand,x,y,LBSP::PATCH_SIZE/2,m_oImgSize); + const size_t n_rand = rand(); + const size_t idx_rand_uchar = m_oImgSize.width*y_rand + x_rand; + const size_t idx_rand_flt32 = idx_rand_uchar*4; + const float fRandMeanLastDist = *((float*)(m_oMeanLastDistFrame.data+idx_rand_flt32)); + const float fRandMeanRawSegmRes = *((float*)(m_oMeanRawSegmResFrame_ST.data+idx_rand_flt32)); + if((n_rand%(bCurrUsing3x3Spread?nLearningRate:(nLearningRate/2+1)))==0 + || (fRandMeanRawSegmRes>GHOSTDET_S_MIN && fRandMeanLastDistm_fCurrLearningRateLowerCap) + *pfCurrLearningRate -= FEEDBACK_T_DECR*(*pfCurrVariationFactor)/std::max(*pfCurrMeanMinDist_LT,*pfCurrMeanMinDist_ST); + if((*pfCurrLearningRate)m_fCurrLearningRateUpperCap) + *pfCurrLearningRate = m_fCurrLearningRateUpperCap; + if(std::max(*pfCurrMeanMinDist_LT,*pfCurrMeanMinDist_ST)>UNSTABLE_REG_RATIO_MIN && m_oBlinksFrame.data[idx_uchar]) + (*pfCurrVariationFactor) += FEEDBACK_V_INCR; + else if((*pfCurrVariationFactor)>FEEDBACK_V_DECR) { + (*pfCurrVariationFactor) -= m_oFGMask_last.data[idx_uchar]?FEEDBACK_V_DECR/4:m_oUnstableRegionMask.data[idx_uchar]?FEEDBACK_V_DECR/2:FEEDBACK_V_DECR; + if((*pfCurrVariationFactor)=4) + ++nNonZeroDescCount; + for(size_t c=0; c<3; ++c) { + anLastIntraDesc[c] = anCurrIntraDesc[c]; + anLastColor[c] = anCurrColor[c]; + } + } + } +#if DISPLAY_SUBSENSE_DEBUG_INFO + std::cout << std::endl; + cv::Point dbgpt(nDebugCoordX,nDebugCoordY); + cv::Mat oMeanMinDistFrameNormalized; m_oMeanMinDistFrame_ST.copyTo(oMeanMinDistFrameNormalized); + cv::circle(oMeanMinDistFrameNormalized,dbgpt,5,cv::Scalar(1.0f)); + cv::resize(oMeanMinDistFrameNormalized,oMeanMinDistFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("d_min(x)",oMeanMinDistFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " d_min(" << dbgpt << ") = " << m_oMeanMinDistFrame_ST.at(dbgpt) << std::endl; + cv::Mat oMeanLastDistFrameNormalized; m_oMeanLastDistFrame.copyTo(oMeanLastDistFrameNormalized); + cv::circle(oMeanLastDistFrameNormalized,dbgpt,5,cv::Scalar(1.0f)); + cv::resize(oMeanLastDistFrameNormalized,oMeanLastDistFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("d_last(x)",oMeanLastDistFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " d_last(" << dbgpt << ") = " << m_oMeanLastDistFrame.at(dbgpt) << std::endl; + cv::Mat oMeanRawSegmResFrameNormalized; m_oMeanRawSegmResFrame_ST.copyTo(oMeanRawSegmResFrameNormalized); + cv::circle(oMeanRawSegmResFrameNormalized,dbgpt,5,cv::Scalar(1.0f)); + cv::resize(oMeanRawSegmResFrameNormalized,oMeanRawSegmResFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("s_avg(x)",oMeanRawSegmResFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " s_avg(" << dbgpt << ") = " << m_oMeanRawSegmResFrame_ST.at(dbgpt) << std::endl; + cv::Mat oMeanFinalSegmResFrameNormalized; m_oMeanFinalSegmResFrame_ST.copyTo(oMeanFinalSegmResFrameNormalized); + cv::circle(oMeanFinalSegmResFrameNormalized,dbgpt,5,cv::Scalar(1.0f)); + cv::resize(oMeanFinalSegmResFrameNormalized,oMeanFinalSegmResFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("z_avg(x)",oMeanFinalSegmResFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " z_avg(" << dbgpt << ") = " << m_oMeanFinalSegmResFrame_ST.at(dbgpt) << std::endl; + cv::Mat oDistThresholdFrameNormalized; m_oDistThresholdFrame.convertTo(oDistThresholdFrameNormalized,CV_32FC1,0.25f,-0.25f); + cv::circle(oDistThresholdFrameNormalized,dbgpt,5,cv::Scalar(1.0f)); + cv::resize(oDistThresholdFrameNormalized,oDistThresholdFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("r(x)",oDistThresholdFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " r(" << dbgpt << ") = " << m_oDistThresholdFrame.at(dbgpt) << std::endl; + cv::Mat oVariationModulatorFrameNormalized; cv::normalize(m_oVariationModulatorFrame,oVariationModulatorFrameNormalized,0,255,cv::NORM_MINMAX,CV_8UC1); + cv::circle(oVariationModulatorFrameNormalized,dbgpt,5,cv::Scalar(255)); + cv::resize(oVariationModulatorFrameNormalized,oVariationModulatorFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("v(x)",oVariationModulatorFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " v(" << dbgpt << ") = " << m_oVariationModulatorFrame.at(dbgpt) << std::endl; + cv::Mat oUpdateRateFrameNormalized; m_oUpdateRateFrame.convertTo(oUpdateRateFrameNormalized,CV_32FC1,1.0f/FEEDBACK_T_UPPER,-FEEDBACK_T_LOWER/FEEDBACK_T_UPPER); + cv::circle(oUpdateRateFrameNormalized,dbgpt,5,cv::Scalar(1.0f)); + cv::resize(oUpdateRateFrameNormalized,oUpdateRateFrameNormalized,DEFAULT_FRAME_SIZE); + cv::imshow("t(x)",oUpdateRateFrameNormalized); + std::cout << std::fixed << std::setprecision(5) << " t(" << dbgpt << ") = " << m_oUpdateRateFrame.at(dbgpt) << std::endl; +#endif //DISPLAY_SUBSENSE_DEBUG_INFO + cv::bitwise_xor(oCurrFGMask,m_oRawFGMask_last,m_oRawFGBlinkMask_curr); + cv::bitwise_or(m_oRawFGBlinkMask_curr,m_oRawFGBlinkMask_last,m_oBlinksFrame); + m_oRawFGBlinkMask_curr.copyTo(m_oRawFGBlinkMask_last); + oCurrFGMask.copyTo(m_oRawFGMask_last); + cv::morphologyEx(oCurrFGMask,m_oFGMask_PreFlood,cv::MORPH_CLOSE,cv::Mat()); + m_oFGMask_PreFlood.copyTo(m_oFGMask_FloodedHoles); + cv::floodFill(m_oFGMask_FloodedHoles,cv::Point(0,0),UCHAR_MAX); + cv::bitwise_not(m_oFGMask_FloodedHoles,m_oFGMask_FloodedHoles); + cv::erode(m_oFGMask_PreFlood,m_oFGMask_PreFlood,cv::Mat(),cv::Point(-1,-1),3); + cv::bitwise_or(oCurrFGMask,m_oFGMask_FloodedHoles,oCurrFGMask); + cv::bitwise_or(oCurrFGMask,m_oFGMask_PreFlood,oCurrFGMask); + cv::medianBlur(oCurrFGMask,m_oFGMask_last,m_nMedianBlurKernelSize); + cv::dilate(m_oFGMask_last,m_oFGMask_last_dilated,cv::Mat(),cv::Point(-1,-1),3); + cv::bitwise_and(m_oBlinksFrame,m_oFGMask_last_dilated_inverted,m_oBlinksFrame); + cv::bitwise_not(m_oFGMask_last_dilated,m_oFGMask_last_dilated_inverted); + cv::bitwise_and(m_oBlinksFrame,m_oFGMask_last_dilated_inverted,m_oBlinksFrame); + m_oFGMask_last.copyTo(oCurrFGMask); + cv::addWeighted(m_oMeanFinalSegmResFrame_LT,(1.0f-fRollAvgFactor_LT),m_oFGMask_last,(1.0/UCHAR_MAX)*fRollAvgFactor_LT,0,m_oMeanFinalSegmResFrame_LT,CV_32F); + cv::addWeighted(m_oMeanFinalSegmResFrame_ST,(1.0f-fRollAvgFactor_ST),m_oFGMask_last,(1.0/UCHAR_MAX)*fRollAvgFactor_ST,0,m_oMeanFinalSegmResFrame_ST,CV_32F); + const float fCurrNonZeroDescRatio = (float)nNonZeroDescCount/m_nKeyPoints; + if(fCurrNonZeroDescRatiocv::saturate_cast(m_nLBSPThresholdOffset+ceil(t*m_fRelLBSPThreshold/4))) + --m_anLBSPThreshold_8bitLUT[t]; + } + else if(fCurrNonZeroDescRatio>LBSPDESC_NONZERO_RATIO_MAX && m_fLastNonZeroDescRatio>LBSPDESC_NONZERO_RATIO_MAX) { + for(size_t t=0; t<=UCHAR_MAX; ++t) + if(m_anLBSPThreshold_8bitLUT[t](m_nLBSPThresholdOffset+UCHAR_MAX*m_fRelLBSPThreshold)) + ++m_anLBSPThreshold_8bitLUT[t]; + } + m_fLastNonZeroDescRatio = fCurrNonZeroDescRatio; + if(m_bLearningRateScalingEnabled) { + cv::resize(oInputImg,m_oDownSampledColorFrame,m_oDownSampledFrameSize,0,0,cv::INTER_AREA); + cv::accumulateWeighted(m_oDownSampledColorFrame,m_oMeanDownSampledLastDistFrame_LT,fRollAvgFactor_LT); + cv::accumulateWeighted(m_oDownSampledColorFrame,m_oMeanDownSampledLastDistFrame_ST,fRollAvgFactor_ST); + size_t nTotColorDiff = 0; + for(int i=0; i1000) + m_bAutoModelResetEnabled = false; + else if(fCurrColorDiffRatio>=FRAMELEVEL_COLOR_DIFF_RESET_THRESHOLD && m_nModelResetCooldown==0) { + m_nFramesSinceLastReset = 0; + refreshModel(0.1f); // reset 10% of the bg model + m_nModelResetCooldown = m_nSamplesForMovingAvgs; + m_oUpdateRateFrame = cv::Scalar(1.0f); + } + else + ++m_nFramesSinceLastReset; + } + else if(fCurrColorDiffRatio>=FRAMELEVEL_COLOR_DIFF_RESET_THRESHOLD*2) { + m_nFramesSinceLastReset = 0; + m_bAutoModelResetEnabled = true; + } + if(fCurrColorDiffRatio>=FRAMELEVEL_COLOR_DIFF_RESET_THRESHOLD/2) { + m_fCurrLearningRateLowerCap = (float)std::max((int)FEEDBACK_T_LOWER>>(int)(fCurrColorDiffRatio/2),1); + m_fCurrLearningRateUpperCap = (float)std::max((int)FEEDBACK_T_UPPER>>(int)(fCurrColorDiffRatio/2),1); + } + else { + m_fCurrLearningRateLowerCap = FEEDBACK_T_LOWER; + m_fCurrLearningRateUpperCap = FEEDBACK_T_UPPER; + } + if(m_nModelResetCooldown>0) + --m_nModelResetCooldown; + } +} + +void MotionSaliencySuBSENSE::getBackgroundImage(cv::OutputArray backgroundImage) const { + CV_Assert(m_bInitialized); + cv::Mat oAvgBGImg = cv::Mat::zeros(m_oImgSize,CV_32FC((int)m_nImgChannels)); + for(size_t s=0; s