Merge pull request #4017 from D00E:known-foreground-mask

In support of Optional Known Foreground Mask for Background Subtractors #4017

Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

I agree to contribute to the project under Apache 2 License.
 x To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible        with OpenCV
 x The PR is proposed to the proper branch
 x There is a reference to the original bug report and related work
    There is accuracy test, performance test and test data in opencv_extra repository, if applicable
 x Patch to opencv_extra has the same branch name.

 The feature is well documented and sample code can be built with the project CMake

Description

Is in support to a pull request regarding, issue https://github.com/opencv/opencv/issues/26476.
Core paired pull request: https://github.com/opencv/opencv/pull/27810
This adds function templates inline with the pure virtual overloaded apply function present for background subtraction. Supporting the build of the pull request:

modules/bgsegm/include/opencv2/bgsegm.hpp

@@ -62,6 +62,32 @@ The class implements the algorithm described in @cite KB2001 .
 class CV_EXPORTS_W BackgroundSubtractorMOG : public BackgroundSubtractor
 {
 public:
+    // BackgroundSubtractor interface
+    /** @brief Computes a foreground mask.
+
+    @param image Next video frame of type CV_8UC(n),CV_8SC(n),CV_16UC(n),CV_16SC(n),CV_32SC(n),CV_32FC(n),CV_64FC(n), where n is 1,2,3,4.
+    @param fgmask The output foreground mask as an 8-bit binary image.
+    @param learningRate The value between 0 and 1 that indicates how fast the background model is
+    learnt. Negative parameter value makes the algorithm to use some automatically chosen learning
+    rate. 0 means that the background model is not updated at all, 1 means that the background model
+    is completely reinitialized from the last frame.
+     */
+
+    CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
+    /** @brief Computes a foreground mask and skips known foreground in evaluation.
+
+    @param image Next video frame of type CV_8UC(n),CV_8SC(n),CV_16UC(n),CV_16SC(n),CV_32SC(n),CV_32FC(n),CV_64FC(n), where n is 1,2,3,4.
+    @param fgmask The output foreground mask as an 8-bit binary image.
+    @param knownForegroundMask The mask for inputting already known foreground, allows model to ignore learning known pixels.
+    @param learningRate The value between 0 and 1 that indicates how fast the background model is
+    learnt. Negative parameter value makes the algorithm to use some automatically chosen learning
+    rate. 0 means that the background model is not updated at all, 1 means that the background model
+    is completely reinitialized from the last frame.
+     */
+
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
     CV_WRAP virtual int getHistory() const = 0;
     CV_WRAP virtual void setHistory(int nframes) = 0;
 
@@ -110,6 +136,22 @@ public:
     is completely reinitialized from the last frame.
      */
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
+    /** @brief Computes a foreground mask with known foreground mask input.
+
+    @param image Next video frame.
+    @param fgmask The output foreground mask as an 8-bit binary image.
+    @param knownForegroundMask The mask for inputting already known foreground.
+    @param learningRate The value between 0 and 1 that indicates how fast the background model is
+    learnt. Negative parameter value makes the algorithm to use some automatically chosen learning
+    rate. 0 means that the background model is not updated at all, 1 means that the background model
+    is completely reinitialized from the last frame.
+
+    @note This method has a default virtual implementation that throws a "not impemented" error.
+    Foreground masking may not be supported by all background subtractors.
+    */
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
     CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE = 0;
 
     /** @brief Returns total number of distinct colors to maintain in histogram.
@@ -210,6 +252,22 @@ class CV_EXPORTS_W BackgroundSubtractorCNT  : public BackgroundSubtractor
 public:
     // BackgroundSubtractor interface
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
+    /** @brief Computes a foreground mask with known foreground mask input.
+
+    @param image Next video frame.
+    @param knownForegroundMask The mask for inputting already known foreground.
+    @param fgmask The output foreground mask as an 8-bit binary image.
+    @param learningRate The value between 0 and 1 that indicates how fast the background model is
+    learnt. Negative parameter value makes the algorithm to use some automatically chosen learning
+    rate. 0 means that the background model is not updated at all, 1 means that the background model
+    is completely reinitialized from the last frame.
+
+    @note This method has a default virtual implementation that throws a "not impemented" error.
+    Foreground masking may not be supported by all background subtractors.
+    */
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
     CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE = 0;
 
     /** @brief Returns number of frames with same pixel color to consider stable.
@@ -269,6 +327,7 @@ class CV_EXPORTS_W BackgroundSubtractorGSOC : public BackgroundSubtractor
 public:
     // BackgroundSubtractor interface
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
 
     CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE = 0;
 };
@@ -280,6 +339,7 @@ class CV_EXPORTS_W BackgroundSubtractorLSBP : public BackgroundSubtractor
 public:
     // BackgroundSubtractor interface
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
 
     CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE = 0;
 };

modules/bgsegm/src/bgfg_gaussmix.cpp

@@ -42,6 +42,7 @@
 
 #include "precomp.hpp"
 #include <float.h>
+#include "opencv2/core/utils/logger.hpp"
 
 // to make sure we can use these short names
 #undef K
@@ -104,6 +105,8 @@ public:
     //! the update operator
     virtual void apply(InputArray image, OutputArray fgmask, double learningRate=0) CV_OVERRIDE;
 
+    virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
+
     //! re-initiaization method
     virtual void initialize(Size _frameSize, int _frameType)
     {
@@ -461,6 +464,15 @@ void BackgroundSubtractorMOGImpl::apply(InputArray _image, OutputArray _fgmask,
         CV_Error( Error::StsUnsupportedFormat, "Only 1- and 3-channel 8-bit images are supported in BackgroundSubtractorMOG" );
 }
 
+void BackgroundSubtractorMOGImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+    if(!_knownForegroundMask.empty())
+    {
+        CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, learningRate);
+}
+
 Ptr<BackgroundSubtractorMOG> createBackgroundSubtractorMOG(int history, int nmixtures,
                                   double backgroundRatio, double noiseSigma)
 {

modules/bgsegm/src/bgfg_gmg.cpp

@@ -51,6 +51,7 @@
 #include "precomp.hpp"
 #include "opencv2/core/utility.hpp"
 #include <limits>
+#include "opencv2/core/utils/logger.hpp"
 
 namespace cv
 {
@@ -97,6 +98,7 @@ public:
      * @param fgmask Output mask image representing foreground and background pixels
      */
     virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1.0) CV_OVERRIDE;
+    virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
 
     /**
      * Releases all inner buffers.
@@ -473,6 +475,15 @@ void BackgroundSubtractorGMGImpl::apply(InputArray _frame, OutputArray _fgmask,
     ++frameNum_;
 }
 
+void BackgroundSubtractorGMGImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double newLearningRate){
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+    if(!_knownForegroundMask.empty())
+    {
+        CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, newLearningRate);
+}
+
 void BackgroundSubtractorGMGImpl::release()
 {
     frameSize_ = Size();

modules/bgsegm/src/bgfg_gsoc.cpp

@@ -53,6 +53,7 @@
 #include <opencv2/calib3d.hpp>
 #include <iostream>
 #include "opencv2/core/cvdef.h"
+#include "opencv2/core/utils/logger.hpp"
 
 namespace cv
 {
@@ -494,6 +495,7 @@ public:
                                  float noiseRemovalThresholdFacFG);
 
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate = -1) CV_OVERRIDE;
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
 
     CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
 
@@ -542,6 +544,7 @@ public:
                                 );
 
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate = -1) CV_OVERRIDE;
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
 
     CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
 
@@ -793,6 +796,15 @@ void BackgroundSubtractorGSOCImpl::apply(InputArray _image, OutputArray _fgmask,
     this->postprocessing(fgMask);
 }
 
+void BackgroundSubtractorGSOCImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+    if(!_knownForegroundMask.empty())
+    {
+        CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, learningRate);
+}
+
 void BackgroundSubtractorGSOCImpl::getBackgroundImage(OutputArray _backgroundImage) const {
     CV_Assert(!backgroundModel.empty());
     const Size sz = backgroundModel->getSize();
@@ -928,6 +940,15 @@ void BackgroundSubtractorLSBPImpl::apply(InputArray _image, OutputArray _fgmask,
     this->postprocessing(fgMask);
 }
 
+void BackgroundSubtractorLSBPImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+    if(!_knownForegroundMask.empty())
+    {
+        CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, learningRate);
+}
+
 void BackgroundSubtractorLSBPImpl::getBackgroundImage(OutputArray _backgroundImage) const {
     CV_Assert(!backgroundModel.empty());
     const Size sz = backgroundModel->getSize();

modules/bgsegm/src/bgfg_subcnt.cpp

@@ -44,6 +44,7 @@
 
 #include "precomp.hpp"
 #include <functional>
+#include "opencv2/core/utils/logger.hpp"
 
 namespace cv
 {
@@ -61,6 +62,8 @@ public:
 
     // BackgroundSubtractor interface
     virtual void apply(InputArray image, OutputArray fgmask, double learningRate) CV_OVERRIDE;
+    virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
+
     virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
 
     int getMinPixelStability() const CV_OVERRIDE;
@@ -409,6 +412,14 @@ void BackgroundSubtractorCNTImpl::apply(InputArray image, OutputArray _fgmask, d
     prevFrame = frame;
 }
 
+void BackgroundSubtractorCNTImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+    if(!_knownForegroundMask.empty())
+    {
+        CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, learningRate);
+}
 
 Ptr<BackgroundSubtractorCNT> createBackgroundSubtractorCNT(int minPixelStability, bool useHistory, int maxStability, bool isParallel)
 {

modules/cudabgsegm/include/opencv2/cudabgsegm.hpp

@@ -84,6 +84,8 @@ public:
     using cv::BackgroundSubtractor::apply;
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream) = 0;
 
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate, Stream& stream) = 0;
+
     using cv::BackgroundSubtractor::getBackgroundImage;
     virtual void getBackgroundImage(OutputArray backgroundImage, Stream& stream) const = 0;
 
@@ -135,6 +137,8 @@ public:
 
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream) = 0;
 
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate, Stream& stream) = 0;
+
     virtual void getBackgroundImage(OutputArray backgroundImage, Stream& stream) const = 0;
 
     CV_WRAP inline void getBackgroundImage(CV_OUT GpuMat &backgroundImage, Stream& stream) {

modules/cudabgsegm/src/mog.cpp

@@ -41,6 +41,7 @@
 //M*/
 
 #include "precomp.hpp"
+#include "opencv2/core/utils/logger.hpp"
 
 using namespace cv;
 using namespace cv::cuda;
@@ -79,6 +80,9 @@ namespace
         void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE;
         void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream) CV_OVERRIDE;
 
+        void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE;
+        void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate, Stream& stream) CV_OVERRIDE;
+
         void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
         void getBackgroundImage(OutputArray backgroundImage, Stream& stream) const CV_OVERRIDE;
 
@@ -131,6 +135,22 @@ namespace
         apply(image, fgmask, learningRate, Stream::Null());
     }
 
+    void MOGImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+        if(!_knownForegroundMask.empty())
+        {
+            CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+        }
+        apply(_image, _fgmask, learningRate, Stream::Null());
+    }
+
+    void MOGImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate, Stream &stream){
+        if(!_knownForegroundMask.empty())
+        {
+            CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+        }
+        apply(_image, _fgmask, learningRate, stream);
+    }
+
     void MOGImpl::apply(InputArray _frame, OutputArray _fgmask, double learningRate, Stream& stream)
     {
         using namespace cv::cuda::device::mog;

modules/cudabgsegm/src/mog2.cpp

@@ -42,6 +42,7 @@
 
 #include "precomp.hpp"
 #include "cuda/mog2.hpp"
+#include "opencv2/core/utils/logger.hpp"
 
 using namespace cv;
 using namespace cv::cuda;
@@ -83,6 +84,9 @@ public:
     void apply(InputArray image, OutputArray fgmask, double learningRate = -1) CV_OVERRIDE;
     void apply(InputArray image, OutputArray fgmask, double learningRate, Stream &stream) CV_OVERRIDE;
 
+    void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate = -1) CV_OVERRIDE;
+    void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate, Stream& stream) CV_OVERRIDE;
+
     void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
     void getBackgroundImage(OutputArray backgroundImage, Stream &stream) const CV_OVERRIDE;
 
@@ -174,6 +178,22 @@ void MOG2Impl::apply(InputArray image, OutputArray fgmask, double learningRate)
     apply(image, fgmask, learningRate, Stream::Null());
 }
 
+void MOG2Impl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+    if(!_knownForegroundMask.empty())
+    {
+        CV_Error( Error::StsNotImplemented, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, learningRate, Stream::Null());
+}
+
+void MOG2Impl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate, Stream &stream){
+    if(!_knownForegroundMask.empty())
+    {
+        CV_Error( Error::StsNotImplemented, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+    }
+    apply(_image, _fgmask, learningRate, stream);
+}
+
 void MOG2Impl::apply(InputArray _frame, OutputArray _fgmask, double learningRate, Stream &stream)
 {
     using namespace cv::cuda::device::mog2;

modules/cudalegacy/include/opencv2/cudalegacy.hpp

@@ -92,6 +92,7 @@ class CV_EXPORTS BackgroundSubtractorGMG : public cv::BackgroundSubtractor
 public:
     using cv::BackgroundSubtractor::apply;
     virtual void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream) = 0;
+    virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate, Stream& stream) = 0;
 
     virtual int getMaxFeatures() const = 0;
     virtual void setMaxFeatures(int maxFeatures) = 0;

modules/cudalegacy/src/fgd.cpp

@@ -55,6 +55,7 @@ Ptr<cuda::BackgroundSubtractorFGD> cv::cuda::createBackgroundSubtractorFGD(const
 
 #include "cuda/fgd.hpp"
 #include "opencv2/imgproc.hpp"
+#include "opencv2/core/utils/logger.hpp"
 
 /////////////////////////////////////////////////////////////////////////
 // FGDParams
@@ -546,6 +547,7 @@ namespace
         ~FGDImpl();
 
         void apply(InputArray image, OutputArray fgmask, double learningRate=-1);
+        void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1);
 
         void getBackgroundImage(OutputArray backgroundImage) const;
 
@@ -588,6 +590,14 @@ namespace
     {
     }
 
+    void FGDImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+        if(!_knownForegroundMask.empty())
+        {
+            CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+        }
+        apply(_image, _fgmask, learningRate);
+    }
+
     void FGDImpl::apply(InputArray _frame, OutputArray fgmask, double)
     {
         GpuMat curFrame = _frame.getGpuMat();

modules/cudalegacy/src/gmg.cpp

@@ -41,6 +41,7 @@
 //M*/
 
 #include "precomp.hpp"
+#include "opencv2/core/utils/logger.hpp"
 
 using namespace cv;
 using namespace cv::cuda;
@@ -73,6 +74,10 @@ namespace
         void apply(InputArray image, OutputArray fgmask, double learningRate=-1);
         void apply(InputArray image, OutputArray fgmask, double learningRate, Stream& stream);
 
+        // Overloaded Background Subtractor Applys featuring  knownForegroundMask parameter
+        void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1);
+        void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate, Stream& stream);
+
         void getBackgroundImage(OutputArray backgroundImage) const;
 
         int getMaxFeatures() const { return maxFeatures_; }
@@ -165,6 +170,22 @@ namespace
         apply(image, fgmask, learningRate, Stream::Null());
     }
 
+    void GMGImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate){
+        if(!_knownForegroundMask.empty())
+        {
+            CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+        }
+        apply(_image, _fgmask, learningRate);
+    }
+
+    void GMGImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate, Stream& stream){
+        if(!_knownForegroundMask.empty())
+        {
+            CV_LOG_WARNING(NULL, "Known Foreground Masking has not been implemented for this specific background subtractor, falling back to subtraction without known foreground");
+        }
+        apply(_image, _fgmask, learningRate, stream);
+    }
+
     void GMGImpl::apply(InputArray _frame, OutputArray _fgmask, double newLearningRate, Stream& stream)
     {
         using namespace cv::cuda::device::gmg;