Correctly using mutex in OpenCL-OpenCV-Realtime-Th

2019-06-11 05:51发布

站内文章 / C++
34 0 0

问题:

Im trying to get a stereo-videostream in realtime via usb-webcams in a GPU-Thread (way faster than to get and process the images via cpu), processing said stream in a second thread to get the faces and control the threads via keyboard in the main-function (will be implemented later). At the moment the code runs properly (shows both Links/Rechts and draws a rectangle around my face) for ~30s and then crashes because of an "...unhandled exception (opencv_core249d.dll)".

Ive tried using mutex-lock-guard to handle the variables "Links"/"Rechts" and faces, but im not sure how to use them in Mat-context, because i didnt find any suitable examples for OpenCV. Also: is it correct how i reference Links/Rechts/faces to the thread/is it the fastest thread-safe-variety?

My mutex-trys are left out of the code. Thanks for any help.

#include <thread>
#include <iostream>
#include <functional>
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/ocl/ocl.hpp"
#include <mutex>

using namespace std;
using namespace cv;

Mat Links = Mat::zeros(960,1080, CV_8UC3);                                          
Mat Rechts = Mat::zeros(960,1080, CV_8UC3);                                         
Mat facesMat =Mat::zeros(960,1080,CV_8UC3);                                         

std::vector<Rect> faces;

void change(Mat& Links, Mat& Rechts)
{
Mat captureFrame;
VideoCapture LeftCam(1);
VideoCapture RightCam(0);
LeftCam.set(CV_CAP_PROP_FRAME_WIDTH,1920);              
LeftCam.set(CV_CAP_PROP_FRAME_HEIGHT,1080);
RightCam.set(CV_CAP_PROP_FRAME_WIDTH,1920);
RightCam.set(CV_CAP_PROP_FRAME_HEIGHT,1080);

FileStorage fs, fs2, fs3, fs4;
Mat map1x, map1y, map2x, map2y; 
Mat mapi1x, mapi1y, mapi2x, mapi2y;
fs.open("mx1.xml", FileStorage::READ);
fs2.open("my1.xml", FileStorage::READ);
fs3.open("mx2.xml", FileStorage::READ);
fs4.open("my2.xml", FileStorage::READ);
fs["mx1"] >> map1x;
fs2["my1"] >> map1y;
fs3["mx2"] >> map2x;
fs4["my2"] >> map2y;
fs.release();
fs2.release();
fs3.release();
fs4.release();

ocl::oclMat map1xocl(map1x), map1yocl(map1y), map2xocl(map2x), map2yocl(map2y);

while (true)
{
    cv::Mat3b frame0, frame1, LeftframeRaw, RightframeRaw, LeftRoi, RightRoi, framecon;

    LeftCam >> frame0;
    RightCam >> frame1;
    LeftRoi = frame0(Rect(420,60,1080,960));                                                                    //960*1080 passend zuschneiden
    RightRoi = frame1(Rect(420,60,1080,960));
    ocl::oclMat Leftr(LeftRoi), Rightr(RightRoi), Leftt, Lefttf, Rightt, Righttf, LeftRemappedocl, RightRemappedocl;
    ocl::transpose(Leftr, Leftt);                                                                               //ACHTUNG: ocl::transpose src darf nicht dst-image sein, sonst gibts starke Bildartefakte!
    ocl::transpose(Rightr, Rightt);
    ocl::flip(Leftt, Lefttf, 1);                                                                                //ACHTUNG: ocl::flip src darf nicht dst-image sein, sonst gibts starke Bildartefakte!
    ocl::flip(Rightt, Rightt, 1);
    ocl::remap(Lefttf, LeftRemappedocl, map1xocl, map1yocl, CV_INTER_LINEAR, BORDER_CONSTANT, 0);
    ocl::remap(Rightt, RightRemappedocl, map2xocl, map2yocl, CV_INTER_LINEAR, BORDER_CONSTANT, 0);


    Links = LeftRemappedocl.clone();                                                                            //kopiere sauber in neues Mat: leer
    Rechts = RightRemappedocl.clone();                                                                          //kopiere sauber in neues Mat: leer
}
}

void facefind(Mat& f, vector<Rect>&e)
{
CascadeClassifier face_cascade;
face_cascade.load("haarcascade_frontalface_alt.xml");

while (true)
{
    Mat Lokalesf;
    Lokalesf = f.clone();
    Mat grau;
    cvtColor(Lokalesf, grau, CV_BGR2GRAY);
    equalizeHist(grau, grau);

    face_cascade.detectMultiScale(grau, faces, 1.1, 3, CASCADE_SCALE_IMAGE|CV_HAAR_SCALE_IMAGE, Size(30,30));

        e=faces;                                                                                        //kopiere sauber in neues vector<rect>
}
}

int main()
{
    Mat stereo;
    std::thread t1(change,std::ref(Links),std::ref(Rechts));
    std::thread t2(facefind,std::ref(Links),std::ref(faces));
    while (true)
    {
        for(int i = 0; i < faces.size(); i++)
        {
            Point pt1(faces[i].x + faces[i].width, faces[i].y + faces[i].height);
            Point pt2(faces[i].x, faces[i].y);

            rectangle(Links, pt1, pt2, cvScalar(0, 255, 0, 0), 1, 8, 0);
        }
        imshow("rechts", Rechts);
        imshow("links", Links);

        waitKey(10);
    }
    t1.join();
    t2.join();
    return 0;
}

Edit: I tried to understand the code from Mr.Worshipme and included my code to make it work. I get two assertion errors regarding size.width/height in cv::imshow and ROI, anyone can me help me out?

#include <memory>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <vector>
#include <cmath>
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/ocl/ocl.hpp"
#include <functional>

using namespace std;
using namespace cv;

typedef unsigned char uchar;
typedef unsigned int uint;

class FaceDetector
{
mutex imageLock, facesLock;
condition_variable imageAqcuired;
bool newImageAvailable;

Mat _img;
Mat _imgToWorkOn;
Mat _faceImages;

bool quit;
void acquireImage()
{

    Mat captureFrame;
    VideoCapture LeftCam(1);
    LeftCam.set(CV_CAP_PROP_FRAME_WIDTH,1920);              
    LeftCam.set(CV_CAP_PROP_FRAME_HEIGHT,1080);

    FileStorage fs, fs2;
    Mat map1x, map1y; 
    Mat mapi1x, mapi1y;
    fs.open("mx1.xml", FileStorage::READ);
    fs2.open("my1.xml", FileStorage::READ);
    fs["mx1"] >> map1x;
    fs2["my1"] >> map1y;
    fs.release();
    fs2.release();

    ocl::oclMat map1xocl(map1x), map1yocl(map1y);

    while (!quit)
    {
        unique_lock<mutex> ulock(imageLock);
        imageAqcuired.wait(ulock,[&](){return !newImageAvailable;}); //only take
//new image after current one was consumed

        cv::Mat3b frame0, frame1, LeftframeRaw, RightframeRaw, LeftRoi,     RightRoi,framecon;

            LeftCam >> frame0;
           LeftRoi=frame0(Rect(420,60,1080,960));                                                                   //960*1080 passend zuschneiden
        ocl::oclMat Leftr(LeftRoi), Leftt, Lefttf, LeftRemappedocl;
        ocl::transpose(Leftr, Leftt);                                                                               //ACHTUNG: ocl::transpose src darf nicht dst-image sein, sonst gibts starke Bildartefakte!
        ocl::flip(Leftt, Lefttf, 1);                                                                                //ACHTUNG: ocl::flip src darf nicht dst-image sein, sonst gibts starke Bildartefakte!
        ocl::remap(Lefttf, LeftRemappedocl, map1xocl, map1yocl, CV_INTER_LINEAR, BORDER_CONSTANT, 0);
        _img = LeftRemappedocl.clone();

        ulock.unlock();
        newImageAvailable = true;
        imageAqcuired.notify_one(); //notify that a new image is available
    }
}
void processImage()
{
    CascadeClassifier face_cascade;
    face_cascade.load("haarcascade_frontalface_alt.xml");

    while (!quit)
    {
        unique_lock<mutex> ulock(imageLock);
        imageAqcuired.wait(ulock,[&](){return newImageAvailable;}); //wait untill a new image is available
        _imgToWorkOn = _img;
        ulock.unlock();
        newImageAvailable = false;
        imageAqcuired.notify_one(); //notify the current image can be replaced by a newer one
        unique_lock<mutex> lockFace(facesLock);

        Mat grau;
        cvtColor(_imgToWorkOn, grau, CV_BGR2GRAY);
        equalizeHist(grau, grau);
        std::vector<Rect> faces;

        face_cascade.detectMultiScale(grau, faces, 1.1, 3, CASCADE_SCALE_IMAGE|CV_HAAR_SCALE_IMAGE, Size(30,30));

        for(int i = 0; i < faces.size(); i++)
        {
            Point pt1(faces[i].x + faces[i].width, faces[i].y + faces[i].height);
            Point pt2(faces[i].x, faces[i].y);

            rectangle(_imgToWorkOn, pt1, pt2, cvScalar(0, 255, 0, 0), 1, 8, 0);
        }
        _faceImages = _imgToWorkOn;

        lockFace.unlock();
    }
}

public:
FaceDetector() : newImageAvailable(false) {}
void start() {
    quit = false;
    thread t1(&FaceDetector::acquireImage,this);
    t1.detach();
    thread t2(&FaceDetector::processImage,this);
    t2.detach();
}
void stop() {
    quit = true;
}
Mat getImage() {
    lock_guard<mutex> lock(imageLock);
    return _img;
}

Mat getProcessedImage() {
    lock_guard<mutex> lock(facesLock);
    return _faceImages;
}
};


int main()
{
Mat img = Mat::zeros(960,1080, CV_8UC3);                                                //notwendig
Mat imgs = Mat::zeros(960,1080, CV_8UC3);                                               //notwendig
bool quit(false);
FaceDetector faceDet;
faceDet.start();
thread input([](bool &quitFlag) { getchar(); quitFlag = true; },ref(quit)); //stop on user press Enter
input.detach();
while (!quit) {
    Mat img = faceDet.getImage();

    imshow("links", img);

    Mat imgs = faceDet.getProcessedImage();

    imshow("gsichter", imgs);

    waitKey(1);
    this_thread::sleep_for(chrono::milliseconds(33)); //no need to show more than 30 fps...
}
faceDet.stop();
return 0;
}

Edit 2: Edited the code again to allow deep copys of the Mat-images as it was suggested (facetracker does nothing at the moment and ocl-mat-stuff/facecascade erased for code-simplicity) and got the code to work, but it behaves strangely. My webcam lights an led when it runs, about every 500ms or so it lights up and shows me an new image via imshow, then it goes back to standby, despite the fact that it should update _img as fast as it can. I dont know how to set up Videocapture in the mutex-environment to run only once to set frame.width and height and then run freely as the webcam-image-buffer. Heres the mentioned code:

#include <memory>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <vector>
#include <cmath>
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/ocl/ocl.hpp"
#include <functional>

using namespace std;
using namespace cv;

typedef unsigned char uchar;
typedef unsigned int uint;

class FaceDetector
{
mutex imageLock, facesLock;
condition_variable imageAqcuired;
bool newImageAvailable;

Mat _img;
Mat _imgToWorkOn;
Mat _faceImages;
bool quit;
int c;
void acquireImage()
{
    while (!quit)
    {

        unique_lock<mutex> ulock(imageLock);
        imageAqcuired.wait(ulock,[&](){return !newImageAvailable;}); //only take new image after current one was consumed
        VideoCapture captureDevice(0);
        if (c = 1)
        {
        captureDevice.set(CV_CAP_PROP_FRAME_WIDTH, 620);
        captureDevice.set(CV_CAP_PROP_FRAME_HEIGHT, 480);
        c = 0;
        }
        Mat captureFrame;
        captureDevice>>captureFrame;
        transpose(captureFrame,captureFrame);
        flip(captureFrame,captureFrame,1);

        _img = captureFrame.clone();

        ulock.unlock();
        newImageAvailable = true;
        imageAqcuired.notify_one(); //notify that a new image is available
    }
}
void processImage()
{
    while (!quit)
    {
        unique_lock<mutex> ulock(imageLock);
        imageAqcuired.wait(ulock,[&](){return newImageAvailable;}); //wait untill a new image is available
        _imgToWorkOn = _img.clone();
        ulock.unlock();
        newImageAvailable = false;
        imageAqcuired.notify_one(); //notify the current image can be replaced by a newer one
        unique_lock<mutex> lockFace(facesLock);
        //arbeit
        lockFace.unlock();
    }
}

public:
FaceDetector() : newImageAvailable(false) {}
void start() {
    quit = false;
    thread t1(&FaceDetector::acquireImage,this);
    t1.detach();
    thread t2(&FaceDetector::processImage,this);
    t2.detach();
}
void stop() {
    quit = true;
}
Mat getImage() {
    if (quit)
        return Mat();
    lock_guard<mutex> lock(imageLock);
    return _img;
}

Mat getProcessedImage() {
    if (quit)
        return Mat();
    lock_guard<mutex> lock(facesLock);
    return _faceImages;
}
};


int main()
{
bool quit(false);
int c = 1;
FaceDetector faceDet;
faceDet.start();
thread input([](bool &quitFlag) { getchar(); quitFlag = true; },ref(quit)); //stop on user press Enter
input.detach();
while (!quit) {


    Mat img = faceDet.getImage();
    Mat imgc = img.clone();

    imshow("links", imgc);

    /*
    imgs = faceDet.getProcessedImage();
    Mat imgsc = imgs.clone();

    imshow("gsichter", imgsc);
    */
    waitKey(30);
    this_thread::sleep_for(chrono::milliseconds(33)); //no need to show more than 30 fps...
}
faceDet.stop();
return 0;
}

回答1:

I made an example code for you to modify for your needs. If any line is not self explanatory, feel welcome to ask...

#include <memory>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <vector>
#include <cmath>



using namespace std;

typedef unsigned char uchar;
typedef unsigned int uint;

class FaceDetector
{

    mutex imageLock, facesLock;
    condition_variable imageAqcuired;
    bool newImageAvailable;
    vector<uchar> _img;
    vector<uchar> _imgToWorkOn;
    vector<uchar> _faceImages;
    bool quit;
    void acquireImage()
    {
        while (!quit)
        {
            unique_lock<mutex> ulock(imageLock);
            imageAqcuired.wait(ulock,[&](){return !newImageAvailable;}); //only take new image after current one was consumed
            //aqcuire image from camera into _img here
            ulock.unlock();
            newImageAvailable = true;
            imageAqcuired.notify_one(); //notify that a new image is available
        }
    }
    void processImage()
    {
        while (!quit)
        {
            unique_lock<mutex> ulock(imageLock);
            imageAqcuired.wait(ulock,[&](){return newImageAvailable;}); //wait untill a new image is available
            _imgToWorkOn = _img;
            ulock.unlock();
            newImageAvailable = false;
            imageAqcuired.notify_one(); //notify the current image can be replaced by a newer one
            unique_lock<mutex> lockFace(facesLock);
            //process _imgToWorkOn, face detect... 
            lockFace.unlock();
        }
    }

public:
    FaceDetector() : _img(100000),_imgToWorkOn(100000), _faceImages(100000), newImageAvailable(false) {}
    void start() {
        quit = false;
        thread t1(&FaceDetector::acquireImage,this);
        t1.detach();
        thread t2(&FaceDetector::processImage,this);
        t2.detach();
    }
    void stop() {
        quit = true;
    }
    vector<uchar> getImage() {
        if (quit)
            return vector<uchar>();
        lock_guard<mutex> lock(imageLock);
        return _img;
    }

    vector<uchar> getProcessedImage() {
        if (quit)
            return vector<uchar>();
        lock_guard<mutex> lock(facesLock);
        return _faceImages;
    }
};


int main()
{
    bool quit(false);
    FaceDetector faceDet;
    faceDet.start();
    thread input([](bool &quitFlag) { getchar(); quitFlag = true; },ref(quit)); //stop on user press Enter
    input.detach();
    while (!quit) {
        vector<uchar> img = faceDet.getImage();
        if (img.size()) 
        {  /*show image*/ }  
        img = faceDet.getProcessedImage();
        if (img.size()) {  /*show image*/ }
        this_thread::sleep_for(chrono::milliseconds(33)); //no need to show more than 30 fps...
    }
    faceDet.stop();
    return 0;
}


标签： c++ multithreading opencv opencl mutex
举报

收藏的人(0)

Ta的文章 更多文章
登录 后发表评论
0条评论
还没有人评论过~