How to use OpenCV face detection in portrait using

问题:

I am trying to use OpenCV face detection using the the byte[] data obtained from the onPreviewFrame() method of the Camera.PreviewCallback

I manage to convert the data into grayscale image using the codes below.

    Mat matNew = new Mat(pHeight, pWidth, CvType.CV_8U);
    matNew.put(0, 0, data);
    Mat matrgb = new Mat();
    Imgproc.cvtColor(matNew, matrgb, Imgproc.COLOR_YUV420sp2RGB, 4);
    Mat matgray = new Mat();
    Imgproc.cvtColor(matrgb, matgray, Imgproc.COLOR_RGB2GRAY, 0);

and I have set android:screenOrientation to "portrait" in the AndroidManifest file.

I am using OpenCV JavaDetector

mJavaDetector.detectMultiScale(matgray, faceDetected, 1.1, 3, 0, new org.opencv.core.Size(0,0), new org.opencv.core.Size(matgray.width(), matgray.height())); and drawing a rectangle over the faces detected using this

 for (Rect rect : faceDetected.toArray()){
  Core.rectangle(matgray, new Point(rect.x, rect.y), 
new Point(rect.x + rect.width, rect.y + rect.height),
new Scalar(0, 255, 0));
                    }

However, in the resulting grayscale mat, face detection only happens when I hold my Android phone in landscape position. It does not work in portrait position.

The transpose and flip suggested in this post How to detect face by portrait mode? doesn't seem to work.

Is there any way to overcome this issue? I have used the Android FaceDetectionListener and that doesn't seem to have problem detecting faces in portrait mode. But, FaceDetectionListener's functions are limited compared to OpenCV.

Any help would be greatly appreciated. Thx.

回答1:

Something like the following works for me with android:screenOrientation="portrait" in AndroidManifest.xml

public Mat onCameraFrame(CvCameraViewFrame inputFrame) {

    mRgba = inputFrame.rgba();
    mGray = inputFrame.gray();

    MatOfRect faces = new MatOfRect();
    Core.flip(mRgba.t(), mRgba, -1);
    Core.flip(mGray.t(), mGray, -1);

    if (mNativeDetector != null)
        mNativeDetector.detect(mGray, faces);

    Rect[] facesArray = faces.toArray();
    for (int i = 0; i < facesArray.length; i++)
        Core.rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(), FACE_RECT_COLOR, 2);

    return mRgba;
}

回答2:

Flip the colored and gray image(Mat) clockwise for face/feature detection to work in portrait mode. At the end of feature detection Logic, you flip counter clockwise the colored image(mRgba Mat) . As illustrated.

 public Mat onCameraFrame(CvCameraViewFrame inputFrame) {

    Core.flip(inputFrame.gray().t(),mGray,1); //rotate clockwise 
    Core.flip(inputFrame.rgba().t(),mRgba,1);
    mRgba=Feature_DetectionNATIVE(mRgba,mGray);
    Core.flip(mRgba.t(),mRgba,0);             //rotate counter clockwise
//this is a solution for  allowing face detection in portrait view if it isn't working at all.
    return mRgba;
 }
public Mat Feature_DetectionNATIVE(Mat mRgba2, final Mat Gray)
{
if (mAbsoluteFaceSize == 0) 
{
    int height = Gray.rows();
    if (Math.round(height * mRelativeFaceSize) > 0)
     {
       mAbsoluteFaceSize = Math.round(height * mRelativeFaceSize);

     }
 mNativeDetector.setMinFaceSize(mAbsoluteFaceSize);

}

MatOfRect faces = new MatOfRect();

if (mDetectorType == JAVA_DETECTOR) 
 {
if (mJavaDetector != null)
  mJavaDetector.detectMultiScale(Gray, faces, 1.1, 2, 2, 
  new Size(mAbsoluteFaceSize, mAbsoluteFaceSize), new Size());
 }
else if (mDetectorType == NATIVE_DETECTOR)
{
if (mNativeDetector != null)
  mNativeDetector.detect(Gray, faces);
 }




Rect[] facesArray = faces.toArray();
for (int i = 0; i < facesArray.length; i++)
{
  Core.rectangle(mRgba2, facesArray[i].tl(), facesArray[i].br(),     FACE_RECT_COLOR, 3);
}
    return mRgba2;
}

After that the camera will show face detection in landscape orientation to fix this you rotate the canvas clockwise by 90 in opencv's CameraBridgeViewBase main class or hack it.(Note this reduces FPS but face detection is still fast)

 protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
 Mat modified;
 if (mListener != null) {
 modified = mListener.onCameraFrame(frame);
 } else {
  modified = frame.rgba();
 }

 boolean bmpValid = true;
 if (modified != null) {
 try {
  Utils.matToBitmap(modified, mCacheBitmap);

  } catch(Exception e) {
 Log.e(TAG, "Mat type: " + modified);
 Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" +   mCacheBitmap.getHeight());
 Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
 bmpValid = false;
  }
 }

 if (bmpValid && mCacheBitmap != null) {
 Canvas canvas = getHolder().lockCanvas();
  if (canvas != null) {
 canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
 Log.d(TAG, "mStretch value: " + mScale);


 canvas=rotateCanvas(canvas,mCacheBitmap);
 getHolder().unlockCanvasAndPost(canvas);
 }

  }
 }

 protected Canvas rotateCanvas(final Canvas canvas, final Bitmap mCacheBitmap)
 { 
 final CountDownLatch latch =new CountDownLatch(1);
 final Mat[] mRgba=new Mat[1];

 new Thread(new Runnable() {
 @Override
 public void run() {

 try {
 Bitmap bitmap = Bitmap.createScaledBitmap(mCacheBitmap, canvas.getHeight(),  canvas.getWidth(), true);
canvas.rotate(90,0,0);
mScale = canvas.getWidth() / (float)bitmap.getHeight();
float scale2 = canvas.getHeight() / (float)bitmap.getWidth();
if(scale2 > mScale){
    mScale = scale2;
 }
if (mScale != 0) {
 canvas.scale(mScale, mScale,0,0);
 }
canvas.drawBitmap(bitmap, 0, -bitmap.getHeight(), null);

}
catch (CvException e) {           e.printStackTrace();}
latch.countDown();//setting //release await() in this thread
}

}).start();

try {  latch.await(); //waits for countDown in the Thread inorder to obtain a value from the thread

} catch (InterruptedException e) {   e.printStackTrace();}

return canvas;


}

This solution works when one is implementing Opencv's CameraBridgeViewBase class (Tested on OpenCV 2.4.9)