How to convert & rotate raw NV21 array image (andr

2019-01-24 23:47发布

站内文章 / Android
50 0 0

问题:

Note: All info in my post only goes for Samsung Galaxy S7 device. I do not know how emulators and other devices behave.

In onImageAvailable I convert continuously each image to a NV21 byte array and forward it to an API expecting raw NV21 format.

This is how I initialize the image reader and receive the images:

private void openCamera() {
    ...
    mImageReader = ImageReader.newInstance(WIDTH, HEIGHT,
            ImageFormat.YUV_420_888, 1); // only 1 for best performance
    mImageReader.setOnImageAvailableListener(
    mOnImageAvailableListener, mBackgroundHandler);
    ...
}

private final ImageReader.OnImageAvailableListener mOnImageAvailableListener
        = new ImageReader.OnImageAvailableListener() {

    @Override
    public void onImageAvailable(ImageReader reader) {
        Image image = reader.acquireLatestImage();
        if (image != null) {
            byte[] data = convertYUV420ToNV21_ALL_PLANES(image); // this image is turned 90 deg using front cam in portrait mode
            byte[] data_rotated = rotateNV21_working(data, WIDTH, HEIGHT, 270);
            ForwardToAPI(data_rotated); // image data is being forwarded to api and received later on
            image.close();
        }
    }
};

The function converting the image to raw NV21 (from here), working fine, the image is (due to android?) turned by 90 degrees when using front cam in portrait mode: (I modified it, slightly according to comments of Alex Cohn)

private byte[] convertYUV420ToNV21_ALL_PLANES(Image imgYUV420) {

    byte[] rez;

    ByteBuffer buffer0 = imgYUV420.getPlanes()[0].getBuffer();
    ByteBuffer buffer1 = imgYUV420.getPlanes()[1].getBuffer();
    ByteBuffer buffer2 = imgYUV420.getPlanes()[2].getBuffer();

    // actually here should be something like each second byte
    // however I simply get the last byte of buffer 2 and the entire buffer 1
    int buffer0_size = buffer0.remaining();
    int buffer1_size = buffer1.remaining(); // / 2 + 1;
    int buffer2_size = 1;//buffer2.remaining(); // / 2 + 1;

    byte[] buffer0_byte = new byte[buffer0_size];
    byte[] buffer1_byte = new byte[buffer1_size];
    byte[] buffer2_byte = new byte[buffer2_size];

    buffer0.get(buffer0_byte, 0, buffer0_size);
    buffer1.get(buffer1_byte, 0, buffer1_size);
    buffer2.get(buffer2_byte, buffer2_size-1, buffer2_size);


    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    try {
        // swap 1 and 2 as blue and red colors are swapped
        outputStream.write(buffer0_byte);
        outputStream.write(buffer2_byte);
        outputStream.write(buffer1_byte);
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    rez = outputStream.toByteArray();

    return rez;
}

Hence "data" needs to be rotated. Using this function (from here), I get a weird 3-times interlaced picture error:

public static byte[] rotateNV21(byte[] input, int width, int height, int rotation) {
    byte[] output = new byte[input.length];
    boolean swap = (rotation == 90 || rotation == 270);
    // **EDIT:** in portrait mode & front cam this needs to be set to true:
    boolean yflip = true;// (rotation == 90 || rotation == 180);
    boolean xflip = (rotation == 270 || rotation == 180);
    for (int x = 0; x < width; x++) {
        for (int y = 0; y < height; y++) {
            int xo = x, yo = y;
            int w = width, h = height;
            int xi = xo, yi = yo;
            if (swap) {
                xi = w * yo / h;
                yi = h * xo / w;
            }
            if (yflip) {
                yi = h - yi - 1;
            }
            if (xflip) {
                xi = w - xi - 1;
            }
            output[w * yo + xo] = input[w * yi + xi];
            int fs = w * h;
            int qs = (fs >> 2);
            xi = (xi >> 1);
            yi = (yi >> 1);
            xo = (xo >> 1);
            yo = (yo >> 1);
            w = (w >> 1);
            h = (h >> 1);
            // adjust for interleave here
            int ui = fs + (w * yi + xi) * 2;
            int uo = fs + (w * yo + xo) * 2;
            // and here
            int vi = ui + 1;
            int vo = uo + 1;
            output[uo] = input[ui];
            output[vo] = input[vi];
        }
    }
    return output;
}

Resulting into this picture:

Note: it is still the same cup, however you see it 3-4 times.

Using another suggested rotate function from here gives the proper result:

public static byte[] rotateNV21_working(final byte[] yuv,
                                final int width,
                                final int height,
                                final int rotation)
{
  if (rotation == 0) return yuv;
  if (rotation % 90 != 0 || rotation < 0 || rotation > 270) {
    throw new IllegalArgumentException("0 <= rotation < 360, rotation % 90 == 0");
  }

  final byte[]  output    = new byte[yuv.length];
  final int     frameSize = width * height;
  final boolean swap      = rotation % 180 != 0;
  final boolean xflip     = rotation % 270 != 0;
  final boolean yflip     = rotation >= 180;

  for (int j = 0; j < height; j++) {
    for (int i = 0; i < width; i++) {
      final int yIn = j * width + i;
      final int uIn = frameSize + (j >> 1) * width + (i & ~1);
      final int vIn = uIn       + 1;

      final int wOut     = swap  ? height              : width;
      final int hOut     = swap  ? width               : height;
      final int iSwapped = swap  ? j                   : i;
      final int jSwapped = swap  ? i                   : j;
      final int iOut     = xflip ? wOut - iSwapped - 1 : iSwapped;
      final int jOut     = yflip ? hOut - jSwapped - 1 : jSwapped;

      final int yOut = jOut * wOut + iOut;
      final int uOut = frameSize + (jOut >> 1) * wOut + (iOut & ~1);
      final int vOut = uOut + 1;

      output[yOut] = (byte)(0xff & yuv[yIn]);
      output[uOut] = (byte)(0xff & yuv[uIn]);
      output[vOut] = (byte)(0xff & yuv[vIn]);
    }
  }
  return output;
}

The result is fine now:

The top image shows the direct stream using a texture view's surface and adding it to the captureRequestBuilder. The bottom image shows the raw image data after rotating.

The questions are:

  • Does this hack in "convertYUV420ToNV21_ALL_PLANES" work on any device/emulator?
  • Why does rotateNV21 not work, while rotateNV21_working works fine.

Edit: The mirror issue is fixed, see code comment. The squeeze issue is fixed, it was caused by the API it gets forwarded. The actual open issue is a proper not too expensive function, converting and rotating an image into raw NV21 working on any device.

回答1:

So, here is the correct code to convert the Image to NV21 byte[]. This will work wehn the imgYUV420 U and V planes have pixelStride=1 (as on emulator) or pixelStride=1 (as on Nexus):

private byte[] convertYUV420ToNV21_ALL_PLANES(Image imgYUV420) {

    assert(imgYUV420.getFormat() == ImageFormat.YUV_420_888);
    Log.d(TAG, "image: " + imgYUV420.getWidth() + "x" + imgYUV420.getHeight() + " " + imgYUV420.getFormat());
    Log.d(TAG, "planes: " + imgYUV420.getPlanes().length);
    for (int nplane = 0; nplane < imgYUV420.getPlanes().length; nplane++) {
        Log.d(TAG, "plane[" + nplane + "]: length " + imgYUV420.getPlanes()[nplane].getBuffer().remaining() + ", strides: " + imgYUV420.getPlanes()[nplane].getPixelStride() + " " + imgYUV420.getPlanes()[nplane].getRowStride());
    }

    byte[] rez = new byte[imgYUV420.getWidth() * imgYUV420.getHeight() * 3 / 2];
    ByteBuffer buffer0 = imgYUV420.getPlanes()[0].getBuffer();
    ByteBuffer buffer1 = imgYUV420.getPlanes()[1].getBuffer();
    ByteBuffer buffer2 = imgYUV420.getPlanes()[2].getBuffer();

    int n = 0;
    assert(imgYUV420.getPlanes()[0].getPixelStride() == 1);
    for (int row = 0; row < imgYUV420.getHeight(); row++) {
        for (int col = 0; col < imgYUV420.getWidth(); col++) {
            rez[n++] = buffer0.get();
        }
    }
    assert(imgYUV420.getPlanes()[2].getPixelStride() == imgYUV420.getPlanes()[1].getPixelStride());
    int stride = imgYUV420.getPlanes()[1].getPixelStride();
    for (int row = 0; row < imgYUV420.getHeight(); row += 2) {
        for (int col = 0; col < imgYUV420.getWidth(); col += 2) {
            rez[n++] = buffer1.get();
            rez[n++] = buffer2.get();
            for (int skip = 1; skip < stride; skip++) {
                if (buffer1.remaining() > 0) {
                    buffer1.get();
                }
                if (buffer2.remaining() > 0) {
                    buffer2.get();
                }
            }
        }
    }

    Log.w(TAG, "total: " + rez.length);
    return rez;
}

As you can see, it is very easy to change this code to produce a rotated image in a single step:

private byte[] rotateYUV420ToNV21(Image imgYUV420) {

    Log.d(TAG, "image: " + imgYUV420.getWidth() + "x" + imgYUV420.getHeight() + " " + imgYUV420.getFormat());
    Log.d(TAG, "planes: " + imgYUV420.getPlanes().length);
    for (int nplane = 0; nplane < imgYUV420.getPlanes().length; nplane++) {
        Log.d(TAG, "plane[" + nplane + "]: length " + imgYUV420.getPlanes()[nplane].getBuffer().remaining() + ", strides: " + imgYUV420.getPlanes()[nplane].getPixelStride() + " " + imgYUV420.getPlanes()[nplane].getRowStride());
    }

    byte[] rez = new byte[imgYUV420.getWidth() * imgYUV420.getHeight() * 3 / 2];
    ByteBuffer buffer0 = imgYUV420.getPlanes()[0].getBuffer();
    ByteBuffer buffer1 = imgYUV420.getPlanes()[1].getBuffer();
    ByteBuffer buffer2 = imgYUV420.getPlanes()[2].getBuffer();

    int width = imgYUV420.getHeight();
    assert(imgYUV420.getPlanes()[0].getPixelStride() == 1);
    for (int row = imgYUV420.getHeight()-1; row >=0; row--) {
        for (int col = 0; col < imgYUV420.getWidth(); col++) {
            rez[col*width+row] = buffer0.get();
        }
    }
    int uv_offset = imgYUV420.getWidth()*imgYUV420.getHeight();
    assert(imgYUV420.getPlanes()[2].getPixelStride() == imgYUV420.getPlanes()[1].getPixelStride());
    int stride = imgYUV420.getPlanes()[1].getPixelStride();
    for (int row = imgYUV420.getHeight() - 2; row >= 0; row -= 2) {
        for (int col = 0; col < imgYUV420.getWidth(); col += 2) {
            rez[uv_offset+col/2*width+row] = buffer1.get();
            rez[uv_offset+col/2*width+row+1] = buffer2.get();
            for (int skip = 1; skip < stride; skip++) {
                if (buffer1.remaining() > 0) {
                    buffer1.get();
                }
                if (buffer2.remaining() > 0) {
                    buffer2.get();
                }
            }
        }
    }

    Log.w(TAG, "total rotated: " + rez.length);
    return rez;
}

I sincerely recommend the site http://rawpixels.net/ to see the actual structure of your raw images.



回答2:

With OpenCV and Android Camera API 2 this task is very fast and you don't need YUV420toNV21 Java conversion, and with OpenCV this convertion is 4x more fast:

Java side:

//Starts a builtin camera with api camera 2
public void startCamera() {
    CameraManager manager = (CameraManager) AppData.getAppContext().getSystemService(Context.CAMERA_SERVICE);
    try {
        String pickedCamera = getCamera(manager);
        manager.openCamera(pickedCamera, cameraStateCallback, null);
        // set image format on YUV
        mImageReader = ImageReader.newInstance(mWidth,mHeight, ImageFormat.YUV_420_888, 4);
        mImageReader.setOnImageAvailableListener(onImageAvailableListener, null);
        Log.d(TAG, "imageReader created");
    } catch (CameraAccessException e) {
        Log.e(TAG, e.getMessage());
    }
}


//Listens for frames and send them to  be processed
protected ImageReader.OnImageAvailableListener onImageAvailableListener = new ImageReader.OnImageAvailableListener() {
    @Override
    public void onImageAvailable(ImageReader reader) {
        Image image = null;
        try {

            image = reader.acquireLatestImage();
            ByteBuffer buffer = image.getPlanes()[0].getBuffer();
            byte[] frameData = new byte[buffer.capacity()];
            buffer.get(frameData);
            // Native process (see below)
            processAndRotateFrame(frameData);
            image.close();

        } catch (Exception e) {
            Logger.e(TAG, "imageReader exception: "+e.getMessage());

        } finally {
            if (image != null) {
                image.close();
            }
        }
    }
};

Native side (NDK or Cmake):

JNIEXPORT jint JNICALL  com_android_mvf_Utils_ProccessAndRotateFrame
    (JNIEnv *env, jobject object, jint width, jint height, jbyteArray frame, jint rotation) {
    // load data from JAVA side
    jbyte *pFrameData = env->GetByteArrayElements(frame, 0);
    // convert array to Mat, for example GRAY or COLOR
    Mat mGray(height, width, cv::IMREAD_GRAYSCALE, (unsigned char *)pFrameData);
    // rotate image
    rotateMat(mGray, rotation);
    int objects = your_function(env, mGray);
    env->ReleaseByteArrayElements(frame, pFrameData, 0);
    return objects;
}

void rotateMat(cv::Mat &matImage, int rotFlag) {
    if (rotFlag != 0 && rotFlag != 360) {
        if (rotFlag == 90) {
            cv::transpose(matImage, matImage);
            cv::flip(matImage, matImage, 1);
        } else if (rotFlag == 270 || rotFlag == -90) {
            cv::transpose(matImage, matImage);
            cv::flip(matImage, matImage, 0);
        } else if (rotFlag == 180) {
            cv::flip(matImage, matImage, -1);
        }
    }
}


标签： android rotation android-camera2 portrait front-camera
举报

收藏的人(0)

Ta的文章 更多文章
登录 后发表评论
0条评论
还没有人评论过~