Get video pixel data from Exoplayer in Android

Question

Is it possible to grab the pixel data (e.g. as RGB byte array) from a running video within the ExoPlayer? Ideally as the real video resolution and not the size as the shown View. I'd want to forward that data to OpenCV for ImageProcessing purposes.

Alternatively I'm looking for a robust (ffmpeg based) Android framework to input videos into OpenCV where the input might be IP-Cameras, local files, online files and online streams.

Any help is appreciated.

Answer 1

WARNING this solution currently only works on my Android 8 emulator because of this.

Ok, here is my solution. I'm using the SimpleExoPlayer with a custom MediaCodecVideoRenderer. I'm not binding the player to a SimpleExoPlayerView because I'm not allowed to if I want to manually grab the image data. Within my custom MediaCodecVideoRenderer I override the processOutputBuffer and use getOutputImage to get a nice standard Android Image. I then send it through my OpenCV code and then transform it back into an Android Bitmap using the OpenCV Utils whenever I need. Careful this code requires API >= 21. Also this codes does not output any audio.

private void startPlayer(Context context, Uri uri) {

    BandwidthMeter bandwidthMeter = new DefaultBandwidthMeter();
    TrackSelection.Factory videoTrackSelectionFactory = new AdaptiveTrackSelection.Factory(bandwidthMeter);
    TrackSelector trackSelector = new DefaultTrackSelector(videoTrackSelectionFactory);

    SimpleExoPlayer player = ExoPlayerFactory.newSimpleInstance((eventHandler, videoRendererEventListener, audioRendererEventListener, textRendererOutput, metadataRendererOutput) -> {
        return new Renderer[]{new CustomMediaCodecVideoRenderer(context, MediaCodecSelector.DEFAULT, 1000, eventHandler, videoRendererEventListener, 100)};
    }, trackSelector);

    DataSource.Factory dataSourceFactory = new DefaultDataSourceFactory(context, Util.getUserAgent(context, context.getPackageName()));
    ExtractorMediaSource videoSource = new ExtractorMediaSource.Factory(dataSourceFactory).createMediaSource(uri);

    LoopingMediaSource loopingMediaSource = new LoopingMediaSource(videoSource);

    player.setPlayWhenReady(true);
    player.prepare(loopingMediaSource);

}

class CustomMediaCodecVideoRenderer extends MediaCodecVideoRenderer {
    CustomMediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector, long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFrameCountToNotify) {
        super(context, mediaCodecSelector, allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFrameCountToNotify);
    }

    @Override
    protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, boolean shouldSkip) throws ExoPlaybackException {


        Image image = codec.getOutputImage(bufferIndex);

        if (image != null) {
            Log.d(MainActivity.class.getName(), "test");

            Mat mat = convertYuv420888ToMat(image, false);
            // TODO do your thing with the OpenCV Mat
        }

        return super.processOutputBuffer(positionUs, elapsedRealtimeUs, codec, buffer, bufferIndex, bufferFlags, bufferPresentationTimeUs, shouldSkip);
    }
}

I found the Image to Mat conversion here.

private Mat convertYuv420888ToMat(Image image, boolean isGreyOnly) {
    int width = image.getWidth();
    int height = image.getHeight();

    Image.Plane yPlane = image.getPlanes()[0];
    int ySize = yPlane.getBuffer().remaining();

    if (isGreyOnly) {
        byte[] data = new byte[ySize];
        yPlane.getBuffer().get(data, 0, ySize);

        Mat greyMat = new Mat(height, width, CvType.CV_8UC1);
        greyMat.put(0, 0, data);

        return greyMat;
    }

    Image.Plane uPlane = image.getPlanes()[1];
    Image.Plane vPlane = image.getPlanes()[2];

    // be aware that this size does not include the padding at the end, if there is any
    // (e.g. if pixel stride is 2 the size is ySize / 2 - 1)
    int uSize = uPlane.getBuffer().remaining();
    int vSize = vPlane.getBuffer().remaining();

    byte[] data = new byte[ySize + (ySize/2)];

    yPlane.getBuffer().get(data, 0, ySize);

    ByteBuffer ub = uPlane.getBuffer();
    ByteBuffer vb = vPlane.getBuffer();

    int uvPixelStride = uPlane.getPixelStride(); //stride guaranteed to be the same for u and v planes
    if (uvPixelStride == 1) {
        uPlane.getBuffer().get(data, ySize, uSize);
        vPlane.getBuffer().get(data, ySize + uSize, vSize);

        Mat yuvMat = new Mat(height + (height / 2), width, CvType.CV_8UC1);
        yuvMat.put(0, 0, data);
        Mat rgbMat = new Mat(height, width, CvType.CV_8UC3);
        Imgproc.cvtColor(yuvMat, rgbMat, Imgproc.COLOR_YUV2RGB_I420, 3);
        yuvMat.release();
        return rgbMat;
    }

    // if pixel stride is 2 there is padding between each pixel
    // converting it to NV21 by filling the gaps of the v plane with the u values
    vb.get(data, ySize, vSize);
    for (int i = 0; i < uSize; i += 2) {
        data[ySize + i + 1] = ub.get(i);
    }

    Mat yuvMat = new Mat(height + (height / 2), width, CvType.CV_8UC1);
    yuvMat.put(0, 0, data);
    Mat rgbMat = new Mat(height, width, CvType.CV_8UC3);
    Imgproc.cvtColor(yuvMat, rgbMat, Imgproc.COLOR_YUV2RGB_NV21, 3);
    yuvMat.release();
    return rgbMat;
}