Most efficient/realtime way to get pixel values fr

Question

There are some discussions on here about similar questions. Like this, but they seem quite outdated, so I thought I'd ask here.

I want to get near-realtime RGB pixel values, or even better, a full image RGB histogram from a camera feed in swift 2.0. I want this to be as quick and up to date as possible (~30 fps or higher ideally)

Can I get this directly from a AVCaptureVideoPreviewLayer or do I need to capture each frame (async, I assume, if the process takes significant time) then extract pixel values from the jpeg/png render?

Some example code, taken from jquave but modified for swift 2.0

import UIKit
import AVFoundation

class ViewController: UIViewController {

let captureSession = AVCaptureSession()
var previewLayer : AVCaptureVideoPreviewLayer?

var captureDevice : AVCaptureDevice?

override func viewDidLoad() {
    super.viewDidLoad()

    // Do any additional setup after loading the view, typically from a nib.
    captureSession.sessionPreset = AVCaptureSessionPresetHigh

    let devices = AVCaptureDevice.devices()

    // Loop through all the capture devices on this phone
    for device in devices {
        // Make sure this particular device supports video
        if (device.hasMediaType(AVMediaTypeVideo)) {
            // Finally check the position and confirm we've got the back camera
            if(device.position == AVCaptureDevicePosition.Back) {
                captureDevice = device as? AVCaptureDevice
                if captureDevice != nil {
                    print("Capture device found")
                    beginSession()
                }
            }
        }
    }
}

func focusTo(value : Float) {
    if let device = captureDevice {
        do {
            try device.lockForConfiguration()
                device.setFocusModeLockedWithLensPosition(value, completionHandler: { (time) -> Void in
                })
            device.unlockForConfiguration()
        } catch {
            //error message
            print("Can't change focus of capture device")
        }
    }
}

func configureDevice() {
    if let device = captureDevice {
        do {
            try device.lockForConfiguration()
                device.focusMode = .Locked
                device.unlockForConfiguration()
        } catch {
            //error message etc.
            print("Capture device not configurable")
        }
    }

}

func beginSession() {

    configureDevice()
    do {
        //try captureSession.addInput(input: captureDevice)
        try captureSession.addInput(AVCaptureDeviceInput(device: captureDevice))
        updateDeviceSettings(0.0, isoValue: 0.0)
    } catch {
        //error message etc.
        print("Capture device not initialisable")
    }
    previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
    self.view.layer.addSublayer(previewLayer!)
    previewLayer?.frame = self.view.layer.frame
    captureSession.startRunning()        
}

func updateDeviceSettings(focusValue : Float, isoValue : Float) {
    if let device = captureDevice {
        do {
            try device.lockForConfiguration()
            device.setFocusModeLockedWithLensPosition(focusValue, completionHandler: { (time) -> Void in
                //
            })

            // Adjust the iso to clamp between minIso and maxIso based on the active format
            let minISO = device.activeFormat.minISO
            let maxISO = device.activeFormat.maxISO
            let clampedISO = isoValue * (maxISO - minISO) + minISO

            device.setExposureModeCustomWithDuration(AVCaptureExposureDurationCurrent, ISO: clampedISO, completionHandler: { (time) -> Void in
                //
            })

            device.unlockForConfiguration()
        } catch {
            //error message etc.
            print("Can't update device settings")
        }

    }
}
}

Answer 1

You don't want an AVCaptureVideoPreviewLayer - that's what you want if you want to display the video. Instead, you want a different output: AVCaptureVideoDataOutput:

https://developer.apple.com/library/ios/documentation/AVFoundation/Reference/AVCaptureVideoDataOutput_Class/index.html#//apple_ref/occ/cl/AVCaptureVideoDataOutput

This gives you direct access to the stream of sample buffers, which you can then get into pixel-space.

Just a note: I don't know what the throughput on current devices is, but I was unable to get a live stream at the highest quality from the iPhone 4S because the GPU<-->CPU pipeline was too slow.