How can I do face detection in realtime just as "Camera" does? like white round shape around and over the face. I use AVCapturSession . I found that the image I saved for facial detection. Below I have attached my current code. it only captures image when I press the button and save it into the photo gallery. some please help me to create real-time round shape over according to the person's face!
code
class CameraFaceRecongnitionVC: UIViewController {
@IBOutlet weak var imgOverlay: UIImageView!
@IBOutlet weak var btnCapture: UIButton!
let captureSession = AVCaptureSession()
let stillImageOutput = AVCaptureStillImageOutput()
var previewLayer : AVCaptureVideoPreviewLayer?
// If we find a device we'll store it here for later use
var captureDevice : AVCaptureDevice?
override func viewDidLoad() {
super.viewDidLoad()
btnCapture.CameraButton()
roundButton.RoundButtonForFaceRecong()
// Do any additional setup after loading the view, typically from a nib.
captureSession.sessionPreset = AVCaptureSessionPresetHigh
if let devices = AVCaptureDevice.devices() as? [AVCaptureDevice] {
// Loop through all the capture devices on this phone
for device in devices {
// Make sure this particular device supports video
if (device.hasMediaType(AVMediaTypeVideo)) {
// Finally check the position and confirm we've got the front camera
if(device.position == AVCaptureDevicePosition.front) {
captureDevice = device
if captureDevice != nil {
print("Capture device found")
beginSession()
}
}
}
}
}
}
@IBAction func actionCameraCapture(_ sender: AnyObject) {
print("Camera button pressed")
saveToCamera()
}
func beginSession() {
do {
try captureSession.addInput(AVCaptureDeviceInput(device: captureDevice))
stillImageOutput.outputSettings = [AVVideoCodecKey:AVVideoCodecJPEG]
if captureSession.canAddOutput(stillImageOutput) {
captureSession.addOutput(stillImageOutput)
}
}
catch {
print("error: \(error.localizedDescription)")
}
guard let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession) else {
print("no preview layer")
return
}
self.view.layer.addSublayer(previewLayer)
previewLayer.frame = self.view.layer.frame
captureSession.startRunning()
// self.view.addSubview(navigationBar)
self.view.addSubview(imgOverlay)
self.view.addSubview(btnCapture)
}
func saveToCamera() {
if let videoConnection = stillImageOutput.connection(withMediaType: AVMediaTypeVideo) {
stillImageOutput.captureStillImageAsynchronously(from: videoConnection, completionHandler: { (CMSampleBuffer, Error) in
if let imageData = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(CMSampleBuffer) {
if let cameraImage = UIImage(data: imageData) {
UIImageWriteToSavedPhotosAlbum(cameraImage, nil, nil, nil)
}
}
})
}
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
// Dispose of any resources that can be recreated.
}
}
Swift 3
I have found a solution using AVFoundation that will create square face tracking in real time on iOS. I have modified some code here.
import UIKit
import AVFoundation
class DetailsView: UIView {
func setup() {
layer.borderColor = UIColor.red.withAlphaComponent(0.7).cgColor
layer.borderWidth = 5.0
}
}
class ViewController: UIViewController {
let stillImageOutput = AVCaptureStillImageOutput()
var session: AVCaptureSession?
var stillOutput = AVCaptureStillImageOutput()
var borderLayer: CAShapeLayer?
let detailsView: DetailsView = {
let detailsView = DetailsView()
detailsView.setup()
return detailsView
}()
lazy var previewLayer: AVCaptureVideoPreviewLayer? = {
var previewLay = AVCaptureVideoPreviewLayer(session: self.session!)
previewLay?.videoGravity = AVLayerVideoGravityResizeAspectFill
return previewLay
}()
lazy var frontCamera: AVCaptureDevice? = {
guard let devices = AVCaptureDevice.devices(withMediaType: AVMediaTypeVideo) as? [AVCaptureDevice] else { return nil }
return devices.filter { .position == .front }.first
}()
let faceDetector = CIDetector(ofType: CIDetectorTypeFace, context: nil, options: [CIDetectorAccuracy : CIDetectorAccuracyLow])
override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
previewLayer?.frame = view.frame
}
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
guard let previewLayer = previewLayer else { return }
view.layer.addSublayer(previewLayer)
view.addSubview(detailsView)
view.bringSubview(toFront: detailsView)
}
override func viewDidLoad() {
super.viewDidLoad()
sessionPrepare()
session?.startRunning()
}
//function to store image
func saveToCamera() {
if let videoConnection = stillImageOutput.connection(withMediaType: AVMediaTypeVideo) {
stillImageOutput.captureStillImageAsynchronously(from: videoConnection, completionHandler: { (CMSampleBuffer, Error) in
if let imageData = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(CMSampleBuffer) {
if let cameraImage = UIImage(data: imageData) {
UIImageWriteToSavedPhotosAlbum(cameraImage, nil, nil, nil)
}
}
})
}
}
}
extension ViewController {
func sessionPrepare() {
session = AVCaptureSession()
guard let session = session, let captureDevice = frontCamera else { return }
session.sessionPreset = AVCaptureSessionPresetPhoto
do {
let deviceInput = try AVCaptureDeviceInput(device: captureDevice)
session.beginConfiguration()
stillImageOutput.outputSettings = [AVVideoCodecKey:AVVideoCodecJPEG]
if session.canAddOutput(stillImageOutput) {
session.addOutput(stillImageOutput)
}
if session.canAddInput(deviceInput) {
session.addInput(deviceInput)
}
let output = AVCaptureVideoDataOutput()
output.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String : NSNumber(value: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)]
output.alwaysDiscardsLateVideoFrames = true
if session.canAddOutput(output) {
session.addOutput(output)
}
session.commitConfiguration()
let queue = DispatchQueue(label: "output.queue")
output.setSampleBufferDelegate(self, queue: queue)
} catch {
print("error with creating AVCaptureDeviceInput")
}
}
}
extension ViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {
let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
let attachments = CMCopyDictionaryOfAttachments(kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate)
let ciImage = CIImage(cvImageBuffer: pixelBuffer!, options: attachments as! [String : Any]?)
let options: [String : Any] = [CIDetectorImageOrientation: exifOrientation(orientation: UIDevice.current.orientation),
CIDetectorSmile: true,
CIDetectorEyeBlink: true]
let allFeatures = faceDetector?.features(in: ciImage, options: options)
let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer)
let cleanAperture = CMVideoFormatDescriptionGetCleanAperture(formatDescription!, false)
guard let features = allFeatures else { return }
for feature in features {
if let faceFeature = feature as? CIFaceFeature {
let faceRect = calculateFaceRect(facePosition: faceFeature.mouthPosition, faceBounds: faceFeature.bounds, clearAperture: cleanAperture)
update(with: faceRect)
}
}
if features.count == 0 {
DispatchQueue.main.async {
self.detailsView.alpha = 0.0
}
}
}
func exifOrientation(orientation: UIDeviceOrientation) -> Int {
switch orientation {
case .portraitUpsideDown:
return 8
case .landscapeLeft:
return 3
case .landscapeRight:
return 1
default:
return 6
}
}
func videoBox(frameSize: CGSize, apertureSize: CGSize) -> CGRect {
let apertureRatio = apertureSize.height / apertureSize.width
let viewRatio = frameSize.width / frameSize.height
var size = CGSize.zero
if (viewRatio > apertureRatio) {
size.width = frameSize.width
size.height = apertureSize.width * (frameSize.width / apertureSize.height)
} else {
size.width = apertureSize.height * (frameSize.height / apertureSize.width)
size.height = frameSize.height
}
var videoBox = CGRect(origin: .zero, size: size)
if (size.width < frameSize.width) {
videoBox.origin.x = (frameSize.width - size.width) / 2.0
} else {
videoBox.origin.x = (size.width - frameSize.width) / 2.0
}
if (size.height < frameSize.height) {
videoBox.origin.y = (frameSize.height - size.height) / 2.0
} else {
videoBox.origin.y = (size.height - frameSize.height) / 2.0
}
return videoBox
}
func calculateFaceRect(facePosition: CGPoint, faceBounds: CGRect, clearAperture: CGRect) -> CGRect {
let parentFrameSize = previewLayer!.frame.size
let previewBox = videoBox(frameSize: parentFrameSize, apertureSize: clearAperture.size)
var faceRect = faceBounds
swap(&faceRect.size.width, &faceRect.size.height)
swap(&faceRect.origin.x, &faceRect.origin.y)
let widthScaleBy = previewBox.size.width / clearAperture.size.height
let heightScaleBy = previewBox.size.height / clearAperture.size.width
faceRect.size.width *= widthScaleBy
faceRect.size.height *= heightScaleBy
faceRect.origin.x *= widthScaleBy
faceRect.origin.y *= heightScaleBy
faceRect = faceRect.offsetBy(dx: 0.0, dy: previewBox.origin.y)
let frame = CGRect(x: parentFrameSize.width - faceRect.origin.x - faceRect.size.width - previewBox.origin.x / 2.0, y: faceRect.origin.y, width: faceRect.width, height: faceRect.height)
return frame
}
}
extension ViewController {
func update(with faceRect: CGRect) {
DispatchQueue.main.async {
UIView.animate(withDuration: 0.2) {
self.detailsView.alpha = 1.0
self.detailsView.frame = faceRect
}
}
}
}
**Edited *******
Swift 4
Apple's own Vision frameworks available to detect face in real time from Swift 4. click the link for document and sample app.
To anyone looking for a working and update example, Apple's website has the best I have found so far: https://developer.apple.com/documentation/vision/tracking_the_user_s_face_in_real_time.
You have to change two lines: in AppDelegate.swift you need to change line 15 to
func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplicationLaunchOptionsKey: Any]?) -> Bool {
And in ViewController.swiftyou need to change line 462 to
let cameraIntrinsicData = CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_CameraIntrinsicMatrix, nil)
Based on this article
Details
- Xcode 10.2.1 (10E1001), Swift 5
Features
This solution allow:
- to check camera access
- to select front or back camera
- if no access to the camera show alert with link to the app settings page
- to highlight face (face rectangle + eyebrows, lips..)
Solution
FaceDetectionService
import UIKit
import AVFoundation
import Vision
class FaceDetectionService: NSObject {
private weak var previewView: UIView?
private weak var faceView: FaceView?
private var cameraIsReadyToUse = false
private let session = AVCaptureSession()
private lazy var cameraPosition = AVCaptureDevice.Position.front
private weak var previewLayer: AVCaptureVideoPreviewLayer?
private lazy var sequenceHandler = VNSequenceRequestHandler()
private lazy var dataOutputQueue = DispatchQueue(label: "FaceDetectionService",
qos: .userInitiated, attributes: [],
autoreleaseFrequency: .workItem)
private var preparingCompletionHandler: ((Bool) -> Void)?
func prepare(previewView: UIView,
cameraPosition: AVCaptureDevice.Position,
completion: ((Bool) -> Void)?) {
self.previewView = previewView
self.preparingCompletionHandler = completion
self.cameraPosition = cameraPosition
checkCameraAccess { allowed in
if allowed { self.setup() }
completion?(allowed)
self.preparingCompletionHandler = nil
}
}
private func setup() {
guard let bounds = previewView?.bounds else { return }
let faceView = FaceView(frame: bounds)
previewView?.addSubview(faceView)
faceView.backgroundColor = .clear
self.faceView = faceView
configureCaptureSession()
}
func start() { if cameraIsReadyToUse { session.startRunning() } }
func stop() { session.stopRunning() }
}
extension FaceDetectionService {
private func askUserForCameraPermission(_ completion: ((Bool) -> Void)?) {
AVCaptureDevice.requestAccess(for: AVMediaType.video) { (allowedAccess) -> Void in
DispatchQueue.main.async { completion?(allowedAccess) }
}
}
private func checkCameraAccess(completion: ((Bool) -> Void)?) {
askUserForCameraPermission { [weak self] allowed in
guard let self = self, let completion = completion else { return }
self.cameraIsReadyToUse = allowed
if allowed {
completion(true)
} else {
self.showDisabledCameraAlert(completion: completion)
}
}
}
private func configureCaptureSession() {
guard let previewView = previewView else { return }
// Define the capture device we want to use
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: cameraPosition) else {
let error = NSError(domain: "", code: 0, userInfo: [NSLocalizedDescriptionKey : "No front camera available"])
show(error: error)
return
}
// Connect the camera to the capture session input
do {
try camera.lockForConfiguration()
defer { camera.unlockForConfiguration() }
if camera.isFocusModeSupported(.continuousAutoFocus) {
camera.focusMode = .continuousAutoFocus
}
if camera.isExposureModeSupported(.continuousAutoExposure) {
camera.exposureMode = .continuousAutoExposure
}
let cameraInput = try AVCaptureDeviceInput(device: camera)
session.addInput(cameraInput)
} catch {
show(error: error as NSError)
return
}
// Create the video data output
let videoOutput = AVCaptureVideoDataOutput()
videoOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
// Add the video output to the capture session
session.addOutput(videoOutput)
let videoConnection = videoOutput.connection(with: .video)
videoConnection?.videoOrientation = .portrait
// Configure the preview layer
let previewLayer = AVCaptureVideoPreviewLayer(session: session)
previewLayer.videoGravity = .resizeAspectFill
previewLayer.frame = previewView.bounds
previewView.layer.insertSublayer(previewLayer, at: 0)
self.previewLayer = previewLayer
}
}
extension FaceDetectionService: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let detectFaceRequest = VNDetectFaceLandmarksRequest(completionHandler: detectedFace)
do {
try sequenceHandler.perform(
[detectFaceRequest],
on: imageBuffer,
orientation: .leftMirrored)
} catch { show(error: error as NSError) }
}
}
extension FaceDetectionService {
private func detectedFace(request: VNRequest, error: Error?) {
guard let previewLayer = previewLayer,
let results = request.results as? [VNFaceObservation],
let result = results.first
else { faceView?.clearAndSetNeedsDisplay(); return }
faceView?.read(result: result, previewLayer: previewLayer)
}
}
// Navigation
extension FaceDetectionService {
private func show(alert: UIAlertController) {
DispatchQueue.main.async {
UIApplication.topViewController?.present(alert, animated: true, completion: nil)
}
}
private func showDisabledCameraAlert(completion: ((Bool) -> Void)?) {
let alertVC = UIAlertController(title: "Enable Camera Access",
message: "Please provide access to your camera",
preferredStyle: .alert)
alertVC.addAction(UIAlertAction(title: "Go to Settings", style: .default, handler: { action in
guard let previewView = self.previewView,
let settingsUrl = URL(string: UIApplication.openSettingsURLString),
UIApplication.shared.canOpenURL(settingsUrl) else { return }
UIApplication.shared.open(settingsUrl) { [weak self] _ in
guard let self = self else { return }
self.prepare(previewView: previewView,
cameraPosition: self.cameraPosition,
completion: self.preparingCompletionHandler)
}
}))
alertVC.addAction(UIAlertAction(title: "Cancel", style: .cancel, handler: { _ in completion?(false) }))
show(alert: alertVC)
}
private func show(error: NSError) {
let alertVC = UIAlertController(title: "Error", message: error.localizedDescription, preferredStyle: .alert)
alertVC.addAction(UIAlertAction(title: "Ok", style: .cancel, handler: nil ))
show(alert: alertVC)
}
}
FaceView
import UIKit
import Vision
import AVFoundation
struct FaceElement {
let points: [CGPoint]
let needToClosePath: Bool
func draw(in context: CGContext) {
if points.isEmpty { return }
context.addLines(between: points)
if needToClosePath { context.closePath() }
context.strokePath()
}
}
class FaceView: UIView {
private var faceElements = [FaceElement]()
private var boundingBox = CGRect.zero
func clearAndSetNeedsDisplay() {
faceElements = []
boundingBox = .zero
DispatchQueue.main.async { [weak self] in self?.setNeedsDisplay() }
}
private func drawElement(context: CGContext, points: [CGPoint], needToClosePath: Bool) {
if !points.isEmpty {
context.addLines(between: points)
if needToClosePath { context.closePath() }
context.strokePath()
}
}
override func draw(_ rect: CGRect) {
super.draw(rect)
guard let context = UIGraphicsGetCurrentContext() else { return }
context.saveGState()
defer { context.restoreGState()}
context.addRect(boundingBox)
UIColor.red.setStroke()
context.strokePath()
UIColor.white.setStroke()
faceElements.forEach { $0.draw(in: context) }
}
func read(result: VNFaceObservation, previewLayer: AVCaptureVideoPreviewLayer) {
defer { DispatchQueue.main.async { [weak self] in self?.setNeedsDisplay() } }
let rect = result.boundingBox
let origin = previewLayer.layerPointConverted(fromCaptureDevicePoint: rect.origin)
let size = previewLayer.layerPointConverted(fromCaptureDevicePoint: rect.size.cgPoint).cgSize
boundingBox = CGRect(origin: origin, size: size)
func addFaceElement(from landmark: VNFaceLandmarkRegion2D?, needToClosePath: Bool) {
guard let normalizedPoints = landmark?.normalizedPoints else { return }
let points = normalizedPoints.compactMap { point -> CGPoint in
let absolute = point.absolutePoint(in: result.boundingBox)
let converted = previewLayer.layerPointConverted(fromCaptureDevicePoint: absolute)
return converted
}
faceElements.append(FaceElement(points: points, needToClosePath: needToClosePath))
}
guard let landmarks = result.landmarks else { return }
faceElements = []
addFaceElement(from: landmarks.leftEye, needToClosePath: true)
addFaceElement(from: landmarks.rightEye, needToClosePath: true)
addFaceElement(from: landmarks.leftEyebrow, needToClosePath: false)
addFaceElement(from: landmarks.rightEyebrow, needToClosePath: false)
addFaceElement(from: landmarks.nose, needToClosePath: false)
addFaceElement(from: landmarks.outerLips, needToClosePath: true)
addFaceElement(from: landmarks.innerLips, needToClosePath: true)
addFaceElement(from: landmarks.faceContour, needToClosePath: false)
}
}
Helpers
import CoreGraphics
func + (left: CGPoint, right: CGPoint) -> CGPoint {
return CGPoint(x: left.x + right.x, y: left.y + right.y)
}
extension CGSize {
var cgPoint: CGPoint { return CGPoint(x: width, y: height) }
}
extension CGPoint {
var cgSize: CGSize { return CGSize(width: x, height: y) }
func absolutePoint(in rect: CGRect) -> CGPoint {
return CGPoint(x: x * rect.size.width, y: y * rect.size.height) + rect.origin
}
}
import UIKit
extension UIApplication {
private class func topViewController(controller: UIViewController? = UIApplication.shared.keyWindow?.rootViewController) -> UIViewController? {
if let navigationController = controller as? UINavigationController {
return topViewController(controller: navigationController.visibleViewController)
}
if let tabController = controller as? UITabBarController {
if let selected = tabController.selectedViewController {
return topViewController(controller: selected)
}
}
if let presented = controller?.presentedViewController {
return topViewController(controller: presented)
}
return controller
}
class var topViewController: UIViewController? { return topViewController() }
}
Usage
private lazy var faceDetectionService = FaceDetectionService()
//....
faceDetectionService.prepare(previewView: previewView, cameraPosition: .front) { [weak self] _ in
self?.faceDetectionService.start()
}
Full sample
import UIKit
class ViewController: UIViewController {
private lazy var faceDetectionService = FaceDetectionService()
private weak var previewView: UIView!
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view.
let previewView = UIView(frame: .zero)
view.addSubview(previewView)
previewView.translatesAutoresizingMaskIntoConstraints = false
previewView.topAnchor.constraint(equalTo: view.topAnchor).isActive = true
previewView.leftAnchor.constraint(equalTo: view.leftAnchor).isActive = true
previewView.rightAnchor.constraint(equalTo: view.rightAnchor).isActive = true
previewView.bottomAnchor.constraint(equalTo: view.bottomAnchor).isActive = true
previewView.layoutIfNeeded()
self.previewView = previewView
faceDetectionService.prepare(previewView: previewView, cameraPosition: .front) { [weak self] _ in
self?.faceDetectionService.start()
}
}
// Ensure that the interface stays locked in Portrait.
override var supportedInterfaceOrientations: UIInterfaceOrientationMask { return .portrait }
// Ensure that the interface stays locked in Portrait.
override var preferredInterfaceOrientationForPresentation: UIInterfaceOrientation { return .portrait }
}
More info
- How to make UIImage using AVCaptureVideoPreviewLayer
- https://developer.apple.com/documentation/vision/tracking_the_user_s_face_in_real_time
