SceneKit - Get the rendered scene from a SCNView a

Question

My SCNView is using Metal as the rendering API and I would like to know if there's a way to grab the rendered scene as a MTLTexture without having to use a separate SCNRenderer? Performance drops when I'm trying to both display the scene via the SCNView and re-rendering the scene offscreen to a MTLTexture via a SCNRenderer (I'm trying to grab the output every frame).

SCNView gives me access to the MTLDevice, MTLRenderCommandEncoder, and MTLCommandQueue that it uses, but not to the underlying MTLRenderPassDescriptor that I would need in order to get the MTLTexture (via renderPassDescriptor.colorAttachments[0].texture)

Some alternatives I tried was trying to use SCNView.snapshot() to get a UIImage and converting it but performance was even worse.

Answer 1

Updated for Swift 4:

Swift 4 doesn't support dispatch_once(), and @objc added to replacement functions. Here's the updated swizzle setup. This is tested working nicely for me.

extension CAMetalLayer {

    // Interface so user can grab this drawable at any time
    private struct nextDrawableExtPropertyData {
        static var _currentSceneDrawable : CAMetalDrawable? = nil
    }
    var currentSceneDrawable : CAMetalDrawable? {
        get {
            return nextDrawableExtPropertyData._currentSceneDrawable
        }
    }

    // The rest of this is just swizzling
    private static let doJustOnce : Any? = {
        print ("***** Doing the doJustOnce *****")
        CAMetalLayer.setupSwizzling()

        return nil
    }()

    public static func enableNextDrawableSwizzle() {
        _ = CAMetalLayer.doJustOnce
    }

    public static func setupSwizzling() {
        print ("***** Doing the setupSwizzling *****")

        let copiedOriginalSelector = #selector(CAMetalLayer.originalNextDrawable)
        let originalSelector = #selector(CAMetalLayer.nextDrawable)
        let swizzledSelector = #selector(CAMetalLayer.newNextDrawable)

        let copiedOriginalMethod = class_getInstanceMethod(self, copiedOriginalSelector)
        let originalMethod = class_getInstanceMethod(self, originalSelector)
        let swizzledMethod = class_getInstanceMethod(self, swizzledSelector)

        let oldImp = method_getImplementation(originalMethod!)
        method_setImplementation(copiedOriginalMethod!, oldImp)

        let newImp = method_getImplementation(swizzledMethod!)
        method_setImplementation(originalMethod!, newImp)

    }


    @objc func newNextDrawable() -> CAMetalDrawable? {
        // After swizzling, originalNextDrawable() actually calls the real nextDrawable()
        let drawable = originalNextDrawable()

        // Save the drawable
        nextDrawableExtPropertyData._currentSceneDrawable = drawable

        return drawable
    }

    @objc func originalNextDrawable() -> CAMetalDrawable? {
        // This is just a placeholder. Implementation will be replaced with nextDrawable.
        // ***** This will never be called *****
        return nil
    }
}

In your AppDelegate:

func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplicationLaunchOptionsKey: Any]?) -> Bool {
    // Override point for customization after application launch.

    // Swizzle
    CAMetalLayer.enableNextDrawableSwizzle()

    return true
}

Updated to add a currentSceneDrawable property to CAMetalLayer, so you can just use layer.currentSceneDrawable to access it, rather than having the extension store it externally.

Answer 2

** Warning: This may not be a proper method for App Store. But it's working.

Step 1: Swap the method of nextDrawable of CAMetalLayer with a new one using swizzling. Save the CAMetalDrawable for each render loop.

extension CAMetalLayer {
  public static func setupSwizzling() {
    struct Static {
      static var token: dispatch_once_t = 0
    }

    dispatch_once(&Static.token) {
      let copiedOriginalSelector = #selector(CAMetalLayer.orginalNextDrawable)
      let originalSelector = #selector(CAMetalLayer.nextDrawable)
      let swizzledSelector = #selector(CAMetalLayer.newNextDrawable)

      let copiedOriginalMethod = class_getInstanceMethod(self, copiedOriginalSelector)
      let originalMethod = class_getInstanceMethod(self, originalSelector)
      let swizzledMethod = class_getInstanceMethod(self, swizzledSelector)

      let oldImp = method_getImplementation(originalMethod)
      method_setImplementation(copiedOriginalMethod, oldImp)
      method_exchangeImplementations(originalMethod, swizzledMethod)
    }
  }


  func newNextDrawable() -> CAMetalDrawable? {
    let drawable = orginalNextDrawable()
    // Save the drawable to any where you want
    AppManager.sharedInstance.currentSceneDrawable = drawable
    return drawable
  }

  func orginalNextDrawable() -> CAMetalDrawable? {
    // This is just a placeholder. Implementation will be replaced with nextDrawable.
    return nil
  }
}

Step 2: Setup the swizzling in AppDelegate: didFinishLaunchingWithOptions

func application(application: UIApplication, didFinishLaunchingWithOptions launchOptions: [NSObject: AnyObject]?) -> Bool {
  CAMetalLayer.setupSwizzling()
  return true
}

Step 3: Disable framebufferOnly for your's SCNView's CAMetalLayer (In order to call getBytes for MTLTexture)

if let metalLayer = scnView.layer as? CAMetalLayer {
  metalLayer.framebufferOnly = false
}

Step 4: In your SCNView's delegate (SCNSceneRendererDelegate), play with the texture

func renderer(renderer: SCNSceneRenderer, didRenderScene scene: SCNScene, atTime time: NSTimeInterval) {
    if let texture = AppManager.sharedInstance.currentSceneDrawable?.texture where !texture.framebufferOnly {
      AppManager.sharedInstance.currentSceneDrawable = nil
      // Play with the texture
    }
}

Step 5 (Optional): You may need to confirm the drawable at CAMetalLayer you are getting is your target. (If more then one CAMetalLayer at the same time)