// // CaptureManager.swift // WeScan // // Created by Boris Emorine on 2/8/18. // Copyright © 2018 WeTransfer. All rights reserved. // import Foundation import AVFoundation @available(iOS 10, *) /// A set of functions that inform the delegate object of the state of the detection. protocol RectangleDetectionDelegateProtocol: NSObjectProtocol { /// Called when the capture of a picture has started. /// /// - Parameters: /// - captureSessionManager: The `CaptureSessionManager` instance that started capturing a picture. func didStartCapturingPicture(for captureSessionManager: CaptureSessionManager) /// Called when a quadrilateral has been detected. /// - Parameters: /// - captureSessionManager: The `CaptureSessionManager` instance that has detected a quadrilateral. /// - quad: The detected quadrilateral in the coordinates of the image. /// - imageSize: The size of the image the quadrilateral has been detected on. func captureSessionManager(_ captureSessionManager: CaptureSessionManager, didDetectQuad quad: Quadrilateral?, _ imageSize: CGSize) /// Called when a picture with or without a quadrilateral has been captured. /// /// - Parameters: /// - captureSessionManager: The `CaptureSessionManager` instance that has captured a picture. /// - picture: The picture that has been captured. /// - quad: The quadrilateral that was detected in the picture's coordinates if any. func captureSessionManager(_ captureSessionManager: CaptureSessionManager, didCapturePicture picture: UIImage, withQuad quad: Quadrilateral?) /// Called when an error occured with the capture session manager. /// - Parameters: /// - captureSessionManager: The `CaptureSessionManager` that encountered an error. /// - error: The encountered error. func captureSessionManager(_ captureSessionManager: CaptureSessionManager, didFailWithError error: Error) } @available(iOS 10, *) /// The CaptureSessionManager is responsible for setting up and managing the AVCaptureSession and the functions related to capturing. final class CaptureSessionManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { private let videoPreviewLayer: AVCaptureVideoPreviewLayer private let captureSession = AVCaptureSession() private let rectangleFunnel = RectangleFeaturesFunnel() weak var delegate: RectangleDetectionDelegateProtocol? private var displayedRectangleResult: RectangleDetectorResult? private var photoOutput = AVCapturePhotoOutput() /// Whether the CaptureSessionManager should be detecting quadrilaterals. private var isDetecting = true /// The number of times no rectangles have been found in a row. private var noRectangleCount = 0 /// The minimum number of time required by `noRectangleCount` to validate that no rectangles have been found. private let noRectangleThreshold = 3 // MARK: Life Cycle init?(videoPreviewLayer: AVCaptureVideoPreviewLayer) { self.videoPreviewLayer = videoPreviewLayer super.init() captureSession.beginConfiguration() captureSession.sessionPreset = AVCaptureSession.Preset.photo photoOutput.isHighResolutionCaptureEnabled = true let videoOutput = AVCaptureVideoDataOutput() videoOutput.alwaysDiscardsLateVideoFrames = true guard let inputDevice = AVCaptureDevice.default(for: AVMediaType.video), let deviceInput = try? AVCaptureDeviceInput(device: inputDevice), captureSession.canAddInput(deviceInput), captureSession.canAddOutput(photoOutput), captureSession.canAddOutput(videoOutput) else { let error = ImageScannerControllerError.inputDevice delegate?.captureSessionManager(self, didFailWithError: error) return } captureSession.addInput(deviceInput) captureSession.addOutput(photoOutput) captureSession.addOutput(videoOutput) videoPreviewLayer.session = captureSession videoPreviewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "video_ouput_queue")) captureSession.commitConfiguration() } // MARK: Capture Session Life Cycle /// Starts the camera and detecting quadrilaterals. internal func start() { let authorizationStatus = AVCaptureDevice.authorizationStatus(for: AVMediaType.video) switch authorizationStatus { case .authorized: self.captureSession.startRunning() isDetecting = true case .notDetermined: AVCaptureDevice.requestAccess(for: AVMediaType.video, completionHandler: { (_) in DispatchQueue.main.async { [weak self] in self?.start() } }) default: let error = ImageScannerControllerError.authorization delegate?.captureSessionManager(self, didFailWithError: error) } } internal func stop() { captureSession.stopRunning() } internal func capturePhoto() { let photoSettings = AVCapturePhotoSettings() photoSettings.isHighResolutionPhotoEnabled = true photoSettings.isAutoStillImageStabilizationEnabled = true if let photoOutputConnection = self.photoOutput.connection(with: .video) { photoOutputConnection.videoOrientation = AVCaptureVideoOrientation(deviceOrientation: UIDevice.current.orientation) ?? AVCaptureVideoOrientation.portrait } photoOutput.capturePhoto(with: photoSettings, delegate: self) } // MARK: - AVCaptureVideoDataOutputSampleBufferDelegate func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard isDetecting == true else { return } guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let videoOutputImage = CIImage(cvPixelBuffer: pixelBuffer) let imageSize = videoOutputImage.extent.size guard let rectangle = RectangleDetector.rectangle(forImage: videoOutputImage) else { DispatchQueue.main.async { [weak self] in guard let strongSelf = self else { return } strongSelf.noRectangleCount += 1 if strongSelf.noRectangleCount > strongSelf.noRectangleThreshold { strongSelf.displayedRectangleResult = nil strongSelf.delegate?.captureSessionManager(strongSelf, didDetectQuad: nil, imageSize) } } return } noRectangleCount = 0 rectangleFunnel.add(rectangle, currentlyDisplayedRectangle: displayedRectangleResult?.rectangle) { (rectangle) in displayRectangleResult(rectangleResult: RectangleDetectorResult(rectangle: rectangle, imageSize: imageSize)) } } @discardableResult private func displayRectangleResult(rectangleResult: RectangleDetectorResult) -> Quadrilateral { displayedRectangleResult = rectangleResult let quad = Quadrilateral(rectangleFeature: rectangleResult.rectangle).toCartesian(withHeight: rectangleResult.imageSize.height) DispatchQueue.main.async { [weak self] in guard let strongSelf = self else { return } strongSelf.delegate?.captureSessionManager(strongSelf, didDetectQuad: quad, rectangleResult.imageSize) } return quad } } @available(iOS 10, *) extension CaptureSessionManager: AVCapturePhotoCaptureDelegate { func photoOutput(_ captureOutput: AVCapturePhotoOutput, didFinishProcessingPhoto photoSampleBuffer: CMSampleBuffer?, previewPhoto previewPhotoSampleBuffer: CMSampleBuffer?, resolvedSettings: AVCaptureResolvedPhotoSettings, bracketSettings: AVCaptureBracketedStillImageSettings?, error: Error?) { if let error = error { delegate?.captureSessionManager(self, didFailWithError: error) return } isDetecting = false delegate?.didStartCapturingPicture(for: self) if let sampleBuffer = photoSampleBuffer, let imageData = AVCapturePhotoOutput.jpegPhotoDataRepresentation(forJPEGSampleBuffer: sampleBuffer, previewPhotoSampleBuffer: nil) { completeImageCapture(with: imageData) } else { let error = ImageScannerControllerError.capture delegate?.captureSessionManager(self, didFailWithError: error) return } } @available(iOS 11.0, *) func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) { if let error = error { delegate?.captureSessionManager(self, didFailWithError: error) return } isDetecting = false delegate?.didStartCapturingPicture(for: self) if let imageData = photo.fileDataRepresentation() { completeImageCapture(with: imageData) } else { let error = ImageScannerControllerError.capture delegate?.captureSessionManager(self, didFailWithError: error) return } } /// Completes the image capture by processing the image, and passing it to the delegate object. /// This function is necessary because the capture functions for iOS 10 and 11 are decoupled. private func completeImageCapture(with imageData: Data) { DispatchQueue.global(qos: .background).async { [weak self] in guard let image = UIImage(data: imageData) else { let error = ImageScannerControllerError.capture DispatchQueue.main.async { guard let strongSelf = self else { return } strongSelf.delegate?.captureSessionManager(strongSelf, didFailWithError: error) } return } var angle: CGFloat = 0.0 switch image.imageOrientation { case .right: angle = CGFloat.pi / 2 case .up: angle = CGFloat.pi default: break } var quad: Quadrilateral? if let displayedRectangleResult = self?.displayedRectangleResult { quad = self?.displayRectangleResult(rectangleResult: displayedRectangleResult) quad = quad?.scale(displayedRectangleResult.imageSize, image.size, withRotationAngle: angle) } DispatchQueue.main.async { guard let strongSelf = self else { return } strongSelf.delegate?.captureSessionManager(strongSelf, didCapturePicture: image, withQuad: quad) } } } } /// Data structure representing the result of the detection of a quadrilateral. fileprivate struct RectangleDetectorResult { /// The detected quadrilateral. let rectangle: CIRectangleFeature /// The size of the image the quadrilateral was detected on. let imageSize: CGSize }