基于 TensorFlow 示例 Swift 录制和播放视频
Posted
技术标签:
【中文标题】基于 TensorFlow 示例 Swift 录制和播放视频【英文标题】：Record and play Video based on TensorFlow example Swift 【发布时间】：2021-07-01 20:09:23 【问题描述】：#DEFINE UPDATE
我意识到我忘记请求录制许可了。现在已经解决了。但是，当我按下“录制按钮”时，我收到错误 Cannot create file。所以当我开始录制时，路径可能有问题？
#UNDEF 更新
我正在开发一个应用程序，我希望在该应用程序中拥有自己的神经网络，并具有开始录制视频的功能。此后我想播放视频并使用来自神经网络的信息。
我在 android 中有一个工作功能，现在我正在尝试为 iPhone 制作类似的东西。作为开始，我使用了来自TensorFlowLite 的ImageClassifierExample。第一个任务是添加一个按钮Record 开始录制视频，然后添加一个按钮Play 播放视频。
我已经实现了这两个功能，但是当我尝试播放视频时，它只是在加载。可能是录制不工作，或者视频播放器不工作（或两者兼而有之）。我已经检查过了，所以路径是一样的。
我对 ios 开发不是很熟悉，所以可以提供一些帮助。
这是我开始的base。
这是我略微采纳的ViewController：
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import AVFoundation
import AVKit
import UIKit
class ViewController: UIViewController
// MARK: Storyboards Connections
@IBOutlet weak var previewView: PreviewView!
@IBOutlet weak var cameraUnavailableLabel: UILabel!
@IBOutlet weak var resumeButton: UIButton!
@IBOutlet weak var bottomSheetView: CurvedView!
@IBOutlet weak var bottomSheetViewBottomSpace: NSLayoutConstraint!
@IBOutlet weak var bottomSheetStateImageView: UIImageView!
// MARK: Constants
private let animationDuration = 0.5
private let collapseTransitionThreshold: CGFloat = -40.0
private let expandThransitionThreshold: CGFloat = 40.0
private let delayBetweenInferencesMs: Double = 1000
// MARK: Instance Variables
// Holds the results at any time
private var result: Result?
private var initialBottomSpace: CGFloat = 0.0
private var previousInferenceTimeMs: TimeInterval = Date.distantPast.timeIntervalSince1970 * 1000
// MARK: Controllers that manage functionality
// Handles all the camera related functionality
private lazy var cameraCapture = CameraFeedManager(previewView: previewView)
private var isRecording = false // <<<----- Mine
private let captureSession: AVCaptureSession = AVCaptureSession()
// Handles all data preprocessing and makes calls to run inference through the `Interpreter`.
private var modelDataHandler: ModelDataHandler? =
ModelDataHandler(modelFileInfo: MobileNet.modelInfo, labelsFileInfo: MobileNet.labelsInfo)
@IBAction func startRecording(_ sender: Any) . // <<<----- Mine
print("Recording pressed")
if (!isRecording)
cameraCapture.startRecording()
else
cameraCapture.stopRecording()
isRecording = !isRecording
// Handles the presenting of results on the screen
private var inferenceViewController: InferenceViewController?
// MARK: View Handling Methods
override func viewDidLoad()
super.viewDidLoad()
guard modelDataHandler != nil else
fatalError("Model set up failed")
#if targetEnvironment(simulator)
previewView.shouldUseClipboardImage = true
NotificationCenter.default.addObserver(self,
selector: #selector(classifyPasteboardImage),
name: UIApplication.didBecomeActiveNotification,
object: nil)
#endif
cameraCapture.delegate = self
addPanGesture()
override func viewWillAppear(_ animated: Bool)
super.viewWillAppear(animated)
changeBottomViewState()
#if !targetEnvironment(simulator)
cameraCapture.checkCameraConfigurationAndStartSession()
#endif
#if !targetEnvironment(simulator)
override func viewWillDisappear(_ animated: Bool)
super.viewWillDisappear(animated)
cameraCapture.stopSession()
#endif
override var preferredStatusBarStyle: UIStatusBarStyle
return .lightContent
func presentUnableToResumeSessionAlert()
let alert = UIAlertController(
title: "Unable to Resume Session",
message: "There was an error while attempting to resume session.",
preferredStyle: .alert
)
alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil))
self.present(alert, animated: true)
// MARK: Storyboard Segue Handlers
override func prepare(for segue: UIStoryboardSegue, sender: Any?)
super.prepare(for: segue, sender: sender)
if segue.identifier == "EMBED"
guard let tempModelDataHandler = modelDataHandler else
return
inferenceViewController = segue.destination as? InferenceViewController
inferenceViewController?.wantedInputHeight = tempModelDataHandler.inputHeight
inferenceViewController?.wantedInputWidth = tempModelDataHandler.inputWidth
inferenceViewController?.maxResults = tempModelDataHandler.resultCount
inferenceViewController?.threadCountLimit = tempModelDataHandler.threadCountLimit
inferenceViewController?.delegate = self
@objc func classifyPasteboardImage()
guard let image = UIPasteboard.general.images?.first else
return
guard let buffer = CVImageBuffer.buffer(from: image) else
return
previewView.image = image
DispatchQueue.global().async
self.didOutput(pixelBuffer: buffer)
deinit
NotificationCenter.default.removeObserver(self)
// MARK: InferenceViewControllerDelegate Methods
extension ViewController: InferenceViewControllerDelegate
func didChangeThreadCount(to count: Int)
if modelDataHandler?.threadCount == count return
modelDataHandler = ModelDataHandler(
modelFileInfo: MobileNet.modelInfo,
labelsFileInfo: MobileNet.labelsInfo,
threadCount: count
)
// MARK: CameraFeedManagerDelegate Methods
extension ViewController: CameraFeedManagerDelegate
func didOutput(pixelBuffer: CVPixelBuffer)
let currentTimeMs = Date().timeIntervalSince1970 * 1000
guard (currentTimeMs - previousInferenceTimeMs) >= delayBetweenInferencesMs else return
previousInferenceTimeMs = currentTimeMs
// Pass the pixel buffer to TensorFlow Lite to perform inference.
result = modelDataHandler?.runModel(onFrame: pixelBuffer)
// Display results by handing off to the InferenceViewController.
DispatchQueue.main.async
let resolution = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
self.inferenceViewController?.inferenceResult = self.result
self.inferenceViewController?.resolution = resolution
self.inferenceViewController?.tableView.reloadData()
// MARK: Session Handling Alerts
func sessionWasInterrupted(canResumeManually resumeManually: Bool)
// Updates the UI when session is interupted.
if resumeManually
self.resumeButton.isHidden = false
else
self.cameraUnavailableLabel.isHidden = false
func sessionInterruptionEnded()
// Updates UI once session interruption has ended.
if !self.cameraUnavailableLabel.isHidden
self.cameraUnavailableLabel.isHidden = true
if !self.resumeButton.isHidden
self.resumeButton.isHidden = false
func sessionRunTimeErrorOccured()
// Handles session run time error by updating the UI and providing a button if session can be manually resumed.
self.resumeButton.isHidden = false
previewView.shouldUseClipboardImage = true
func presentCameraPermissionsDeniedAlert()
let alertController = UIAlertController(title: "Camera Permissions Denied", message: "Camera permissions have been denied for this app. You can change this by going to Settings", preferredStyle: .alert)
let cancelAction = UIAlertAction(title: "Cancel", style: .cancel, handler: nil)
let settingsAction = UIAlertAction(title: "Settings", style: .default) (action) in
UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil)
alertController.addAction(cancelAction)
alertController.addAction(settingsAction)
present(alertController, animated: true, completion: nil)
previewView.shouldUseClipboardImage = true
func presentVideoConfigurationErrorAlert()
let alert = UIAlertController(title: "Camera Configuration Failed", message: "There was an error while configuring camera.", preferredStyle: .alert)
alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil))
self.present(alert, animated: true)
previewView.shouldUseClipboardImage = true
// MARK: Bottom Sheet Interaction Methods
extension ViewController
// MARK: Bottom Sheet Interaction Methods
/**
This method adds a pan gesture to make the bottom sheet interactive.
*/
private func addPanGesture()
let panGesture = UIPanGestureRecognizer(target: self, action: #selector(ViewController.didPan(panGesture:)))
bottomSheetView.addGestureRecognizer(panGesture)
/** Change whether bottom sheet should be in expanded or collapsed state.
*/
private func changeBottomViewState()
guard let inferenceVC = inferenceViewController else
return
if bottomSheetViewBottomSpace.constant == inferenceVC.collapsedHeight - bottomSheetView.bounds.size.height
bottomSheetViewBottomSpace.constant = 0.0
else
bottomSheetViewBottomSpace.constant = inferenceVC.collapsedHeight - bottomSheetView.bounds.size.height
setImageBasedOnBottomViewState()
/**
Set image of the bottom sheet icon based on whether it is expanded or collapsed
*/
private func setImageBasedOnBottomViewState()
if bottomSheetViewBottomSpace.constant == 0.0
bottomSheetStateImageView.image = UIImage(named: "down_icon")
else
bottomSheetStateImageView.image = UIImage(named: "up_icon")
/**
This method responds to the user panning on the bottom sheet.
*/
@objc func didPan(panGesture: UIPanGestureRecognizer)
// Opens or closes the bottom sheet based on the user's interaction with the bottom sheet.
let translation = panGesture.translation(in: view)
switch panGesture.state
case .began:
initialBottomSpace = bottomSheetViewBottomSpace.constant
translateBottomSheet(withVerticalTranslation: translation.y)
case .changed:
translateBottomSheet(withVerticalTranslation: translation.y)
case .cancelled:
setBottomSheetLayout(withBottomSpace: initialBottomSpace)
case .ended:
translateBottomSheetAtEndOfPan(withVerticalTranslation: translation.y)
setImageBasedOnBottomViewState()
initialBottomSpace = 0.0
default:
break
/**
This method sets bottom sheet translation while pan gesture state is continuously changing.
*/
private func translateBottomSheet(withVerticalTranslation verticalTranslation: CGFloat)
let bottomSpace = initialBottomSpace - verticalTranslation
guard bottomSpace <= 0.0 && bottomSpace >= inferenceViewController!.collapsedHeight - bottomSheetView.bounds.size.height else
return
setBottomSheetLayout(withBottomSpace: bottomSpace)
/**
This method changes bottom sheet state to either fully expanded or closed at the end of pan.
*/
private func translateBottomSheetAtEndOfPan(withVerticalTranslation verticalTranslation: CGFloat)
// Changes bottom sheet state to either fully open or closed at the end of pan.
let bottomSpace = bottomSpaceAtEndOfPan(withVerticalTranslation: verticalTranslation)
setBottomSheetLayout(withBottomSpace: bottomSpace)
/**
Return the final state of the bottom sheet view (whether fully collapsed or expanded) that is to be retained.
*/
private func bottomSpaceAtEndOfPan(withVerticalTranslation verticalTranslation: CGFloat) -> CGFloat
// Calculates whether to fully expand or collapse bottom sheet when pan gesture ends.
var bottomSpace = initialBottomSpace - verticalTranslation
var height: CGFloat = 0.0
if initialBottomSpace == 0.0
height = bottomSheetView.bounds.size.height
else
height = inferenceViewController!.collapsedHeight
let currentHeight = bottomSheetView.bounds.size.height + bottomSpace
if currentHeight - height <= collapseTransitionThreshold
bottomSpace = inferenceViewController!.collapsedHeight - bottomSheetView.bounds.size.height
else if currentHeight - height >= expandThransitionThreshold
bottomSpace = 0.0
else
bottomSpace = initialBottomSpace
return bottomSpace
/**
This method layouts the change of the bottom space of bottom sheet with respect to the view managed by this controller.
*/
func setBottomSheetLayout(withBottomSpace bottomSpace: CGFloat)
view.setNeedsLayout()
bottomSheetViewBottomSpace.constant = bottomSpace
view.setNeedsLayout()
CameraFeedManager:
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import UIKit
import AVFoundation
// MARK: CameraFeedManagerDelegate Declaration
protocol CameraFeedManagerDelegate: AnyObject
/**
This method delivers the pixel buffer of the current frame seen by the device's camera.
*/
func didOutput(pixelBuffer: CVPixelBuffer)
/**
This method initimates that the camera permissions have been denied.
*/
func presentCameraPermissionsDeniedAlert()
/**
This method initimates that there was an error in video configurtion.
*/
func presentVideoConfigurationErrorAlert()
/**
This method initimates that a session runtime error occured.
*/
func sessionRunTimeErrorOccured()
/**
This method initimates that the session was interrupted.
*/
func sessionWasInterrupted(canResumeManually resumeManually: Bool)
/**
This method initimates that the session interruption has ended.
*/
func sessionInterruptionEnded()
/**
This enum holds the state of the camera initialization.
*/
enum CameraConfiguration
case success
case failed
case permissionDenied
/**
This class manages all camera related functionality
*/
class CameraFeedManager: NSObject, AVCaptureFileOutputRecordingDelegate
func fileOutput(_ output: AVCaptureFileOutput, didFinishRecordingTo outputFileURL: URL, from connections: [AVCaptureConnection], error: Error?) // << --- Mine
print("Video recorded to: " + outputFileURL.absoluteString)
// MARK: Camera Related Instance Variables
private let session: AVCaptureSession = AVCaptureSession()
private let previewView: PreviewView
private let sessionQueue = DispatchQueue(label: "sessionQueue")
private var cameraConfiguration: CameraConfiguration = .failed
private lazy var videoDataOutput = AVCaptureVideoDataOutput()
private var movieDataOutput = AVCaptureMovieFileOutput() // << --- Mine
private var isSessionRunning = false
// MARK: CameraFeedManagerDelegate
weak var delegate: CameraFeedManagerDelegate?
// MARK: Initializer
init(previewView: PreviewView)
self.previewView = previewView
super.init()
// Initializes the session
session.sessionPreset = .high
self.previewView.session = session
self.previewView.previewLayer.connection?.videoOrientation = .portrait
self.previewView.previewLayer.videoGravity = .resizeAspectFill
self.attemptToConfigureSession()
// MARK: Session Start and End methods
/**
This method starts an AVCaptureSession based on whether the camera configuration was successful.
*/
func checkCameraConfigurationAndStartSession()
sessionQueue.async
switch self.cameraConfiguration
case .success:
self.addObservers()
self.startSession()
case .failed:
DispatchQueue.main.async
self.delegate?.presentVideoConfigurationErrorAlert()
case .permissionDenied:
DispatchQueue.main.async
self.delegate?.presentCameraPermissionsDeniedAlert()
/**
This method stops a running an AVCaptureSession.
*/
func stopSession()
self.removeObservers()
sessionQueue.async
if self.session.isRunning
self.session.stopRunning()
self.isSessionRunning = self.session.isRunning
/**
This method resumes an interrupted AVCaptureSession.
*/
func resumeInterruptedSession(withCompletion completion: @escaping (Bool) -> ())
sessionQueue.async
self.startSession()
DispatchQueue.main.async
completion(self.isSessionRunning)
/**
This method starts the AVCaptureSession
**/
private func startSession()
self.session.startRunning()
self.isSessionRunning = self.session.isRunning
// MARK: Session Configuration Methods.
/**
This method requests for camera permissions and handles the configuration of the session and stores the result of configuration.
*/
private func attemptToConfigureSession()
switch AVCaptureDevice.authorizationStatus(for: .video)
case .authorized:
self.cameraConfiguration = .success
case .notDetermined:
self.sessionQueue.suspend()
self.requestCameraAccess(completion: (granted) in
self.sessionQueue.resume()
)
case .denied:
self.cameraConfiguration = .permissionDenied
default:
break
self.sessionQueue.async
self.configureSession()
/**
This method requests for camera permissions.
*/
private func requestCameraAccess(completion: @escaping (Bool) -> ())
AVCaptureDevice.requestAccess(for: .video) (granted) in
if !granted
self.cameraConfiguration = .permissionDenied
else
self.cameraConfiguration = .success
completion(granted)
/**
This method handles all the steps to configure an AVCaptureSession.
*/
private func configureSession()
guard cameraConfiguration == .success else
return
session.beginConfiguration()
// Tries to add an AVCaptureDeviceInput.
guard addVideoDeviceInput() == true else
self.session.commitConfiguration()
self.cameraConfiguration = .failed
return
// Tries to add an AVCaptureVideoDataOutput.
guard addVideoDataOutput() else
self.session.commitConfiguration()
self.cameraConfiguration = .failed
return
session.commitConfiguration()
self.cameraConfiguration = .success
func startRecording() . // << --- Mine
self.session.addOutput(movieDataOutput)
guard let homeDirectory = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask).first else return
let url = URL(fileURLWithPath: homeDirectory.absoluteString + "/mymovie.mov")
movieDataOutput.startRecording(to: url , recordingDelegate: self)
func stopRecording() // <<< -- Mine
self.movieDataOutput.stopRecording()
self.session.removeOutput(movieDataOutput)
/**
This method tries to an AVCaptureDeviceInput to the current AVCaptureSession.
*/
private func addVideoDeviceInput() -> Bool
/**Tries to get the default back camera.
*/
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back) else
return false
do
let videoDeviceInput = try AVCaptureDeviceInput(device: camera)
if session.canAddInput(videoDeviceInput)
session.addInput(videoDeviceInput)
return true
else
return false
catch
fatalError("Cannot create video device input")
/**
This method tries to an AVCaptureVideoDataOutput to the current AVCaptureSession.
*/
private func addVideoDataOutput() -> Bool
let sampleBufferQueue = DispatchQueue(label: "sampleBufferQueue")
videoDataOutput.setSampleBufferDelegate(self, queue: sampleBufferQueue)
videoDataOutput.alwaysDiscardsLateVideoFrames = true
videoDataOutput.videoSettings = [ String(kCVPixelBufferPixelFormatTypeKey) : kCMPixelFormat_32BGRA]
if session.canAddOutput(videoDataOutput)
session.addOutput(videoDataOutput)
videoDataOutput.connection(with: .video)?.videoOrientation = .portrait
return true
return false
// MARK: Notification Observer Handling
private func addObservers()
NotificationCenter.default.addObserver(self, selector: #selector(CameraFeedManager.sessionRuntimeErrorOccured(notification:)), name: NSNotification.Name.AVCaptureSessionRuntimeError, object: session)
NotificationCenter.default.addObserver(self, selector: #selector(CameraFeedManager.sessionWasInterrupted(notification:)), name: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session)
NotificationCenter.default.addObserver(self, selector: #selector(CameraFeedManager.sessionInterruptionEnded), name: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session)
private func removeObservers()
NotificationCenter.default.removeObserver(self, name: NSNotification.Name.AVCaptureSessionRuntimeError, object: session)
NotificationCenter.default.removeObserver(self, name: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session)
NotificationCenter.default.removeObserver(self, name: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session)
// MARK: Notification Observers
@objc func sessionWasInterrupted(notification: Notification)
if let userInfoValue = notification.userInfo?[AVCaptureSessionInterruptionReasonKey] as AnyObject?,
let reasonIntegerValue = userInfoValue.integerValue,
let reason = AVCaptureSession.InterruptionReason(rawValue: reasonIntegerValue)
print("Capture session was interrupted with reason \(reason)")
var canResumeManually = false
if reason == .videoDeviceInUseByAnotherClient
canResumeManually = true
else if reason == .videoDeviceNotAvailableWithMultipleForegroundApps
canResumeManually = false
self.delegate?.sessionWasInterrupted(canResumeManually: canResumeManually)
@objc func sessionInterruptionEnded(notification: Notification)
self.delegate?.sessionInterruptionEnded()
@objc func sessionRuntimeErrorOccured(notification: Notification)
guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? AVError else
return
print("Capture session runtime error: \(error)")
if error.code == .mediaServicesWereReset
sessionQueue.async
if self.isSessionRunning
self.startSession()
else
DispatchQueue.main.async
self.delegate?.sessionRunTimeErrorOccured()
else
self.delegate?.sessionRunTimeErrorOccured()
/**
AVCaptureVideoDataOutputSampleBufferDelegate
*/
extension CameraFeedManager: AVCaptureVideoDataOutputSampleBufferDelegate
/** This method delegates the CVPixelBuffer of the frame seen by the camera currently.
*/
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection)
// Converts the CMSampleBuffer to a CVPixelBuffer.
let pixelBuffer: CVPixelBuffer? = CMSampleBufferGetImageBuffer(sampleBuffer)
guard let imagePixelBuffer = pixelBuffer else
return
// Delegates the pixel buffer to the ViewController.
delegate?.didOutput(pixelBuffer: imagePixelBuffer)
玩家控制器：
import Foundation
import UIKit
import AVFoundation
import AVKit
class PlayerController : UIViewController
override func viewDidLoad()
super.viewDidLoad()
override func viewDidAppear(_ animated: Bool)
super.viewDidAppear(animated)
guard let homeDirectory = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask).first else return
let url = URL(fileURLWithPath: homeDirectory.absoluteString + "/mymovie.mov")
print(url.absoluteString)
let player = AVPlayer(url: url) // video path coming from above function
let playerViewController = AVPlayerViewController()
playerViewController.player = player
self.present(playerViewController, animated: true)
playerViewController.player!.play()
【问题讨论】：
【参考方案1】：解决方案是使用以下方法创建路径：
private func documentDirectory() -> String
let documentDirectory = NSSearchPathForDirectoriesInDomains(.documentDirectory,
.userDomainMask,
true)
return documentDirectory[0]
private func append(toPath path: String,
withPathComponent pathComponent: String) -> String?
if var pathURL = URL(string: path)
pathURL.appendPathComponent(pathComponent)
return pathURL.absoluteString
return nil
和
guard let path = append(toPath: documentDirectory(), withPathComponent: "movie_test.mov") else return
【讨论】：
以上是关于基于 TensorFlow 示例 Swift 录制和播放视频的主要内容，如果未能解决你的问题，请参考以下文章
在 MacOS 上使用 VLCKit 在本地录制流 - 寻找示例
RK3588平台开发系列讲解（AUDIO篇）基于alsa api的音频播放/录制流程