无法使用 firebase ML 套件检测眨眼事件

Posted 2023-03-16

【中文标题】无法使用 firebase ML 套件检测眨眼事件

【英文标题】：Not able to detect eye blink event with firebase ML kit

【发布时间】：2020-02-08 14:11:32

【问题描述】：

我正在使用 Firebase ML-Kit 进行人脸检测，现在我想实现眨眼，但我无法检测到眨眼。我总是得到 leftEyeOpenProbability 和 rightEyeOpenProbability 1.0。我正在使用与https://github.com/firebase/quickstart-ios/tree/master/mlvision相同的以下代码

请帮助我如何实现眨眼功能。

func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) 

    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else 
        print("Failed to get image buffer from sample buffer.")
        return
    
    let visionImage = VisionImage(buffer: sampleBuffer)
    let metadata = VisionImageMetadata()
    let visionOrientation = visionImageOrientation(from:imageOrientation(fromDevicePosition: .front))
    metadata.orientation = visionOrientation
    visionImage.metadata = metadata
    let imageWidth = CGFloat(CVPixelBufferGetWidth(imageBuffer))
    let imageHeight = CGFloat(CVPixelBufferGetHeight(imageBuffer))
    detectFacesOnDevice(in: visionImage, width: imageWidth, height: imageHeight)


public func visionImageOrientation(
    from imageOrientation: UIImage.Orientation
    ) -> VisionDetectorImageOrientation 
    switch imageOrientation 
    case .up:
        return .topLeft
    case .down:
        return .bottomRight
    case .left:
        return .leftBottom
    case .right:
        return .rightTop
    case .upMirrored:
        return .topRight
    case .downMirrored:
        return .bottomLeft
    case .leftMirrored:
        return .leftTop
    case .rightMirrored:
        return .rightBottom
    


public  func imageOrientation(
    fromDevicePosition devicePosition: AVCaptureDevice.Position = .back
    ) -> UIImage.Orientation 
    var deviceOrientation = UIDevice.current.orientation
    if deviceOrientation == .faceDown || deviceOrientation == .faceUp ||
        deviceOrientation == .unknown 
        deviceOrientation = currentUIOrientation()
    
    switch deviceOrientation 
    case .portrait:
        return devicePosition == .front ? .leftMirrored : .right
    case .landscapeLeft:
        return devicePosition == .front ? .downMirrored : .up
    case .portraitUpsideDown:
        return devicePosition == .front ? .rightMirrored : .left
    case .landscapeRight:
        return devicePosition == .front ? .upMirrored : .down
    case .faceDown, .faceUp, .unknown:
        return .up
    


private func currentUIOrientation() -> UIDeviceOrientation 
    let deviceOrientation =  () -> UIDeviceOrientation in
        switch UIApplication.shared.statusBarOrientation 
        case .landscapeLeft:
            return .landscapeRight
        case .landscapeRight:
            return .landscapeLeft
        case .portraitUpsideDown:
            return .portraitUpsideDown
        case .portrait, .unknown:
            return .portrait
        
    
    guard Thread.isMainThread else 
        var currentOrientation: UIDeviceOrientation = .portrait
        DispatchQueue.main.sync 
            currentOrientation = deviceOrientation()
        
        return currentOrientation
    
    return deviceOrientation()


    private func detectFacesOnDevice(in image: VisionImage, width: CGFloat, height: CGFloat) 
    let options = VisionFaceDetectorOptions()
    options.performanceMode = .accurate
    options.landmarkMode = .all
    options.isTrackingEnabled = true
    options.contourMode = .all

    let faceDetector = vision.faceDetector(options: options)


    faceDetector.process(image, completion:  features, error in
        if let error = error 
            print(error.localizedDescription)
            return
        

        guard error == nil, let features = features, !features.isEmpty else 
            //self.removeDetectionAnnotations()
            print("On-Device face detector returned no results.")
            return
        
        //self.removeDetectionAnnotations()
        for face in features 

            let angleY = face.headEulerAngleY
            let angleZ = face.headEulerAngleZ
            let smilingProbability = face.smilingProbability
            let leftEyeOpenProbability = face.leftEyeOpenProbability
            let rightEyeOpenProbability = face.rightEyeOpenProbability

            self.infoLabel.text = "right eye open probablity-- \(rightEyeOpenProbability)"


        
    )

   

【参考方案1】：

必须设置options.classificationMode = .all 才能检测leftEyeOpenProbability 和rightEyeOpenProbability？

在以下文档中搜索classificationMode：

https://firebase.google.com/docs/ml-kit/ios/detect-faces