如何将 YUV 帧（来自 OTVideoFrame）转换为 CVPixelBuffer

Posted 2023-03-27

【中文标题】如何将 YUV 帧（来自 OTVideoFrame）转换为 CVPixelBuffer

【英文标题】：How to convert YUV frames (from OTVideoFrame) to CVPixelBuffer

【发布时间】：2017-10-03 18:12:23

【问题描述】：

我需要将 YUV 帧转换为从 OTVideoFrame 类获得的 CVPixelBuffer

此类在视频帧中提供一个平面数组，其中包含 y,u,v 帧的三个元素，每个元素的索引为 0,1,2。 p>

@property（非原子，保留）NSPointerArray *planes

视频帧的format

@property（非原子，保留）OTVideoFormat *format

包含框架的宽度、高度、字节数等属性

我需要为收到的OTVideoFrame形式的图像添加过滤器，我已经尝试过这些答案：

How to convert from YUV to CIImage for ios Create CVPixelBuffer from YUV with IOSurface backed

这两个链接在 Objective-C 中有解决方案，但我想快速完成。第二个链接中的答案之一是 swift 但缺少有关答案所引用的 YUVFrame 结构的一些信息。

我收到的格式是 NV12

这是我到目前为止一直在尝试做的事情，但我不知道下一步该怎么做：-

 /**
 * Calcualte the size of each plane from OTVideoFrame.
 *
 * @param frame The frame to render.
 * @return tuple containing three elements for size of each plane
 */
fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
        -> (ySize: Int, uSize: Int, vSize: Int)
            guard let frameFormat = frame.format
                else 
                    return (0, 0 ,0)
            
            let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
            return (baseSize, baseSize / 4, baseSize / 4)
    

/**
 * Renders a frame to the video renderer.
 *
 * @param frame The frame to render.
 */
func renderVideoFrame(_ frame: OTVideoFrame) 


    let planeSize = calculatePlaneSize(forFrame: frame)
    let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
    let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
    let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)

    memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
    memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
    memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)

    let yStride = frame.format!.bytesPerRow.object(at: 0) as! Int
    // multiply chroma strides by 2 as bytesPerRow represents 2x2 subsample
    let uStride = frame.format!.bytesPerRow.object(at: 1) as! Int
    let vStride = frame.format!.bytesPerRow.object(at: 2) as! Int

    let width = frame.format!.imageWidth
    let height = frame.format!.imageHeight

    var pixelBuffer: CVPixelBuffer? = nil
    var err: CVReturn;


    err = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
    if (err != 0) 
        NSLog("Error at CVPixelBufferCreate %d", err)
        fatalError()
    

从这两个链接的指导下，我尝试创建像素缓冲区，但每次都卡住了，因为此后的 Objective-C 代码转换与我们在 Swift 3 中的转换不同。

【参考方案1】：

对于那些正在寻找快速解决方案的人，我使用了swift Accelerate 使用vImageConvert_AnyToAny(_:_:_:_:_:) 函数。

import Foundation
import Accelerate
import UIKit
import OpenTok
class Accelerater


    var infoYpCbCrToARGB = vImage_YpCbCrToARGB()
    init() 
        _ = configureYpCbCrToARGBInfo()
    

    func configureYpCbCrToARGBInfo() -> vImage_Error 
        print("Configuring")
        var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 0,
                                                 CbCr_bias: 128,
                                                 YpRangeMax: 255,
                                                 CbCrRangeMax: 255,
                                                 YpMax: 255,
                                                 YpMin: 1,
                                                 CbCrMax: 255,
                                                 CbCrMin: 0)

        let error = vImageConvert_YpCbCrToARGB_GenerateConversion(
            kvImage_YpCbCrToARGBMatrix_ITU_R_601_4!,
            &pixelRange,
            &infoYpCbCrToARGB,
            kvImage420Yp8_Cb8_Cr8,
            kvImageARGB8888,
            vImage_Flags(kvImagePrintDiagnosticsToConsole))



        print("Configration done \(error)")
        return error
    

    public func convertFrameVImageYUV(toUIImage frame: OTVideoFrame, flag: Bool) -> UIImage 

        var result: UIImage? = nil
        let width = frame.format?.imageWidth ?? 0
        let height = frame.format?.imageHeight ?? 0
        var pixelBuffer: CVPixelBuffer? = nil
        _ = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_32BGRA, nil, &pixelBuffer)

        _ = convertFrameVImageYUV(frame, to: pixelBuffer)
        var ciImage: CIImage? = nil
        if let pixelBuffer = pixelBuffer 
            ciImage = CIImage(cvPixelBuffer: pixelBuffer)
        

        let temporaryContext = CIContext(options: nil)
        var uiImage: CGImage? = nil
        if let ciImage = ciImage 
            uiImage = temporaryContext.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixelBuffer!), height: CVPixelBufferGetHeight(pixelBuffer!)))
        

        if let uiImage = uiImage 
            result = UIImage(cgImage: uiImage)
        
        CVPixelBufferUnlockBaseAddress(pixelBuffer!, [])
        return result!

    

    func convertFrameVImageYUV(_ frame: OTVideoFrame, to pixelBufferRef: CVPixelBuffer?) -> vImage_Error
        let start  = CFAbsoluteTimeGetCurrent()
        if pixelBufferRef == nil 
            print("No PixelBuffer refrance found")
            return vImage_Error(kvImageInvalidParameter)
        

        let width = frame.format?.imageWidth ?? 0
        let height = frame.format?.imageHeight ?? 0
        let subsampledWidth = frame.format!.imageWidth/2
        let subsampledHeight = frame.format!.imageHeight/2
        print("subsample height \(subsampledHeight) \(subsampledWidth)")
        let planeSize = calculatePlaneSize(forFrame: frame)

        print("ysize : \(planeSize.ySize) \(planeSize.uSize) \(planeSize.vSize)")
        let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
        let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
        let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)

        memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
        memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
        memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)

        let yStride = frame.format!.bytesPerRow.object(at: 0) as! Int
        // multiply chroma strides by 2 as bytesPerRow represents 2x2 subsample
        let uStride = frame.format!.bytesPerRow.object(at: 1) as! Int
        let vStride = frame.format!.bytesPerRow.object(at: 2) as! Int

        var yPlaneBuffer = vImage_Buffer(data: yPlane, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: yStride)

        var uPlaneBuffer = vImage_Buffer(data: uPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: uStride)




        var vPlaneBuffer = vImage_Buffer(data: vPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: vStride)
        CVPixelBufferLockBaseAddress(pixelBufferRef!, .readOnly)
        let pixelBufferData = CVPixelBufferGetBaseAddress(pixelBufferRef!)
        let rowBytes = CVPixelBufferGetBytesPerRow(pixelBufferRef!)
        var destinationImageBuffer = vImage_Buffer()
        destinationImageBuffer.data = pixelBufferData
        destinationImageBuffer.height = vImagePixelCount(height)
        destinationImageBuffer.width = vImagePixelCount(width)
        destinationImageBuffer.rowBytes = rowBytes

        var permuteMap: [UInt8] = [3, 2, 1, 0] // BGRA
        let convertError = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&yPlaneBuffer, &uPlaneBuffer, &vPlaneBuffer, &destinationImageBuffer, &infoYpCbCrToARGB, &permuteMap, 255, vImage_Flags(kvImagePrintDiagnosticsToConsole))

        CVPixelBufferUnlockBaseAddress(pixelBufferRef!, [])


        yPlane.deallocate()
        uPlane.deallocate()
        vPlane.deallocate()
        let end = CFAbsoluteTimeGetCurrent()
        print("Decoding time \((end-start)*1000)")
        return convertError

    
    fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
        -> (ySize: Int, uSize: Int, vSize: Int)
    
        guard let frameFormat = frame.format
            else 
                return (0, 0 ,0)
        
        let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
        return (baseSize, baseSize / 4, baseSize / 4)
    

在iPhone7上测试的性能，一帧转换不到一毫秒。

【参考方案2】：

这对我有用（我已经采用了你的功能并对其进行了一些更改）：

func createPixelBufferWithVideoFrame(_ frame: OTVideoFrame) -> CVPixelBuffer? 
    if let fLock = frameLock 
        fLock.lock()

        let planeSize = calculatePlaneSize(forFrame: frame)

        let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
        let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
        let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)

        memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
        memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
        memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)

        let width = frame.format!.imageWidth
        let height = frame.format!.imageHeight

        var pixelBuffer: CVPixelBuffer? = nil
        var err: CVReturn;

        err = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
        if (err != 0) 
            NSLog("Error at CVPixelBufferCreate %d", err)
            return nil
        

        if let pixelBuffer = pixelBuffer 
            CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
            let yPlaneTo = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)
            memcpy(yPlaneTo, yPlane, planeSize.ySize)

            let uvRow: Int = planeSize.uSize*2/Int(width)

            let halfWidth: Int = Int(width)/2

            if let uPlaneTo = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) 
                let uvPlaneTo = uPlaneTo.bindMemory(to: GLubyte.self, capacity: Int(uvRow*halfWidth*2))

                for i in 0..<uvRow 
                    for j in 0..<halfWidth 
                        let dataIndex: Int = Int(i) * Int(halfWidth) + Int(j)
                        let uIndex: Int = (i * Int(width)) + Int(j) * 2
                        let vIndex: Int = uIndex + 1

                        uvPlaneTo[uIndex] = uPlane[dataIndex]
                        uvPlaneTo[vIndex] = vPlane[dataIndex]

                    
                

            

        

        fLock.unlock()

        return pixelBuffer
    
    return nil