iOS 加速：将亮度和色度缓冲区放入单个 CVPixelBuffer

Question

我正在将相机 output 420YpCbCr8BiPlanarFullRange转换为ARGB8888以执行一些图像处理。 我需要使用 webRTC 将结果转换回420YpCbCr8BiPlanarFullRange到 stream ：

func convertTo420Yp8(source: inout vImage_Buffer) -> CVPixelBuffer? {
    let lumaWidth = source.width
    let lumaHeight = source.height
    
    let chromaWidth = source.width
    let chromaHeight = source.height / 2
    
    guard var lumaDestination = try? vImage_Buffer(
        width: Int(lumaWidth),
        height: Int(lumaHeight),
        bitsPerPixel: 8
    ) else {
        return nil
    }
    
    guard var chromaDestination = try? vImage_Buffer(
        width: Int(chromaWidth),
        height: Int(chromaHeight),
        bitsPerPixel: 8
    ) else {
        return nil
    }
    
    defer {
        lumaDestination.free()
        chromaDestination.free()
    }
    
    var error = kvImageNoError
    
    error = vImageConvert_ARGB8888To420Yp8_CbCr8(
        &source,
        &lumaDestination,
        &chromaDestination,
        &infoARGBtoYpCbCr,
        nil,
        vImage_Flags(kvImagePrintDiagnosticsToConsole)
    )
    
    guard error == kvImageNoError else {
        return nil
    }
    
    var pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
    var planeWidths = [Int(lumaWidth), Int(chromaWidth)]
    var planeHeights = [Int(chromaHeight), Int(chromaHeight)]
    var bytesPerRows = [Int(1 * lumaWidth), Int(2 * chromaWidth)]
    var baseAddresses: [UnsafeMutableRawPointer?] = [lumaDestination.data, chromaDestination.data]
    var outputPixelBuffer: CVPixelBuffer?
    
    let status = CVPixelBufferCreateWithPlanarBytes(
        kCFAllocatorDefault,
        Int(lumaWidth),
        Int(lumaHeight),
        pixelFormat,
        nil,
        0,
        2,
        &baseAddresses,
        &planeWidths,
        &planeHeights,
        &bytesPerRows,
        nil,
        nil,
        nil,
        &outputPixelBuffer
    )
    
    if status == noErr {
        print("converted to CVPixelBuffer")
    }
    return outputPixelBuffer
}

vImageConvert_ARGB8888To420Yp8_CbCr8产生两个缓冲区：色度和亮度。 CVPixelBufferCreateWithPlanarBytes返回noErr状态，但 Chroma 和 Luma 数据不在缓冲区中 - 查询时平面地址nil 。 知道我做错了什么吗？

Answer 1

您需要锁定CVPixelBuffer才能访问基址。 所以，这有效：

    let cvPixelBuffer = convertTo420Yp8(source: vImageBuffer)!
    
    CVPixelBufferLockBaseAddress(cvPixelBuffer,
                                 CVPixelBufferLockFlags.readOnly)
    
    print(CVPixelBufferGetBaseAddressOfPlane(cvPixelBuffer, 0))
    print(CVPixelBufferGetBaseAddressOfPlane(cvPixelBuffer, 1))
    
    CVPixelBufferUnlockBaseAddress(cvPixelBuffer,
                                   CVPixelBufferLockFlags.readOnly)

我还可以建议您将bytesPerRow更改为：

var bytesPerRows = [lumaDestination.rowBytes, chromaDestination.rowBytes]

有时，vImage 会为缓冲区添加额外的填充以提高性能。

Answer 2

对了， CVPixelBuffer需要加锁。 我的代码中还有一些其他问题：

• 未正确设置平面的宽度和高度
• vImage_Buffer memory 需要正确释放

我在这里发布工作代码以供参考：

func convertTo420Yp8(source: inout vImage_Buffer) -> CVPixelBuffer? {
    let lumaWidth = source.width
    let lumaHeight = source.height
    
    let chromaWidth = source.width
    let chromaHeight = source.height / 2
    
    guard var lumaDestination = try? vImage_Buffer(
        width: Int(lumaWidth),
        height: Int(lumaHeight),
        bitsPerPixel: 8
    ) else {
        return nil
    }
    
    guard var chromaDestination = try? vImage_Buffer(
        width: Int(chromaWidth),
        height: Int(chromaHeight),
        bitsPerPixel: 8
    ) else {
        return nil
    }
    
    var error = kvImageNoError
    
    error = vImageConvert_ARGB8888To420Yp8_CbCr8(
        &source,
        &lumaDestination,
        &chromaDestination,
        &infoARGBtoYpCbCr,
        nil,
        vImage_Flags(kvImagePrintDiagnosticsToConsole)
    )
    
    guard error == kvImageNoError else {
        return nil
    }
    
    var pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
    var planeWidths = [Int(lumaWidth), Int(chromaWidth)]
    var planeHeights = [Int(lumaHeight), Int(chromaHeight)]
    var bytesPerRows = [lumaDestination.rowBytes, chromaDestination.rowBytes]
    var baseAddresses: [UnsafeMutableRawPointer?] = [lumaDestination.data, chromaDestination.data]
    var outputPixelBuffer: CVPixelBuffer?
    
    let status = CVPixelBufferCreateWithPlanarBytes(
        kCFAllocatorDefault,
        Int(lumaWidth),
        Int(lumaHeight),
        pixelFormat,
        nil,
        0,
        2,
        &baseAddresses,
        &planeWidths,
        &planeHeights,
        &bytesPerRows,
        { releaseRefCon, dataPtr, dataSize, numberOfPlanes, planeAddresses  in
            planeAddresses?[0]?.deallocate()
            planeAddresses?[1]?.deallocate()
        },
        nil,
        nil,
        &outputPixelBuffer
    )
    if status == noErr {
        print("converted to CVPixelBuffer")
    }
    return outputPixelBuffer
}