NV12 到 RGB32 与 Microsoft Media Foundation

Posted 2023-02-16

【中文标题】NV12 到 RGB32 与 Microsoft Media Foundation

【英文标题】：NV12 to RGB32 with Microsoft Media Foundation

【发布时间】：2018-06-08 00:20:12

【问题描述】：

我正在尝试通过 Media FOundation 获得一些我在 YT 上工作的相机预览。我已经列举了我的设备。我可以从我的相机获得预览，但我的数据格式是 NV12。我需要 RGB32 格式的数据。这是包含设备信息的类。

class Media : public IMFSourceReaderCallback 

    CRITICAL_SECTION criticalSection;
    long referenceCount;
    WCHAR                   *wSymbolicLink;
    UINT32                  cchSymbolicLink;
    IMFSourceReader* sourceReader;


public:
    LONG stride;
    int bytesPerPixel;
    GUID videoFormat;
    UINT height;
    UINT width;
    WCHAR deviceNameString[2048];
    BYTE* rawData;

    HRESULT CreateCaptureDevice();
    HRESULT SetSourceReader(IMFActivate *device);
    HRESULT IsMediaTypeSupported(IMFMediaType* type);
    HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride);
    HRESULT Close();
    Media();
    ~Media();   

    // the class must implement the methods from IUnknown 
    STDMETHODIMP QueryInterface(REFIID iid, void** ppv);
    STDMETHODIMP_(ULONG) AddRef();
    STDMETHODIMP_(ULONG) Release();

    //  the class must implement the methods from IMFSourceReaderCallback 
    STDMETHODIMP OnReadSample(HRESULT status, DWORD streamIndex, DWORD streamFlags, LONGLONG timeStamp, IMFSample *sample);
    STDMETHODIMP OnEvent(DWORD, IMFMediaEvent *);
    STDMETHODIMP OnFlush(DWORD);

;

这是创建设备的方法：

HRESULT Media::CreateCaptureDevice()

    HRESULT hr = S_OK;

    //this is important!!
    hr = CoInitializeEx(NULL, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE);//COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE);

    UINT32 count = 0;
    IMFAttributes *attributes = NULL;
    IMFActivate **devices = NULL;

    if (FAILED(hr))  CLEAN_ATTRIBUTES() 
    // Create an attribute store to specify enumeration parameters.
    hr = MFCreateAttributes(&attributes, 1);

    if (FAILED(hr))  CLEAN_ATTRIBUTES() 

    //The attribute to be requested is devices that can capture video
    hr = attributes->SetGUID(
        MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
        MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID
    );
    if (FAILED(hr))  CLEAN_ATTRIBUTES() 
    //Enummerate the video capture devices
    hr = MFEnumDeviceSources(attributes, &devices, &count);

    if (FAILED(hr))  CLEAN_ATTRIBUTES() 
    //if there are any available devices
    if (count > 0)
    
        /*If you actually need to select one of the available devices
        this is the place to do it. For this example the first device
        is selected
        */
        //Get a source reader from the first available device
        SetSourceReader(devices[0]);

        WCHAR *nameString = NULL;
        // Get the human-friendly name of the device
        UINT32 cchName;
        hr = devices[0]->GetAllocatedString(
            MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME,
            &nameString, &cchName);

        if (SUCCEEDED(hr))
        
            //allocate a byte buffer for the raw pixel data
            bytesPerPixel = abs(stride) / width;
            rawData = new BYTE[width*height * bytesPerPixel];
            wcscpy(deviceNameString,nameString);
        
        CoTaskMemFree(nameString);
    

    //clean
    CLEAN_ATTRIBUTES()

这是设置设备的方法：

HRESULT Media::SetSourceReader(IMFActivate *device)

    HRESULT hr = S_OK;

    IMFMediaSource *source = NULL;
    IMFAttributes *attributes = NULL;
    IMFMediaType *mediaType = NULL;

    EnterCriticalSection(&criticalSection);

    hr = device->ActivateObject(__uuidof(IMFMediaSource), (void**)&source);

    //get symbolic link for the device
    if(SUCCEEDED(hr))
        hr = device->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &wSymbolicLink, &cchSymbolicLink);
    //Allocate attributes
    if (SUCCEEDED(hr))
        hr = MFCreateAttributes(&attributes, 2);
    //get attributes
    if (SUCCEEDED(hr))
        hr = attributes->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, TRUE);
    // Set the callback pointer.
    if (SUCCEEDED(hr))
        hr = attributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK,this);
    //Create the source reader
    if (SUCCEEDED(hr))
        hr = MFCreateSourceReaderFromMediaSource(source,attributes,&sourceReader);
    // Try to find a suitable output type.
    if (SUCCEEDED(hr))
    
        for (DWORD i = 0; ; i++)
        
            hr = sourceReader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,i,&mediaType);
            if (FAILED(hr))  break; 

            hr = IsMediaTypeSupported(mediaType);
            if (FAILED(hr))  break; 
            //Get width and height
            MFGetAttributeSize(mediaType, MF_MT_FRAME_SIZE, &width, &height);
            if (mediaType) 
             mediaType->Release(); mediaType = NULL; 

            if (SUCCEEDED(hr))// Found an output type.
                break;
        
    
    if (SUCCEEDED(hr))
    
        // Ask for the first sample.
        hr = sourceReader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,   0, NULL, NULL,NULL,NULL);
    

    if (FAILED(hr))
    
        if (source)
        
            source->Shutdown(); 
        
        Close();
    
    if (source)  source->Release(); source = NULL; 
    if (attributes)  attributes->Release(); attributes = NULL; 
    if (mediaType)  mediaType->Release(); mediaType = NULL; 

    LeaveCriticalSection(&criticalSection);
    return hr;

【问题讨论】：

网络摄像头通常支持 RGB 输出。您可能希望枚举设备媒体类型，在预览流上选择并设置最合适的 RGB32 类型（宽度/高度/fps）。

【参考方案1】：

Video Processor MFT 可以处理颜色转换。设置和使用相当简单，因为它是一个同步变换。 See this link for more information and examples.

【参考方案2】：

通常当yuv(nv12)转rgb(rgb32)时：

Byte* p_rgb_Data = (((Height / 2) + Height) * width) * 3;// 没有alpha

编辑

我的回答是不正确的，你想从相机里得到 rgb 格式，对吧？

您必须从 SourceReader 中枚举 MediaType：

enumerating_output_formats

HRESULT EnumerateTypesForStream(IMFSourceReader *pReader, DWORD wStreamIndex)

HRESULT hr = S_OK;
DWORD dwMediaTypeIndex = 0;

while (SUCCEEDED(hr))

    IMFMediaType *pType = NULL;
    hr = pReader->GetNativeMediaType(dwStreamIndex, dwMediaTypeIndex, &pType);
    if (hr == MF_E_NO_MORE_TYPES)
    
        hr = S_OK;
        break;
    
    else if (SUCCEEDED(hr))
    
        // Examine the media type. (Not shown.)

        pType->Release();
    
    ++dwMediaTypeIndex;

return hr;

当MediaType为RGB时，设置MediaType：

setting_output_formats

hr = pReader->SetCurrentMediaType(dwStreamIndex, pMediaType);

编辑...

是的，来自：

Byte* p_rgb_Data = (((Height / 2) + Height) * width) * 3;// 没有alpha

-> 应该是 Byte* p_rgb_Size = (((Height / 2) + Height) * width) * 3;// 没有alpha

我错过了转换。类似的东西：

BYTE GetR(const int bY, int const bU)

    int iR = bY + (int)(1.402f * bU);
    iR = iR > 255 ? 255 : iR < 0 ? 0 : iR;

    return iR;

BYTE GetG(const int bY, const int bU, const int bV)

    int iG = bY - (int)(0.344f * bV + 0.714f * bU);
    iG = iG > 255 ? 255 : iG < 0 ? 0 : iG;

    return iG;

BYTE GetB(const int bY, const int bV)

    int iB = bY + (int)(1.772f * bV);
    iB = iB > 255 ? 255 : iB < 0 ? 0 : iB;

    return iB;

如果您想在 NV12 缓冲区和 RGB 缓冲区之间完成转换，请询问。

【讨论】：

你能解释一下吗？没有来自 NV12 的任何数据样本，数据怎么可能只是高度和宽度的组合。我很抱歉，但我就是不明白。还是谢谢你！

并非如此。我已经尝试检查媒体类型的子类型，但只有适用于我的相机的格式是 NV12 和 YUY2，所以我正在尝试从 NV12 转换为 RGB32。我是通过枚举媒体类型并检查子类型 == MF_VIDEOFORMAT_XXX 并且只有 NV12 和 YUY2 工作的 :(