Media Foundation網絡攝像頭視頻H264編碼/解碼在播放時會產生偽影

Question

我有一個解決方案，我使用Media Foundation的h264編碼器從網絡攝像頭編碼視頻（YUY2）樣本。 然后我通過TCP連接將其發送到另一個應用程序，該應用程序使用Media Foundation的h264解碼器將流解碼為YUY2格式。 解碼后，使用DirectX在屏幕上顯示視頻樣本/圖像。

問題是在關鍵幀之間，視頻圖像會產生越來越多的偽像。 收到關鍵幀時，偽像消失。

我將TCP連接從示波器中刪除，並在編碼后立即進行解碼，但我仍然有困擾我的工件。

這是從網絡攝像頭接收樣本的回調方法：

//-------------------------------------------------------------------
// OnReadSample
//
// Called when the IMFMediaSource::ReadSample method completes.
//-------------------------------------------------------------------

HRESULT CPreview::OnReadSample(
    HRESULT hrStatus,
    DWORD /* dwStreamIndex */,
    DWORD dwStreamFlags,
    LONGLONG llTimestamp,
    IMFSample *pSample      // Can be NULL
    )
{
    HRESULT hr = S_OK;
    IMFMediaBuffer *pBuffer = NULL;

    EnterCriticalSection(&m_critsec);

    if (FAILED(hrStatus))
    {
        hr = hrStatus;
    }

    if (SUCCEEDED(hr))
    {
        if (pSample)
        {
            IMFSample *pEncodedSample = NULL;
            hr = m_pCodec->EncodeSample(pSample, &pEncodedSample);
            if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT || pEncodedSample == NULL)
            {
                hr = m_pReader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
                LeaveCriticalSection(&m_critsec);
                return S_OK;
            }

            LONGLONG llEncodedSampleTimeStamp = 0;
            LONGLONG llEncodedSampleDuration = 0;
            pEncodedSample->GetSampleTime(&llEncodedSampleTimeStamp);
            pEncodedSample->GetSampleDuration(&llEncodedSampleDuration);

            pBuffer = NULL;
            hr = pEncodedSample->GetBufferByIndex(0, &pBuffer);
            if (hr != S_OK)
            {
                hr = m_pReader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
                LeaveCriticalSection(&m_critsec);
                return hr;
            }

            BYTE *pOutBuffer = NULL;
            DWORD dwMaxLength, dwCurrentLength;
            hr = pBuffer->Lock(&pOutBuffer, &dwMaxLength, &dwCurrentLength);
            if (hr != S_OK)
            {
                hr = m_pReader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
                LeaveCriticalSection(&m_critsec);
                return hr;
            }
            // Send encoded webcam data to connected clients
            //SendData(pOutBuffer, dwCurrentLength, llEncodedSampleTimeStamp, llEncodedSampleDuration);

            pBuffer->Unlock();
            SafeRelease(&pBuffer);

            IMFSample *pDecodedSample = NULL;           
            m_pCodec->DecodeSample(pEncodedSample, &pDecodedSample);
            if (pDecodedSample != NULL)
            {
                pDecodedSample->SetSampleTime(llTimestamp);
                pDecodedSample->SetSampleTime(llTimestamp - llLastSampleTimeStamp);
                llLastSampleTimeStamp = llTimestamp;
                hr = pDecodedSample->GetBufferByIndex(0, &pBuffer);
                //hr = pSample->GetBufferByIndex(0, &pBuffer);

                // Draw the frame.
                if (SUCCEEDED(hr))
                {
                    hr = m_draw.DrawFrame(pBuffer);
                }
                SafeRelease(&pDecodedSample);
            }

            SafeRelease(&pBuffer);
            SafeRelease(&pEncodedSample);           
        }
    }

    // Request the next frame.
    if (SUCCEEDED(hr))
    {
        hr = m_pReader->ReadSample(
            (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
            0,
            NULL,   // actual
            NULL,   // flags
            NULL,   // timestamp
            NULL    // sample
            );
    }

    if (FAILED(hr))
    {
        NotifyError(hr);
    }
    SafeRelease(&pBuffer);

    LeaveCriticalSection(&m_critsec);
    return hr;
}

這是編碼器/解碼器初始化代碼：

    HRESULT Codec::InitializeEncoder()
    {   
        IMFMediaType *pMFTInputMediaType = NULL, *pMFTOutputMediaType = NULL;
        IUnknown *spTransformUnk = NULL;    
        DWORD mftStatus = 0;
        UINT8 blob[] = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x1e, 0x96, 0x54, 0x05, 0x01,
            0xe9, 0x80, 0x80, 0x40, 0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x3c, 0x80 };

        CoInitializeEx(NULL, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE);
        MFStartup(MF_VERSION);

        // Create H.264 encoder.
        CHECK_HR(CoCreateInstance(CLSID_CMSH264EncoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IUnknown, (void**)&spTransformUnk), "Failed to create H264 encoder MFT.\n");

        CHECK_HR(spTransformUnk->QueryInterface(IID_PPV_ARGS(&pEncoderTransform)), "Failed to get IMFTransform interface from H264 encoder MFT object.\n");

        // Transform output type
        MFCreateMediaType(&pMFTOutputMediaType);
        pMFTOutputMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
        pMFTOutputMediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
        pMFTOutputMediaType->SetUINT32(MF_MT_AVG_BITRATE, 500000);
        CHECK_HR(MFSetAttributeSize(pMFTOutputMediaType, MF_MT_FRAME_SIZE, 640, 480), "Failed to set frame size on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTOutputMediaType, MF_MT_FRAME_RATE, 30, 1), "Failed to set frame rate on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTOutputMediaType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1), "Failed to set aspect ratio on H264 MFT out type.\n");
        pMFTOutputMediaType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_MixedInterlaceOrProgressive);
        pMFTOutputMediaType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);

        // Special attributes for H264 transform, if needed
        /*CHECK_HR(pMFTOutputMediaType->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base), "Failed to set profile on H264 MFT out type.\n");
        CHECK_HR(pMFTOutputMediaType->SetUINT32(MF_MT_MPEG2_LEVEL, eAVEncH264VLevel4), "Failed to set level on H264 MFT out type.\n");
        CHECK_HR(pMFTOutputMediaType->SetUINT32(MF_MT_MAX_KEYFRAME_SPACING, 10), "Failed to set key frame interval on H264 MFT out type.\n");
        CHECK_HR(pMFTOutputMediaType->SetUINT32(CODECAPI_AVEncCommonQuality, 100), "Failed to set H264 codec qulaity.\n");
        CHECK_HR(pMFTOutputMediaType->SetUINT32(CODECAPI_AVEncMPVGOPSize, 1), "Failed to set CODECAPI_AVEncMPVGOPSize = 1\n");*/
        CHECK_HR(pEncoderTransform->SetOutputType(0, pMFTOutputMediaType, 0), "Failed to set output media type on H.264 encoder MFT.\n");

        // Transform input type
        MFCreateMediaType(&pMFTInputMediaType);
        pMFTInputMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
        pMFTInputMediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_YUY2);
        CHECK_HR(MFSetAttributeSize(pMFTInputMediaType, MF_MT_FRAME_SIZE, 640, 480), "Failed to set frame size on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTInputMediaType, MF_MT_FRAME_RATE, 30, 1), "Failed to set frame rate on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTInputMediaType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1), "Failed to set aspect ratio on H264 MFT out type.\n");
        CHECK_HR(pEncoderTransform->SetInputType(0, pMFTInputMediaType, 0), "Failed to set input media type on H.264 encoder MFT.\n");

        CHECK_HR(pEncoderTransform->GetInputStatus(0, &mftStatus), "Failed to get input status from H.264 MFT.\n");
        if (MFT_INPUT_STATUS_ACCEPT_DATA != mftStatus)
        {
            printf("E: pEncoderTransform->GetInputStatus() not accept data.\n");
            goto done;
        }

        CHECK_HR(pEncoderTransform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, NULL), "Failed to process FLUSH command on H.264 MFT.\n");
        CHECK_HR(pEncoderTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL), "Failed to process BEGIN_STREAMING command on H.264 MFT.\n");
        CHECK_HR(pEncoderTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, NULL), "Failed to process START_OF_STREAM command on H.264 MFT.\n");

        return S_OK;

    done:

        SafeRelease(&pMFTInputMediaType);
        SafeRelease(&pMFTOutputMediaType);

        return S_FALSE;
    }

    HRESULT Codec::InitializeDecoder()
    {
        IUnknown *spTransformUnk = NULL;
        IMFMediaType *pMFTOutputMediaType = NULL;
        IMFMediaType *pMFTInputMediaType = NULL;
        DWORD mftStatus = 0;

        // Create H.264 decoder.
        CHECK_HR(CoCreateInstance(CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IUnknown, (void**)&spTransformUnk), "Failed to create H264 decoder MFT.\n");

        // Query for the IMFTransform interface 
        CHECK_HR(spTransformUnk->QueryInterface(IID_PPV_ARGS(&pDecoderTransform)), "Failed to get IMFTransform interface from H264 decoder MFT object.\n");

        // Create input mediatype for the decoder
        MFCreateMediaType(&pMFTInputMediaType);
        pMFTInputMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
        pMFTInputMediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
        CHECK_HR(MFSetAttributeSize(pMFTInputMediaType, MF_MT_FRAME_SIZE, 640, 480), "Failed to set frame size on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTInputMediaType, MF_MT_FRAME_RATE, 30, 1), "Failed to set frame rate on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTInputMediaType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1), "Failed to set aspect ratio on H264 MFT out type.\n");
        pMFTInputMediaType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_MixedInterlaceOrProgressive);
        pMFTInputMediaType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
        CHECK_HR(pDecoderTransform->SetInputType(0, pMFTInputMediaType, 0), "Failed to set input media type on H.264 encoder MFT.\n");

        CHECK_HR(pDecoderTransform->GetInputStatus(0, &mftStatus), "Failed to get input status from H.264 MFT.\n");
        if (MFT_INPUT_STATUS_ACCEPT_DATA != mftStatus)
        {
            printf("E: pDecoderTransform->GetInputStatus() not accept data.\n");
            goto done;
        }

        // Create outmedia type for the decoder
        MFCreateMediaType(&pMFTOutputMediaType);
        pMFTOutputMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
        pMFTOutputMediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_YUY2);
        CHECK_HR(MFSetAttributeSize(pMFTOutputMediaType, MF_MT_FRAME_SIZE, 640, 480), "Failed to set frame size on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTOutputMediaType, MF_MT_FRAME_RATE, 30, 1), "Failed to set frame rate on H264 MFT out type.\n");
        CHECK_HR(MFSetAttributeRatio(pMFTOutputMediaType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1), "Failed to set aspect ratio on H264 MFT out type.\n");
        CHECK_HR(pDecoderTransform->SetOutputType(0, pMFTOutputMediaType, 0), "Failed to set output media type on H.264 decoder MFT.\n");

        CHECK_HR(pDecoderTransform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, NULL), "Failed to process FLUSH command on H.264 MFT.\n");
        CHECK_HR(pDecoderTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL), "Failed to process BEGIN_STREAMING command on H.264 MFT.\n");
        CHECK_HR(pDecoderTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, NULL), "Failed to process START_OF_STREAM command on H.264 MFT.\n");

        return S_OK;

    done:

        SafeRelease(&pMFTInputMediaType);
        SafeRelease(&pMFTOutputMediaType);

        return S_FALSE;
    }

這是實際的解碼/編碼器部分：

HRESULT Codec::EncodeSample(IMFSample *pSample, IMFSample **ppEncodedSample)
{
    return TransformSample(pEncoderTransform, pSample, ppEncodedSample);
}

HRESULT Codec::DecodeSample(IMFSample *pSample, IMFSample **ppEncodedSample)
{
    return TransformSample(pDecoderTransform, pSample, ppEncodedSample);
}

HRESULT Codec::TransformSample(IMFTransform *pTransform, IMFSample *pSample, IMFSample **ppSampleOut)
{
    IMFSample *pOutSample = NULL;
    IMFMediaBuffer *pBuffer = NULL;
    DWORD mftOutFlags;
    pTransform->ProcessInput(0, pSample, 0);
    CHECK_HR(pTransform->GetOutputStatus(&mftOutFlags), "H264 MFT GetOutputStatus failed.\n");

    // Note: Decoder does not return MFT flag MFT_OUTPUT_STATUS_SAMPLE_READY, so we just need to rely on S_OK return
    if (pTransform == pEncoderTransform && mftOutFlags == S_OK)
    {
        return S_OK;
    }
    else if (pTransform == pEncoderTransform && mftOutFlags == MFT_OUTPUT_STATUS_SAMPLE_READY ||
        pTransform == pDecoderTransform && mftOutFlags == S_OK)
    {
        DWORD processOutputStatus = 0;
        MFT_OUTPUT_DATA_BUFFER outputDataBuffer;
        MFT_OUTPUT_STREAM_INFO StreamInfo;
        pTransform->GetOutputStreamInfo(0, &StreamInfo);

        CHECK_HR(MFCreateSample(&pOutSample), "Failed to create MF sample.\n");
        CHECK_HR(MFCreateMemoryBuffer(StreamInfo.cbSize, &pBuffer), "Failed to create memory buffer.\n");
        if (pTransform == pEncoderTransform)
            CHECK_HR(pBuffer->SetCurrentLength(StreamInfo.cbSize), "Failed SetCurrentLength.\n");
        CHECK_HR(pOutSample->AddBuffer(pBuffer), "Failed to add sample to buffer.\n");      
        outputDataBuffer.dwStreamID = 0;
        outputDataBuffer.dwStatus = 0;
        outputDataBuffer.pEvents = NULL;
        outputDataBuffer.pSample = pOutSample;

        HRESULT hr = pTransform->ProcessOutput(0, 1, &outputDataBuffer, &processOutputStatus);
        if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT)
        {
            SafeRelease(&pBuffer);
            SafeRelease(&pOutSample);
            return hr;
        }

        LONGLONG llVideoTimeStamp, llSampleDuration;
        pSample->GetSampleTime(&llVideoTimeStamp);
        pSample->GetSampleDuration(&llSampleDuration);
        CHECK_HR(outputDataBuffer.pSample->SetSampleTime(llVideoTimeStamp), "Error setting MFT sample time.\n");
        CHECK_HR(outputDataBuffer.pSample->SetSampleDuration(llSampleDuration), "Error setting MFT sample duration.\n");        
        if (pTransform == pEncoderTransform)
        {
            IMFMediaBuffer *pMediaBuffer = NULL;
            DWORD dwBufLength;
            CHECK_HR(pOutSample->ConvertToContiguousBuffer(&pMediaBuffer), "ConvertToContiguousBuffer failed.\n");
            CHECK_HR(pMediaBuffer->GetCurrentLength(&dwBufLength), "Get buffer length failed.\n");

            WCHAR *strDebug = new WCHAR[256];
            wsprintf(strDebug, L"Encoded sample ready: time %I64d, sample duration %I64d, sample size %i.\n", llVideoTimeStamp, llSampleDuration, dwBufLength);
            OutputDebugString(strDebug);
            SafeRelease(&pMediaBuffer);
        }
        else if (pTransform == pDecoderTransform)
        {
            IMFMediaBuffer *pMediaBuffer = NULL;
            DWORD dwBufLength;
            CHECK_HR(pOutSample->ConvertToContiguousBuffer(&pMediaBuffer), "ConvertToContiguousBuffer failed.\n");
            CHECK_HR(pMediaBuffer->GetCurrentLength(&dwBufLength), "Get buffer length failed.\n");

            WCHAR *strDebug = new WCHAR[256];
            wsprintf(strDebug, L"Decoded sample ready: time %I64d, sample duration %I64d, sample size %i.\n", llVideoTimeStamp, llSampleDuration, dwBufLength);
            OutputDebugString(strDebug);
            SafeRelease(&pMediaBuffer);
        }

        // Decoded sample out
        *ppSampleOut = pOutSample;

        //SafeRelease(&pMediaBuffer);
        SafeRelease(&pBuffer);

        return S_OK;
    }

done:
    SafeRelease(&pBuffer);
    SafeRelease(&pOutSample);

    return S_FALSE;
}

我已經搜索了很長一段時間的解決方案，發現一個問題與我的問題非常相似，但是對於不同的API，它對我沒有幫助。 關鍵幀之間的FFMPEG解碼工件

最誠摯的問候，Toni Riikonen

Answer 1

這聽起來像質量/比特率問題。

pMFTOutputMediaType->SetUINT32(MF_MT_AVG_BITRATE, 500000); 

對於比特率，500kbps的值太低，您可以嘗試使用5,10或20Mbps之類的更大值。

我可以建議：

1. 由於您自己創建H264編碼器，因此可以查詢ICodecAPI並嘗試不同的設置。 即，CODECAPI_AVEncCommonRateControlMode，CODECAPI_AVEncCommonQuality，CODECAPI_AVEncAdaptiveMode，CODECAPI_AVEncCommonQualityVsSpeed，CODECAPI_AVEncVideoEncodeQP。

2. 您也可以嘗試創建硬件H264編碼器並使用IMFDXGIDeviceManager（Windows 8及更高版本？）

Answer 2

這個問題似乎有答案，但我仍想分享我的經驗。 希望能幫助遇到類似問題的人。

在解碼H264時我也遇到了類似的工件問題。 但是，在我的情況下，流來自視頻捕獲設備，並且在從流開始30-60秒后，工件不會消失。

在我看來，我認為正常設置的解碼器由於低延遲而無法解碼直播。 因此，我嘗試啟用CODECAPI_AVLowLatencyMode，它可以將解碼/編碼模式設置為低延遲，以進行實時通信或實時捕獲。 （要獲得更多詳細信息，請參閱MS https://msdn.microsoft.com/zh-tw/library/windows/desktop/hh447590(v=vs.85).aspx中的以下鏈接。幸運的是，問題一直存在解決了，解碼器正常工作。

雖然我們的問題有點不同，但您可以嘗試在您的情況下啟用/禁用CODECAPI_AVLowLatencyMode，我希望您也可以獲得好消息。

Answer 3

我這里的游戲有點晚了，但我可以確認來自主頁的答案是正確的解決方案。 我也遇到了同樣的問題，但我只使用了此示例代碼的解碼器部分。 我正在閱讀一個MP4文件，看到關鍵幀之間的增加的文物。 一旦我收到一個關鍵幀，圖像看起來很好，然后逐漸變得更糟。 這是我在Codec :: InitializeDecoder（）中添加的代碼：

// Set CODECAPI_AVLowLatencyMode
ICodecAPI *mpCodecAPI = NULL;
hr = pDecoderTransform->QueryInterface(IID_PPV_ARGS(&mpCodecAPI));
CHECK_HR(hr, "Failed to get ICodecAPI.\n");

VARIANT var;
var.vt = VT_BOOL;
var.boolVal = VARIANT_TRUE;
hr = mpCodecAPI->SetValue(&CODECAPI_AVLowLatencyMode, &var);
CHECK_HR(hr, "Failed to enable low latency mode.\n");

添加這些更改后，程序運行得更好！ 感謝GitHub上的這個軟件為我提供了必要的代碼： https ： //github.com/GameTechDev/ChatHeads/blob/master/VideoStreaming/EncodeTransform.cpp

Answer 4

這聽起來像是IP（B）幀排序問題。

編碼幀順序與解碼幀順序不同。 我沒有測試你的代碼，但我認為編碼器以編碼順序提供幀，你需要在渲染之前重新排序幀。