如何编码 1 张图像并使用 ffmpeg h264_nvenc 接收它

Question

I am trying to create a real-time stream of my screen using ffmpeg and encoder h264_nvenc but there's 5-6 frames delay between my mouse move and mouse move on the translation(first 5-6 frames are black)

I need to somehow encode 1 image of my screen using h264_nvenc and receive it as valid frame(packet) or maybe decrease delay to 1-2 frames

Maybe there are some other methods to get the image right away

UPD: I realised that my latency is because of encoder and decoder frames delay so is there any way to completely remove this frame delays?(Added decoder init function and decode function)

I've done some tests and managed to receive image right away with decoder and encoder flush method but cant do this the second time

Encode function:

static void encode(AVCodecContext* enc_ctx, AVFrame* frame, AVPacket* pkt, AVPacket** new_packet)
{
    int ret;

    ret = avcodec_send_frame(enc_ctx, frame);
    if (ret < 0) {
        fprintf(stderr, "Error sending a frame for encoding\n");
        exit(1);
    }

    int size = 0;

    while (ret >= 0) {
        ret = avcodec_receive_packet(enc_ctx, pkt);
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
        {
            //printf("count: %i size: %i\n", count, size);
            return;
        }
        else if (ret < 0) {
            fprintf(stderr, "Error during encoding\n");
            exit(1);
        }
        size += pkt->size;

        *new_packet = av_packet_clone(pkt);
        
        av_packet_unref(pkt);
    }
}

FFMpeg encoder initialization:

void InitializeFFmpegEncoder()
{
    const AVCodec* codec;
    int i, ret, x, y;
    FILE* f;

    // Encoder

    codec = avcodec_find_encoder_by_name("h264_nvenc"); // hevc_nvenc h264_nvenc
    if (!codec) {
        fprintf(stderr, "Codec '%s' not found\n", "hevc");
        exit(1);
    }

    /*codec = avcodec_find_encoder(AV_CODEC_ID_H265);
    if (!codec) {
        fprintf(stderr, "Codec '%s' not found\n", "hevc");
        exit(1);
    }*/
    cout << "Encoder codec name: " << codec->name << endl;

    cout << "1" << endl;

    enc_c = avcodec_alloc_context3(codec);
    if (!enc_c) {
        fprintf(stderr, "Could not allocate video codec context\n");
        exit(1);
    }
    cout << "2" << endl;
    pkt = av_packet_alloc();
    if (!pkt)
        exit(1);

    cout << "3" << endl;

    /* put sample parameters */
    enc_c->bit_rate = 192000000;
    /* resolution must be a multiple of two */
    enc_c->width = 1920;
    enc_c->height = 1080;
    /* frames per second */
    enc_c->time_base = AVRational(1, 1);
    //enc_c->framerate = AVRational(60, 1);

    /* emit one intra frame every ten frames
     * check frame pict_type before passing frame
     * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I
     * then gop_size is ignored and the output of encoder
     * will always be I frame irrespective to gop_size
     */
    enc_c->gop_size = 1;
    enc_c->max_b_frames = 0;
    enc_c->pix_fmt = AV_PIX_FMT_BGR0;
    enc_c->keyint_min = 0;

    if (codec->id == AV_CODEC_ID_H264)
    {
        //av_opt_set(enc_c->priv_data, "preset", "p1", 0);
        //av_opt_set(enc_c->priv_data, "tune", "ull", 0);
        //av_opt_set(enc_c->priv_data, "zerolatency", "1", 0);
        //av_opt_set(enc_c->priv_data, "preset", "p1", 0);
        //av_opt_set(enc_c->priv_data, "tune", "ull", 0);
        //av_opt_set(enc_c->priv_data, "strict_gop", "1", 0);
        //av_opt_set(enc_c->priv_data, "preset", "lossless", 0);
        //av_opt_set(enc_c->priv_data, "zerolatency", "1", 0);
    }

    cout << "4" << endl;

    /* open it */
    ret = avcodec_open2(enc_c, codec, NULL);
    if (ret < 0) {
        printf("Could not open codec: %s\n", av_make_error_string((char[64])(0), 64, ret));
        exit(1);
    }

    cout << "5" << endl;

    frame = av_frame_alloc();
    if (!frame) {
        fprintf(stderr, "Could not allocate video frame\n");
        exit(1);
    }
    frame->format = AV_PIX_FMT_BGR0; // AV_PIX_FMT_YUV444P AV_PIX_FMT_ARGB
    frame->width = enc_c->width;
    frame->height = enc_c->height;

    cout << "6" << endl;

    ret = av_frame_get_buffer(frame, 0);
    if (ret < 0) {
        fprintf(stderr, "Could not allocate the video frame data\n");
        exit(1);
    }

    cout << "7" << endl;
}

Decoder init function:

int InitializeFFmpegDecoder()
{
    int ret;
    const AVCodec* decoder = NULL;
    enum AVHWDeviceType type;
    int i;

    decoder = avcodec_find_decoder_by_name("h264_cuvid"); // h264_cuvid hevc_cuvid
    if (!decoder) {
        fprintf(stderr, "Codec not found\n");
        exit(1);
    }

    cout << "Decoder name: " << decoder->name << endl;

    if (!(decoder_ctx = avcodec_alloc_context3(decoder)))
        return AVERROR(ENOMEM);
    decoder_ctx->get_format = ffmpeg_GetFormat;
    decoder_ctx->gop_size = 0;
    decoder_ctx->max_b_frames = 0;
    decoder_ctx->keyint_min = 0;
    decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;

    //decoder_ctx->get_format = get_format;

    if ((ret = avcodec_open2(decoder_ctx, decoder, NULL)) < 0) {
        fprintf(stderr, "Failed to open codec for stream \n");
        return -1;
    }

    if (ctx == NULL)
    {
        ctx = sws_getContext(1920, 1080,
            AV_PIX_FMT_NV12, 1920, 1080, // AV_PIX_FMT_YUV420P AV_PIX_FMT_NV12
            AV_PIX_FMT_BGR0, 0, 0, 0, 0);
    }
}

Decode function:

static void decode(AVCodecContext* dec_ctx, AVFrame* frame, AVPacket* pkt, char* bgra_image)
{
    int ret;

    ret = avcodec_send_packet(dec_ctx, pkt);
    if (ret < 0) {
        fprintf(stderr, "Error sending a packet for decoding\n");
        exit(1);
    }

    while (ret >= 0) {
        ret = avcodec_receive_frame(dec_ctx, frame);
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
            return;
        else if (ret < 0) {
            fprintf(stderr, "Error during decoding\n");
            exit(1);
        }

        char* outData[1] = { bgra_image }; // RGB24 have one plane
        int outLinesize[1] = { 4 * 1920 }; // RGB stride
        sws_scale(ctx, frame->data, frame->linesize, 0, 1080, (uint8_t* const*)outData, outLinesize);

        //*new_frame = av_frame_clone(frame);

        //av_frame_ref()

        //break;
    }
}

Answer 1

You can check av_seek_frame(av_format_ctx, video_stream_index, timestamp, AVSEEK_FLAG_BACKWARD) out for your decoder. Go to the frame you want to encode then encode that frame.

Encoding a single frame works just like in the loop after you seek to the frame you want to seek, note that av_seek seeks iFrames.

Answer 2

For h264_nvenc set option delay to 0
But for decoder h264_cuvid I don't know how to make 0 delay stream so I switched to DXVA2 and set max_b_frames to 0