How to get h264 frames via gstreamer

Question

I'm familiar with ffmpeg, but not with GStreamer. I know how to get a H264 frame through ffmpeg, for example, I can get a H264 frame through AVPacket. But I don't know how to use GStreamer to get a frame of h264. I don't intend to save the H264 data directly as a local file because I need to do other processing. Can anyone give me some sample code? I'll be very grateful. Here's what I learned from other people's code.

#include <stdio.h>
#include <string.h>
#include <fstream>
#include <unistd.h>
#include <gst/gst.h>
#include <gst/app/gstappsrc.h>

typedef struct {
    GstPipeline *pipeline;
    GstAppSrc  *src;
    GstElement *filter1;
    GstElement *encoder;
    GstElement *filter2;
    GstElement *parser;
    GstElement *qtmux;
    GstElement *sink;

    GstClockTime timestamp;
    guint sourceid;
} gst_app_t;

static gst_app_t gst_app;

int main()
{
    gst_app_t *app = &gst_app;
    GstStateChangeReturn state_ret;
    gst_init(NULL, NULL); //Initialize Gstreamer
    app->timestamp = 0; //Set timestamp to 0

    //Create pipeline, and pipeline elements
    app->pipeline = (GstPipeline*)gst_pipeline_new("mypipeline");
    app->src    =   (GstAppSrc*)gst_element_factory_make("appsrc", "mysrc");
    app->filter1 =  gst_element_factory_make ("capsfilter", "myfilter1");
    app->encoder =  gst_element_factory_make ("omxh264enc", "myomx");
    app->filter2 =  gst_element_factory_make ("capsfilter", "myfilter2");
    app->parser =   gst_element_factory_make("h264parse"  , "myparser");
    app->qtmux =    gst_element_factory_make("qtmux"      , "mymux");
    app->sink =     gst_element_factory_make ("filesink"  , NULL);
    
    if( !app->pipeline || 
        !app->src      || !app->filter1 || 
        !app->encoder  || !app->filter2 || 
        !app->parser   || !app->qtmux    || 
        !app->sink    )  {
        printf("Error creating pipeline elements!\n");
        exit(2);
    }

    //Attach elements to pipeline
    gst_bin_add_many(
        GST_BIN(app->pipeline), 
        (GstElement*)app->src,
        app->filter1,   
        app->encoder,
        app->filter2,   
        app->parser,
        app->qtmux,
        app->sink,
        NULL);

    //Set pipeline element attributes
    g_object_set (app->src, "format", GST_FORMAT_TIME, NULL);
    GstCaps *filtercaps1 = gst_caps_new_simple ("video/x-raw",
        "format", G_TYPE_STRING, "I420",
        "width", G_TYPE_INT, 1280,
        "height", G_TYPE_INT, 720,
        "framerate", GST_TYPE_FRACTION, 1, 1,
        NULL);
    g_object_set (G_OBJECT (app->filter1), "caps", filtercaps1, NULL);
    GstCaps *filtercaps2 = gst_caps_new_simple ("video/x-h264",
        "stream-format", G_TYPE_STRING, "byte-stream",
        NULL);
    g_object_set (G_OBJECT (app->filter2), "caps", filtercaps2, NULL);
    g_object_set (G_OBJECT (app->sink), "location", "output.h264", NULL);

    //Link elements together
    g_assert( gst_element_link_many(
        (GstElement*)app->src, 
        app->filter1,
        app->encoder,
        app->filter2,
        app->parser,
        app->qtmux,
        app->sink,
        NULL ) );

    //Play the pipeline
    state_ret = gst_element_set_state((GstElement*)app->pipeline, GST_STATE_PLAYING);
    g_assert(state_ret == GST_STATE_CHANGE_ASYNC);

    //Get a pointer to the test input
    FILE *testfile = fopen("test.yuv", "rb");   
    g_assert(testfile != NULL);

    //Push the data from buffer to gstpipeline 100 times
    for(int i = 0; i < 100; i++) {
        char* filebuffer = (char*)malloc (1382400); //Allocate memory for framebuffer
        if (filebuffer == NULL) {printf("Memory error\n"); exit (2);} //Errorcheck
        size_t bytesread = fread(filebuffer, 1 , (1382400), testfile); //Read to filebuffer
        //printf("File Read: %zu bytes\n", bytesread);

        GstBuffer *pushbuffer; //Actual databuffer
        GstFlowReturn ret; //Return value
        pushbuffer = gst_buffer_new_wrapped (filebuffer, 1382400); //Wrap the data

        //Set frame timestamp
        GST_BUFFER_PTS      (pushbuffer) = app->timestamp;
        GST_BUFFER_DTS      (pushbuffer) = app->timestamp;  
        GST_BUFFER_DURATION (pushbuffer) = gst_util_uint64_scale_int (1, GST_SECOND, 1);
        app->timestamp += GST_BUFFER_DURATION (pushbuffer);
        //printf("Frame is at %lu\n", app->timestamp);

        ret = gst_app_src_push_buffer( app->src, pushbuffer); //Push data into pipeline

        g_assert(ret ==  GST_FLOW_OK);
    }
    usleep(100000);
    
    //Declare end of stream
    gst_app_src_end_of_stream (GST_APP_SRC (app->src));
    printf("End Program.\n");

return 0;

}

Here is a link to the source of the code link

Answer 1

Your example serves for the purpose of feeding the data from application to the GStreamer with a hope to encode with x264 and the result goes to file.

What you need (I am guessing here) is to read data from file - lets say movie.mp4 and get the decoded data into your application (?)

I believe you have two options:

1, Use appsink instead of filesink and feed the data from file using filesrc. So if you need also other processing beside grabbing the h264 frames (like playing or sending via network), you would have to use tee to split the pipeline into two output branches like example gst-launch below. One branch of output pipeline would go to for example windowed output - autovideosink and the other part would go to your application.

To demonstrate this split and still show you what is really happening I will use debugging element identity which is able to dump data which goes through it. This way you will learn to use this handy tool for experiments and verification that you know what you are doing. This is not the solution you need.

gst-launch-1.0 -q filesrc location= movie.mp4. qtdemux name=qt. video/x-h264 ! h264parse ! tee name=t t. ! queue ! avdec_h264 ! videoconvert ! autovideosink t. ! queue ! identity dump=1 ! fakesink sync=true

This pipeline plays the video into window (autovideosink) and the other branch of tee goes to the debugging element called identity which dumps the frame in hexdump manner (with addresses, character representation and everything). So what you see in the stdout of the gst-launch are actual h264 frames (but you do not see boundaries or anything.. its just really raw dump).

To understand the gst-launch syntax (mainly the aliases with name= ) check this part of the documentation.

In real code you would not use identity and fakesink but instead you would link there just appsink and connect the appsink signals to callbacks in your C source code.

There are nice examples for this, I will not attempt to give you complete solution. This example demonstrate how to get samples out of appsink. The important bits are:

/* The appsink has received a buffer */
static GstFlowReturn new_sample (GstElement *sink, CustomData *data) {
  GstSample *sample;

  /* Retrieve the buffer */
  g_signal_emit_by_name (sink, "pull-sample", &sample);
  if (sample) {
    /* The only thing we do in this example is print a * to indicate a received buffer */
    g_print ("*");
    gst_sample_unref (sample);
    return GST_FLOW_OK;
  }

  return GST_FLOW_ERROR;
}

// somewhere in main()
// construction and linkage of elements
g_signal_connect (data.app_sink, "new-sample", G_CALLBACK (new_sample), &data);

2, Second solution is to use pad probe registered for buffers only. Pad probe is a way to register a callback on any pad of any element in pipeline and tell GStreamer in what information are you interested in on that probe. You can ask it to call the callback upon every event, or any downstream event, or on any buffer going through that probe. In the callback which pad probe calls you will extract the buffer and the actuall data in that buffer.

Again there are many examples on how to use pad probes. One very nice example containing logic of almost exactly what you need can be found here

The important bits:

static GstPadProbeReturn
cb_have_data (GstPad          *pad,
              GstPadProbeInfo *info,
              gpointer         user_data)
{
// ... the code for writing the buffer data somewhere ..
}

// ... later in main()
pad = gst_element_get_static_pad (src, "src");
gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_BUFFER,
      (GstPadProbeCallback) cb_have_data, NULL, NULL);