android上的openGL ES 2.0，YUV到RGB和用ffMpeg渲染

Question

My renderer dies 1~2 frames later when video shows after.

FATAL ERROR 11 : blabla...(Exactly occurs in glDrawElements (Y part))

I think problem is 'glPixelStorei' or 'GL_RGB', 'GL_LUMINANCE' but.. I don't get it.

My rendering way :

1. Decode data that got from network, (SDK Getting-> NDK Decoding), Enqueueing.

2. Dequeueing another threads (of course synchronized) get ready to setup OpenGL ES 2.0.(SDK)

3. When onDrawFrame, onSurfaceCreated, onSurfaceChanged methods are called, it shrink down to NDK. (My Renderer source in NDK will attach below.)

4. Rendering.

As you know, Fragment shader is using for conversion. My Data is YUV 420p (pix_fmt_YUV420p) (12bit per pixel)

Here is my entire source.

I haven't any knowledge about OpenGL ES before, this is first time.

Please let me know what am I do improving performance.

and What am I use parameters in 'glTexImage2D', 'glTexSubImage2D', 'glRenderbufferStorage'???? GL_LUMINANCE? GL_RGBA? GL_RGB? (GL_LUMINANCE is using now)

void Renderer::set_draw_frame(JNIEnv* jenv, jbyteArray yData, jbyteArray uData, jbyteArray vData)
{
    for (int i = 0; i < 3; i++) {
        if (yuv_data_[i] != NULL) {
            free(yuv_data_[i]);
        }
    }

  int YSIZE = -1;
  int USIZE = -1;
  int VSIZE = -1;

  if (yData != NULL) {
        YSIZE = (int)jenv->GetArrayLength(yData);
    LOG_DEBUG("YSIZE : %d", YSIZE);
        yuv_data_[0] = (unsigned char*)malloc(sizeof(unsigned char) * YSIZE);
    memset(yuv_data_[0], 0, YSIZE);
        jenv->GetByteArrayRegion(yData, 0, YSIZE, (jbyte*)yuv_data_[0]);
    yuv_data_[0] = reinterpret_cast<unsigned char*>(yuv_data_[0]);
    } else {
        YSIZE = (int)jenv->GetArrayLength(yData);
        yuv_data_[0] = (unsigned char*)malloc(sizeof(unsigned char) * YSIZE);
    memset(yuv_data_[0], 1, YSIZE);
  }

    if (uData != NULL) {
        USIZE = (int)jenv->GetArrayLength(uData);
    LOG_DEBUG("USIZE : %d", USIZE);
        yuv_data_[1] = (unsigned char*)malloc(sizeof(unsigned char) * USIZE);
    memset(yuv_data_[1], 0, USIZE);
        jenv->GetByteArrayRegion(uData, 0, USIZE, (jbyte*)yuv_data_[1]);
    yuv_data_[1] = reinterpret_cast<unsigned char*>(yuv_data_[1]);
    } else {
        USIZE = YSIZE/4;
        yuv_data_[1] = (unsigned char*)malloc(sizeof(unsigned char) * USIZE);
    memset(yuv_data_[1], 1, USIZE);
  }

    if (vData != NULL) {
        VSIZE = (int)jenv->GetArrayLength(vData);
    LOG_DEBUG("VSIZE : %d", VSIZE);
        yuv_data_[2] = (unsigned char*)malloc(sizeof(unsigned char) * VSIZE);
    memset(yuv_data_[2], 0, VSIZE);
        jenv->GetByteArrayRegion(vData, 0, VSIZE, (jbyte*)yuv_data_[2]);
    yuv_data_[2] = reinterpret_cast<unsigned char*>(yuv_data_[2]);
    } else {
        VSIZE = YSIZE/4;
        yuv_data_[2] = (unsigned char*)malloc(sizeof(unsigned char) * VSIZE);
    memset(yuv_data_[2], 1, VSIZE);
  }

    glClearColor(1.0F, 1.0F, 1.0F, 1.0F);
    check_gl_error("glClearColor");
    glClear(GL_COLOR_BUFFER_BIT);
    check_gl_error("glClear");
}

void Renderer::draw_frame()
{
  // Binding created FBO
  glBindFramebuffer(GL_FRAMEBUFFER, frame_buffer_object_);
  check_gl_error("glBindFramebuffer");
    // Add program to OpenGL environment
    glUseProgram(program_object_);
    check_gl_error("glUseProgram");

  for (int i = 0; i < 3; i++) {
    LOG_DEBUG("Success");
      //Bind texture
      glActiveTexture(GL_TEXTURE0 + i);
      check_gl_error("glActiveTexture");
      glBindTexture(GL_TEXTURE_2D, yuv_texture_id_[i]);
      check_gl_error("glBindTexture");
      glUniform1i(yuv_texture_object_[i], i);
      check_gl_error("glBindTexture");
    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, stream_yuv_width_[i], stream_yuv_height_[i], GL_RGBA, GL_UNSIGNED_BYTE, yuv_data_[i]);
      check_gl_error("glTexSubImage2D");
  }

  LOG_DEBUG("Success");
    // Load vertex information
    glVertexAttribPointer(position_object_, 2, GL_FLOAT, GL_FALSE, kStride, kVertexInformation);
    check_gl_error("glVertexAttribPointer");
    // Load texture information
    glVertexAttribPointer(texture_position_object_, 2, GL_SHORT, GL_FALSE, kStride, kTextureCoordinateInformation);
    check_gl_error("glVertexAttribPointer");

LOG_DEBUG("9");
    glEnableVertexAttribArray(position_object_);
    check_gl_error("glEnableVertexAttribArray");
    glEnableVertexAttribArray(texture_position_object_);
    check_gl_error("glEnableVertexAttribArray");

  // Back to window buffer
  glBindFramebuffer(GL_FRAMEBUFFER, 0);
  check_gl_error("glBindFramebuffer");
  LOG_DEBUG("Success");
    // Draw the Square
    glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_SHORT, kIndicesInformation);
    check_gl_error("glDrawElements");
}

void Renderer::setup_render_to_texture()
{
    glGenFramebuffers(1, &frame_buffer_object_);
    check_gl_error("glGenFramebuffers");
    glBindFramebuffer(GL_FRAMEBUFFER, frame_buffer_object_);
    check_gl_error("glBindFramebuffer");
    glGenRenderbuffers(1, &render_buffer_object_);
    check_gl_error("glGenRenderbuffers");
    glBindRenderbuffer(GL_RENDERBUFFER, render_buffer_object_);
    check_gl_error("glBindRenderbuffer");
    glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA4, stream_yuv_width_[0], stream_yuv_height_[0]);
    check_gl_error("glRenderbufferStorage");
    glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, render_buffer_object_);
    check_gl_error("glFramebufferRenderbuffer");
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id_[0], 0);
    check_gl_error("glFramebufferTexture2D");  
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id_[1], 0);
    check_gl_error("glFramebufferTexture2D");  
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id_[2], 0);
    check_gl_error("glFramebufferTexture2D");  

  glBindFramebuffer(GL_FRAMEBUFFER, 0);
    check_gl_error("glBindFramebuffer");

    GLint status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
    if (status != GL_FRAMEBUFFER_COMPLETE) {
        print_log("renderer.cpp", "setup_graphics", "FBO setting fault.", LOGERROR);
        LOG_ERROR("%d\n", status);
        return;
    }
}

void Renderer::setup_yuv_texture() 
{
    // Use tightly packed data
    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
    check_gl_error("glPixelStorei");

  for (int i = 0; i < 3; i++) {
    if (yuv_texture_id_[i]) {
      glDeleteTextures(1, &yuv_texture_id_[i]);
      check_gl_error("glDeleteTextures");
    }
      glActiveTexture(GL_TEXTURE0+i);
      check_gl_error("glActiveTexture"); 
      // Generate texture object
      glGenTextures(1, &yuv_texture_id_[i]);
      check_gl_error("glGenTextures");
      glBindTexture(GL_TEXTURE_2D, yuv_texture_id_[i]);
      check_gl_error("glBindTexture");
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
      check_gl_error("glTexParameteri");
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
      check_gl_error("glTexParameteri");
      glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
      check_gl_error("glTexParameterf");
      glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
      check_gl_error("glTexParameterf"); 
    glEnable(GL_TEXTURE_2D);
    check_gl_error("glEnable");
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, maximum_yuv_width_[i], maximum_yuv_height_[i], 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
      check_gl_error("glTexImage2D");
  }
}

void Renderer::setup_graphics()
{
    print_gl_string("Version", GL_VERSION);
    print_gl_string("Vendor", GL_VENDOR);
    print_gl_string("Renderer", GL_RENDERER);
    print_gl_string("Extensions", GL_EXTENSIONS);

    program_object_ = create_program(kVertexShader, kFragmentShader);
    if (!program_object_) {
        print_log("renderer.cpp", "setup_graphics", "Could not create program.", LOGERROR);
        return;
    }

    position_object_ = glGetAttribLocation(program_object_, "vPosition");
    check_gl_error("glGetAttribLocation");
    texture_position_object_ = glGetAttribLocation(program_object_, "vTexCoord");
    check_gl_error("glGetAttribLocation");

    yuv_texture_object_[0] = glGetUniformLocation(program_object_, "yTexture");
    check_gl_error("glGetUniformLocation");
  yuv_texture_object_[1] = glGetUniformLocation(program_object_, "uTexture");
    check_gl_error("glGetUniformLocation");
    yuv_texture_object_[2] = glGetUniformLocation(program_object_, "vTexture");
    check_gl_error("glGetUniformLocation");

  setup_yuv_texture();
    setup_render_to_texture();

  glViewport(0, 0, stream_yuv_width_[0], stream_yuv_height_[0]);//736, 480);//1920, 1080);//maximum_yuv_width_[0], maximum_yuv_height_[0]);
  check_gl_error("glViewport");
}

GLuint Renderer::create_program(const char* vertex_source, const char* fragment_source)
{
    GLuint vertexShader = load_shader(GL_VERTEX_SHADER, vertex_source);
    if (!vertexShader) {
        return 0;
    }

    GLuint pixelShader = load_shader(GL_FRAGMENT_SHADER, fragment_source);
    if (!pixelShader) {
        return 0;
    }

    GLuint program = glCreateProgram();
    if (program) {
        glAttachShader(program, vertexShader);
        check_gl_error("glAttachShader");
        glAttachShader(program, pixelShader);
        check_gl_error("glAttachShader");
        glLinkProgram(program);
        /* Get a Status */
        GLint linkStatus = GL_FALSE;
        glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
        if (linkStatus != GL_TRUE) {
            GLint bufLength = 0;
            glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength);
            if (bufLength) {
                char* buf = (char*) malloc(bufLength);
                if (buf) {
                    glGetProgramInfoLog(program, bufLength, NULL, buf);
                    print_log("renderer.cpp", "create_program", "Could not link program.", LOGERROR);
                    LOG_ERROR("%s\n", buf);
                    free(buf);
                }
            }
            glDeleteProgram(program);
            program = 0;
        }
    }
    return program;
}

GLuint Renderer::load_shader(GLenum shaderType, const char* pSource)
{
    GLuint shader = glCreateShader(shaderType);
        if (shader) {
            glShaderSource(shader, 1, &pSource, NULL);
            glCompileShader(shader);
            /* Get a Status */
            GLint compiled = 0;
            glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
            if (!compiled) {
                GLint infoLen = 0;
                glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);
                if (infoLen) {
                    char* buf = (char*) malloc(infoLen);
                    if (buf) {
                        glGetShaderInfoLog(shader, infoLen, NULL, buf);
                        print_log("renderer.cpp", "load_shader", "Could not link program.", LOGERROR);
                                  LOG_ERROR("%d :: %s\n", shaderType, buf);
                        free(buf);
                    }
                    glDeleteShader(shader);
                    shader = 0;
                }
            }
        }
    return shader;
}


void Renderer::onDrawFrame(JNIEnv* jenv, jbyteArray yData, jbyteArray uData, jbyteArray vData)
{
    set_draw_frame(jenv, yData, uData, vData);
    draw_frame();
    return;
}

void Renderer::setSize(int stream_width, int stream_height) {
  stream_yuv_width_[0] = stream_width;
  stream_yuv_width_[1] = stream_width/2;
  stream_yuv_width_[2] = stream_width/2;
  stream_yuv_height_[0] = stream_height;
  stream_yuv_height_[1] = stream_height/2;
  stream_yuv_height_[2] = stream_height/2;
}

void Renderer::onSurfaceChanged(int width, int height)
{
  mobile_yuv_width_[0] = width;
  mobile_yuv_width_[1] = width/2;
  mobile_yuv_width_[2] = width/2; 
  mobile_yuv_height_[0] = height;
  mobile_yuv_height_[1] = height/2;
  mobile_yuv_height_[2] = height/2;

  maximum_yuv_width_[0] = 1920;
  maximum_yuv_width_[1] = 1920/2;
  maximum_yuv_width_[2] = 1920/2;
  maximum_yuv_height_[0] = 1080;
  maximum_yuv_height_[1] = 1080/2;
  maximum_yuv_height_[2] = 1080/2;

  // If stream size not setting, default size D1
  //if (stream_yuv_width_[0] == 0) {
    stream_yuv_width_[0] = 736;
    stream_yuv_width_[1] = 736/2;
    stream_yuv_width_[2] = 736/2;
    stream_yuv_height_[0] = 480;
    stream_yuv_height_[1] = 480/2;
    stream_yuv_height_[2] = 480/2;
  //}

    setup_graphics();
    return;
}

Here is my Fragment, Vertex source and coordinates :

static const char kVertexShader[] =
    "attribute vec4 vPosition;      \n"
      "attribute vec2 vTexCoord;        \n"
      "varying vec2 v_vTexCoord;        \n"
    "void main() {                        \n"
        "gl_Position = vPosition;       \n"
        "v_vTexCoord = vTexCoord;       \n"
    "}                                          \n";

static const char kFragmentShader[] =
        "precision mediump float;               \n"
        "varying vec2 v_vTexCoord;          \n"
        "uniform sampler2D yTexture;        \n"
        "uniform sampler2D uTexture;        \n"
        "uniform sampler2D vTexture;        \n"
        "void main() {                      \n"
            "float y=texture2D(yTexture, v_vTexCoord).r;\n"
            "float u=texture2D(uTexture, v_vTexCoord).r - 0.5;\n"
            "float v=texture2D(vTexture, v_vTexCoord).r - 0.5;\n"
            "float r=y + 1.13983 * v;\n"
            "float g=y - 0.39465 * u - 0.58060 * v;\n"
            "float b=y + 2.03211 * u;\n"
            "gl_FragColor = vec4(r, g, b, 1.0);\n"
        "}\n";

static const GLfloat kVertexInformation[] =
{
         -1.0f, 1.0f,           // TexCoord 0 top left
         -1.0f,-1.0f,           // TexCoord 1 bottom left
          1.0f,-1.0f,           // TexCoord 2 bottom right
          1.0f, 1.0f            // TexCoord 3 top right
};
static const GLshort kTextureCoordinateInformation[] =
{
          0, 0,         // TexCoord 0 top left
          0, 1,         // TexCoord 1 bottom left
          1, 1,         // TexCoord 2 bottom right
          1, 0          // TexCoord 3 top right
};
static const GLuint kStride = 0;//COORDS_PER_VERTEX * 4;
static const GLshort kIndicesInformation[] =
{
    0, 1, 2, 
    0, 2, 3
};

Answer 1

You can't call glTexSubImage2D() and expect it to keep up with video frame rates. This is a very common mistake. glTexSubImage2D() and glTexImage2D() are too slow for video.

You should be using the EGL Image extensions instead. There is an example of this for Android here:

http://software.intel.com/en-us/articles/using-opengl-es-to-accelerate-apps-with-legacy-2d-guis