Use OpenGL ES2.0 to render video on GLSurfaceView, but the texture is gray

Question

I am using OpenGLES to render videos. The decoded YUV420 video frame are mapped as textures to a GLSurfaceView. The YUV420 frame is correctly decoded. Everytime there is an available frame, drawFrame() will be called and the frame is supposed to be rendered. Howerver, the video frame did not show up at all, and the texture is gray all the time. My opengl code is here:

    static const char VERTEX_SHADER[] =
            "varying vec2 interp_tc;\n"
                    "attribute vec4 in_pos;\n"
                    "attribute vec2 in_tc;\n"
                    "void main() {\n"
                    "  gl_Position = in_pos;\n"
                    "  interp_tc = in_tc;\n"
                    "}\n";

    static const char FRAGMENT_SHADER[] =
            "precision mediump float;\n"
                    "varying vec2 interp_tc;\n"
                    "uniform sampler2D y_tex;\n"
                    "uniform sampler2D u_tex;\n"
                    "uniform sampler2D v_tex;\n"
                    "void main() {\n"
                    "  float y = 1.164 * (texture2D(y_tex, interp_tc).r - 0.0625);\n"
                    "  float u = texture2D(u_tex, interp_tc).r - 0.5;\n"
                    "  float v = texture2D(v_tex, interp_tc).r - 0.5;\n"
                    "  gl_FragColor = vec4(y + 1.596 * v, "
                    "                      y - 0.391 * u - 0.813 * v, "
                    "                      y + 2.018 * u, "
                    "                      1.0);\n"
                    "}\n";


    const GLfloat TEXTURE_VERTICES[] = {
            -1.0f, 1.0f,
            -1.0f, -1.0f,
            1.0f, 1.0f,
            1.0f, -1.0f,};

    const char* TEXTURE_UNIFORMS[] = {"y_tex", "u_tex", "v_tex"};
    GLuint yuvTextures[3];

    void addShader(int type, const char* source, int program) {
        int result[1] = {GL_FALSE};
        int shader = glCreateShader(type);
        glShaderSource(shader, 1, &source, NULL);
        glCompileShader(shader);
        GLint compiled = 0;
        glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
        glAttachShader(program, shader);
        glDeleteShader(shader);
    }


    bool GLRenderer::init()
    {
        EGLBoolean returnValue;
        EGLint majorVersion;
        EGLint minorVersion;
        EGLConfig myConfig = {0};
        EGLint numConfig = 0;
        EGLint context_attribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
        EGLint s_configAttribs[] = {
                EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
                EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
                EGL_NONE };
        const EGLint RGBA8888ConfigAttr[] = {
                EGL_BUFFER_SIZE,     24,
                EGL_BLUE_SIZE,       8,
                EGL_GREEN_SIZE,      8,
                EGL_RED_SIZE,        8,
                EGL_DEPTH_SIZE,      0,
                EGL_SURFACE_TYPE,    EGL_WINDOW_BIT,
                EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
                EGL_NONE
        };

        dpy = eglGetDisplay(EGL_DEFAULT_DISPLAY);
        checkEglError("eglGetDisplay");
        if (dpy == EGL_NO_DISPLAY) {
            LOGI("eglGetDisplay returned EGL_NO_DISPLAY.\n");
            return false;
        }

        returnValue = eglInitialize(dpy, &majorVersion, &minorVersion);
        checkEglError("eglInitialize", returnValue);
        if (returnValue != EGL_TRUE) {
            LOGI("eglInitialize failed\n");
            return false;
        }

        returnValue = eglChooseConfig(dpy, RGBA8888ConfigAttr, &myConfig, 1, &numConfig);
        checkEglError("eglChooseConfig", returnValue);
        if (returnValue != EGL_TRUE || numConfig != 1) {
            LOGI("eglInitialize failed\n");
            return false;
        }

        surface = eglCreateWindowSurface(dpy, myConfig, static_cast<EGLNativeWindowType>(_window), NULL);
        checkEglError("eglCreateWindowSurface");
        if (surface == EGL_NO_SURFACE) {
            if(DEBUG) {
                memset(tmp,0,sizeof(tmp));
                sprintf(tmp,"eglCreateWindowSurface error! \n");
                LogGL(tmp);
            }
            return false;
        }

        context = eglCreateContext(dpy, myConfig, EGL_NO_CONTEXT, context_attribs);
        checkEglError("eglCreateContext");
        if (context == EGL_NO_CONTEXT) {
            if(DEBUG) {
                memset(tmp,0,sizeof(tmp));
                sprintf(tmp,"eglCreateContext error \n");
                LogGL(tmp);
            }
            return false;
        }

        returnValue = eglMakeCurrent(dpy, surface, surface, context);
        checkEglError("eglMakeCurrent", returnValue);
        if (returnValue != EGL_TRUE) {
            return false;
        }

        eglQuerySurface(dpy, surface, EGL_WIDTH, &w);
        checkEglError("eglQuerySurface");
        eglQuerySurface(dpy, surface, EGL_HEIGHT, &h);
        checkEglError("eglQuerySurface");

        if(!setupGraphics(w, h)) {
            fprintf(stderr, "Could not set up graphics.\n");
            return false;
        }

        if(!setupTextures()) {
            fprintf(stderr, "Could not set up Textures.\n");
            return false;
        }

        eglSwapInterval(dpy,0);

        return true;
    }


    bool GLRenderer::setupGraphics(int w, int h) {
        gProgram = glCreateProgram();
        if (!gProgram) {
            return false;
        }
        addShader(GL_VERTEX_SHADER, VERTEX_SHADER, gProgram);
        addShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER, gProgram);

        glLinkProgram(gProgram);
        GLint linkStatus = GL_FALSE;
        glGetProgramiv(gProgram, GL_LINK_STATUS, &linkStatus);
        if (linkStatus != GL_TRUE) {
            if(DEBUG) {
                memset(tmp,0,sizeof(tmp));
                sprintf(tmp,"glGetProgramiv error \n");
                LogGL(tmp);
            }
        }
        glUseProgram(gProgram);
        gvPositionHandle = glGetAttribLocation(gProgram, "in_pos");
        glEnableVertexAttribArray(gvPositionHandle);
        glVertexAttribPointer(
                gvPositionHandle, 2, GL_FLOAT, false, 0, TEXTURE_VERTICES);
        gYuvTexSamplerHandle = glGetAttribLocation(gProgram, "in_tc");
        glEnableVertexAttribArray(gYuvTexSamplerHandle);

        glViewport(0, 0, w, h);
        checkGlError("glViewport");

        return true;
    }

    bool GLRenderer::setupTextures() {
        glGenTextures(3, yuvTextures);
        for (int i = 0; i < 3; i++)  {
            glActiveTexture(GL_TEXTURE0 + i);
            glUniform1i(glGetUniformLocation(gProgram, TEXTURE_UNIFORMS[i]), i);
            glBindTexture(GL_TEXTURE_2D, yuvTextures[i]);
            glTexParameterf(GL_TEXTURE_2D,
                            GL_TEXTURE_MIN_FILTER, GL_LINEAR);
            glTexParameterf(GL_TEXTURE_2D,
                            GL_TEXTURE_MAG_FILTER, GL_LINEAR);
            glTexParameterf(GL_TEXTURE_2D,
                            GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
            glTexParameterf(GL_TEXTURE_2D,
                            GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        }
        //checkNoGLES2Error();
        return true;
    }

    void GLRenderer::drawFrame(int YStride, int UVStride, int iWidth, int iHeight, const char* YData, const char* UData, const char* VData) {
        glUseProgram(gProgram);
        glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
        glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT);
        glDisable(GL_DEPTH_TEST);
        glUseProgram(gProgram);

        int yuvStrides[3];
        yuvStrides[0] = YStride;
        yuvStrides[1] = UVStride;
        yuvStrides[2] = UVStride;

        float crop = (float) iWidth / yuvStrides[0];
        GLfloat textureCoords[] = {
                0.0f, 0.0f,
                0.0f, 1.0f,
                crop, 0.0f,
                crop, 1.0f,};

        glVertexAttribPointer(
                gYuvTexSamplerHandle, 2, GL_FLOAT, false, 0, textureCoords);
        glVertexAttribPointer(
                gvPositionHandle, 2, GL_FLOAT, false, 0, TEXTURE_VERTICES);
        glEnableVertexAttribArray(gvPositionHandle);
        glEnableVertexAttribArray(gYuvTexSamplerHandle);

        eglQuerySurface(dpy, surface, EGL_WIDTH, &w);
        checkEglError("eglQuerySurface");
        eglQuerySurface(dpy, surface, EGL_HEIGHT, &h);
        checkEglError("eglQuerySurface");
        glViewport(0, 0, w, h);

        char *yuvPlanes[3];

        yuvPlanes[0] = (char *)YData;
        yuvPlanes[1] = (char *)UData;
        yuvPlanes[2] = (char *)VData;
        for (int i = 0; i < 3; i++) {
            int h = (i == 0) ? iHeight : (iHeight + 1) / 2;
            glActiveTexture(GL_TEXTURE0 + i);
            glBindTexture(GL_TEXTURE_2D, yuvTextures[i]);
            glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
            glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, yuvStrides[i],
                         h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuvPlanes[i]);
        }

        glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);

        eglSwapBuffers(dpy, surface);
        checkEglError("eglSwapBuffers");
    }

Answer 1

If you are rendering a SurfaceTexture texture, you should use the GL_TEXTURE_EXTERNAL_OES texture target and samplerExternalOES sampler.

As written here: SurfaceTexture :

The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the GL_OES_EGL_image_external OpenGL ES extension. This limits how the texture may be used. Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture must declare its use of this extension using, for example, an "#extension GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using the samplerExternalOES GLSL sampler type.

One more thing - you should also use GL_TEXTURE_EXTERNAL_OES for glTexParameterf .