OpenGL draw (color) things oddly, fragment shader messes things up?

Question

I wrote a small program to simply load a Wavefront OBJ mesh file and display the geometry with proper lighting (Phong shading), however the rendering seems problematic, it's weird. When i simply change my fragment shader to color things evenly (without lighting) it's OK, but whenever i push some norm calculations into my code, problems start to appear.

I'm almost assure that the problem comes from fragment shader and/or normals, as the geometry coordinates are consistent (turned to GL_POINT, points were properly positioned), only the coloring doesn't work properly.

I'll put my code here; also i'll upload the result of rendering. i'll be grateful to anybody who could figure out what's the problem.

Also here's the OBJ file which is a Skyrim helmet!: helmet.obj

Note: I use math32 (included in GLTools) library for perspective and other matrix calcs, which is available here: GLTools.zip

Render function:

void render(HDC hdc)
{
    //Some matrix calculations start

    unsigned int tTime=GetTickCount();
    float angle=(tLastTime!=0)?(float(GetTickCount()-tLastTime)/(1.0f/speed)):(0);
    if(tLastTime==0) tLastTime=GetTickCount();

    float projection_matrix[16],rotation_matrix[16],MV_matrix[16],temp_matrix[16];

    m3dRotationMatrix44(rotation_matrix,angle,2.0f,4.0f,2.0f);
    Rotate(MV_matrix,angle,2.0f,4.0f,2.0f,CenterX,CenterY,CenterZ);

    m3dTranslationMatrix44(temp_matrix,-CenterX,-CenterY,(-CenterZ)-40.0f);

    Multiply(temp_matrix,MV_matrix);

    PerspectiveMatrix(projection_matrix,60.0f,1.0f,0.0001f,1000.0f);

    //Some matrix calculations end

    glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT|GL_STENCIL_BUFFER_BIT);

    glUseProgramObjectARB(pShaderProgram);

    glUniformMatrix4fvARB(uniMVMatrix,1,false,MV_matrix);
    glUniformMatrix4fvARB(uniProjectionMatrix,1,false,projection_matrix);
    glUniformMatrix4fvARB(uniRotationMatrix,1,false,rotation_matrix);
    glUniform3fvARB(uniLightPosition,1,light_position);

    glBindVertexArray(objVertexArray);

    glDrawArrays(GL_TRIANGLES,0,nVertices);

    glBindVertexArray(0);

    SwapBuffers(hdc);

    return;
}

main function (context creation/initialization):

int WINAPI WinMain(HINSTANCE hInstance,HINSTANCE hPrevInstance,LPSTR lpCmdLine,int nShowCmd)
{
    WNDCLASSEX wndcls;

    wndcls.cbSize=sizeof(WNDCLASSEX);
    wndcls.cbClsExtra=0;
    wndcls.cbWndExtra=0;
    wndcls.hbrBackground=(HBRUSH)(COLOR_WINDOW+1);
    wndcls.style=CS_HREDRAW|CS_VREDRAW|CS_OWNDC;
    wndcls.hCursor=(HCURSOR)LoadCursor(NULL,IDC_ARROW);
    wndcls.hIcon=LoadIcon(hInstance,IDI_APPLICATION);
    wndcls.hIconSm=NULL;
    wndcls.hInstance=hInstance;
    wndcls.lpfnWndProc=(WNDPROC)WndProc;
    wndcls.lpszClassName="Win32Class";
    wndcls.lpszMenuName=NULL;

    RegisterClassEx(&wndcls);
    HWND hwnd=CreateWindowEx(0,"Win32Class","OpenGL Projekt",WS_OVERLAPPEDWINDOW,CW_USEDEFAULT,CW_USEDEFAULT,1280,720,NULL,NULL,hInstance,NULL);

    HDC hdc=GetDC(hwnd);

    PIXELFORMATDESCRIPTOR pfd=
    {
        sizeof(PIXELFORMATDESCRIPTOR),
        1,
        PFD_DRAW_TO_WINDOW|PFD_SUPPORT_OPENGL|PFD_DOUBLEBUFFER,
        PFD_TYPE_RGBA,
        32,
        0,0,0,0,0,0,
        0,
        0,
        0,
        0,0,0,0,
        24,
        8,
        0,
        PFD_MAIN_PLANE,
        0,
        0,0,0
    };

    int pf=ChoosePixelFormat(hdc,&pfd);

    SetPixelFormat(hdc,pf,&pfd);
    HGLRC hglrc=wglCreateContext(hdc);
    wglMakeCurrent(hdc,hglrc);

    setup_extensions(); // using wglGetProcAddress

    wglMakeCurrent(hdc,NULL);
    DestroyWindow(hwnd); // that was just for getting function pointers!


    //Real window/context:
    hwnd=CreateWindowEx(0,"Win32Class","OpenGL Projekt",WS_OVERLAPPED|WS_SYSMENU|WS_MINIMIZEBOX|WS_CAPTION,CW_USEDEFAULT,CW_USEDEFAULT,700,722,NULL,NULL,hInstance,NULL);
    hdc=GetDC(hwnd);

    const int attribs[]={
        WGL_DRAW_TO_WINDOW_ARB,1,
        WGL_SUPPORT_OPENGL_ARB,1,
        WGL_DOUBLE_BUFFER_ARB,1,
        WGL_PIXEL_TYPE_ARB,WGL_TYPE_RGBA_ARB,
        WGL_COLOR_BITS_ARB,32,
        WGL_DEPTH_BITS_ARB,24,
        WGL_STENCIL_BITS_ARB,8,
        WGL_SAMPLE_BUFFERS_ARB,1, // MSAA enabled
        WGL_SAMPLES_ARB,8, // MSAA 8x
        0};
    int wpf[3];
    unsigned int nwpf;

    wglChoosePixelFormatARB(hdc,attribs,NULL,3,wpf,&nwpf);
    SetPixelFormat(hdc,wpf[0],&pfd);

    const int attribs_cc[]={
        WGL_CONTEXT_MAJOR_VERSION_ARB,3,
        WGL_CONTEXT_MINOR_VERSION_ARB,3,
        WGL_CONTEXT_PROFILE_MASK_ARB,WGL_CONTEXT_CORE_PROFILE_BIT_ARB,
        0
    };

    hglrc=wglCreateContextAttribsARB(hdc,NULL,attribs_cc);
    wglMakeContextCurrentARB(hdc,hdc,hglrc);

    setup();

    ShowWindow(hwnd,nShowCmd);

    MSG msg;
    bool done=false;

    while(!done)
    {
        if(PeekMessage(&msg,NULL,0,0,PM_REMOVE))
            if(msg.message==WM_QUIT)
                done=true;
            else
            {
                TranslateMessage(&msg);
                DispatchMessage(&msg);
            }
        else
            render(hdc);
    }

    return (int)msg.wParam;
}

Setup:

void setup()
{
    glClearColor(0.11,0.11,0.11,1.0);
    glEnable(GL_MULTISAMPLE_ARB);
    glEnable(GL_DEPTH_TEST);
    glDisable(GL_BLEND);

    glPolygonMode(GL_FRONT_AND_BACK,GL_FILL);

    //Shader loading start

    FILE *f;
    int fsize,status;
    char *shadersrc[1];
    unsigned int shVertexShader=0,shFragmentShader=0;

    shVertexShader=glCreateShaderObjectARB(GL_VERTEX_SHADER_ARB);
    shFragmentShader=glCreateShaderObjectARB(GL_FRAGMENT_SHADER_ARB);

    f=fopen("VShader","rb");
    fseek(f,0,SEEK_END);
    fsize=ftell(f);
    fseek(f,0,SEEK_SET);
    shadersrc[0]=new char[fsize];
    fread(shadersrc[0],sizeof(char),fsize,f);

    glShaderSourceARB(shVertexShader,1,(const char**)shadersrc,&fsize);

    free(shadersrc[0]);
    fclose(f);

    f=fopen("FShader","rb");
    fseek(f,0,SEEK_END);
    fsize=ftell(f);
    fseek(f,0,SEEK_SET);
    shadersrc[0]=new char[fsize];
    fread(shadersrc[0],sizeof(char),fsize,f);

    glShaderSourceARB(shFragmentShader,1,(const char**)shadersrc,&fsize);

    free(shadersrc[0]);
    fclose(f);

    glCompileShaderARB(shVertexShader);

    glGetObjectParameterivARB(shVertexShader,GL_OBJECT_COMPILE_STATUS_ARB,&status);
    if(!status)
    {
        char errinfo[512];

        glGetInfoLogARB(shVertexShader,511,NULL,errinfo);

        MessageBox(NULL,errinfo,"Error",MB_OK);

        exit(EXIT_FAILURE);
    }

    glCompileShaderARB(shFragmentShader);

    glGetObjectParameterivARB(shFragmentShader,GL_OBJECT_COMPILE_STATUS_ARB,&status);
    if(!status)
    {
        char errinfo[512];

        glGetInfoLogARB(shFragmentShader,511,NULL,errinfo);

        MessageBox(NULL,errinfo,"Error",MB_OK);

        exit(EXIT_FAILURE);
    }

    pShaderProgram=glCreateProgramObjectARB();

    glAttachObjectARB(pShaderProgram,shVertexShader);
    glAttachObjectARB(pShaderProgram,shFragmentShader);

    glBindAttribLocationARB(pShaderProgram,0,"vertices");
    glBindAttribLocationARB(pShaderProgram,1,"normals");

    glLinkProgramARB(pShaderProgram);

    glGetObjectParameterivARB(pShaderProgram,GL_OBJECT_LINK_STATUS_ARB,&status);
    if(!status)
    {
        char errinfo[512];

        glGetInfoLogARB(pShaderProgram,511,NULL,errinfo);

        MessageBox(NULL,errinfo,"Error",MB_OK);

        exit(EXIT_FAILURE);
    }

    glDeleteObjectARB(shVertexShader);
    glDeleteObjectARB(shFragmentShader);

    uniMVMatrix=glGetUniformLocationARB(pShaderProgram,"MV_matrix");
    uniProjectionMatrix=glGetUniformLocationARB(pShaderProgram,"projection_matrix");
    uniRotationMatrix=glGetUniformLocationARB(pShaderProgram,"rotation_matrix");
    uniLightPosition=glGetUniformLocationARB(pShaderProgram,"light_position");

    //Shader loading end

    //Loading from OBJ file start

    if(!LoadOBJFile(OBJFile,&vertices,&normals,&nVertices))
    {
        char msg[700];
        sprintf(msg,"Cannot open file \"%s\".",OBJFile);
        MessageBox(NULL,msg,"Error",MB_OK);
        exit(EXIT_FAILURE);
    }

    //Loading from OBJ file end

    GetCenter(vertices,nVertices,&CenterX,&CenterY,&CenterZ);

    light_position[0]=0.0f;
    light_position[0]=0.0f;
    light_position[0]=4000.0f;

    glGenVertexArrays(1,&objVertexArray);
    glBindVertexArray(objVertexArray);

    glGenBuffers(1,&buffVertexArray);
    glBindBuffer(GL_ARRAY_BUFFER,buffVertexArray);
    glBufferData(GL_ARRAY_BUFFER,sizeof(float)*3*nVertices,vertices,GL_DYNAMIC_DRAW);
    glGenBuffers(1,&buffNormalArray);
    glBindBuffer(GL_ARRAY_BUFFER,buffNormalArray);
    glBufferData(GL_ARRAY_BUFFER,sizeof(float)*3*nVertices,normals,GL_DYNAMIC_DRAW);

    glEnableVertexAttribArrayARB(0);
    glEnableVertexAttribArrayARB(1);

    glBindBuffer(GL_ARRAY_BUFFER,buffVertexArray);
    glVertexAttribPointerARB(0,3,GL_FLOAT,false,0,NULL);
    glBindBuffer(GL_ARRAY_BUFFER,buffNormalArray);
    glVertexAttribPointerARB(1,3,GL_FLOAT,false,0,NULL);

    glBindVertexArray(0);

    return;
}

Vertex shader:

#version 330 core

uniform mat4 MV_matrix;
uniform mat4 projection_matrix;
uniform mat4 rotation_matrix;

in vec3 vertices;
in vec3 normals;

smooth out vec3 f_vertices;
smooth out vec3 f_normals;

void main(void)
{
    gl_Position=projection_matrix*MV_matrix*vec4(vertices,1.0);

    f_vertices=(MV_matrix*vec4(vertices,1.0)).xyz;

    f_normals=(rotation_matrix*vec4(normals,1.0)).xyz;

    return;
}

Fragment shader:

#version 330 core

uniform vec3 light_position;

smooth in vec3 f_vertices;
smooth in vec3 f_normals;

out vec4 fragcolor;

void main(void)
{
    vec3 to_light_dir=light_position-f_vertices;

    float diff=min(1.0,(dot(normalize(f_normals),normalize(to_light_dir))));

    fragcolor=diff*vec4(0.5,0.5,0.5,1.0);
}

OBJ file loader:

#include <string>
#include <fstream>
#include <vector>
#include "math3d.h"

struct vertex
{
    float x;
    float y;
    float z;
    float w;
};

struct normal
{
    float x;
    float y;
    float z;
};

struct texcoord
{
    float u;
    float v;
    float w;
};

struct face
{
    unsigned int vertices[3];
    unsigned int normals[3];
    unsigned int texcoords[3];
};

using namespace std;

bool LoadOBJFile(const char *path,float **vertices,float **normals,unsigned int *num_vertices)
{
    char id[1024];
    float fa[9];
    int nc;
    vertex tvertex;
    normal tnormal;
    texcoord ttexcoord;
    face tface;
    bool gotnormal=false;

    vector<vertex> _v;
    vector<normal> _vn;
    vector<texcoord> _vt;
    vector<face> _f;

    ifstream f(path);

    if(!f.is_open()) return false;

    for(string line;getline(f,line);)
    {
        if(line.c_str()[0]=='#' || line.c_str()[0]=='\x0A' || line.c_str()[0]=='\x0D' || line.size()==0) continue;

        sscanf(line.c_str(),"%s",id);

        if(strcmp(id,"v")==0)
        {
            if(sscanf(line.c_str(),"%s %f %f %f %f",id,fa,fa+1,fa+2,fa+3)==5)
            {
                tvertex.x=fa[0];
                tvertex.y=fa[1];
                tvertex.z=fa[2];
                tvertex.w=fa[3];
            }else if(sscanf(line.c_str(),"%s %f %f %f",id,fa,fa+1,fa+2)==4)
            {
                tvertex.x=fa[0];
                tvertex.y=fa[1];
                tvertex.z=fa[2];
                tvertex.w=1.0f;
            }else
            {
                return false;
            }
            _v.push_back(tvertex);
        }else if(strcmp(id,"vn")==0)
        {
            if(sscanf(line.c_str(),"%s %f %f %f",id,fa,fa+1,fa+2)==4)
            {
                tnormal.x=fa[0];
                tnormal.y=fa[1];
                tnormal.z=fa[2];
            }else
            {
                return false;
            }
            _vn.push_back(tnormal);
        }else if(strcmp(id,"vn")==0)
        {
            if(sscanf(line.c_str(),"%s %f %f %f",id,fa,fa+1,fa+2)==4)
            {
                ttexcoord.u=fa[0];
                ttexcoord.v=fa[1];
                ttexcoord.w=fa[2];
            }else if(sscanf(line.c_str(),"%s %f %f",id,fa,fa+1)==3)
            {
                ttexcoord.u=fa[0];
                ttexcoord.v=fa[1];
                ttexcoord.w=0.0f;
            }else if(sscanf(line.c_str(),"%s %f %f %f",id,fa)==2)
            {
                ttexcoord.u=fa[0];
                ttexcoord.v=0.0f;
                ttexcoord.w=0.0f;
            }else
            {
                return false;
            }
            _vt.push_back(ttexcoord);
        }else if(strcmp(id,"f")==0)
        {
            if(sscanf(line.c_str(),"%s %f/%f/%f %f/%f/%f %f/%f/%f",id,fa,fa+1,fa+2,fa+3,fa+4,fa+5,fa+6,fa+7,fa+8)==10)
            {
                for(int j=0;j<3;j++)
                {
                    tface.vertices[j]=fa[j*3];
                    tface.normals[j]=fa[j*3+2];
                    tface.texcoords[j]=fa[j*3+1];
                }
            }else if(sscanf(line.c_str(),"%s %f//%f %f//%f %f//%f",id,fa,fa+2,fa+3,fa+5,fa+6,fa+8)==7)
            {
                for(int j=0;j<3;j++)
                {
                    tface.vertices[j]=fa[j*3];
                    tface.normals[j]=fa[j*3+2];
                    tface.texcoords[j]=0;
                }
            }else if(sscanf(line.c_str(),"%s %f/%f %f/%f %f/%f",id,fa,fa+1,fa+3,fa+4,fa+6,fa+7)==7)
            {
                for(int j=0;j<3;j++)
                {
                    tface.vertices[j]=fa[j*3];
                    tface.normals[j]=0;
                    tface.texcoords[j]=fa[j*3+1];
                }
            }else if(sscanf(line.c_str(),"%s %f %f %f",id,fa,fa+3,fa+6)==4)
            {
                for(int j=0;j<3;j++)
                {
                    tface.vertices[j]=fa[j*3];
                    tface.normals[j]=0;
                    tface.texcoords[j]=0;
                }
            }else
            {
                return false;
            }
            _f.push_back(tface);
        }
    }

    *num_vertices=3*(_f.size());

    *vertices=new float[(*num_vertices)*3];
    *normals=new float[(*num_vertices)*3];

    if(_vn.size() && _f[0].normals[0])
        gotnormal=true;

    for(unsigned int i=0;i<(*num_vertices)*3;i+=3)
    {
        int iFc=int(float(i)/9.0f);
        int iVr=(i/3)%3;

        (*vertices)[i]=_v[_f[iFc].vertices[iVr]-1].x;
        (*vertices)[i+1]=_v[_f[iFc].vertices[iVr]-1].y;
        (*vertices)[i+2]=_v[_f[iFc].vertices[iVr]-1].z;

        if(gotnormal)
        {
            (*normals)[i]=_vn[_f[iFc].normals[iVr]-1].x;
            (*normals)[i+1]=_vn[_f[iFc].normals[iVr]-1].y;
            (*normals)[i+2]=_vn[_f[iFc].normals[iVr]-1].z;
        }
    }

    for(unsigned int i=0;i<(*num_vertices)*3 && !gotnormal;i+=9)
    {
        M3DVector3f norm;

        m3dFindNormal(norm,(*vertices)+i,(*vertices)+(i+3),(*vertices)+(i+6));

        for(unsigned short j=0;j<9;j+=3)
        {
            (*normals)[i+j]=norm[0];
            (*normals)[i+j+1]=norm[1];
            (*normals)[i+j+2]=norm[2];
        }
    }

    _v.~vector();
    _f.~vector();
    _vt.~vector();
    _vn.~vector();

    return true;
}

Result without lighting (diff=1.0):

Result with lighting (animated version below):

Animated version: 1.avi (5.94MB)

Answer 1

In your fragment shader, the dot product of two normalized vectors can't be greater than 1, so clamping it to be not greater than 1 is not necessary:

float diff=min(1.0,(dot(normalize(f_normals),normalize(to_light_dir))));

Instead you can clamp it to be not less than 0:

float diff=max(0.0,(dot(normalize(f_normals),normalize(to_light_dir))));

Anyway, I think that something else causes the problem. It doesn't look like wrong normals, it may be rather some z-fighting. Try setting your near and far planes closer to each other. Or maybe your geometry is to thin? Have you enabled back-face culling? It may solve the problem as well. You should also double check your OBJ loader to make sure it's 100% correct, especially at reading normals.