Even though the shadow map is rendered correctly, shadow calculations are incorrect

Question

I am trying to render basic shadows produced by a single directional light source in Vulkan.

The problem is I am getting results that don't make sense (at least to me).

Shadow Map Output

This is the result produced by the depth pass rendered onto a quad that resides in world space

The depth values seem to be correct. As the lower right parts of the geometry are tinted to a darker color as expected because they're closer to the light view. No precision or clipping issues appear to happen either.

The shader code used to render the debug quad is as follows:

# VERT
#version 450 core

layout(location = 0) in vec3 a_Position;
layout(location = 1) in vec2 a_UV;
layout(location = 2) in vec3 a_Normal;
layout(location = 3) in vec3 a_Tangent;
layout(location = 4) in vec3 a_Bitangent;

layout(location = 0) out vec2 v_UV;

layout(binding = 0) uniform ModelMat
{
    mat4 Matrix;
} Model;

layout(binding = 1) uniform ProjMatrix
{
    mat4 Matrix;
} Proj;

layout(binding = 2) uniform viewMatrix
{
    mat4 Matrix;
} View;

void main()
{
    v_UV = a_UV;
    gl_Position = Proj.Matrix * View.Matrix * Model.Matrix * vec4(a_Position, 1.0);
}

# FRAGMENT
#version 450 core

layout(location = 0) out vec4 FragColor;
layout(location = 0) in vec2 v_UV;

layout(binding = 3) uniform sampler2D depthMap;

void main()
{
    float depthValue = texture(depthMap, v_UV).r;
    //FragColor = vec4(vec3(depthValue), 1.0);
    if(depthValue < 1.0)
    {
        depthValue = (depthValue / 1.0) * 0.2;
    }
    FragColor =  vec4(vec3(depthValue, 0.0, 0.0), 1.0);
    
}

A few things to note here:

1. Attributes other than "a_Position" and "a_UV" are not being used but I am passing them to the shader anyway. Will do a cleanup later. I am trying to make the shadows work first.

2. I am not storing the MVP matrix inside a single uniform buffer, but rather, pass them individually. Because I don't want to pass matrices that don't change every frame over and over again.

Rendering the closest depth value of the shadow map onto the geometry

I've already tried a few debugging techniques I've learned from other discussions.

I rendered the sampled values from the shadow map onto the model and this provided me with the following results:

Model .

From the opposite angle.

Close up from inside the model.

• Projection and View matrices:

float near_plane = 20.0f;
float far_plane = 60.0f;
glm::mat4 lightView = glm::lookAt(directionalLightPosition, glm::vec3(0.0f), glm::vec3(0.0f, 1.0f, 0.0f));
glm::mat4 lightProjectionMatrix = glm::ortho(-40.0f, 40.0f, -40.0f, 40.0f, near_plane, far_plane);

• Directional light pos:

glm::vec3 directionalLightPosition = glm::vec3(30.0f, 30.0f, 30.0f);

• Fragment Shader (with the closest depth value being output):

#version 430 core

// INs
layout(location = 0) in vec3 v_Pos;
layout(location = 1) in vec2 v_UV;
layout(location = 2) in vec3 v_Normal;
layout(location = 3) in vec4 v_FragPosLightSpace;
layout(location = 4) in vec3 v_DirLightPos;
layout(location = 5) in mat3 v_TBN;



layout(binding = 5) uniform CameraPosition
{
    vec3 pos;
} camPos;

layout(binding = 6) uniform PointLightPosition
{
    vec3 pos;
} pointLightPos;

layout(binding = 7) uniform sampler2D u_DiffuseSampler;
layout(binding = 8) uniform sampler2D u_NormalSampler;
layout(binding = 9) uniform sampler2D u_RoughnessMetallicSampler;
layout(binding = 10) uniform sampler2D u_DirectionalShadowMap;


layout(location = 0) out vec4 FragColor;
void main()
{
 // BLINN PHONG
    vec3 N = texture(u_NormalSampler, v_UV).rgb;
    N = N * 2.0 - 1.0;
    N = normalize(v_TBN * N);
    
    vec3 color = texture(u_DiffuseSampler, v_UV).rgb;
    vec3 normal = N;
    vec3 lightColor = vec3(0.3);
    // ambient
    vec3 ambient = 0.05 * lightColor;
    
    // diffuse
    vec3 lightDir = normalize(v_DirLightPos - v_Pos);
    float diff = max(dot(lightDir, normal), 0.0);
    vec3 diffuse = diff * lightColor;
    // specular
    vec3 viewDir = normalize(camPos.pos - v_Pos);
    vec3 reflectDir = reflect(-lightDir, normal);
    float spec = 0.0;
    vec3 halfwayDir = normalize(lightDir + viewDir);  
    spec = pow(max(dot(normal, halfwayDir), 0.0), 64.0);
    vec3 specular = spec * lightColor;    
    // calculate shadow
    

    //SHADOW
    //vec3 projCoords = v_FragPosLightSpace.xyz / v_FragPosLightSpace.w;
    vec3 projCoords = v_FragPosLightSpace.xyz;
    vec3 altCoords = projCoords * 0.5 + 0.5;
    float closestDepth = texture(u_DirectionalShadowMap, altCoords.xy).r; 
    float bias = 0.005;
    float currentDepth = projCoords.z;
    float shadow = (currentDepth - bias) > closestDepth  ? 1.0 : 0.0;


    vec3 lighting = (ambient + (1.0 - shadow) * (diffuse + specular)) * color;    
    
    FragColor = vec4(vec3(closestDepth), 1.0);
}

: Note: Uncommenting the line //vec3 projCoords = v_FragPosLightSpace.xyz / v_FragPosLightSpace.w; and commenting out the line vec3 projCoords = v_FragPosLightSpace.xyz; makes the model go completely white (1.0f depth).

• Vert Shader:

#version 450

// INs
layout(location = 0) in vec3 a_Position;
layout(location = 1) in vec2 a_UV;
layout(location = 2) in vec3 a_Normal;
layout(location = 3) in vec3 a_Tangent;
layout(location = 4) in vec3 a_Bitangent;

//OUTs
layout(location = 0) out vec3 v_Pos;
layout(location = 1) out vec2 v_UV;
layout(location = 2) out vec3 v_Normal;
layout(location = 3) out vec4 v_FragPosLightSpace;
layout(location = 4) out vec3 v_DirLightPos;
layout(location = 5) out mat3 v_TBN;

layout(binding = 0) uniform ModelMatrix
{
    mat4 ModelMat;
} Model;

layout(binding = 1) uniform ViewMatrix
{
    mat4 ViewMat;
} View;

layout(binding = 2) uniform ProjectionMatrix
{
    mat4 ProjMat;
} Proj;

layout(binding = 3) uniform DirectionalLightPosition
{
    vec3 pos;
} dirLightPos;

layout(binding = 4) uniform light
{
    mat4 Matrix;
} lightViewProj;


void main()
{

    v_UV        = a_UV;
    v_Pos       = vec3(Model.ModelMat * vec4(a_Position, 1.0));
    v_Normal    = mat3(Model.ModelMat) * a_Normal;   

    mat3 normalMatrix = transpose(mat3(Model.ModelMat));

    vec3 T      = normalize(normalMatrix * a_Tangent);
    vec3 N      = normalize(normalMatrix * a_Normal);
    vec3 B      = normalize(normalMatrix * a_Bitangent);

    T           = normalize(T - dot(T, N) * N);

    v_TBN       = mat3(T, B, N);
    gl_Position = Proj.ProjMat * View.ViewMat * Model.ModelMat * vec4(a_Position, 1.0);

    v_FragPosLightSpace = (lightViewProj.Matrix * Model.ModelMat) * vec4(v_Pos, 1.0);

    
    v_DirLightPos = dirLightPos.pos;

My thought process here:

• I expect the colors on the geometry to resemble the shadow map. But for some reason half of the image abruptly turns into darker colors. This phenomenon can better be seen in the third image where the camera is placed inside the hallway of the model. You can see the exact point where the shadow map breaks and draws a line cutting the model in the middle.

• The interesting part is, half of the image looks to have the correct values.

Scene with normal lighting calculations

Another shot from a different angle

Other places to look for the culprit

• I am a little suspicious about my render pass. Here is the code:

VkAttachmentDescription depthAttachmentDescription;
        VkAttachmentReference depthAttachmentRef;

        depthAttachmentDescription.format = VK_FORMAT_D16_UNORM;
        depthAttachmentDescription.samples = VK_SAMPLE_COUNT_1_BIT;
        depthAttachmentDescription.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
        depthAttachmentDescription.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
        depthAttachmentDescription.flags = 0;
        depthAttachmentDescription.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
        depthAttachmentDescription.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
        depthAttachmentDescription.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
        depthAttachmentDescription.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL;

        depthAttachmentRef.attachment = 0;
        depthAttachmentRef.layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;

        VkSubpassDescription subpass{};
        subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
        subpass.colorAttachmentCount = 0;
        subpass.pColorAttachments = 0;
        subpass.pDepthStencilAttachment = &depthAttachmentRef;

        std::array<VkSubpassDependency, 2> dependencies;

        dependencies[0].srcSubpass = VK_SUBPASS_EXTERNAL;
        dependencies[0].dstSubpass = 0;
        dependencies[0].srcStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
        dependencies[0].dstStageMask = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT;
        dependencies[0].srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
        dependencies[0].dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
        dependencies[0].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;

        dependencies[1].srcSubpass = 0;
        dependencies[1].dstSubpass = VK_SUBPASS_EXTERNAL;
        dependencies[1].srcStageMask = VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
        dependencies[1].dstStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
        dependencies[1].srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
        dependencies[1].dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
        dependencies[1].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;

        VkRenderPassCreateInfo renderPassInfo{};
        renderPassInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO;
        renderPassInfo.attachmentCount = 1;
        renderPassInfo.pAttachments = &depthAttachmentDescription;
        renderPassInfo.subpassCount = 1;
        renderPassInfo.pSubpasses = &subpass;
        renderPassInfo.dependencyCount = static_cast<uint32_t>(dependencies.size());
        renderPassInfo.pDependencies = dependencies.data();

        VkRenderPass renderPass;

        ASSERT(vkCreateRenderPass(VulkanApplication::GetVKDevice(), &renderPassInfo, nullptr, &renderPass) == VK_SUCCESS, "Failed to create a render pass.");

• It looks to me either as if the shadow map is sampled with incorrect coordinates or a small Vulkan error like choosing the format of the shadow map VK_FORMAT_D16_UNORM ?

• Could the shadow map be flipped due to Vulkan? Sampling it with inverted UVs would help? Even though I've already tried this but maybe I am doing something wrong.

Any pointers are appreciated. And go easy on me I am very new at this. Thank you!

EDIT

I managed to debug the situation a little bit, things are looking quite better now. But Shadows still don't work.

What I've changed:

• This line v_FragPosLightSpace = (lightViewProj.Matrix * Model.ModelMat) * vec4(v_Pos, 1.0); in my vertex shader was incorrect. I removed the extra * Model.ModelMat multiplication.

• The pipeline I am using to render the shadow map was interpreting the shadow map incorrectly. Specifically the viewport part:

VkViewport viewport{};
viewport.x = 0.0f;
viewport.y = (float)m_CI.ViewportHeight;
viewport.width = (float)m_CI.ViewportWidth;
viewport.height = -(float)m_CI.ViewportHeight;
viewport.minDepth = 0.0f;
viewport.maxDepth = 1.0f;

For some reason, I tried to flip the the viewport and render the shadow pass like that. The above code has been converted to the following one:

VkViewport viewport{};
viewport.x = 0.0f;
viewport.y = 0.0f;
viewport.width = (float)m_CI.ViewportWidth;
viewport.height = (float)m_CI.ViewportHeight;
viewport.minDepth = 0.0f;
viewport.maxDepth = 1.0f;

I am now flipping the depth map with a matrix multiplication while setting up the projection matrix:

glm::mat4 clip = glm::mat4(
1.0f, 0.0f, 0.0f, 0.0f,
0.0f, -1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.5f, 0.0f,
0.0f, 0.0f, 0.5f, 1.0f);

glm::mat4 lightProjectionMatrix = clip * glm::ortho(-20.0f, 20.0f, -20.0f, 20.0f, near_plane, far_plane);

This takes care of the orientation / upside-down problems of the shadow map.

Answer 1

I've managed to make the shadows work. I am going to post an answer here in case anyone does the same mistakes as me. Vulkan is a different beast and the community could use all the information they can get.

So here it goes:

Mistake / Change number 1) I don't know how much of an effect this made but I've changed the projection matrix from orthographic to perspective. The whole setup for the MVP is as follows:

glm::vec3 directionalLightPosition = glm::vec3(200.0f, 000.0f, 0.0f);
float near_plane = 50.0f;
float far_plane = 1000.0f;


glm::mat4 lightView = glm::lookAt(directionalLightPosition, glm::vec3(0.0f), glm::vec3(0.0f, 1.0f, 0.0f));
glm::mat4 lightProjectionMatrix = glm::perspective(glm::radians(45.0f), 1.0f, near_plane, far_plane);
glm::mat4 lightModel = glm::mat4(1.0f);
glm::mat4 depthMVP;
depthMVP = lightProjectionMatrix * lightView * lightModel;

Mistake number 2) This is a Vulkan related one. I was passing incorrect data to my vertex shader. I had the following layout in my vert shader:

layout(binding = 0) uniform ModelMatrix
{
    mat4 ModelMat;
} Model;

layout(binding = 1) uniform ViewMatrix
{
    mat4 ViewMat;
} View;

layout(binding = 2) uniform ProjectionMatrix
{
    mat4 ProjMat;
} Proj;

layout(binding = 3) uniform DirectionalLightPosition
{
    vec3 pos;
} dirLightPos;

layout(binding = 4) uniform light
{
    mat4 Matrix;
} lightViewProj;

Bindings number 3 and 4 were swapped for some reason in the host. I was trying to pass the data of binding 3 to 4 and vice versa. This was obviously a devastating error. It was a miracle the program had given me some images that resembled a depth map which I shared in my question.

Mistake / Change number 3) Vulkan's Y axis is flipped and Z range is halved. I failed to take this into consideration while writing my shaders. I quickly looked at Sascha Willems' shadow mapping example and I've seen that he multiplies the MVP matrix of the directional light with a bias matrix.

My vertex shader looks like this now:

#version 450

// INs
layout(location = 0) in vec3 a_Position;
layout(location = 1) in vec2 a_UV;
layout(location = 2) in vec3 a_Normal;
layout(location = 3) in vec3 a_Tangent;
layout(location = 4) in vec3 a_Bitangent;

//OUTs
layout(location = 0) out vec3 v_Pos;
layout(location = 1) out vec2 v_UV;
layout(location = 2) out vec3 v_Normal;
layout(location = 3) out vec4 v_FragPosLightSpace;
layout(location = 4) out vec3 v_DirLightPos;
layout(location = 5) out mat3 v_TBN;

layout(binding = 0) uniform ModelMatrix
{
    mat4 ModelMat;
} Model;

layout(binding = 1) uniform ViewMatrix
{
    mat4 ViewMat;
} View;

layout(binding = 2) uniform ProjectionMatrix
{
    mat4 ProjMat;
} Proj;

layout(binding = 3) uniform depthMVP
{
    mat4 Matrix;
} lightMVP;

layout(binding = 4) uniform DirectionalLightPosition
{
    vec3 pos;
} dirLightPos;


const mat4 bias = mat4( 
  0.5, 0.0, 0.0, 0.0,
  0.0, 0.5, 0.0, 0.0,
  0.0, 0.0, 1.0, 0.0,
  0.5, 0.5, 0.0, 1.0 );

void main()
{
    v_UV                = a_UV;
    v_Pos               = vec3(Model.ModelMat * vec4(a_Position, 1.0));
    v_Normal            = mat3(Model.ModelMat) * a_Normal;   

    mat3 normalMatrix   = transpose(mat3(Model.ModelMat));

    vec3 T              = normalize(normalMatrix * a_Tangent);
    vec3 N              = normalize(normalMatrix * a_Normal);
    vec3 B              = normalize(normalMatrix * a_Bitangent);

    T                   = normalize(T - dot(T, N) * N);

    v_TBN               = mat3(T, B, N);
    v_FragPosLightSpace = bias * lightMVP.Matrix * Model.ModelMat * vec4(a_Position, 1.0);
    v_DirLightPos       = dirLightPos.pos;

    gl_Position = Proj.ProjMat * View.ViewMat * Model.ModelMat * vec4(a_Position, 1.0);
}

• Notice the part v_FragPosLightSpace = bias * lightMVP.Matrix * Model.ModelMat * vec4(a_Position, 1.0);

The fragment shader is straight forward. I am not going to share the entire thing because I've switched to PBR and it is lengthy, but the part that matters is as follows:

float shadow = 1.0;
vec4 shadowCoords = v_FragPosLightSpace / v_FragPosLightSpace.w;
if( texture( u_DirectionalShadowMap, shadowCoords.xy ).r < shadowCoords.z - 0.005 )
{
    shadow = 0.0;
}

• Notice there is no depth range transformation happening here because Vulkan's Z range is already in between [0, 1], unlike OpenGL. We just need a perspective division and the rest is basic shadow calculations.