WebGL Stencils, How to use a 2D sprite's transparency as a mask?

Question

if (statuseffect) {
           // Clearing the stencil buffer
           gl.clearStencil(0);
           gl.clear(gl.STENCIL_BUFFER_BIT);
       

           gl.stencilFunc(gl.ALWAYS, 1, 1);
           gl.stencilOp(gl.REPLACE, gl.REPLACE, gl.REPLACE);
       

            gl.colorMask(false, false, false, false); 
            
           gl.enable(gl.STENCIL_TEST);

           // Renders the mask through gl.drawArrays L111
           drawImage(statuseffectmask.texture, lerp(-725, 675, this.Transtion_Value), 280, 128 * 4, 32 * 4)
       
           // Telling the stencil now to draw/keep only pixels that equals 1 - which we set earlier
           gl.stencilFunc(gl.EQUAL, 1, 1);
           gl.stencilOp(gl.KEEP, gl.KEEP, gl.KEEP);
           
           // enabling back the color buffer
           gl.colorMask(true, true, true, true);
      
       
           drawImage(statuseffect.texture, lerp(-725, 675, this.Transtion_Value), 280, 128 * 4, 32 * 4)

  
           gl.disable(gl.STENCIL_TEST);
        }

Im trying to get something to work like this Where it gets the transparency of the sprite, and then draws a sprite in areas where there is no transparency, thank you.

Answer 1

It's not clear why you want to use the stencil for this. Normally you'd just setup blending and use the transparency to blend .

If you really wanted to use the stencil you'd need to make a shader that calls discard if the transparency (alpha) is less then some value in order to make the stencil get set only where the sprite is not transparent

precision highp float;

varying vec2 v_texcoord;

uniform sampler2D u_texture;
uniform float u_alphaTest;

void main() {
  vec4 color = texture2D(u_texture, v_texcoord);
  if (color.a < u_alphaTest) {
    discard;  // don't draw this pixel
  }
  gl_FragColor = color;
}

But the thing is that would already draw the texture transparently without using the stencil.

 const m4 = twgl.m4; const gl = document.querySelector('canvas').getContext('webgl'); const vs = ` attribute vec4 position; attribute vec2 texcoord; uniform mat4 u_matrix; varying vec2 v_texcoord; void main() { gl_Position = u_matrix * position; v_texcoord = texcoord; } `; const fs = ` precision highp float; varying vec2 v_texcoord; uniform sampler2D u_texture; uniform float u_alphaTest; void main() { vec4 color = texture2D(u_texture, v_texcoord); if (color.a < u_alphaTest) { discard; // don't draw this pixel } gl_FragColor = color; } `; // compile shaders, link program, look up locations const programInfo = twgl.createProgramInfo(gl, [vs, fs]); // make buffers for positions and texcoords for a plane // calls gl.createBuffer, gl.bindBuffer, gl.bufferData const bufferInfo = twgl.primitives.createXYQuadBufferInfo(gl); const texture = makeSpriteTexture(gl, '🌲', 128); function render(time) { time *= 0.001; // convert to seconds gl.useProgram(programInfo.program); // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo); const mat = m4.ortho(-150, 150, 75, -75, -1, 1); m4.translate(mat, [Math.cos(time) * 20, Math.sin(time) * 20, 0], mat); m4.scale(mat, [64, 64, 1], mat); // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX twgl.setUniformsAndBindTextures(programInfo, { u_texture: texture, u_alphaTest: 0.5, u_matrix: mat, }); // calls gl.drawArrays or gl.drawElements twgl.drawBufferInfo(gl, bufferInfo); requestAnimationFrame(render); } requestAnimationFrame(render); // just so we don't have to download an image // we'll make our own using a canvas and the 2D API function makeSpriteTexture(gl, str, size) { const canvas = document.createElement('canvas'); canvas.width = size; canvas.height = size; const ctx = canvas.getContext('2d'); ctx.font = `${size * 3 / 4}px sans-serif`; ctx.textAlign = 'center'; ctx.textBaseline = 'middle'; ctx.fillText(str, size / 2, size / 2); const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.texImage2D( gl.TEXTURE_2D, 0, // mip level gl.RGBA, // internal format gl.RGBA, // format gl.UNSIGNED_BYTE, // type, canvas); // let's assume we used power of 2 dimensions gl.generateMipmap(gl.TEXTURE_2D); return tex; }

 canvas { background: url(https://i.imgur.com/v38pV.jpg) no-repeat center center; background-size: cover; }

 <canvas></canvas> <script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Otherwise if you really want to use the stencil now that the code is discarding some pixels it should work and your code was correct. note the code below doesn't clear the stencil because it defaults to being cleared every frame

 const m4 = twgl.m4; const gl = document.querySelector('canvas').getContext('webgl', {stencil: true}); const vs = ` attribute vec4 position; attribute vec2 texcoord; uniform mat4 u_matrix; varying vec2 v_texcoord; void main() { gl_Position = u_matrix * position; v_texcoord = texcoord; } `; const fs = ` precision highp float; varying vec2 v_texcoord; uniform sampler2D u_texture; uniform float u_alphaTest; void main() { vec4 color = texture2D(u_texture, v_texcoord); if (color.a < u_alphaTest) { discard; // don't draw this pixel } gl_FragColor = color; } `; // compile shaders, link program, look up locations const programInfo = twgl.createProgramInfo(gl, [vs, fs]); // make buffers for positions and texcoords for a plane // calls gl.createBuffer, gl.bindBuffer, gl.bufferData const bufferInfo = twgl.primitives.createXYQuadBufferInfo(gl); const starTexture = makeSpriteTexture(gl, '✱', 128); const bugTexture = makeSpriteTexture(gl, '🐞', 128); function render(time) { time *= 0.001; // convert to seconds gl.useProgram(programInfo.program); // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo); const mat = m4.ortho(-150, 150, 75, -75, -1, 1); m4.translate(mat, [Math.cos(time) * 20, 0, 0], mat); m4.scale(mat, [64, 64, 1], mat); gl.enable(gl.STENCIL_TEST); // set the stencil to 1 everwhere we draw a pixel gl.stencilFunc(gl.ALWAYS, 1, 0xFF); gl.stencilOp(gl.REPLACE, gl.REPLACE, gl.REPLACE); // don't render pixels gl.colorMask(false, false, false, false); // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX twgl.setUniformsAndBindTextures(programInfo, { u_texture: starTexture, u_alphaTest: 0.5, u_matrix: mat, }); // calls gl.drawArrays or gl.drawElements twgl.drawBufferInfo(gl, bufferInfo); // only draw if the stencil = 1 gl.stencilFunc(gl.EQUAL, 1, 0xFF); gl.stencilOp(gl.KEEP, gl.KEEP, gl.KEEP); // render pixels gl.colorMask(true, true, true, true); m4.ortho(-150, 150, 75, -75, -1, 1, mat); m4.translate(mat, [0, Math.cos(time * 1.1) * 20, 0], mat); m4.scale(mat, [64, 64, 1], mat); // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX twgl.setUniformsAndBindTextures(programInfo, { u_texture: bugTexture, u_alphaTest: 0, // draw all pixels (but stencil will prevent some) u_matrix: mat, }); // calls gl.drawArrays or gl.drawElements twgl.drawBufferInfo(gl, bufferInfo); requestAnimationFrame(render); } requestAnimationFrame(render); // just so we don't have to download an image // we'll make our own using a canvas and the 2D API function makeSpriteTexture(gl, str, size) { const canvas = document.createElement('canvas'); canvas.width = size; canvas.height = size; const ctx = canvas.getContext('2d'); ctx.font = `${size * 3 / 4}px sans-serif`; ctx.textAlign = 'center'; ctx.textBaseline = 'middle'; ctx.fillText(str, size / 2, size / 2); const tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); gl.texImage2D( gl.TEXTURE_2D, 0, // mip level gl.RGBA, // internal format gl.RGBA, // format gl.UNSIGNED_BYTE, // type, canvas); // let's assume we used power of 2 dimensions gl.generateMipmap(gl.TEXTURE_2D); return tex; }

 canvas { background: url(https://i.imgur.com/v38pV.jpg) no-repeat center center; background-size: cover; }

 <canvas></canvas> <script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Let me also point out that that this is also probably better done using alpha blending, passing both textures into a single shader and passing in another matrix or other uniforms to apply one texture's alpha ot the other. This would be more flexible as you could can blend across all values of 0 to 1 where as with the stencil you can only mask 0 or 1 period.

My point isn't to say "don't use the stencil" but rather that there are times where it's best and times where it's not. Only you can know for your situation which solution to choose.