webgl how to draw many cubes

Question

I am trying to draw 5161 cubes using webGL. The problem is not all cubes are drawn. Upon some searching, I think its because I am passing too many vertices in one VBO call. You can take a look at jsfiddle here: http://jsfiddle.net/n5fjhe21/ . You can move around with QWERASDF and arrows keys but it isnt well implemented right now.

My draw call used to look like this:

function render(){
    gl.uniformMatrix4fv(u_matrixLoc, false, new Float32Array(pMatrix));
    gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
    gl.drawElements(gl.TRIANGLES, data.triangles.length, gl.UNSIGNED_SHORT, 0);
}

So I would do is data.pushData() once and render as needed; It was fast. glObject is an array of Cubes.

data.pushData = function(){
// pushData once then call drawElements on every render call doesnt work as I hit some kind of limit;
// not all cubes are drawn; I think the draw calls must be split up;
data.vertices = [];
data.uv = [];
data.triangles = [];
var vertexOffset = 0;

glObjects.forEach(function pushingObject(o){
    data.vertices.push.apply(data.vertices,o.vertices);
    data.uv.push.apply(data.uv,o.uv);
    o.triangles.forEach(function pushingTriangles(index){
        data.triangles.push(index+vertexOffset);
    });

    vertexOffset += o.vertices.length/3; // change to component length later
});

    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data.vertices),gl.DYNAMIC_DRAW );
    gl.bindBuffer(gl.ARRAY_BUFFER, uvBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data.uv),gl.STATIC_DRAW);
    gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, triangleBuffer);
    gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(data.triangles), gl.DYNAMIC_DRAW );
};

But the problem (I think) is that I am passing in too many vertices at once. So I tried to merge pushData and render together:

data.render = function(){
    data.vertices = [];
    data.uv = [];
    data.triangles = [];
    var vertexOffset = 0;

    glObjects.forEach(function pushingObject(o){
        if (vertexOffset + o.vertices.length > 65536){
            vertexOffset = 0;
            glDraw();
            data.vertices.length = 0;
            data.uv.length = 0;
            data.triangles.length = 0;
        }

        data.vertices.push.apply(data.vertices,o.vertices);
        data.uv.push.apply(data.uv,o.uv);
        o.triangles.forEach(function pushingTriangles(index){
            data.triangles.push(index+vertexOffset);
        });

        vertexOffset += o.vertices.length/3; // change to component length later
    });

    glDraw();

    function glDraw(){
        gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
        gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data.vertices),gl.STATIC_DRAW);
        gl.bindBuffer(gl.ARRAY_BUFFER, uvBuffer);
        gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data.uv),gl.STATIC_DRAW);
        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, triangleBuffer);
        gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(data.triangles), gl.STATIC_DRAW);
        gl.drawElements(gl.TRIANGLES, data.triangles.length, gl.UNSIGNED_SHORT, 0);
    }
};

But this isnt fast enough because as I learnt, passing in new bufferData is slow. So my question is, what does one do in this situation? I was unable to locate any webgl resource that deal with this. My feeling leans towards creating multiple VBO objects but I want to make sure I am going in the right direction first. And as a follow up question, suppose if one need to draw many cubes all with unique position (x,y,z) and orientation (rX,rY,rZ), how does one go about implementing it? Thanks in advance.

Answer 1

Ok I solved my problem and I'll leave this here for stragglers:

Basically, I had the right idea in that I need to use multiple draw calls as each indexed draw (drawElements) can only refer to 2^16 elements in a VBO. The flaw in my first implementation is that I actually tried to reconstruct a new big typedArray made of multiple cube vertices in every render call. Needless to say, that is very slow. So instead of that, I really should have only created the typedArray/buffer once. To overcome the 2^16 element reference limitation, all I have to do is to separate the one bigass typedArray into manageable sizes, and this is exactly what this new version of pushData does:

data.pushData = function(){
    // ensure each vertex attribute has less than 2^16 vertices because that is how many that be be referenced each time
    // with gl.drawElements call

    function newChunk(){
        return {
            vertices: [],
            uv: [],
            triangles: []
        }
    }
    var chunk = newChunk();

    var vertexOffset = 0;

    glObjects.forEach(function pushingVerts(o){
        if (vertexOffset + o.vertices.length > 65536){
            vertexOffset = 0;
            data.chunks.push(chunk);
            chunk = newChunk();
        }

        chunk.vertices.push.apply(chunk.vertices,o.vertices);
        chunk.uv.push.apply(chunk.uv,o.uv);
        o.triangles.forEach(function pushingTriangles(index){
            chunk.triangles.push(index+vertexOffset);
        });

        vertexOffset += o.vertices.length/3; // change to component length later
    });

    data.chunks.push(chunk);

    data.chunks.forEach(function toTypeArray(c){
        c.vertices = new Float32Array(c.vertices);
        c.uv = new Float32Array(c.uv);
        c.triangles = new Uint16Array(c.triangles);
    });

    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, sizeofFloat * 65536*3,gl.DYNAMIC_DRAW);
    gl.bindBuffer(gl.ARRAY_BUFFER, uvBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, sizeofFloat * 65536*2,gl.DYNAMIC_DRAW);
    gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, triangleBuffer);
    gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, sizeofFloat * 65536, gl.DYNAMIC_DRAW);
    // for some reason only allocating sizeofUnsignedShort * 65536 is not enough.

    return data.chunks;
}; 

Then for render its simply:

data.renderChunks = function(){

    data.chunks.forEach(function renderChunk(c){
        gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
        gl.bufferSubData(gl.ARRAY_BUFFER,  0, c.vertices);
        gl.bindBuffer(gl.ARRAY_BUFFER, uvBuffer);
        gl.bufferSubData(gl.ARRAY_BUFFER,  0, c.uv);
        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, triangleBuffer);
        gl.bufferSubData(gl.ELEMENT_ARRAY_BUFFER, 0, c.triangles);
        gl.drawElements(gl.TRIANGLES, c.triangles.length, gl.UNSIGNED_SHORT, 0);
    });
};

Also I changed from using gl.bufferData to gl.bufferSubData to avoid the overhead of constructing a new buffer.

And with this I can now draw 60,000 cubes (at least): http://jsfiddle.net/n5fjhe21/1/