Minimize mouse input lag on GL desktop app?

Question

I think this is a common problem which relates to the OpenGL pipeline and how it queues rendered frames for display.

How it looks like

An extreme example of this can be seen in this video on Android.

Mouse latency is present on the simplest desktop apps. You'll see it's really noticeable if you run one of the small apps I wrote with GLFW in C++ :

#include <GL/glew.h>
#include <GLFW/glfw3.h>

#include <stdlib.h>
#include <stdio.h>

const float box_size = 20;

static const struct
{
    float x, y;
} vertices[4] =
{
    { -box_size, -box_size},
    {  box_size, -box_size},
    {  box_size,  box_size},
    { -box_size,  box_size}
};

static const char* vertex_shader_text =
"#version 110\n"
"attribute vec2 vPos;\n"
"varying vec3 color;\n"
"uniform vec2 vMouse;\n"
"uniform vec2 vWindow;\n"
"void main()\n"
"{\n"
"    gl_Position = vec4(vPos/vWindow+vMouse, 0.0, 1.0);\n"
"    color = vec3(1.0, 1.0, 0.);\n"
"}\n";

static const char* fragment_shader_text =
"#version 110\n"
"varying vec3 color;\n"
"void main()\n"
"{\n"
"    gl_FragColor = vec4(color, 1.0);\n"
"}\n";

static void error_callback(int error, const char* description)
{
    fprintf(stderr, "Error: %s\n", description);
}

int main(void)
{
    GLFWwindow* window;
    GLuint vertex_buffer, vertex_shader, fragment_shader, program;
    GLint mouse_location, vpos_location, window_location;

    glfwSetErrorCallback(error_callback);

    if (!glfwInit())
        exit(EXIT_FAILURE);

    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 2);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0);

    window = glfwCreateWindow(500, 500, "Square Follows Mouse - GLFW", NULL, NULL);
    if (!window)
    {
        glfwTerminate();
        exit(EXIT_FAILURE);
    }

    glfwMakeContextCurrent(window);

    GLenum err = glewInit();
    if (GLEW_OK != err)
    {
        /* Problem: glewInit failed, something is seriously wrong. */
        fprintf(stderr, "Error: %s\n", glewGetErrorString(err));
        glfwTerminate();
        exit(EXIT_FAILURE);
    }
    glfwSwapInterval(1);

    // NOTE: OpenGL error checks have been omitted for brevity

    glGenBuffers(1, &vertex_buffer);
    glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer);
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);

    vertex_shader = glCreateShader(GL_VERTEX_SHADER);
    glShaderSource(vertex_shader, 1, &vertex_shader_text, NULL);
    glCompileShader(vertex_shader);

    fragment_shader = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(fragment_shader, 1, &fragment_shader_text, NULL);
    glCompileShader(fragment_shader);

    program = glCreateProgram();
    glAttachShader(program, vertex_shader);
    glAttachShader(program, fragment_shader);
    glLinkProgram(program);

    vpos_location = glGetAttribLocation(program, "vPos");
    mouse_location = glGetUniformLocation(program, "vMouse");
    window_location = glGetUniformLocation(program, "vWindow");

    glEnableVertexAttribArray(vpos_location);
    glVertexAttribPointer(vpos_location, 2, GL_FLOAT, GL_FALSE,
                          sizeof(vertices[0]), (void*) 0);

    while (!glfwWindowShouldClose(window))
    {
        float ratio;
        int width, height;

        glfwGetFramebufferSize(window, &width, &height);
        ratio = width / (float) height;

        glViewport(0, 0, width, height);
        glClear(GL_COLOR_BUFFER_BIT);

        glUseProgram(program);

        double mouse_x, mouse_y;
        glfwGetCursorPos(window, &mouse_x, &mouse_y);
        glUniform2f(mouse_location, mouse_x/width*2-1, -mouse_y/height*2+1);
        glUniform2f(window_location, (float)width, (float)height);

        glDrawArrays(GL_POLYGON, 0, 4);

        glfwSwapBuffers(window);
        glfwPollEvents();
    }

    glfwDestroyWindow(window);

    glfwTerminate();
    exit(EXIT_SUCCESS);
}

...or with GLUT in C :

#include <GL/glut.h>

int window_w, window_h = 0;
float mouse_x, mouse_y = 0.0;
float box_size = 0.02;

void display(void)
{
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

  glBegin(GL_POLYGON);
  glVertex3f(mouse_x+box_size, mouse_y+box_size, 0.0);
  glVertex3f(mouse_x-box_size, mouse_y+box_size, 0.0);
  glVertex3f(mouse_x-box_size, mouse_y-box_size, 0.0);
  glVertex3f(mouse_x+box_size, mouse_y-box_size, 0.0);
  glEnd();

  glutSwapBuffers();
}

void motion(int x, int y)
{
  mouse_x = (float)x/window_w - 0.5;
  mouse_y = -(float)y/window_h + 0.5;
  glutPostRedisplay();
}

void reshape(int w, int h)
{
  window_w = w;
  window_h = h;
  glViewport(0, 0, w, h);
  glMatrixMode(GL_PROJECTION);
  glLoadIdentity();
  gluOrtho2D(-.5, .5, -.5, .5);
}

int main(int argc, char **argv)
{
  glutInit(&argc, argv);
  glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH);
  glutInitWindowSize(500, 500);
  glutCreateWindow("Square Follows Mouse - GLUT");
  glutPassiveMotionFunc(motion);
  glutReshapeFunc(reshape);
  glutDisplayFunc(display);
  glutMainLoop();
  return 0;
}

There's precompiled binaries here too (Linux x64).

The 1st and 3rd gifs at the bottom of this post are screencasts of the aforementioned GLFW app.

What I think it is

The problem here must be display latency, ie the time between when the app renders a frame and when the monitor lights up the pixels. The goal is to minimize it.

Why I think it is

Given that vertical sync is on and your rendering is simple and completes in much less than the frame period, this display latency is usually two frames. This is because the application is triple-buffered: one buffer is being displayed, one is the front buffer to be displayed on the next flip, and one is the back buffer being drawn into by the application. The app renders the next frame as soon as the back buffer becomes available. If instead the app waited and rendered the frame at about half a frame before displaying it, this latency could be less than 8.3ms instead of 33.3-25.0ms (at 60fps).

I confirmed this by executing a sleep function every frame for 17ms (a tiny bit more than one frame). This way the display jitters every second or so, but mouse latency is significantly smaller, because the frames are sent for display sooner, because the queue is 'starved', ie there are no pre-rendered frames. The 2nd and 4th gifs below show this. If you put this app into full-screen mode, the latency is almost imperceptible from the OS cursor.

So the problem becomes how to synchronize frame rendering to start at a specific time (eg T-8ms) relative to when it's displayed on the monitor (T). For example, half a frame before T or as much as we estimate the rendering will take.

Is there a common way to solve this?

What I found

• I could only find an analogous question on Android here , which shows how to shave half a frame period off the two-frame latency, but only on Android.
• And another for a desktop app here , but the solution there is to only render frames when there are mouse events. Which reduces latency for the first frame or two when the mouse starts moving, but the frame queue quickly fills up and two-frame latency appears again.

I couldn't even find GL functions for querying whether rendering is falling behind the frame consumption by the monitor. Neither a function to block until the front and back buffers swap ( the docs say its glFinish , but in my experiments it always returns much sooner than when the back buffer becomes available). An action on the frame buffer (specifically CopyTexImage2D) does seem to block until buffers swap and could be used for synchronization, but there's probably other problems that will emerge from synchronizing in such a roundabout way.

Any function that can return some status on this triple-buffer queue and how much of it is consumed could be very helpful for implementing this kind of synchronization.

Images

Same gifs, just slowed down and trimmed:

Answer 1

Replace the input handling with something native, in glfw input reckognition is done via callback procedures, they costinitially less performance than polling but they have a latency involved you are experiencing. Callbacks are janky, you want to poll directly from the system on windows via WinAPI which is ultra easy and on Linux its X11 not as easy as WinAPI but doable. Be Aware that you can decide the Rate of the poll.

It seems like you are working on Linux How can I get the current mouse (pointer) position co-ordinates in X