JNI YUV_420_888 to RGBA_8888 conversion

Question

Currently I'm building an app to do real-time image processing and then display. The first step is to try displaying the original preview using Camera2 API and ANativeWindow API. I pass the y, u, v channels through JNI separately and do YUV2RGB conversion following the Wikipedia article , but got wrong color output running on Google Pixel - 7.1.0 - API 25 - 1080x1920 on Genymotion :

Implementation of ImageReader.OnImageAvailableListener :

private ImageReader.OnImageAvailableListener mOnImageAvailableListener = new ImageReader.OnImageAvailableListener() {

    @Override
    public void onImageAvailable(ImageReader reader) {
        // get the newest frame
        Image image = reader.acquireNextImage();

        if (image == null) {
            return;
        }

        Image.Plane Y_plane = image.getPlanes()[0];
        int Y_rowStride = Y_plane.getRowStride();
        Image.Plane U_plane = image.getPlanes()[1];
        int U_rowStride = U_plane.getRowStride();
        Image.Plane V_plane = image.getPlanes()[2];
        int V_rowStride = V_plane.getRowStride();
        JNIUtils.RGBADisplay(image.getWidth(), image.getHeight(), Y_rowStride, Y_plane.getBuffer(), U_rowStride, U_plane.getBuffer(), V_rowStride, V_plane.getBuffer(), surface);
        image.close();
    }
};

JNI:

public static native void RGBADisplay(int srcWidth, int srcHeight, int Y_rowStride, ByteBuffer Y_Buffer, int U_rowStride, ByteBuffer U_Buffer, int V_rowStride, ByteBuffer V_Buffer, Surface surface);

C++:

const uint8_t NUM_128 = 128;
const uint8_t NUM_255 = 255;

JNIEXPORT void JNICALL Java_tau_camera2demo_JNIUtils_RGBADisplay(
        JNIEnv *env,
        jobject obj,
        jint srcWidth,
        jint srcHeight,
        jint Y_rowStride,
        jobject Y_Buffer,
        jint U_rowStride,
        jobject U_Buffer,
        jint V_rowStride,
        jobject V_Buffer,
        jobject surface) {


    uint8_t *srcYPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(Y_Buffer));
    uint8_t *srcUPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(U_Buffer));
    uint8_t *srcVPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(V_Buffer));

    ANativeWindow * window = ANativeWindow_fromSurface(env, surface);
    ANativeWindow_acquire(window);
    ANativeWindow_Buffer buffer;

    //set output size and format
    //only 3 formats are available:
    //WINDOW_FORMAT_RGBA_8888(DEFAULT), WINDOW_FORMAT_RGBX_8888, WINDOW_FORMAT_RGB_565
    ANativeWindow_setBuffersGeometry(window, 0, 0, WINDOW_FORMAT_RGBA_8888);
    if (int32_t err = ANativeWindow_lock(window, &buffer, NULL)) {
        LOGE("ANativeWindow_lock failed with error code: %d\n", err);
        ANativeWindow_release(window);
    }

    //convert YUV_420_888 to RGBA_8888 and display
    uint8_t * outPtr = reinterpret_cast<uint8_t *>(buffer.bits);
    for (size_t y = 0; y < srcHeight; y++)
    {
        uint8_t * Y_rowPtr = srcYPtr + y * Y_rowStride;
        uint8_t * U_rowPtr = srcUPtr + (y >> 1) * U_rowStride;
        uint8_t * V_rowPtr = srcVPtr + (y >> 1) * V_rowStride;
        for (size_t x = 0; x < srcWidth; x++)
        {
            //from Wikipedia article YUV:
            //Integer operation of ITU-R standard for YCbCr(8 bits per channel) to RGB888
            //Y-Y, U-Cb, V-Cr
            //R = Y + V + (V >> 2) + (V >> 3) + (V >> 5);
            //G = Y - ((U >> 2) + (U >> 4) + (U >> 5)) - ((V >> 1) + (V >> 3) + (V >> 4) + (V >> 5));
            //B = Y + U + (U >> 1) + (U >> 2) + (U >> 6);
            uint8_t Y = Y_rowPtr[x];
            uint8_t U = U_rowPtr[(x >> 1)] - NUM_128;
            uint8_t V = V_rowPtr[(x >> 1)] - NUM_128;
            *(outPtr++) = Y + V + (V >> 2) + (V >> 3) + (V >> 5); //R
            *(outPtr++) = Y - ((U >> 2) + (U >> 4) + (U >> 5)) - ((V >> 1) + (V >> 3) + (V >> 4) + (V >> 5)); //G
            *(outPtr++) = Y + U + (U >> 1) + (U >> 2) + (U >> 6); //B
            *(outPtr++) = NUM_255; // gamma for RGBA_8888
        }
    }

    ANativeWindow_unlockAndPost(window);
    ANativeWindow_release(window);
}

The whole demo could be found here on Github: https://github.com/Fung-yuantao/android-camera2demo

UPDATE :

Added the following code after the line calling JNIUtils.RGBADisplay :

        Log.d(TAG, "Y plane pixel stride: " + Y_plane.getPixelStride());
        Log.d(TAG, "U plane pixel stride: " + U_plane.getPixelStride());
        Log.d(TAG, "V plane pixel stride: " + V_plane.getPixelStride());

In Logcat:

09-07 06:40:02.576 5376-5392/tau.camera2demo D/Camera2Demo: Y plane pixel stride: 1
09-07 06:40:02.576 5376-5392/tau.camera2demo D/Camera2Demo: U plane pixel stride: 1
09-07 06:40:02.576 5376-5392/tau.camera2demo D/Camera2Demo: V plane pixel stride: 1

The image format should be planar according to the answer from alijandro.

Answer 1

The image output format for YUV_420_888 might be planar(I420, YV12) or semiplanar(NV12, NV21) format, from the documentation here .

So how to know it's planar or semi-planar format?

I guess you can find by image.getPlanes()[1].getPixelStride() . If it's 2 , the image format is semi-planar format and has the following bit pattern.

YYYYYYYY UVUVUVUV ...

In my test environment, the output image format from ImageReader is semi-planar.

For semi-planar, we only need to handle the first two planar.

Change your code like following.

ANativeWindow_setBuffersGeometry(window, srcWidth, srcHeight, WINDOW_FORMAT_RGBA_8888);
if (int32_t err = ANativeWindow_lock(window, &buffer, NULL)) {
    LOGE("ANativeWindow_lock failed with error code: %d\n", err);
    ANativeWindow_release(window);
}

//convert YUV_420_888 to RGBA_888 and display
uint8_t * outPtr = reinterpret_cast<uint8_t *>(buffer.bits);
for (size_t y = 0; y < srcHeight; y++)
{
    uint8_t * Y_rowPtr = srcYPtr + y * Y_rowStride;
    uint8_t * UV_rowPtr = srcUPtr + (y >> 1) * Y_rowStride;
    // uint8_t * V_rowPtr = srcVPtr + (y >> 1) * Y_rowStride / 4;
    for (size_t x = 0; x < srcWidth; x++)
    {
        uint8_t Y = Y_rowPtr[x];
        size_t uIndex = x & 0xfffffffe;
        uint8_t U = UV_rowPtr[uIndex];
        uint8_t V = UV_rowPtr[uIndex + 1];
        double R = ((Y-16) * 1.164 + (V-128) * 1.596);
        double G = ((Y-16) * 1.164 - (U-128) * 0.392 - (V-128) * 0.813);
        double B = ((Y-16) * 1.164 + (U-128) * 2.017);
        *(outPtr++) = (uint8_t) (R > 255 ? 255 : (R < 0 ? 0 : R));
        *(outPtr++) = (uint8_t) (G > 255 ? 255 : (G < 0 ? 0 : G));
        *(outPtr++) = (uint8_t) (B > 255 ? 255 : (B < 0 ? 0 : B));
        *(outPtr++) = NUM_255; // gamma for RGBA_8888
    }
}

For your planar output format, try use the YUV2RGB transform method below.

for (size_t y = 0; y < srcHeight; y++)
{
    uint8_t * Y_rowPtr = srcYPtr + y * Y_rowStride;
    uint8_t * U_rowPtr = srcUPtr + (y >> 1) * Y_rowStride / 2;
    uint8_t * V_rowPtr = srcVPtr + (y >> 1) * Y_rowStride / 2;
    for (size_t x = 0; x < srcWidth; x++)
    {
        uint8_t Y = Y_rowPtr[x];
        uint8_t U = U_rowPtr[(x >> 1)];
        uint8_t V = V_rowPtr[(x >> 1)];
        double R = ((Y-16) * 1.164 + (V-128) * 1.596);
        double G = ((Y-16) * 1.164 - (U-128) * 0.392 - (V-128) * 0.813);
        double B = ((Y-16) * 1.164 + (U-128) * 2.017);
        *(outPtr++) = (uint8_t) (R > 255 ? 255 : (R < 0 ? 0 : R));
        *(outPtr++) = (uint8_t) (G > 255 ? 255 : (G < 0 ? 0 : G));
        *(outPtr++) = (uint8_t) (B > 255 ? 255 : (B < 0 ? 0 : B));
        *(outPtr++) = NUM_255; // gamma for RGBA_8888
    }
}