MediaCodec raw H264 decoding issue - times out and decoder.dequeueOutputBuffer always return -1

Question

Firstly, thanks for taking time to read this. I need some help or insights as I'm facing decoding issue with h264 frames. I'm posting this question as most of other related posts to this do not provide clear steps. I've shared the code for decoder thread which has got the main logic. I get the sps, pps and data from network and thus I make this mConfigured flag as true when sps and pps are received and respective arrays are fed. I can provide more info if needed. PS - Please excuse for the coding standards as it is still a POC in development.

decoder.dequeueOutputBuffer >> always return -1

//below function called from network when new packet received.

public void decodeAndPlayVideoFrame(byte[] encodedData, int frameType) {
    if (frameType == 1) {
        SPS = encodedData;
    } else if (frameType == 2) {
        PPS = encodedData;
    } else if (frameType == 0) {
        Log.d("EncodeDecode", "enqueueing frame no: " + (frameID));

        try {
            Frame frame = new Frame(frameID);
            int totalDataLength = encodedData.length + SPS.length + PPS.length;
            frame.frameData = new byte[totalDataLength];
            System.arraycopy(SPS, 0, frame.frameData, 0, SPS.length);
            System.arraycopy(PPS, 0, frame.frameData, SPS.length, PPS.length);
            System.arraycopy(encodedData, 0, frame.frameData, SPS.length + PPS.length, encodedData.length);

            queue.add(frame);
            frameID++;
        } catch (NullPointerException e) {
            Log.e("EncodeDecode", "frame is null");
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
            Log.e("EncodeDecode", "problem inserting in the queue");
            e.printStackTrace();
        } catch (IllegalStateException e) {
            Log.e("EncodeDecode", "problem inserting in the queue");
            e.printStackTrace();
        }
        Log.d("EncodeDecode", "frame enqueued. queue size now: " + queue.size());

    }
}

// Player thread starts as screen launches and process packets fed into queue.

  private class PlayerThread extends Thread {
    //private MediaExtractor extractor;
    private MediaCodec decoder;
    private Surface surface;
    private boolean mConfigured;

    public PlayerThread(Surface surface) {
        this.surface = surface;
    }

    private void initCodec() throws IOException {

        MediaFormat mediaFormat = null;

        decoder = MediaCodec.createDecoderByType("video/avc");
        mediaFormat = MediaFormat.createVideoFormat("video/avc",
                320,
                240);

        try {

            decoder.configure(mediaFormat,
                    surface,
                    null,
                    0);
            frameID = 0;
            mConfigured = true;
            decoder.start();
            Log.d("EncodeDecode", "DECODER_THREAD:: decoder.start() called");
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    @Override
    public void run() {
        while (SPS == null || PPS == null || SPS.length == 0 || PPS.length == 0) {
            try {
                Log.d("EncodeDecode", "DECODER_THREAD:: sps,pps not ready yet");
                sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();

            }
        }

        if (!mConfigured) {
            try {
                initCodec();
            } catch (IOException e) {
                e.printStackTrace();
            }

        }
       int i = 0;
        ByteBuffer[] inputBuffers = decoder.getInputBuffers();
        ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
        while (!Thread.interrupted()) {
            Frame currentFrame = null;
            try {
                Log.d("EncodeDecode", "DECODER_THREAD:: calling queue.take(), if there is no frame in the queue it will wait");
                currentFrame = queue.take();
            } catch (InterruptedException e) {
                Log.e("EncodeDecode", "DECODER_THREAD:: interrupted while PlayerThread was waiting for the next frame");
                e.printStackTrace();
            }

            if (currentFrame == null)
                Log.e("EncodeDecode", "DECODER_THREAD:: null frame dequeued");
            else
                Log.d("EncodeDecode", "DECODER_THREAD:: " + currentFrame.id + " no frame dequeued");

            if (currentFrame != null && currentFrame.frameData != null && currentFrame.frameData.length != 0) {
                Log.d("EncodeDecode", "DECODER_THREAD:: decoding frame no: " + i + " , dataLength = " + currentFrame.frameData.length);

                int inIndex = decoder.dequeueInputBuffer(-1);

                if (inIndex >= 0) {
                    Log.d("EncodeDecode", "DECODER_THREAD:: sample size: " + currentFrame.frameData.length + "->" + currentFrame.frameData[0] +  "to" + currentFrame.frameData[currentFrame.frameData.length-1]);

                    ByteBuffer buffer = inputBuffers[inIndex];
                    buffer.clear();
                    buffer.put(currentFrame.frameData);
                    decoder.queueInputBuffer(inIndex, 0, currentFrame.frameData.length, 0, 0);

                    MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
                    int outIndex = decoder.dequeueOutputBuffer(info, 0);

                    switch (outIndex) {
                        case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                            Log.e("EncodeDecode", "DECODER_THREAD:: INFO_OUTPUT_BUFFERS_CHANGED");
                            outputBuffers = decoder.getOutputBuffers();
                            break;
                        case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                            Log.e("EncodeDecode", "DECODER_THREAD:: New format " + decoder.getOutputFormat());
                            break;
                        case MediaCodec.INFO_TRY_AGAIN_LATER:
                            Log.e("EncodeDecode", "DECODER_THREAD:: dequeueOutputBuffer timed out!");
                            break;
                        default:
                            Log.d("EncodeDecode", "DECODER_THREAD:: decoded SUCCESSFULLY!!!");
                            ByteBuffer outbuffer = outputBuffers[outIndex];
                            decoder.releaseOutputBuffer(outIndex, true);
                            break;
                    }

                    try {
                        Thread.sleep(250);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                    i++;
                }
            }
        }

        decoder.stop();
        decoder.release();
        mConfigured = false;

    }
}

Answer 1

I do not know how do you get SPS and PPS and how do you check them...anyway you have to set SPS and PPS to mediaformat before starting the decodec, in your code, you can do something like :

   ....
   while (SPS == null || PPS == null || SPS.length == 0 || PPS.length == 0) {
        try {
            Log.d("EncodeDecode", "DECODER_THREAD:: sps,pps not ready yet");
            sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();

        }
    }

   if (!mConfigured) {
        try {
            // init with SPS and PPS
            initCodec(sps,pps);
        } catch (IOException e) {
            e.printStackTrace();
        }

    }
   .....

And then set codec specific datas in initCodec before starting the decodec :

private void initCodec(byte[] sps, byte[] pps) throws IOException {

        MediaFormat mediaFormat = null;

        decoder = MediaCodec.createDecoderByType("video/avc");
        mediaFormat = MediaFormat.createVideoFormat("video/avc",
                320,
                240);
        // set the codec specific datas cds-0 = SPS, cds-1 = PPS
        mediaFormat.setByteBuffer("cds-0", ByteBuffer.wrap(sps));
        mediaFormat.setByteBuffer("cds-1", ByteBuffer.wrap(pps));
        try {

            decoder.configure(mediaFormat,
                    surface,
                    null,
                    0);
            frameID = 0;
            mConfigured = true;
            decoder.start();
            Log.d("EncodeDecode", "DECODER_THREAD:: decoder.start() called");
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
    ...........

Update: From the doc

...Many decoders require the actual compressed data stream to be preceded by "codec specific data", ie setup data used to initialize the codec such as PPS/SPS in the case of AVC video or code tables in the case of vorbis audio. The class MediaExtractor provides codec specific data as part of the returned track format in entries named "csd-0", "csd-1" ...

These buffers can be submitted directly after start() or flush() by specifying the flag BUFFER_FLAG_CODEC_CONFIG. However, if you configure the codec with a MediaFormat containing these keys, they will be automatically submitted by MediaCodec directly after start. > Therefore, the use of BUFFER_FLAG_CODEC_CONFIG flag is discouraged and is recommended only for advanced users .