Get H264 ip camera stream using mediacodec

Question

I'm trying to get the stream produced by a camera on my network. The main problem is that the stream produced by this camera is an H264 stream. I've look around to find the best way to manage this kind of stream and I've found that mediacodec is the most suitable class. Unfortunately the documentation for this API is very poor, but despite this i've found some resources where i get the base info on this class and I've write the following code.

public class DecodeActivity extends Activity implements SurfaceHolder.Callback {

private final String sampleStreamAddress = "http://192.168.0.240:80";
private PlayerThread mPlayer = null;

String mimeType = "video/avc";
MediaFormat format = MediaFormat.createVideoFormat(mimeType, 640, 360);

final byte[] header_sps = { 0, 0, 0, 1, 103, 100, 0, 40, -84, 52, -59, 1,
        -32, 17, 31, 120, 11, 80, 16, 16, 31, 0, 0, 3, 3, -23, 0, 0, -22,
        96, -108 };
final byte[] header_pps = { 0, 0, 0, 1, 104, -18, 60, -128 };

private static final String tag = DecodeActivity.class.getSimpleName();

@Override
protected void onCreate(final Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    final SurfaceView sv = new SurfaceView(this);
    sv.getHolder().addCallback(this);
    setContentView(sv);

    format.setByteBuffer("csd-0", ByteBuffer.wrap(header_sps));
    format.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));

}

@Override
protected void onDestroy() {
    super.onDestroy();
}

@Override
public void surfaceCreated(final SurfaceHolder holder) {

    Log.i(tag, "surfaceCreated");

    if (mPlayer == null) {
        mPlayer = new PlayerThread(holder.getSurface());
        mPlayer.start();
    }
}

@Override
public void surfaceChanged(final SurfaceHolder holder, final int format,
        final int width, final int height) {

    Log.i(tag, "surfaceChanged");

    if (mPlayer == null) {
        mPlayer = new PlayerThread(holder.getSurface());
        mPlayer.start();
    }
}

@Override
public void surfaceDestroyed(final SurfaceHolder holder) {

    Log.i(tag, "surfaceDestroyed");

    if (mPlayer != null) {
        mPlayer.interrupt();
    }
}

private class PlayerThread extends Thread {
    private MediaCodec decoder;
    private final Surface surface;

    public PlayerThread(final Surface surface) {
        this.surface = surface;
    }

    @Override
    public void run() {

        try {

            Log.i(tag, "running PlayerThrad");

            decoder = MediaCodec.createDecoderByType(format.toString());
            decoder.configure(format, surface, null, 0);

            decoder.start();

            final ByteBuffer[] inputBuffers = decoder.getInputBuffers();
            ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
            final BufferInfo info = new BufferInfo();
            boolean isEOS = false;
            final long startMs = System.currentTimeMillis();

            //
            HttpResponse res;
            final DefaultHttpClient httpclient = new DefaultHttpClient();

            while (!Thread.interrupted()) {
                if (!isEOS) {

                    final int inIndex = decoder.dequeueInputBuffer(10000);
                    if (inIndex >= 0) {
                        final byte buffer2[] = new byte[18800 * 8 * 8 * 8];
                        final ByteBuffer buffer = inputBuffers[inIndex];

                        final HttpParams httpParameters = new BasicHttpParams();
                        HttpConnectionParams.setConnectionTimeout(
                                httpParameters, 200);
                        httpclient.setParams(httpParameters);
                        Log.d(tag, " URL : " + sampleStreamAddress);
                        res = httpclient.execute(new HttpGet(URI
                                .create(sampleStreamAddress)));

                        final DataInputStream mjpegInputStream = new DataInputStream(
                                res.getEntity().getContent());

                        final int sampleSize = mjpegInputStream.read(
                                buffer2, 0, 18800 * 4);

                        buffer.clear();
                        buffer.put(buffer2, 0, sampleSize);
                        buffer.clear();

                        if (sampleSize < 0) {
                            // We shouldn't stop the playback at this point,
                            // just pass the EOS
                            // flag to decoder, we will get it again from
                            // the
                            // dequeueOutputBuffer
                            Log.d("DecodeActivity",
                                    "InputBuffer BUFFER_FLAG_END_OF_STREAM");
                            decoder.queueInputBuffer(inIndex, 0, 0, 0,
                                    MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                            isEOS = true;
                        } else {
                            decoder.queueInputBuffer(inIndex, 0,
                                    sampleSize, 0, 0);
                            // extractor.advance();
                        }
                    }
                }

                final int outIndex = decoder.dequeueOutputBuffer(info,
                        10000);
                switch (outIndex) {
                case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                    Log.d("DecodeActivity", "INFO_OUTPUT_BUFFERS_CHANGED");
                    outputBuffers = decoder.getOutputBuffers();
                    break;
                case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                    Log.d("DecodeActivity",
                            "New format " + decoder.getOutputFormat());
                    break;
                case MediaCodec.INFO_TRY_AGAIN_LATER:
                    Log.d("DecodeActivity",
                            "dequeueOutputBuffer timed out!");
                    break;
                default:
                    final ByteBuffer buffer = outputBuffers[outIndex];
                    Log.v("DecodeActivity",
                            "We can't use this buffer but render it due to the API limit, "
                                    + buffer);

                    // We use a very simple clock to keep the video FPS, or
                    // the
                    // video
                    // playback will be too fast
                    while (info.presentationTimeUs / 1000 > System
                            .currentTimeMillis() - startMs) {
                        try {
                            sleep(10);
                        } catch (final InterruptedException e) {
                            e.printStackTrace();
                            break;
                        }
                    }
                    decoder.releaseOutputBuffer(outIndex, true);
                    break;
                }

                // All decoded frames have been rendered, we can stop
                // playing
                // now
                if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                    Log.d("DecodeActivity",
                            "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
                    break;
                }
            }

            decoder.stop();
            decoder.release();
        } catch (final Exception e) {
            e.printStackTrace();
        }
    }
}

}

The problem is that when I try to run this activity i get the following error:

(20736): Unable to instantiate a decoder for type '{csd-1=java.nio.ReadWriteHeapByteBuffer, status: capacity=8 position=0 limit=8, height=360, mime=video/avc, csd-0=java.nio.ReadWriteHeapByteBuffer, status: capacity=31 position=0 limit=31, width=640}'.

E/MediaCodec(20736): Codec reported an error. (omx error 0x80001003, internalError -2147483648)

F/libc (20736): Fatal signal 11 (SIGSEGV) at 0x00000008 (code=1), thread 20752 (Thread-324)

Is someone able to point me in the right direction? Cause I can't see what I'm doing wrong. Thanks in advance

Answer 1

Pretty sure the mediaextractor is missing. With ur solution you miss to parse out headerinformations, in which the decoder is not interested in.

This guy goes into the right direction, but still has no solution.

Android using MediaExtractor and MediaCodec from a socket (streaming mpeg)

Next you didnt instantiate the decoder correctly. Replace line

decoder = MediaCodec.createDecoderByType(format.toString());

with

decoder = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME));

If i come up with a concrete solution, i will post it.