如何使用 MediaCodec 将位图编码为视频？

Question

I would like to encode a set of Bitmaps that I have into an h264. Is this possible via MediaEncoder? I have written some code in order to do it, but the output cannot be played in any media player I have tried. Here's some of the code that I primarily borrowed from other sources that I found on Stackoverflow.

mMediaCodec = MediaCodec.createEncoderByType("video/avc");
mMediaFormat = MediaFormat.createVideoFormat("video/avc", 320, 240);
mMediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 125000);
mMediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 15);
mMediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar);
mMediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5);
mMediaCodec.configure(mMediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mMediaCodec.start();
mInputBuffers = mMediaCodec.getInputBuffers();

ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
image.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream); // image is the bitmap
byte[] input = byteArrayOutputStream.toByteArray();

int inputBufferIndex = mMediaCodec.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
    ByteBuffer inputBuffer = mInputBuffers[inputBufferIndex];
    inputBuffer.clear();
    inputBuffer.put(input);
    mMediaCodec.queueInputBuffer(inputBufferIndex, 0, input.length, 0, 0);
}

What should I adjust?

Answer 1

I have modified the code provided by abalta to accept bitmaps in realtime (ie you don't already need to have the bitmaps saved to disc). It also has a performance improvement since you don't need to write then read the bitmaps from disc. I also increased the TIMEOUT_USEC from the original example which fixed some timeout related errorsng I was having.

Hopefully this helps someone. I spent a long time trying to do this without having to pack a large third party library into my app (ex ffmpeg), so I really appreciate abalta's answer.

I am using rxjava, so you will need this in your app's build.gradle dependencies:

implementation 'io.reactivex.rxjava2:rxandroid:2.0.2'

If you are trying to write to external storage you will need the external storage permission defined in your manifest:

<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />

and either manually toggle the permission on in the system Settings app for your app, or add permission request handling for it to your activity.

And here is the class:

import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.util.Log;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;

import io.reactivex.Completable;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.schedulers.Schedulers;

public class BitmapToVideoEncoder {
    private static final String TAG = BitmapToVideoEncoder.class.getSimpleName();

    private IBitmapToVideoEncoderCallback mCallback;
    private File mOutputFile;
    private Queue<Bitmap> mEncodeQueue = new ConcurrentLinkedQueue();
    private MediaCodec mediaCodec;
    private MediaMuxer mediaMuxer;

    private Object mFrameSync = new Object();
    private CountDownLatch mNewFrameLatch;

    private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
    private static int mWidth;
    private static int mHeight;
    private static final int BIT_RATE = 16000000;
    private static final int FRAME_RATE = 30; // Frames per second

    private static final int I_FRAME_INTERVAL = 1;

    private int mGenerateIndex = 0;
    private int mTrackIndex;
    private boolean mNoMoreFrames = false;
    private boolean mAbort = false;

    public interface IBitmapToVideoEncoderCallback {
        void onEncodingComplete(File outputFile);
    }

    public BitmapToVideoEncoder(IBitmapToVideoEncoderCallback callback) {
        mCallback = callback;
    }

    public boolean isEncodingStarted() {
        return (mediaCodec != null) && (mediaMuxer != null) && !mNoMoreFrames && !mAbort;
    }

    public int getActiveBitmaps() {
        return mEncodeQueue.size();
    }

    public void startEncoding(int width, int height, File outputFile) {
        mWidth = width;
        mHeight = height;
        mOutputFile = outputFile;

        String outputFileString;
        try {
            outputFileString = outputFile.getCanonicalPath();
        } catch (IOException e) {
            Log.e(TAG, "Unable to get path for " + outputFile);
            return;
        }

        MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
        if (codecInfo == null) {
            Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
            return;
        }
        Log.d(TAG, "found codec: " + codecInfo.getName());
        int colorFormat;
        try {
            colorFormat = selectColorFormat(codecInfo, MIME_TYPE);
        } catch (Exception e) {
            colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
        }

        try {
            mediaCodec = MediaCodec.createByCodecName(codecInfo.getName());
        } catch (IOException e) {
            Log.e(TAG, "Unable to create MediaCodec " + e.getMessage());
            return;
        }

        MediaFormat mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
        mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL);
        mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
        mediaCodec.start();
        try {
            mediaMuxer = new MediaMuxer(outputFileString, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
        } catch (IOException e) {
            Log.e(TAG,"MediaMuxer creation failed. " + e.getMessage());
            return;
        }

        Log.d(TAG, "Initialization complete. Starting encoder...");

        Completable.fromAction(() -> encode())
                .subscribeOn(Schedulers.io())
                .observeOn(AndroidSchedulers.mainThread())
                .subscribe();
    }

    public void stopEncoding() {
        if (mediaCodec == null || mediaMuxer == null) {
            Log.d(TAG, "Failed to stop encoding since it never started");
            return;
        }
        Log.d(TAG, "Stopping encoding");

        mNoMoreFrames = true;

        synchronized (mFrameSync) {
            if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
                mNewFrameLatch.countDown();
            }
        }
    }

    public void abortEncoding() {
        if (mediaCodec == null || mediaMuxer == null) {
            Log.d(TAG, "Failed to abort encoding since it never started");
            return;
        }
        Log.d(TAG, "Aborting encoding");

        mNoMoreFrames = true;
        mAbort = true;
        mEncodeQueue = new ConcurrentLinkedQueue(); // Drop all frames

        synchronized (mFrameSync) {
            if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
                mNewFrameLatch.countDown();
            }
        }
    }

    public void queueFrame(Bitmap bitmap) {
        if (mediaCodec == null || mediaMuxer == null) {
            Log.d(TAG, "Failed to queue frame. Encoding not started");
            return;
        }

        Log.d(TAG, "Queueing frame");
        mEncodeQueue.add(bitmap);

        synchronized (mFrameSync) {
            if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
                mNewFrameLatch.countDown();
            }
        }
    }

    private void encode() {

        Log.d(TAG, "Encoder started");

        while(true) {
            if (mNoMoreFrames && (mEncodeQueue.size() ==  0)) break;

            Bitmap bitmap = mEncodeQueue.poll();
            if (bitmap ==  null) {
                synchronized (mFrameSync) {
                    mNewFrameLatch = new CountDownLatch(1);
                }

                try {
                    mNewFrameLatch.await();
                } catch (InterruptedException e) {}

                bitmap = mEncodeQueue.poll();
            }

            if (bitmap == null) continue;

            byte[] byteConvertFrame = getNV21(bitmap.getWidth(), bitmap.getHeight(), bitmap);

            long TIMEOUT_USEC = 500000;
            int inputBufIndex = mediaCodec.dequeueInputBuffer(TIMEOUT_USEC);
            long ptsUsec = computePresentationTime(mGenerateIndex, FRAME_RATE);
            if (inputBufIndex >= 0) {
                final ByteBuffer inputBuffer = mediaCodec.getInputBuffer(inputBufIndex);
                inputBuffer.clear();
                inputBuffer.put(byteConvertFrame);
                mediaCodec.queueInputBuffer(inputBufIndex, 0, byteConvertFrame.length, ptsUsec, 0);
                mGenerateIndex++;
            }
            MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo();
            int encoderStatus = mediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
            if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
                // no output available yet
                Log.e(TAG, "No output from encoder available");
            } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                // not expected for an encoder
                MediaFormat newFormat = mediaCodec.getOutputFormat();
                mTrackIndex = mediaMuxer.addTrack(newFormat);
                mediaMuxer.start();
            } else if (encoderStatus < 0) {
                Log.e(TAG, "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
            } else if (mBufferInfo.size != 0) {
                ByteBuffer encodedData = mediaCodec.getOutputBuffer(encoderStatus);
                if (encodedData == null) {
                    Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
                } else {
                    encodedData.position(mBufferInfo.offset);
                    encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
                    mediaMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
                    mediaCodec.releaseOutputBuffer(encoderStatus, false);
                }
            }
        }

        release();

        if (mAbort) {
            mOutputFile.delete();
        } else {
            mCallback.onEncodingComplete(mOutputFile);
        }
    }

    private void release() {
        if (mediaCodec != null) {
            mediaCodec.stop();
            mediaCodec.release();
            mediaCodec = null;
            Log.d(TAG,"RELEASE CODEC");
        }
        if (mediaMuxer != null) {
            mediaMuxer.stop();
            mediaMuxer.release();
            mediaMuxer = null;
            Log.d(TAG,"RELEASE MUXER");
        }
    }

    private static MediaCodecInfo selectCodec(String mimeType) {
        int numCodecs = MediaCodecList.getCodecCount();
        for (int i = 0; i < numCodecs; i++) {
            MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
            if (!codecInfo.isEncoder()) {
                continue;
            }
            String[] types = codecInfo.getSupportedTypes();
            for (int j = 0; j < types.length; j++) {
                if (types[j].equalsIgnoreCase(mimeType)) {
                    return codecInfo;
                }
            }
        }
        return null;
    }

    private static int selectColorFormat(MediaCodecInfo codecInfo,
                                         String mimeType) {
        MediaCodecInfo.CodecCapabilities capabilities = codecInfo
                .getCapabilitiesForType(mimeType);
        for (int i = 0; i < capabilities.colorFormats.length; i++) {
            int colorFormat = capabilities.colorFormats[i];
            if (isRecognizedFormat(colorFormat)) {
                return colorFormat;
            }
        }
        return 0; // not reached
    }

    private static boolean isRecognizedFormat(int colorFormat) {
        switch (colorFormat) {
            // these are the formats we know how to handle for
            case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
            case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
            case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
            case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
            case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
                return true;
            default:
                return false;
        }
    }

    private byte[] getNV21(int inputWidth, int inputHeight, Bitmap scaled) {

        int[] argb = new int[inputWidth * inputHeight];

        scaled.getPixels(argb, 0, inputWidth, 0, 0, inputWidth, inputHeight);

        byte[] yuv = new byte[inputWidth * inputHeight * 3 / 2];
        encodeYUV420SP(yuv, argb, inputWidth, inputHeight);

        scaled.recycle();

        return yuv;
    }

    private void encodeYUV420SP(byte[] yuv420sp, int[] argb, int width, int height) {
        final int frameSize = width * height;

        int yIndex = 0;
        int uvIndex = frameSize;

        int a, R, G, B, Y, U, V;
        int index = 0;
        for (int j = 0; j < height; j++) {
            for (int i = 0; i < width; i++) {

                a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
                R = (argb[index] & 0xff0000) >> 16;
                G = (argb[index] & 0xff00) >> 8;
                B = (argb[index] & 0xff) >> 0;


                Y = ((66 * R + 129 * G + 25 * B + 128) >> 8) + 16;
                U = ((-38 * R - 74 * G + 112 * B + 128) >> 8) + 128;
                V = ((112 * R - 94 * G - 18 * B + 128) >> 8) + 128;


                yuv420sp[yIndex++] = (byte) ((Y < 0) ? 0 : ((Y > 255) ? 255 : Y));
                if (j % 2 == 0 && index % 2 == 0) {
                    yuv420sp[uvIndex++] = (byte) ((U < 0) ? 0 : ((U > 255) ? 255 : U));
                    yuv420sp[uvIndex++] = (byte) ((V < 0) ? 0 : ((V > 255) ? 255 : V));

                }

                index++;
            }
        }
    }

    private long computePresentationTime(long frameIndex, int framerate) {
        return 132 + frameIndex * 1000000 / framerate;
    }
}

Usage is something like:

BitmapToVideoEncoder bitmapToVideoEncoder = new BitmapToVideoEncoder(new IBitmapToVideoEncoderCallback() {
    @Override
    public void onEncodingComplete(File outputFile) {
        Toast.makeText(this,  "Encoding complete!", Toast.LENGTH_LONG).show();
    }
});

bitmapToVideoEncoder.startEncoding(getWidth(), getHeight(), new File("some_path"));
bitmapToVideoEncoder.queueFrame(bitmap1);
bitmapToVideoEncoder.queueFrame(bitmap2);
bitmapToVideoEncoder.queueFrame(bitmap3);
bitmapToVideoEncoder.queueFrame(bitmap4);
bitmapToVideoEncoder.queueFrame(bitmap5);
bitmapToVideoEncoder.stopEncoding();

And if your recording is interrupted (ex Activity is pausing) you can abort and it will delete the file (since it would be corrupt anyway). Alternatively just call stopEncoding and it will properly close the file so it is not corrupt:

bitmapToVideoEncoder.abortEncoding();

There is also a getActiveBitmaps() function to see how big the queue is (if the queue gets to big you can run out of memory). Also here is some code to efficiently create a bitmap from a view so you can queue it up (my app takes periodic screenshots and encodes them into a video):

View view = some_view;
final Bitmap bitmap = Bitmap.createBitmap(view.getWidth(), view.getHeight(),
        Bitmap.Config.ARGB_8888);

// Create a handler thread to offload the processing of the image.
final HandlerThread handlerThread = new HandlerThread("PixelCopier");
handlerThread.start();

PixelCopy.request(view, bitmap, (copyResult) -> {
    bitmapToVideoEncoder.queueFrame(bitmap);
}, new Handler(handlerThread.getLooper()));

Answer 2

The output of MediaCodec is a raw H.264 elementary stream. I've found that the Totem media player for Linux is able to play them.

What you really want to do is convert the output to a .mp4 file. ( Update: ) Android 4.3 (API 18) introduced the MediaMuxer class, which provides a way to convert the raw data (plus an optional audio stream) to a .mp4 file.

The layout of data in the ByteBuffer is, as of Android 4.3, device-dependent. (In fact, not all devices support COLOR_FormatYUV420Planar -- some prefer a semi-planar variant.) I can't tell you the exact layout without knowing your device, but I can tell you that it wants uncompressed data, so passing in compressed PNG data is not going to work.

( Update: ) Android 4.3 also allows Surface input to MediaCodec encoders, so anything you can render with OpenGL ES can be recorded. For an example, see the EncodeAndMuxTest sample here .

Answer 3

I used following steps to convert my bitmaps to video file.

Step 1: Preparing

I have prepared encoder like this. I use MediaMuxer to create a mp4 file.

    private void prepareEncoder() {
    try {
        mBufferInfo = new MediaCodec.BufferInfo();

        mediaCodec = MediaCodec.createEncoderByType(MIME_TYPE);
        mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, WIDTH, HEIGHT);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, calcBitRate());
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
        if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.LOLLIPOP) {
            mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar);
        }else{
            mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
        }
        //2130708361, 2135033992, 21
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);

        final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE_AUDIO, SAMPLE_RATE, 1);
        audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
        audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO);
        audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
        audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);

        mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
        mediaCodec.start();

        mediaCodecForAudio = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
        mediaCodecForAudio.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
        mediaCodecForAudio.start();

        try {
            String outputPath = new File(Environment.getExternalStorageDirectory(),
                    "test.mp4").toString();
            mediaMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
        } catch (IOException ioe) {
            throw new RuntimeException("MediaMuxer creation failed", ioe);
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}

Step 2: Buffering

I have created runnable for buffering.

private void bufferEncoder() {
        runnable = new Runnable() {
            @Override
            public void run() {
                prepareEncoder();
                try {
                    while (mRunning) {
                        encode();
                    }
                    encode();
                } finally {
                    release();
                }
            }
        };
        Thread thread = new Thread(runnable);
        thread.start();
    }

Step 3: Encoding

This is the most important part that you have missed. In this part, I have prepared input buffer before output. When input buffers queued, output buffers are ready to encoding.

public void encode() {
            while (true) {
                if (!mRunning) {
                    break;
                }
                int inputBufIndex = mediaCodec.dequeueInputBuffer(TIMEOUT_USEC);
                long ptsUsec = computePresentationTime(generateIndex);
                if (inputBufIndex >= 0) {
                    Bitmap image = loadBitmapFromView(captureImageView);
                    image = Bitmap.createScaledBitmap(image, WIDTH, HEIGHT, false);
                    byte[] input = getNV21(WIDTH, HEIGHT, image);
                    final ByteBuffer inputBuffer = mediaCodec.getInputBuffer(inputBufIndex);
                    inputBuffer.clear();
                    inputBuffer.put(input);
                    mediaCodec.queueInputBuffer(inputBufIndex, 0, input.length, ptsUsec, 0);
                    generateIndex++;
                }
                int encoderStatus = mediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
                if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
                    // no output available yet
                    Log.d("CODEC", "no output from encoder available");
                } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                    // not expected for an encoder
                    MediaFormat newFormat = mediaCodec.getOutputFormat();
                    mTrackIndex = mediaMuxer.addTrack(newFormat);
                    mediaMuxer.start();
                } else if (encoderStatus < 0) {
                    Log.i("CODEC", "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
                } else if (mBufferInfo.size != 0) {
                    ByteBuffer encodedData = mediaCodec.getOutputBuffer(encoderStatus);
                    if (encodedData == null) {
                        Log.i("CODEC", "encoderOutputBuffer " + encoderStatus + " was null");
                    } else {
                        encodedData.position(mBufferInfo.offset);
                        encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
                        mediaMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
                        mediaCodec.releaseOutputBuffer(encoderStatus, false);
                    }
                }
            }
        }
    }

Step 4: Releasing

Finally, if we finished encoding then release the muxer and encoder.

private void release() {
        if (mediaCodec != null) {
            mediaCodec.stop();
            mediaCodec.release();
            mediaCodec = null;
            Log.i("CODEC", "RELEASE CODEC");
        }
        if (mediaMuxer != null) {
            mediaMuxer.stop();
            mediaMuxer.release();
            mediaMuxer = null;
            Log.i("CODEC", "RELEASE MUXER");
        }
    }

I hope this will helps to you.

Answer 4

1. Encoders output is "raw" h264, so you can set filename extension to "h264" and play it with mplayer, ie mplayer ./your_output.h264
2. One more thing: you says to encoder that frame will be in COLOR_FormatYUV420Planar color format but it looks like you give him PNG content, so output file will probably contain color mess. I think you should convert PNG to yuv420 (with this, for example, https://code.google.com/p/libyuv/ ) before feeding it to encoder.