Android - Processing of Camera Data using Renderscript & Camera2 API
Question
in the following I want to show pieces of my custom camera app code. My goal is to apply filters on the incoming video frames and output them. For that, I use Renderscript and Camera2.
Here is my MainActivity.java (because it is a bit long I deleted the methods dealing with getting the camera permissions):
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
private static final int REQUEST_CAMERA_PERMISSION_RESULT = 0;
private TextureView mTextureView;
private Button mButton;
private CameraDevice mCameraDevice;
private String mCameraId;
private HandlerThread mBackgroundHandlerThread;
private Handler mBackgroundHandler;
private Size mPreviewSize;
private CaptureRequest.Builder mCaptureRequestBuilder;
private RsSurfaceRenderer mRenderer;
private Surface mPreviewSurface;
private Surface mProcessingNormalSurface;
private RsCameraPreviewRenderer cameraPreviewRenderer;
private RenderScript rs;
private List<Surface> mSurfaces;
private Toast rendererNameToast;
private String rendererName;
private int currentRendererIndex = 0;
private static List<Class<? extends RsRenderer>> rendererTypes;
static {
rendererTypes = new ArrayList<>();
rendererTypes.add(DefaultRsRenderer.class);
rendererTypes.add(GreyscaleRsRenderer.class);
rendererTypes.add(SharpenRenderer.class);
rendererTypes.add(BlurRsRenderer.class);
rendererTypes.add(ColorFrameRenderer.class);
rendererTypes.add(HueRotationRenderer.class);
rendererTypes.add(TrailsRenderer.class);
rendererTypes.add(AcidRenderer.class);
}
private static final SparseIntArray ORIENTATIONS =
new SparseIntArray();
static {
ORIENTATIONS.append(Surface.ROTATION_0, 0);
ORIENTATIONS.append(Surface.ROTATION_90, 90);
ORIENTATIONS.append(Surface.ROTATION_180, 180);
ORIENTATIONS.append(Surface.ROTATION_270, 270);
}
private TextureView.SurfaceTextureListener mSurfaceTextureListener = new TextureView.SurfaceTextureListener() {
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
setupCamera(width, height);
connectCamera();
}
@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture surfaceTexture, int width, int height) {
}
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surfaceTexture) {
return false;
}
@Override
public void onSurfaceTextureUpdated(SurfaceTexture surfaceTexture) {
}
};
private CameraDevice.StateCallback mCameraDeviceStateCallback = new CameraDevice.StateCallback() {
@Override
public void onOpened(@NonNull CameraDevice cameraDevice) {
mCameraDevice = cameraDevice;
startPreview();
}
@Override
public void onDisconnected(@NonNull CameraDevice cameraDevice) {
cameraDevice.close();
mCameraDevice = null;
}
@Override
public void onError(@NonNull CameraDevice cameraDevice, int i) {
cameraDevice.close();
mCameraDevice = null;
}
};
private CameraCaptureSession.StateCallback mCameraCaptureSessionStateCallback = new CameraCaptureSession.StateCallback() {
@Override
public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) {
try {
cameraCaptureSession.setRepeatingRequest(mCaptureRequestBuilder.build(), null, mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
@Override
public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) {
Toast.makeText(getApplicationContext(), "Unable to setup camera preview", Toast.LENGTH_SHORT).show();
}
};
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
mTextureView = (TextureView) findViewById(R.id.preview);
mButton= (Button) findViewById(R.id.next_button);
mButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View view) {
cycleRendererType();
updateRsRenderer();
if (rendererNameToast != null) {
rendererNameToast.cancel();
}
rendererNameToast =
Toast.makeText(MainActivity.this, rendererName, Toast.LENGTH_LONG);
rendererNameToast.show();
}
});
rs = RenderScript.create(this);
warmUpInBackground(rs);
}
private void cycleRendererType() {
currentRendererIndex++;
if (currentRendererIndex == rendererTypes.size()) {
currentRendererIndex = 0;
}
}
private void updateRsRenderer() {
try {
RsRenderer renderer = rendererTypes.get(currentRendererIndex).newInstance();
rendererName = renderer.getName();
cameraPreviewRenderer.setRsRenderer(renderer);
} catch (InstantiationException | IllegalAccessException e) {
throw new RuntimeException(
"Unable to create renderer for index " + currentRendererIndex +
", make sure it has a no-arg constructor please.", e);
}
}
@Override
protected void onResume() {
super.onResume();
startBackgroundThread();
if(mTextureView.isAvailable()){
setupCamera(mTextureView.getWidth(), mTextureView.getHeight());
connectCamera();
}
else{
mTextureView.setSurfaceTextureListener(mSurfaceTextureListener);
}
}
@Override
public void onWindowFocusChanged(boolean hasFocus){
super.onWindowFocusChanged(hasFocus);
View decorView = getWindow().getDecorView();
if(hasFocus){
decorView.setSystemUiVisibility(
View.SYSTEM_UI_FLAG_LAYOUT_STABLE
| View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
| View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN
| View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION
| View.SYSTEM_UI_FLAG_FULLSCREEN
| View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY
);
}
}
private void connectCamera(){
CameraManager cameraManager = (CameraManager) getSystemService(Context.CAMERA_SERVICE);
try{
if(Build.VERSION.SDK_INT >= Build.VERSION_CODES.M){
if(ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED){
cameraManager.openCamera(mCameraId, mCameraDeviceStateCallback, mBackgroundHandler);
}
else{
if(shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)){
Toast.makeText(this, "Video app required access to camera", Toast.LENGTH_SHORT).show();
}
requestPermissions(new String[] {Manifest.permission.CAMERA}, REQUEST_CAMERA_PERMISSION_RESULT);
}
}
else{
cameraManager.openCamera(mCameraId, mCameraDeviceStateCallback, mBackgroundHandler);
}
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
@Override
protected void onPause() {
closeCamera();
stopBackgroundThread();
super.onPause();
}
private void closeCamera(){
if(mCameraDevice != null){
mCameraDevice.close();
mCameraDevice = null;
}
}
private void startPreview(){
SurfaceTexture surfaceTexture = mTextureView.getSurfaceTexture();
surfaceTexture.setDefaultBufferSize(mPreviewSize.getWidth(), mPreviewSize.getHeight());
mPreviewSurface = new Surface(surfaceTexture);
if (mRenderer == null) {
mRenderer = createNewRendererForCurrentType(mPreviewSize);
}
if (mPreviewSurface == null)
return;
/*
* leads us to rgbOutAlloc.setSurface(outputSurface) whereas outputSurface = mPreviewSurface
*
* setSurface(Surface):
* Associate a Surface with this Allocation. This operation is only valid for Allocations with USAGE_IO_OUTPUT.
*
* rgbOutAlloc is an RGBA_8888 allocation that can act as a Surface producer.
* */
mRenderer.setOutputSurface(mPreviewSurface);
/*
* leads us to yuvInAlloc.getSurface()
*
* getSurface():
* Returns the handle to a raw buffer that is being managed by the screen compositor.
* This operation is only valid for Allocations with USAGE_IO_INPUT.
*
* HERE:
* Get the Surface that the camera will push frames to. This is the Surface from our yuv
* input allocation. It will recieve a callback when a frame is available from the camera.
* */
mProcessingNormalSurface = mRenderer.getInputSurface();
List<Surface> cameraOutputSurfaces = new ArrayList<>();
cameraOutputSurfaces.add(mProcessingNormalSurface);
mSurfaces = cameraOutputSurfaces;
try {
mCaptureRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
mCaptureRequestBuilder.addTarget(mProcessingNormalSurface);
mCameraDevice.createCaptureSession(
mSurfaces,
mCameraCaptureSessionStateCallback,
mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
private void setupCamera(int width, int height){
CameraManager cameraManager = (CameraManager) getSystemService(Context.CAMERA_SERVICE);
try{
for(String cameraId: cameraManager.getCameraIdList()){
CameraCharacteristics cameraCharacteristics =
cameraManager.getCameraCharacteristics(cameraId);
if(cameraCharacteristics.get(CameraCharacteristics.LENS_FACING)==
CameraCharacteristics.LENS_FACING_FRONT){
continue;
}
StreamConfigurationMap map =
cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
int deviceOrientation = getWindowManager().getDefaultDisplay().getRotation();
int totalRotation = sensorToDeviceRotation(cameraCharacteristics, deviceOrientation);
boolean swapRotation = totalRotation == 90 || totalRotation == 270;
int rotatedWidth = width;
int rotatedHeigth = height;
if(swapRotation){
rotatedWidth = height;
rotatedHeigth = width;
}
mPreviewSize = chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class), rotatedWidth, rotatedHeigth);
//mTextureView.setRotation(90);
mCameraId = cameraId;
return;
}
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
private RsSurfaceRenderer createNewRendererForCurrentType(Size outputSize) {
if (cameraPreviewRenderer == null) {
cameraPreviewRenderer =
new RsCameraPreviewRenderer(rs, outputSize.getWidth(), outputSize.getHeight());
}
updateRsRenderer();
return cameraPreviewRenderer;
}
private void startBackgroundThread(){
mBackgroundHandlerThread = new HandlerThread("Camera2VideoImage");
mBackgroundHandlerThread.start();
mBackgroundHandler = new Handler(mBackgroundHandlerThread.getLooper());
}
private void stopBackgroundThread(){
mBackgroundHandlerThread.quitSafely();
try {
mBackgroundHandlerThread.join();
mBackgroundHandlerThread = null;
mBackgroundHandler = null;
} catch (InterruptedException e) {
e.printStackTrace();
}
}
private static int sensorToDeviceRotation(CameraCharacteristics c, int deviceOrientation){
int sensorOrientation = c.get(CameraCharacteristics.SENSOR_ORIENTATION);
// get device orientation in degrees
deviceOrientation = ORIENTATIONS.get(deviceOrientation);
// calculate desired JPEG orientation relative to camera orientation to make
// the image upright relative to the device orientation
return (sensorOrientation + deviceOrientation + 360) % 360;
}
static class CompareSizesByArea implements Comparator<Size>{
@Override
public int compare(Size lhs, Size rhs) {
// we cast here to ensure the multiplications won't
// overflow
return Long.signum((long) lhs.getWidth() * lhs.getHeight() /
(long) rhs.getWidth() * rhs.getHeight());
}
}
private static Size chooseOptimalSize(Size[] choices, int width, int height){
// Collect the supported resolutions that are the least as big as the preview
// Surface
List<Size> bigEnough = new ArrayList<Size>();
for(Size option: choices){
if(option.getHeight() == option.getWidth() * height/width && option.getWidth() >= width && option.getHeight()>=height){
bigEnough.add(option);
}
}
// pick the smallest of those, assuming we found any
if(bigEnough.size() > 0){
return Collections.min(bigEnough, new CompareSizesByArea());
}
else{
Log.e(TAG, "Couldn't find any suitable preview size");
return choices[0];
}
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
if(requestCode == REQUEST_CAMERA_PERMISSION_RESULT){
if(grantResults[0] != PackageManager.PERMISSION_GRANTED){
Toast.makeText(
getApplicationContext(),
"Application will not run without camera service",
Toast.LENGTH_SHORT).show();
}
}
}
/**
* These are custom kernels that are AoT compiled on the very first launch so we want to make
* sure that happens outside of a render loop and also not in the UI thread.
*/
public static void warmUpInBackground(RenderScript rs) {
new Thread(() -> {
Log.i(TAG, "RS warmup start...");
long start = System.currentTimeMillis();
try {
ScriptC_color_frame color_frame = new ScriptC_color_frame(rs);
ScriptC_set_alpha set_alpha = new ScriptC_set_alpha(rs);
ScriptC_to_grey to_grey = new ScriptC_to_grey(rs);
} catch (Exception e) {
e.printStackTrace();
}
Log.i(TAG, "RS warmup end, " + (System.currentTimeMillis() - start) + " ms");
}).start();
}
}
Now, the following class contains all the methods used to create the Renderscript Allocations. It also has methods for binding the allocations to the two surfaces we have seen in the startPreview() method of the MainActivity.java class above. It also starts a render thread which processes incoming frames to yuvInAlloc .
public class RsCameraPreviewRenderer
implements RsSurfaceRenderer, Allocation.OnBufferAvailableListener, Runnable {
private static final String TAG = "RsCameraPreviewRenderer";
private final RenderScript rs;
private final Allocation yuvInAlloc;
private final Allocation rgbInAlloc;
private final Allocation rgbOutAlloc;
private final ScriptIntrinsicYuvToRGB yuvToRGBScript;
@Nullable
private final HandlerThread renderThread;
// all guarded by "this"
private Handler renderHandler;
private RsRenderer rsRenderer;
private int nFramesAvailable;
private boolean outputSurfaceIsSet;
/**
* @param rs
* @param x
* @param y
*/
public RsCameraPreviewRenderer(RenderScript rs, int x, int y) {
this(rs, new DefaultRsRenderer(), x, y);
}
/**
* @param rs
* @param rsRenderer
* @param x
* @param y
*/
public RsCameraPreviewRenderer(RenderScript rs, RsRenderer rsRenderer, int x, int y) {
this(rs, rsRenderer, x, y, null);
}
/**
* @param rs
* @param rsRenderer
* @param x
* @param y
* @param renderHandler
*/
public RsCameraPreviewRenderer(RenderScript rs,
RsRenderer rsRenderer,
int x,
int y,
Handler renderHandler) {
this.rs = rs;
this.rsRenderer = rsRenderer;
if (renderHandler == null) {
this.renderThread = new HandlerThread(TAG);
this.renderThread.start();
this.renderHandler = new Handler(renderThread.getLooper());
} else {
this.renderThread = null;
this.renderHandler = renderHandler;
}
Log.i(TAG,
"Setting up RsCameraPreviewRenderer with " + rsRenderer.getName() + " (" + x + "," +
y + ")");
/*
* Create an YUV allocation that can act as a Surface consumer. This lets us call
* Allocation#getSurface(), set a Allocation.OnBufferAvailableListener
* callback to be notified when a frame is ready, and call Allocation#ioReceive() to
* latch a frame and access its yuv pixel data.
*
* The yuvFormat should be the value ImageFormat#YUV_420_888, ImageFormat#NV21 or maybe
* ImageFormat#YV12.
*
* @param rs RenderScript context
* @param x width in pixels
* @param y height in pixels
* @param yuvFormat yuv pixel format
* @return a YUV Allocation with USAGE_IO_INPUT
* */
yuvInAlloc = RsUtil.createYuvIoInputAlloc(rs, x, y, ImageFormat.YUV_420_888);
yuvInAlloc.setOnBufferAvailableListener(this);
/**
* Create a sized RGBA_8888 Allocation to use with scripts.
*
* @param rs RenderScript context
* @param x width in pixels
* @param y height in pixels
* @return an RGBA_8888 Allocation
*/
rgbInAlloc = RsUtil.createRgbAlloc(rs, x, y);
/**
* Create an RGBA_8888 allocation that can act as a Surface producer. This lets us call
* Allocation#setSurface(Surface) and call Allocation#ioSend(). If
* you wanted to read the data from this Allocation, do so before calling ioSend(), because
* after, the data is undefined.
*
* @param rs rs context
* @param x width in pixels
* @param y height in pixels
* @return an RGBA_8888 Allocation with USAGE_IO_INPUT
*/
rgbOutAlloc = RsUtil.createRgbIoOutputAlloc(rs, x, y);
yuvToRGBScript = ScriptIntrinsicYuvToRGB.create(rs, Element.RGBA_8888(rs));
yuvToRGBScript.setInput(yuvInAlloc);
}
@Override
@AnyThread
public synchronized void setRsRenderer(RsRenderer rsRenderer) {
if (isRunning()) {
this.rsRenderer = rsRenderer;
}
}
/**
* Check if this renderer is still running or has been shutdown.
*
* @return true if we're running, else false
*/
@Override
@AnyThread
public synchronized boolean isRunning() {
if (renderHandler == null) {
Log.w(TAG, "renderer was already shut down");
return false;
}
return true;
}
/**
* Set the output surface to consume the stream of edited camera frames. This is probably
* from a SurfaceView or TextureView. Please make sure it's valid.
*
* @param outputSurface a valid surface to consume a stream of edited frames from the camera
*/
@AnyThread
@Override
public synchronized void setOutputSurface(Surface outputSurface) {
if (isRunning()) {
if (!outputSurface.isValid()) {
throw new IllegalArgumentException("output was invalid");
}
rgbOutAlloc.setSurface(outputSurface);
outputSurfaceIsSet = true;
Log.d(TAG, "output surface was set");
}
}
/**
* Get the Surface that the camera will push frames to. This is the Surface from our yuv
* input allocation. It will recieve a callback when a frame is available from the camera.
*
* @return a surface that consumes yuv frames from the camera preview, or null renderer is
* shutdown
*/
@AnyThread
@Override
public synchronized Surface getInputSurface() {
return isRunning() ? yuvInAlloc.getSurface() : null;
}
/**
* Callback for when the camera has a new frame. We want to handle this on the render thread
* specific thread, so we'll increment nFramesAvailable and post a render request.
*/
@Override
public synchronized void onBufferAvailable(Allocation a) {
if (isRunning()) {
if (!outputSurfaceIsSet) {
Log.e(TAG, "We are getting frames from the camera but we never set the view " +
"surface to render to");
return;
}
nFramesAvailable++;
renderHandler.post(this);
}
}
/**
* Render a frame on the render thread. Everything is async except for ioSend() will block
* until the rendering completes. If we wanted to time it, make sure to log the time after
* that call.
*/
@WorkerThread
@Override
public void run() {
RsRenderer renderer;
int nFrames;
synchronized (this) {
if (!isRunning()) {
return;
}
renderer = rsRenderer;
nFrames = nFramesAvailable;
nFramesAvailable = 0;
renderHandler.removeCallbacks(this);
}
for (int i = 0; i < nFrames; i++) {
/*
* Receive the latest input into the Allocation.
* This operation is only valid if USAGE_IO_INPUT is set on the Allocation.
* */
yuvInAlloc.ioReceive();
}
yuvToRGBScript.forEach(rgbInAlloc);
/*
* Render an edit to an input Allocation and write it to an output allocation. This must
* always overwrite the out Allocation. This is called once for a Bitmap, and once per frame
* for stream rendering.
* */
renderer.renderFrame(rs, rgbInAlloc, rgbOutAlloc);
/*
* Send a buffer to the output stream. The contents of the Allocation will be undefined after this
* operation. This operation is only valid if USAGE_IO_OUTPUT is set on the Allocation.
* */
rgbOutAlloc.ioSend();
}
/**
* Shut down the renderer when you're finished.
*/
@Override
@AnyThread
public void shutdown() {
synchronized (this) {
if (!isRunning()) {
Log.d(TAG, "requesting shutdown...");
renderHandler.removeCallbacks(this);
renderHandler.postAtFrontOfQueue(() -> {
Log.i(TAG, "shutting down");
synchronized (this) {
yuvInAlloc.destroy();
rgbInAlloc.destroy();
rgbOutAlloc.destroy();
yuvToRGBScript.destroy();
if (renderThread != null) {
renderThread.quitSafely();
}
}
});
renderHandler = null;
}
}
}
}
I scrambled most of the code from the following projects:
- https://github.com/googlesamples/android-HdrViewfinder
- https://github.com/lydia-schiff/hella-renderscript (great help)
- https://github.com/googlesamples/android-Camera2Basic
- https://github.com/googlesamples/android-Camera2Video
My problem is that the output is shown as in the following GIF: It is rotated somehow. Not how I expected it. So, why is this happening? I could not find an answer. Can someone help ?
1 answers
solution1
2 2019-10-30 05:47:12
This is because YUV format only output in the default orientation. The sensorToDeviceRotation() you have only rotates for JPEG outputs. You need to handle the YUV rotation by yourself, either create a function in your Activity to rotate the bitmap or use other script to rotate in RenderScript. The later is recommenced as it is faster.
I found a sample to use rotate script here RenderScript
I did not test this, but it should do the work if you research on it a bit more. Hope this helps.
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.
Related Question
Camera zoom setting using Camera2 API
ImageReader callback is not working in android camera2 api?
Android camera2 api galaxy s7
How to use Android's Camera2 API
Android Camera2 API stretching the preview
Android Camera2 API Image Color space
Understanding the button click in camera2 API in Camera2Basic in Android
How to get the frame using camera2 api
Problems with LED Flashlight by using Camera2 Api?
Using OpenCV in Java Camera2 API
Related Tags