/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.devcamera;
import android.content.Context;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
import android.hardware.camera2.CameraAccessException;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CameraManager;
import android.hardware.camera2.CameraMetadata;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult;
import android.hardware.camera2.params.Face;
import android.hardware.camera2.params.InputConfiguration;
import android.media.Image;
import android.media.ImageReader;
import android.media.ImageWriter;
import android.media.MediaActionSound;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.SystemClock;
import android.util.Log;
import android.util.Size;
import android.view.Surface;
import android.media.Image.Plane;
import java.nio.ByteBuffer;
import java.nio.BufferUnderflowException;
import java.lang.IndexOutOfBoundsException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import javax.microedition.khronos.opengles.GL10;
/**
* Api2Camera : a camera2 implementation
*
* The goal here is to make the simplest possible API2 camera,
* where individual streams and capture options (e.g. edge enhancement,
* noise reduction, face detection) can be toggled on and off.
*
*/
public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvailableListener {
private static final String TAG = "DevCamera_API2";
// Nth frame to log; put 10^6 if you don't want logging.
private static int LOG_NTH_FRAME = 30;
// Log dropped frames. There are a log on Angler MDA32.
private static boolean LOG_DROPPED_FRAMES = true;
// IMPORTANT: Only one of these can be true:
private static boolean SECOND_YUV_IMAGEREADER_STREAM = true;
private static boolean SECOND_SURFACE_TEXTURE_STREAM = false;
// Enable raw stream if available.
private static boolean RAW_STREAM_ENABLE = true;
// Use JPEG ImageReader and YUV ImageWriter if reprocessing is available
private static final boolean USE_REPROCESSING_IF_AVAIL = true;
// Whether we are continuously taking pictures, or not.
boolean mIsBursting = false;
// Last total capture result
TotalCaptureResult mLastTotalCaptureResult;
// ImageReader/Writer buffer sizes.
private static final int YUV1_IMAGEREADER_SIZE = 8;
private static final int YUV2_IMAGEREADER_SIZE = 8;
private static final int RAW_IMAGEREADER_SIZE = 8;
private static final int IMAGEWRITER_SIZE = 2;
private CameraInfoCache mCameraInfoCache;
private CameraManager mCameraManager;
private CameraCaptureSession mCurrentCaptureSession;
private MediaActionSound mMediaActionSound = new MediaActionSound();
MyCameraCallback mMyCameraCallback;
// Generally everything running on this thread & this module is *not thread safe*.
private HandlerThread mOpsThread;
private Handler mOpsHandler;
private HandlerThread mInitThread;
private Handler mInitHandler;
private HandlerThread mJpegListenerThread;
private Handler mJpegListenerHandler;
Context mContext;
boolean mCameraIsFront;
SurfaceTexture mSurfaceTexture;
Surface mSurfaceTextureSurface;
private boolean mFirstFrameArrived;
private ImageReader mYuv1ImageReader;
private int mYuv1ImageCounter;
// Handle to last received Image: allows ZSL to be implemented.
private Image mYuv1LastReceivedImage = null;
// Time at which reprocessing request went in (right now we are doing one at a time).
private long mReprocessingRequestNanoTime;
private ImageReader mJpegImageReader;
private ImageReader mYuv2ImageReader;
private int mYuv2ImageCounter;
private ImageReader mRawImageReader;
private int mRawImageCounter;
private boolean mIsDepthCloudSupported = false;
private ImageReader mDepthCloudImageReader;
private int mDepthCloudImageCounter = 0;
private static int STORE_NTH_DEPTH_CLOUD = 30;
private static boolean DEPTH_CLOUD_STORE_ENABLED = false;
// Starting the preview requires each of these 3 to be true/non-null:
volatile private Surface mPreviewSurface;
volatile private CameraDevice mCameraDevice;
volatile boolean mAllThingsInitialized = false;
/**
* Constructor.
*/
public Api2Camera(Context context, boolean useFrontCamera) {
mContext = context;
mCameraIsFront = useFrontCamera;
mCameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE);
mCameraInfoCache = new CameraInfoCache(mCameraManager, useFrontCamera);
// Create thread and handler for camera operations.
mOpsThread = new HandlerThread("CameraOpsThread");
mOpsThread.start();
mOpsHandler = new Handler(mOpsThread.getLooper());
// Create thread and handler for slow initialization operations.
// Don't want to use camera operations thread because we want to time camera open carefully.
mInitThread = new HandlerThread("CameraInitThread");
mInitThread.start();
mInitHandler = new Handler(mInitThread.getLooper());
mInitHandler.post(new Runnable() {
@Override
public void run() {
InitializeAllTheThings();
mAllThingsInitialized = true;
Log.v(TAG, "STARTUP_REQUIREMENT ImageReader initialization done.");
tryToStartCaptureSession();
}
});
// Set initial Noise and Edge modes.
if (mCameraInfoCache.isHardwareLevelAtLeast(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3)) {
// YUV streams.
if (mCameraInfoCache.supportedModesContains(mCameraInfoCache.noiseModes,
CameraCharacteristics.NOISE_REDUCTION_MODE_ZERO_SHUTTER_LAG)) {
mCaptureNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_ZERO_SHUTTER_LAG;
} else {
mCaptureNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_FAST;
}
if (mCameraInfoCache.supportedModesContains(mCameraInfoCache.edgeModes,
CameraCharacteristics.EDGE_MODE_ZERO_SHUTTER_LAG)) {
mCaptureEdgeMode = CameraCharacteristics.EDGE_MODE_ZERO_SHUTTER_LAG;
} else {
mCaptureEdgeMode = CameraCharacteristics.EDGE_MODE_FAST;
}
// Reprocessing.
mReprocessingNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_HIGH_QUALITY;
mReprocessingEdgeMode = CameraCharacteristics.EDGE_MODE_HIGH_QUALITY;
}
if (null != mCameraInfoCache.getDepthCloudSize()) {
mIsDepthCloudSupported = true;
}
}
// Ugh, why is this stuff so slow?
private void InitializeAllTheThings() {
// Thread to handle returned JPEGs.
mJpegListenerThread = new HandlerThread("CameraJpegThread");
mJpegListenerThread.start();
mJpegListenerHandler = new Handler(mJpegListenerThread.getLooper());
// Create ImageReader to receive JPEG image buffers via reprocessing.
mJpegImageReader = ImageReader.newInstance(
mCameraInfoCache.getYuvStream1Size().getWidth(),
mCameraInfoCache.getYuvStream1Size().getHeight(),
ImageFormat.JPEG,
2);
mJpegImageReader.setOnImageAvailableListener(mJpegImageListener, mJpegListenerHandler);
// Create ImageReader to receive YUV image buffers.
mYuv1ImageReader = ImageReader.newInstance(
mCameraInfoCache.getYuvStream1Size().getWidth(),
mCameraInfoCache.getYuvStream1Size().getHeight(),
ImageFormat.YUV_420_888,
YUV1_IMAGEREADER_SIZE);
mYuv1ImageReader.setOnImageAvailableListener(mYuv1ImageListener, mOpsHandler);
if (mIsDepthCloudSupported) {
mDepthCloudImageReader = ImageReader.newInstance(
mCameraInfoCache.getDepthCloudSize().getWidth(),
mCameraInfoCache.getDepthCloudSize().getHeight(),
ImageFormat.DEPTH_POINT_CLOUD, 2);
mDepthCloudImageReader.setOnImageAvailableListener(mDepthCloudImageListener, mOpsHandler);
}
if (SECOND_YUV_IMAGEREADER_STREAM) {
// Create ImageReader to receive YUV image buffers.
mYuv2ImageReader = ImageReader.newInstance(
mCameraInfoCache.getYuvStream2Size().getWidth(),
mCameraInfoCache.getYuvStream2Size().getHeight(),
ImageFormat.YUV_420_888,
YUV2_IMAGEREADER_SIZE);
mYuv2ImageReader.setOnImageAvailableListener(mYuv2ImageListener, mOpsHandler);
}
if (SECOND_SURFACE_TEXTURE_STREAM) {
int[] textures = new int[1];
// generate one texture pointer and bind it as an external texture.
GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textures[0]);
// No mip-mapping with camera source.
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_MIN_FILTER,
GL10.GL_LINEAR);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);
// Clamp to edge is only option.
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);
int texture_id = textures[0];
mSurfaceTexture = new SurfaceTexture(texture_id);
mSurfaceTexture.setDefaultBufferSize(320, 240);
mSurfaceTexture.setOnFrameAvailableListener(this);
mSurfaceTextureSurface = new Surface(mSurfaceTexture);
}
if (RAW_STREAM_ENABLE && mCameraInfoCache.rawAvailable()) {
// Create ImageReader to receive thumbnail sized YUV image buffers.
mRawImageReader = ImageReader.newInstance(
mCameraInfoCache.getRawStreamSize().getWidth(),
mCameraInfoCache.getRawStreamSize().getHeight(),
mCameraInfoCache.getRawFormat(),
RAW_IMAGEREADER_SIZE);
mRawImageReader.setOnImageAvailableListener(mRawImageListener, mOpsHandler);
}
// Load click sound.
mMediaActionSound.load(MediaActionSound.SHUTTER_CLICK);
}
public void setCallback(MyCameraCallback callback) {
mMyCameraCallback = callback;
}
public void triggerAFScan() {
Log.v(TAG, "AF trigger");
issuePreviewCaptureRequest(true);
}
public void setCAF() {
Log.v(TAG, "run CAF");
issuePreviewCaptureRequest(false);
}
public void takePicture() {
mMediaActionSound.play(MediaActionSound.SHUTTER_CLICK);
mOpsHandler.post(new Runnable() {
@Override
public void run() {
runReprocessing();
}
});
}
public void onFrameAvailable (SurfaceTexture surfaceTexture) {
Log.v(TAG, " onFrameAvailable(SurfaceTexture)");
}
public void setBurst(boolean go) {
// if false to true transition.
if (go && !mIsBursting) {
takePicture();
}
mIsBursting = go;
}
public boolean isRawAvailable() {
return mCameraInfoCache.rawAvailable();
}
public boolean isReprocessingAvailable() {
return mCameraInfoCache.isYuvReprocessingAvailable();
}
@Override
public Size getPreviewSize() {
return mCameraInfoCache.getPreviewSize();
}
@Override
public float[] getFieldOfView() {
return mCameraInfoCache.getFieldOfView();
}
@Override
public int getOrientation() {
return mCameraInfoCache.sensorOrientation();
}
@Override
public void openCamera() {
// If API2 FULL mode is not available, display toast
if (!mCameraInfoCache.isCamera2FullModeAvailable()) {
mMyCameraCallback.noCamera2Full();
}
Log.v(TAG, "Opening camera " + mCameraInfoCache.getCameraId());
mOpsHandler.post(new Runnable() {
@Override
public void run() {
CameraTimer.t_open_start = SystemClock.elapsedRealtime();
try {
mCameraManager.openCamera(mCameraInfoCache.getCameraId(), mCameraStateCallback, null);
} catch (CameraAccessException e) {
Log.e(TAG, "Unable to openCamera().");
}
}
});
}
@Override
public void closeCamera() {
// TODO: We are stalling main thread now which is bad.
Log.v(TAG, "Closing camera " + mCameraInfoCache.getCameraId());
if (mCameraDevice != null) {
try {
mCurrentCaptureSession.abortCaptures();
} catch (CameraAccessException e) {
Log.e(TAG, "Could not abortCaptures().");
}
mCameraDevice.close();
}
mCurrentCaptureSession = null;
Log.v(TAG, "Done closing camera " + mCameraInfoCache.getCameraId());
}
public void startPreview(final Surface surface) {
Log.v(TAG, "STARTUP_REQUIREMENT preview Surface ready.");
mPreviewSurface = surface;
tryToStartCaptureSession();
}
private CameraDevice.StateCallback mCameraStateCallback = new LoggingCallbacks.DeviceStateCallback() {
@Override
public void onOpened(CameraDevice camera) {
CameraTimer.t_open_end = SystemClock.elapsedRealtime();
mCameraDevice = camera;
Log.v(TAG, "STARTUP_REQUIREMENT Done opening camera " + mCameraInfoCache.getCameraId() +
". HAL open took: (" + (CameraTimer.t_open_end - CameraTimer.t_open_start) + " ms)");
super.onOpened(camera);
tryToStartCaptureSession();
}
};
private void tryToStartCaptureSession() {
if (mCameraDevice != null && mAllThingsInitialized && mPreviewSurface != null) {
mOpsHandler.post(new Runnable() {
@Override
public void run() {
// It used to be: this needed to be posted on a Handler.
startCaptureSession();
}
});
}
}
// Create CameraCaptureSession. Callback will start repeating request with current parameters.
private void startCaptureSession() {
CameraTimer.t_session_go = SystemClock.elapsedRealtime();
Log.v(TAG, "Configuring session..");
List<Surface> outputSurfaces = new ArrayList<Surface>(4);
outputSurfaces.add(mPreviewSurface);
Log.v(TAG, " .. added SurfaceView " + mCameraInfoCache.getPreviewSize().getWidth() +
" x " + mCameraInfoCache.getPreviewSize().getHeight());
outputSurfaces.add(mYuv1ImageReader.getSurface());
Log.v(TAG, " .. added YUV ImageReader " + mCameraInfoCache.getYuvStream1Size().getWidth() +
" x " + mCameraInfoCache.getYuvStream1Size().getHeight());
if (mIsDepthCloudSupported) {
outputSurfaces.add(mDepthCloudImageReader.getSurface());
Log.v(TAG, " .. added Depth cloud ImageReader");
}
if (SECOND_YUV_IMAGEREADER_STREAM) {
outputSurfaces.add(mYuv2ImageReader.getSurface());
Log.v(TAG, " .. added YUV ImageReader " + mCameraInfoCache.getYuvStream2Size().getWidth() +
" x " + mCameraInfoCache.getYuvStream2Size().getHeight());
}
if (SECOND_SURFACE_TEXTURE_STREAM) {
outputSurfaces.add(mSurfaceTextureSurface);
Log.v(TAG, " .. added SurfaceTexture");
}
if (RAW_STREAM_ENABLE && mCameraInfoCache.rawAvailable()) {
outputSurfaces.add(mRawImageReader.getSurface());
Log.v(TAG, " .. added Raw ImageReader " + mCameraInfoCache.getRawStreamSize().getWidth() +
" x " + mCameraInfoCache.getRawStreamSize().getHeight());
}
if (USE_REPROCESSING_IF_AVAIL && mCameraInfoCache.isYuvReprocessingAvailable()) {
outputSurfaces.add(mJpegImageReader.getSurface());
Log.v(TAG, " .. added JPEG ImageReader " + mCameraInfoCache.getJpegStreamSize().getWidth() +
" x " + mCameraInfoCache.getJpegStreamSize().getHeight());
}
try {
if (USE_REPROCESSING_IF_AVAIL && mCameraInfoCache.isYuvReprocessingAvailable()) {
InputConfiguration inputConfig = new InputConfiguration(mCameraInfoCache.getYuvStream1Size().getWidth(),
mCameraInfoCache.getYuvStream1Size().getHeight(), ImageFormat.YUV_420_888);
mCameraDevice.createReprocessableCaptureSession(inputConfig, outputSurfaces,
mSessionStateCallback, null);
Log.v(TAG, " Call to createReprocessableCaptureSession complete.");
} else {
mCameraDevice.createCaptureSession(outputSurfaces, mSessionStateCallback, null);
Log.v(TAG, " Call to createCaptureSession complete.");
}
} catch (CameraAccessException e) {
Log.e(TAG, "Error configuring ISP.");
}
}
ImageWriter mImageWriter;
private CameraCaptureSession.StateCallback mSessionStateCallback = new LoggingCallbacks.SessionStateCallback() {
@Override
public void onReady(CameraCaptureSession session) {
Log.v(TAG, "capture session onReady(). HAL capture session took: (" + (SystemClock.elapsedRealtime() - CameraTimer.t_session_go) + " ms)");
mCurrentCaptureSession = session;
issuePreviewCaptureRequest(false);
if (session.isReprocessable()) {
mImageWriter = ImageWriter.newInstance(session.getInputSurface(), IMAGEWRITER_SIZE);
mImageWriter.setOnImageReleasedListener(
new ImageWriter.OnImageReleasedListener() {
@Override
public void onImageReleased(ImageWriter writer) {
Log.v(TAG, "ImageWriter.OnImageReleasedListener onImageReleased()");
}
}, null);
Log.v(TAG, "Created ImageWriter.");
}
super.onReady(session);
}
};
// Variables to hold capture flow state.
private boolean mCaptureYuv1 = false;
private boolean mCaptureYuv2 = false;
private boolean mCaptureRaw = false;
private int mCaptureNoiseMode = CaptureRequest.NOISE_REDUCTION_MODE_FAST;
private int mCaptureEdgeMode = CaptureRequest.EDGE_MODE_FAST;
private boolean mCaptureFace = false;
// Variables to hold reprocessing state.
private int mReprocessingNoiseMode = CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY;
private int mReprocessingEdgeMode = CaptureRequest.EDGE_MODE_HIGH_QUALITY;
public void setCaptureFlow(Boolean yuv1, Boolean yuv2, Boolean raw10, Boolean nr, Boolean edge, Boolean face) {
if (yuv1 != null) mCaptureYuv1 = yuv1;
if (yuv2 != null) mCaptureYuv2 = yuv2;
if (raw10 != null) mCaptureRaw = raw10 && RAW_STREAM_ENABLE;
if (nr) {
mCaptureNoiseMode = getNextMode(mCaptureNoiseMode, mCameraInfoCache.noiseModes);
}
if (edge) {
mCaptureEdgeMode = getNextMode(mCaptureEdgeMode, mCameraInfoCache.edgeModes);
}
if (face != null) mCaptureFace = face;
mMyCameraCallback.setNoiseEdgeText(
"NR " + noiseModeToString(mCaptureNoiseMode),
"Edge " + edgeModeToString(mCaptureEdgeMode)
);
if (mCurrentCaptureSession != null) {
issuePreviewCaptureRequest(false);
}
}
public void setReprocessingFlow(Boolean nr, Boolean edge) {
if (nr) {
mReprocessingNoiseMode = getNextMode(mReprocessingNoiseMode, mCameraInfoCache.noiseModes);
}
if (edge) {
mReprocessingEdgeMode = getNextMode(mReprocessingEdgeMode, mCameraInfoCache.edgeModes);
}
mMyCameraCallback.setNoiseEdgeTextForReprocessing(
"NR " + noiseModeToString(mReprocessingNoiseMode),
"Edge " + edgeModeToString(mReprocessingEdgeMode)
);
}
public void issuePreviewCaptureRequest(boolean AFtrigger) {
CameraTimer.t_burst = SystemClock.elapsedRealtime();
Log.v(TAG, "issuePreviewCaptureRequest...");
try {
CaptureRequest.Builder b1 = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
b1.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_USE_SCENE_MODE);
b1.set(CaptureRequest.CONTROL_SCENE_MODE, CameraMetadata.CONTROL_SCENE_MODE_FACE_PRIORITY);
if (AFtrigger) {
b1.set(CaptureRequest.CONTROL_AF_MODE, CameraMetadata.CONTROL_AF_MODE_AUTO);
} else {
b1.set(CaptureRequest.CONTROL_AF_MODE, CameraMetadata.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
}
b1.set(CaptureRequest.NOISE_REDUCTION_MODE, mCaptureNoiseMode);
b1.set(CaptureRequest.EDGE_MODE, mCaptureEdgeMode);
b1.set(CaptureRequest.STATISTICS_FACE_DETECT_MODE, mCaptureFace ? mCameraInfoCache.bestFaceDetectionMode() : CaptureRequest.STATISTICS_FACE_DETECT_MODE_OFF);
Log.v(TAG, " .. NR=" + mCaptureNoiseMode + " Edge=" + mCaptureEdgeMode + " Face=" + mCaptureFace);
if (mCaptureYuv1) {
b1.addTarget(mYuv1ImageReader.getSurface());
Log.v(TAG, " .. YUV1 on");
}
if (mCaptureRaw) {
b1.addTarget(mRawImageReader.getSurface());
}
b1.addTarget(mPreviewSurface);
if (mIsDepthCloudSupported && !mCaptureYuv1 && !mCaptureYuv2 && !mCaptureRaw) {
b1.addTarget(mDepthCloudImageReader.getSurface());
}
if (mCaptureYuv2) {
if (SECOND_SURFACE_TEXTURE_STREAM) {
b1.addTarget(mSurfaceTextureSurface);
}
if (SECOND_YUV_IMAGEREADER_STREAM) {
b1.addTarget(mYuv2ImageReader.getSurface());
}
Log.v(TAG, " .. YUV2 on");
}
if (AFtrigger) {
b1.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_START);
mCurrentCaptureSession.capture(b1.build(), mCaptureCallback, mOpsHandler);
b1.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_IDLE);
}
mCurrentCaptureSession.setRepeatingRequest(b1.build(), mCaptureCallback, mOpsHandler);
} catch (CameraAccessException e) {
Log.e(TAG, "Could not access camera for issuePreviewCaptureRequest.");
}
}
void runReprocessing() {
if (mYuv1LastReceivedImage == null) {
Log.e(TAG, "No YUV Image available.");
return;
}
mImageWriter.queueInputImage(mYuv1LastReceivedImage);
Log.v(TAG, " Sent YUV1 image to ImageWriter.queueInputImage()");
try {
CaptureRequest.Builder b1 = mCameraDevice.createReprocessCaptureRequest(mLastTotalCaptureResult);
// Todo: Read current orientation instead of just assuming device is in native orientation
b1.set(CaptureRequest.JPEG_ORIENTATION, mCameraInfoCache.sensorOrientation());
b1.set(CaptureRequest.JPEG_QUALITY, (byte) 95);
b1.set(CaptureRequest.NOISE_REDUCTION_MODE, mReprocessingNoiseMode);
b1.set(CaptureRequest.EDGE_MODE, mReprocessingEdgeMode);
b1.addTarget(mJpegImageReader.getSurface());
mCurrentCaptureSession.capture(b1.build(), mReprocessingCaptureCallback, mOpsHandler);
mReprocessingRequestNanoTime = System.nanoTime();
} catch (CameraAccessException e) {
Log.e(TAG, "Could not access camera for issuePreviewCaptureRequest.");
}
mYuv1LastReceivedImage = null;
Log.v(TAG, " Reprocessing request submitted.");
}
/*********************************
* onImageAvailable() processing *
*********************************/
ImageReader.OnImageAvailableListener mYuv1ImageListener =
new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader) {
Image img = reader.acquireLatestImage();
if (img == null) {
Log.e(TAG, "Null image returned YUV1");
return;
}
if (mYuv1LastReceivedImage != null) {
mYuv1LastReceivedImage.close();
}
mYuv1LastReceivedImage = img;
if (++mYuv1ImageCounter % LOG_NTH_FRAME == 0) {
Log.v(TAG, "YUV1 buffer available, Frame #=" + mYuv1ImageCounter + " w=" + img.getWidth() + " h=" + img.getHeight() + " time=" + img.getTimestamp());
}
}
};
ImageReader.OnImageAvailableListener mDepthCloudImageListener =
new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader)
throws BufferUnderflowException, IndexOutOfBoundsException {
Image img = reader.acquireLatestImage();
if (img == null) {
Log.e(TAG, "Null image returned Depth");
return;
}
Plane[] planes = img.getPlanes();
if (0 < planes.length) {
if (DEPTH_CLOUD_STORE_ENABLED) {
if ((mDepthCloudImageCounter % STORE_NTH_DEPTH_CLOUD) == 0) {
ByteBuffer b = planes[0].getBuffer();
MediaSaver.saveDepth(mContext, b);
}
}
} else {
Log.e(TAG, "Depth buffer with empty planes!");
}
img.close();
mDepthCloudImageCounter++;
}
};
ImageReader.OnImageAvailableListener mJpegImageListener =
new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader) {
Image img = reader.acquireLatestImage();
if (img == null) {
Log.e(TAG, "Null image returned JPEG");
return;
}
Image.Plane plane0 = img.getPlanes()[0];
final ByteBuffer buffer = plane0.getBuffer();
long dt = System.nanoTime() - mReprocessingRequestNanoTime;
Log.v(TAG, String.format("JPEG buffer available, w=%d h=%d time=%d size=%d dt=%.1f ms ISO=%d",
img.getWidth(), img.getHeight(), img.getTimestamp(), buffer.capacity(), 0.000001 * dt, mLastIso));
// Save JPEG on the utility thread,
final byte[] jpegBuf;
if (buffer.hasArray()) {
jpegBuf = buffer.array();
} else {
jpegBuf = new byte[buffer.capacity()];
buffer.get(jpegBuf);
}
mMyCameraCallback.jpegAvailable(jpegBuf, img.getWidth(), img.getHeight());
img.close();
// take (reprocess) another picture right away if bursting.
if (mIsBursting) {
takePicture();
}
}
};
ImageReader.OnImageAvailableListener mYuv2ImageListener =
new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader) {
Image img = reader.acquireLatestImage();
if (img == null) {
Log.e(TAG, "Null image returned YUV2");
} else {
if (++mYuv2ImageCounter % LOG_NTH_FRAME == 0) {
Log.v(TAG, "YUV2 buffer available, Frame #=" + mYuv2ImageCounter + " w=" + img.getWidth() + " h=" + img.getHeight() + " time=" + img.getTimestamp());
}
img.close();
}
}
};
ImageReader.OnImageAvailableListener mRawImageListener =
new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader) {
final Image img = reader.acquireLatestImage();
if (img == null) {
Log.e(TAG, "Null image returned RAW");
} else {
if (++mRawImageCounter % LOG_NTH_FRAME == 0) {
Image.Plane plane0 = img.getPlanes()[0];
final ByteBuffer buffer = plane0.getBuffer();
Log.v(TAG, "Raw buffer available, Frame #=" + mRawImageCounter + "w=" + img.getWidth()
+ " h=" + img.getHeight()
+ " format=" + CameraDeviceReport.getFormatName(img.getFormat())
+ " time=" + img.getTimestamp()
+ " size=" + buffer.capacity()
+ " getRowStride()=" + plane0.getRowStride());
}
img.close();
}
}
};
/*************************************
* CaptureResult metadata processing *
*************************************/
private CameraCaptureSession.CaptureCallback mCaptureCallback = new LoggingCallbacks.SessionCaptureCallback() {
@Override
public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result) {
if (!mFirstFrameArrived) {
mFirstFrameArrived = true;
long now = SystemClock.elapsedRealtime();
long dt = now - CameraTimer.t0;
long camera_dt = now - CameraTimer.t_session_go + CameraTimer.t_open_end - CameraTimer.t_open_start;
long repeating_req_dt = now - CameraTimer.t_burst;
Log.v(TAG, "App control to first frame: (" + dt + " ms)");
Log.v(TAG, "HAL request to first frame: (" + repeating_req_dt + " ms) " + " Total HAL wait: (" + camera_dt + " ms)");
mMyCameraCallback.receivedFirstFrame();
mMyCameraCallback.performanceDataAvailable((int) dt, (int) camera_dt, null);
}
publishFrameData(result);
// Used for reprocessing.
mLastTotalCaptureResult = result;
super.onCaptureCompleted(session, request, result);
}
};
// Reprocessing capture completed.
private CameraCaptureSession.CaptureCallback mReprocessingCaptureCallback = new LoggingCallbacks.SessionCaptureCallback() {
@Override
public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result) {
Log.v(TAG, "Reprocessing onCaptureCompleted()");
}
};
private static double SHORT_LOG_EXPOSURE = Math.log10(1000000000 / 10000); // 1/10000 second
private static double LONG_LOG_EXPOSURE = Math.log10(1000000000 / 10); // 1/10 second
public int FPS_CALC_LOOKBACK = 15;
private LinkedList<Long> mFrameTimes = new LinkedList<Long>();
private void publishFrameData(TotalCaptureResult result) {
// Faces.
final Face[] faces = result.get(CaptureResult.STATISTICS_FACES);
NormalizedFace[] newFaces = new NormalizedFace[faces.length];
if (faces.length > 0) {
int offX = mCameraInfoCache.faceOffsetX();
int offY = mCameraInfoCache.faceOffsetY();
int dX = mCameraInfoCache.activeAreaWidth() - 2 * offX;
int dY = mCameraInfoCache.activeAreaHeight() - 2 * offY;
if (mCameraInfoCache.IS_NEXUS_6 && mCameraIsFront) {
// Front camera on Nexus 6 is currently 16 x 9 cropped to 4 x 3.
// TODO: Generalize this.
int cropOffset = dX / 8;
dX -= 2 * cropOffset;
offX += cropOffset;
}
int orientation = mCameraInfoCache.sensorOrientation();
for (int i = 0; i < faces.length; ++i) {
newFaces[i] = new NormalizedFace(faces[i], dX, dY, offX, offY);
if (mCameraIsFront && orientation == 90) {
newFaces[i].mirrorInY();
}
if (mCameraIsFront && orientation == 270) {
newFaces[i].mirrorInX();
}
if (!mCameraIsFront && orientation == 270) {
newFaces[i].mirrorInX();
newFaces[i].mirrorInY();
}
}
}
// Normalized lens and exposure coordinates.
double rm = Math.log10(result.get(CaptureResult.SENSOR_EXPOSURE_TIME));
float normExposure = (float) ((rm - SHORT_LOG_EXPOSURE) / (LONG_LOG_EXPOSURE - SHORT_LOG_EXPOSURE));
float normLensPos = (mCameraInfoCache.getDiopterHi() - result.get(CaptureResult.LENS_FOCUS_DISTANCE)) / (mCameraInfoCache.getDiopterHi() - mCameraInfoCache.getDiopterLow());
mLastIso = result.get(CaptureResult.SENSOR_SENSITIVITY);
// Update frame arrival history.
mFrameTimes.add(result.get(CaptureResult.SENSOR_TIMESTAMP));
if (mFrameTimes.size() > FPS_CALC_LOOKBACK) {
mFrameTimes.removeFirst();
}
// Frame drop detector
{
float frameDuration = result.get(CaptureResult.SENSOR_FRAME_DURATION);
if (mFrameTimes.size() > 1) {
long dt = result.get(CaptureResult.SENSOR_TIMESTAMP) - mFrameTimes.get(mFrameTimes.size()-2);
if (dt > 3 * frameDuration / 2 && LOG_DROPPED_FRAMES) {
float drops = (dt * 1f / frameDuration) - 1f;
Log.e(TAG, String.format("dropped %.2f frames", drops));
mMyCameraCallback.performanceDataAvailable(null, null, drops);
}
}
}
// FPS calc.
float fps = 0;
if (mFrameTimes.size() > 1) {
long dt = mFrameTimes.getLast() - mFrameTimes.getFirst();
fps = (mFrameTimes.size() - 1) * 1000000000f / dt;
fps = (float) Math.floor(fps + 0.1); // round to nearest whole number, ish.
}
// Do callback.
if (mMyCameraCallback != null) {
mMyCameraCallback.frameDataAvailable(newFaces, normExposure, normLensPos, fps,
(int) mLastIso, result.get(CaptureResult.CONTROL_AF_STATE), result.get(CaptureResult.CONTROL_AE_STATE), result.get(CaptureResult.CONTROL_AWB_STATE));
} else {
Log.v(TAG, "mMyCameraCallbacks is null!!.");
}
}
long mLastIso = 0;
/*********************
* UTILITY FUNCTIONS *
*********************/
/**
* Return the next mode after currentMode in supportedModes, wrapping to
* start of mode list if currentMode is last. Returns currentMode if it is not found in
* supportedModes.
*
* @param currentMode
* @param supportedModes
* @return next mode after currentMode in supportedModes
*/
private int getNextMode(int currentMode, int[] supportedModes) {
boolean getNext = false;
for (int m : supportedModes) {
if (getNext) {
return m;
}
if (m == currentMode) {
getNext = true;
}
}
if (getNext) {
return supportedModes[0];
}
// Can't find mode in list
return currentMode;
}
private static String edgeModeToString(int mode) {
switch (mode) {
case CaptureRequest.EDGE_MODE_OFF:
return "OFF";
case CaptureRequest.EDGE_MODE_FAST:
return "FAST";
case CaptureRequest.EDGE_MODE_HIGH_QUALITY:
return "HiQ";
case CaptureRequest.EDGE_MODE_ZERO_SHUTTER_LAG:
return "ZSL";
}
return Integer.toString(mode);
}
private static String noiseModeToString(int mode) {
switch (mode) {
case CaptureRequest.NOISE_REDUCTION_MODE_OFF:
return "OFF";
case CaptureRequest.NOISE_REDUCTION_MODE_FAST:
return "FAST";
case CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY:
return "HiQ";
case CaptureRequest.NOISE_REDUCTION_MODE_MINIMAL:
return "MIN";
case CaptureRequest.NOISE_REDUCTION_MODE_ZERO_SHUTTER_LAG:
return "ZSL";
}
return Integer.toString(mode);
}
}