安卓Camera2用ImageReader获取NV21源码分析
Posted 清霜之辰
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了安卓Camera2用ImageReader获取NV21源码分析相关的知识，希望对你有一定的参考价值。
以前如何得到Camera预览流回调
- 可以通过如下方法，得到一路预览回调流 Camera#setPreviewCallbackWithBuffer(Camera.PreviewCallback)，
- 可以通过如下方法，设置回调数据的格式，比如
ImageFormat.NV21Camera.Parameters#setPreviewForma - 从
API21开始，android.hardware.Camera相关API 已经被谷歌废弃，
当我们切换到Camera2相关后，如何得到一路NV21回调流呢？
如何使用Camera2得到预览流的回调
一般我们使用的方法如下，
private final ImageReader.OnImageAvailableListener mOnImageAvailableListener = reader ->
Image image = reader.acquireNextImage();
// do sth with reader
image.close();
;
mImageReader = ImageReader.newInstance(mPreviewSize.getWidth(), mPreviewSize.getHeight(),
ImageFormat.YUV_420_888, 1);
mImageReader.setOnImageAvailableListener(mOnImageAvailableListener, mCameraHandler);
// 看起来是不是跟Camera2拍照很相似，关键在配流时需要添加如下代码
mPreviewRequestBuilder.addTarget(mImageReader.getSurface());
这样我们就可以通过 Image 读取回调的内容，内容是啥呢？
The order of planes in the array returned by Image#getPlanes() is guaranteed such that plane #0 is always Y, plane #1 is always U (Cb), and plane #2 is always V (Cr).
Image#getPlanes 返回三个数组，plane 0 是Y，plane 1 是U (Cb)，plane 2 是 V (Cr)
我们可以把他们转给NV21格式，如何转换呢？
可以参考如下答案
stackoverflow camera2-conversion-from-yuv-420-888-to-nv21
是否可以通过ImageReader直接得到NV21回调呢
上面我们设置的是YUV_420_888，是否可以直接改成NV21呢?
不行，在 ImageReader 我们可以看到如下，会直接闪退
if (format == ImageFormat.NV21)
throw new IllegalArgumentException( "NV21 format is not supported")
ImageReader数据流回调的流程源码分析
相关数据流源码分析如下
private static void postEventFromNative(Object selfRef)
....
if (executor != null && listener != null && isReaderValid)
executor.execute(new Runnable()
@Override
public void run()
listener.onImageAvailable(ir);
);
// frameworks/base/media/jni/android_media_ImageReader.cpp 中部分源码如下
static struct
jfieldID mNativeContext;
jmethodID postEventFromNative;
gImageReaderClassInfo;
gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
ALOGV("%s: frame available", __FUNCTION__);
bool needsDetach = false;
JNIEnv* env = getJNIEnv(&needsDetach);
if (env != NULL)
env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
else
ALOGW("onFrameAvailable event will not posted");
if (needsDetach)
detachJNI();
// frameworks/native/libs/gui/ConsumerBase.cpp
void ConsumerBase::onFrameAvailable(const BufferItem& item)
CB_LOGV("onFrameAvailable");
sp<FrameAvailableListener> listener;
// scope for the lock
Mutex::Autolock lock(mFrameAvailableMutex);
listener = mFrameAvailableListener.promote();
if (listener != nullptr)
CB_LOGV("actually calling onFrameAvailable");
listener->onFrameAvailable(item);
// APP 通过 ImageReader 设置 OnImageAvailableListener 回调拿到 android.media.ImageReader
ImageReader.acquireNextImage() 拿到 android.media.Image
public Image acquireNextImage()
// Initialize with reader format, but can be overwritten by native if the image
// format is different from the reader format.
SurfaceImage si = new SurfaceImage(mFormat);
int status = acquireNextSurfaceImage(si);
return si;
private int acquireNextSurfaceImage(SurfaceImage si)
....
status = nativeImageSetup(si);
....
// frameworks.base\\media\\jni\\android_media_ImageReader.cpp
"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup
static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image)
JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
BufferItem* buffer = ctx->getBufferItem();
status_t res = bufferConsumer->acquireBuffer(buffer, 0);
ctx->returnBufferItem(buffer);
List<BufferItem*> mBuffers;
void JNIImageReaderContext::returnBufferItem(BufferItem* buffer)
buffer->mGraphicBuffer = nullptr;
mBuffers.push_back(buffer);
frameworks/native/libs/gui/include/gui/BufferItem.h
android.media.ImageReader$SurfaceImage
@Override
public Plane[] getPlanes()
throwISEIfImageIsInvalid();
if (mPlanes == null)
// mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat);
mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat);
// Shallow copy is fine.
return mPlanes.clone();
"nativeCreatePlanes", "(IIJ)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", (void*)Image_createSurfacePlanes ,
static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
int numPlanes, int readerFormat, uint64_t ndkReaderUsage)
jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, /*initial_element*/NULL);
LockedImage lockedImg = LockedImage();
Image_getLockedImage(env, thiz, &lockedImg, ndkReaderUsage); // 把前面的 BufferItem 转为 LockedImage
for (int i = 0; i < numPlanes; i++)
if (!Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
&pData, &dataSize, &pixelStride, &rowStride))
return NULL;
byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
// Finally, create this SurfacePlane.
jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
static void ImageReader_classInit(JNIEnv* env, jclass clazz)
gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
"(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
static bool Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride)
status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
pixelStride, rowStride);
if (res != OK)
jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
"Pixel format: 0x%x is unsupported", buffer->flexFormat);
return false;
return true;
// frameworks/base/media/jni/android_media_Utils.cpp
status_t lockImageFromBuffer(BufferItem* bufferItem, uint32_t inUsage,
int fenceFd, LockedImage* outputImage)
...
status_t res = lockImageFromBuffer(bufferItem->mGraphicBuffer, inUsage, bufferItem->mCrop,
fenceFd, outputImage);
...
status_t lockImageFromBuffer(sp<GraphicBuffer> buffer, uint32_t inUsage,
const Rect& rect, int fenceFd, LockedImage* outputImage)
ALOGV("%s: Try to lock the GraphicBuffer", __FUNCTION__);
if (buffer == nullptr || outputImage == nullptr)
ALOGE("Input BufferItem or output LockedImage is NULL!");
return BAD_VALUE;
if (isFormatOpaque(buffer->getPixelFormat()))
ALOGE("Opaque format buffer is not lockable!");
return BAD_VALUE;
void* pData = NULL;
android_ycbcr ycbcr = android_ycbcr();
status_t res;
int format = buffer->getPixelFormat();
int flexFormat = format;
if (isPossiblyYUV(format))
res = buffer->lockAsyncYCbCr(inUsage, rect, &ycbcr, fenceFd);
if (res != OK)
ALOGW("lockAsyncYCbCr failed with error %d (format = 0x%x)", res, format);
pData = ycbcr.y;
flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
// lockAsyncYCbCr for YUV is unsuccessful.
if (pData == NULL)
res = buffer->lockAsync(inUsage, rect, &pData, fenceFd);
if (res != OK)
ALOGE("Lock buffer failed!");
return res;
if (isPossibly10BitYUV(format)
&& OK == extractP010Gralloc4PlaneLayout(buffer, pData, format, outputImage))
ALOGV("%s: Successfully locked the P010 image", __FUNCTION__);
return OK;
outputImage->data = reinterpret_cast<uint8_t*>(pData);
outputImage->width = buffer->getWidth();
outputImage->height = buffer->getHeight();
outputImage->format = format;
outputImage->flexFormat = flexFormat;
outputImage->stride =
(ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();
outputImage->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb);
outputImage->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr);
outputImage->chromaStride = static_cast<uint32_t>(ycbcr.cstride);
outputImage->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step);
ALOGV("%s: Successfully locked the image from the GraphicBuffer", __FUNCTION__);
// Crop, transform, scalingMode, timestamp, and frameNumber should be set by caller,
// and cann't be set them here.
return OK;
status_t getLockedImageInfo(LockedImage* buffer, int idx,
int32_t containerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride)
ALOGV("%s", __FUNCTION__);
LOG_ALWAYS_FATAL_IF(buffer == NULL, "Input buffer is NULL!!!");
LOG_ALWAYS_FATAL_IF(base == NULL, "base is NULL!!!");
LOG_ALWAYS_FATAL_IF(size == NULL, "size is NULL!!!");
LOG_ALWAYS_FATAL_IF(pixelStride == NULL, "pixelStride is NULL!!!");
LOG_ALWAYS_FATAL_IF(rowStride == NULL, "rowStride is NULL!!!");
LOG_ALWAYS_FATAL_IF((idx >= IMAGE_MAX_NUM_PLANES) || (idx < 0), "idx (%d) is illegal", idx);
ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
uint32_t dataSize, ySize, cSize, cStride;
uint32_t pStride = 0, rStride = 0;
uint8_t *cb, *cr;
uint8_t *pData = NULL;
int bytesPerPixel = 0;
dataSize = ySize = cSize = cStride = 0;
int32_t fmt = buffer->flexFormat;
bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, containerFormat);
fmt = applyFormatOverrides(fmt, containerFormat);
switch (fmt)
case HAL_PIXEL_FORMAT_YCbCr_420_888:
// Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.
if (buffer->width % 2 != 0)
ALOGE("YCbCr_420_888: width (%d) should be a multiple of 2", buffer->width);
return BAD_VALUE;
if (buffer->height % 2 != 0)
ALOGE("YCbCr_420_888: height (%d) should be a multiple of 2", buffer->height);
return BAD_VALUE;
if (buffer->width <= 0)
ALOGE("YCbCr_420_888: width (%d) should be a > 0", buffer->width);
return BAD_VALUE;
if (buffer->height <= 0)
ALOGE("YCbCr_420_888: height (%d) should be a > 0", buffer->height);
return BAD_VALUE;
pData =
(idx == 0) ?
buffer->data :
(idx == 1) ?
buffer->dataCb :
buffer->dataCr;
// only map until last pixel
if (idx == 0)
pStride = 1;
rStride = buffer->stride;
dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
else
pStride = buffer->chromaStep;
rStride = buffer->chromaStride;
dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
buffer->chromaStep * (buffer->width / 2 - 1) + 1;
break;
// NV21
case HAL_PIXEL_FORMAT_YCrCb_420_SP:
// Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.
if (buffer->width % 2 != 0)
ALOGE("YCrCb_420_SP: width (%d) should be a multiple of 2", buffer->width);
return BAD_VALUE;
if (buffer->height % 2 != 0)
ALOGE("YCrCb_420_SP: height (%d) should be a multiple of 2", buffer->height);
return BAD_VALUE;
if (buffer->width <= 0)
ALOGE("YCrCb_420_SP: width (%d) should be a > 0", buffer->width);
return BAD_VALUE;
if (buffer->height <= 0)
AL以上是关于安卓Camera2用ImageReader获取NV21源码分析的主要内容，如果未能解决你的问题，请参考以下文章
Android Camera2 ImageReader 图像格式 YUV
使用Android NDK Camera2获取预览帧的正确方法是啥
Android camera2 输出到 ImageReader 格式 YUV_420_888 仍然很慢
Android Camera2 ImageReader 大小在 Android 5.0 Galaxy S5 上不正确