Camera: Add support for new camera zoom API

- Add NDK API spec for the new zoom API
The new zoom API combines optical and digital zoom, and supports both
zoom-out and zoom-in with more precision.
- Add new NDK API to specify separate zoom ratio ranges for different
bokeh modes.
- Add ZoomRatioMapper in camera service to convert between
control.zoomRation to and from scaler.cropRegion.

Test: Camera CTS/ITS/CtsVerifier/ZoomRatioTest
Bug: 130025314
Change-Id: I4c7d867f840b5720bc73bb0485e8a9a93d2276b5

camera/ndk/impl/ACameraMetadata.cpp

@@ -480,6 +480,7 @@ ACameraMetadata::isCaptureRequestTag(const uint32_t tag) {
         case ACAMERA_CONTROL_POST_RAW_SENSITIVITY_BOOST:
         case ACAMERA_CONTROL_ENABLE_ZSL:
         case ACAMERA_CONTROL_BOKEH_MODE:
+        case ACAMERA_CONTROL_ZOOM_RATIO:
         case ACAMERA_EDGE_MODE:
         case ACAMERA_FLASH_MODE:
         case ACAMERA_HOT_PIXEL_MODE:

services/camera/libcameraservice/Android.bp

@@ -56,7 +56,9 @@ cc_library_shared {
         "device3/StatusTracker.cpp",
         "device3/Camera3BufferManager.cpp",
         "device3/Camera3StreamSplitter.cpp",
+        "device3/CoordinateMapper.cpp",
         "device3/DistortionMapper.cpp",
+        "device3/ZoomRatioMapper.cpp",
         "gui/RingBufferConsumer.cpp",
         "utils/CameraThreadState.cpp",
         "hidl/AidlCameraDeviceCallbacks.cpp",

services/camera/libcameraservice/common/CameraProviderManager.cpp

@@ -40,6 +40,7 @@
 #include <utils/Trace.h>
 
 #include "api2/HeicCompositeStream.h"
+#include "device3/ZoomRatioMapper.h"
 
 namespace android {
 
@@ -232,6 +233,15 @@ bool CameraProviderManager::hasFlashUnit(const std::string &id) const {
     return deviceInfo->hasFlashUnit();
 }
 
+bool CameraProviderManager::supportNativeZoomRatio(const std::string &id) const {
+    std::lock_guard<std::mutex> lock(mInterfaceMutex);
+
+    auto deviceInfo = findDeviceInfoLocked(id);
+    if (deviceInfo == nullptr) return false;
+
+    return deviceInfo->supportNativeZoomRatio();
+}
+
 status_t CameraProviderManager::getResourceCost(const std::string &id,
         CameraResourceCost* cost) const {
     std::lock_guard<std::mutex> lock(mInterfaceMutex);
@@ -2035,6 +2045,13 @@ CameraProviderManager::ProviderInfo::DeviceInfo3::DeviceInfo3(const std::string&
                 __FUNCTION__, strerror(-res), res);
     }
 
+    res = camera3::ZoomRatioMapper::overrideZoomRatioTags(
+            &mCameraCharacteristics, &mSupportNativeZoomRatio);
+    if (OK != res) {
+        ALOGE("%s: Unable to override zoomRatio related tags: %s (%d)",
+                __FUNCTION__, strerror(-res), res);
+    }
+
     camera_metadata_entry flashAvailable =
             mCameraCharacteristics.find(ANDROID_FLASH_INFO_AVAILABLE);
     if (flashAvailable.count == 1 &&

services/camera/libcameraservice/common/CameraProviderManager.h

@@ -186,6 +186,11 @@ class CameraProviderManager : virtual public hidl::manager::V1_0::IServiceNotifi
      */
     bool hasFlashUnit(const std::string &id) const;
 
+    /**
+     * Return true if the camera device has native zoom ratio support.
+     */
+    bool supportNativeZoomRatio(const std::string &id) const;
+
     /**
      * Return the resource cost of this camera device
      */
@@ -416,6 +421,7 @@ class CameraProviderManager : virtual public hidl::manager::V1_0::IServiceNotifi
             sp<ProviderInfo> mParentProvider;
 
             bool hasFlashUnit() const { return mHasFlashUnit; }
+            bool supportNativeZoomRatio() const { return mSupportNativeZoomRatio; }
             virtual status_t setTorchMode(bool enabled) = 0;
             virtual status_t getCameraInfo(hardware::CameraInfo *info) const = 0;
             virtual bool isAPI1Compatible() const = 0;
@@ -449,10 +455,11 @@ class CameraProviderManager : virtual public hidl::manager::V1_0::IServiceNotifi
                     mIsLogicalCamera(false), mResourceCost(resourceCost),
                     mStatus(hardware::camera::common::V1_0::CameraDeviceStatus::PRESENT),
                     mParentProvider(parentProvider), mHasFlashUnit(false),
-                    mPublicCameraIds(publicCameraIds) {}
+                    mSupportNativeZoomRatio(false), mPublicCameraIds(publicCameraIds) {}
             virtual ~DeviceInfo();
         protected:
-            bool mHasFlashUnit;
+            bool mHasFlashUnit; // const after constructor
+            bool mSupportNativeZoomRatio; // const after constructor
             const std::vector<std::string>& mPublicCameraIds;
 
             template<class InterfaceT>

services/camera/libcameraservice/device3/Camera3Device.cpp

@@ -133,6 +133,7 @@ status_t Camera3Device::initialize(sp<CameraProviderManager> manager, const Stri
         session->close();
         return res;
     }
+    mSupportNativeZoomRatio = manager->supportNativeZoomRatio(mId.string());
 
     std::vector<std::string> physicalCameraIds;
     bool isLogical = manager->isLogicalCamera(mId.string(), &physicalCameraIds);
@@ -147,15 +148,27 @@ status_t Camera3Device::initialize(sp<CameraProviderManager> manager, const Stri
                 return res;
             }
 
-            if (DistortionMapper::isDistortionSupported(mPhysicalDeviceInfoMap[physicalId])) {
-                mDistortionMappers[physicalId].setupStaticInfo(mPhysicalDeviceInfoMap[physicalId]);
+            bool usePrecorrectArray =
+                    DistortionMapper::isDistortionSupported(mPhysicalDeviceInfoMap[physicalId]);
+            if (usePrecorrectArray) {
+                res = mDistortionMappers[physicalId].setupStaticInfo(
+                        mPhysicalDeviceInfoMap[physicalId]);
                 if (res != OK) {
                     SET_ERR_L("Unable to read camera %s's calibration fields for distortion "
                             "correction", physicalId.c_str());
                     session->close();
                     return res;
                 }
             }
+
+            res = mZoomRatioMappers[physicalId].initZoomRatioTags(
+                    &mPhysicalDeviceInfoMap[physicalId],
+                    mSupportNativeZoomRatio, usePrecorrectArray);
+            if (res != OK) {
+                SET_ERR_L("Failed to initialize camera %s's zoomRatio tags: %s (%d)",
+                        physicalId.c_str(), strerror(-res), res);
+                return res;
+            }
         }
     }
 
@@ -331,13 +344,23 @@ status_t Camera3Device::initializeCommonLocked() {
         }
     }
 
-    if (DistortionMapper::isDistortionSupported(mDeviceInfo)) {
+    bool usePrecorrectArray = DistortionMapper::isDistortionSupported(mDeviceInfo);
+    if (usePrecorrectArray) {
         res = mDistortionMappers[mId.c_str()].setupStaticInfo(mDeviceInfo);
         if (res != OK) {
             SET_ERR_L("Unable to read necessary calibration fields for distortion correction");
             return res;
         }
     }
+
+    res = mZoomRatioMappers[mId.c_str()].initZoomRatioTags(&mDeviceInfo,
+            mSupportNativeZoomRatio, usePrecorrectArray);
+    if (res != OK) {
+        SET_ERR_L("Failed to initialize zoomRatio tags: %s (%d)",
+                strerror(-res), res);
+        return res;
+    }
+
     return OK;
 }
 
@@ -2124,6 +2147,15 @@ status_t Camera3Device::createDefaultRequest(int templateId,
         set_camera_metadata_vendor_id(rawRequest, mVendorTagId);
         mRequestTemplateCache[templateId].acquire(rawRequest);
 
+        // Override the template request with zoomRatioMapper
+        res = mZoomRatioMappers[mId.c_str()].initZoomRatioInTemplate(
+                &mRequestTemplateCache[templateId]);
+        if (res != OK) {
+            CLOGE("Failed to update zoom ratio for template %d: %s (%d)",
+                    templateId, strerror(-res), res);
+            return res;
+        }
+
         *request = mRequestTemplateCache[templateId];
         mLastTemplateId = templateId;
     }
@@ -3146,14 +3178,15 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber,
         int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
         bool hasAppCallback, nsecs_t maxExpectedDuration,
         std::set<String8>& physicalCameraIds, bool isStillCapture,
-        bool isZslCapture, const SurfaceMap& outputSurfaces) {
+        bool isZslCapture, const std::set<std::string>& cameraIdsWithZoom,
+        const SurfaceMap& outputSurfaces) {
     ATRACE_CALL();
     Mutex::Autolock l(mInFlightLock);
 
     ssize_t res;
     res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput,
             hasAppCallback, maxExpectedDuration, physicalCameraIds, isStillCapture, isZslCapture,
-            outputSurfaces));
+            cameraIdsWithZoom, outputSurfaces));
     if (res < 0) return res;
 
     if (mInFlightMap.size() == 1) {
@@ -3504,7 +3537,7 @@ void Camera3Device::sendCaptureResult(CameraMetadata &pendingMetadata,
         CaptureResultExtras &resultExtras,
         CameraMetadata &collectedPartialResult,
         uint32_t frameNumber,
-        bool reprocess, bool zslStillCapture,
+        bool reprocess, bool zslStillCapture, const std::set<std::string>& cameraIdsWithZoom,
         const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas) {
     ATRACE_CALL();
     if (pendingMetadata.isEmpty())
@@ -3573,19 +3606,39 @@ void Camera3Device::sendCaptureResult(CameraMetadata &pendingMetadata,
             mDistortionMappers[mId.c_str()].correctCaptureResult(&captureResult.mMetadata);
     if (res != OK) {
         SET_ERR("Unable to correct capture result metadata for frame %d: %s (%d)",
-                frameNumber, strerror(res), res);
+                frameNumber, strerror(-res), res);
         return;
     }
+
+    // Fix up result metadata to account for zoom ratio availabilities between
+    // HAL and app.
+    bool zoomRatioIs1 = cameraIdsWithZoom.find(mId.c_str()) == cameraIdsWithZoom.end();
+    res = mZoomRatioMappers[mId.c_str()].updateCaptureResult(
+            &captureResult.mMetadata, zoomRatioIs1);
+    if (res != OK) {
+        SET_ERR("Failed to update capture result zoom ratio metadata for frame %d: %s (%d)",
+                frameNumber, strerror(-res), res);
+        return;
+    }
+
     for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
         String8 cameraId8(physicalMetadata.mPhysicalCameraId);
-        if (mDistortionMappers.find(cameraId8.c_str()) == mDistortionMappers.end()) {
-            continue;
+        if (mDistortionMappers.find(cameraId8.c_str()) != mDistortionMappers.end()) {
+            res = mDistortionMappers[cameraId8.c_str()].correctCaptureResult(
+                    &physicalMetadata.mPhysicalCameraMetadata);
+            if (res != OK) {
+                SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)",
+                        frameNumber, strerror(-res), res);
+                return;
+            }
         }
-        res = mDistortionMappers[cameraId8.c_str()].correctCaptureResult(
-                &physicalMetadata.mPhysicalCameraMetadata);
+
+        zoomRatioIs1 = cameraIdsWithZoom.find(cameraId8.c_str()) == cameraIdsWithZoom.end();
+        res = mZoomRatioMappers[cameraId8.c_str()].updateCaptureResult(
+                &physicalMetadata.mPhysicalCameraMetadata, zoomRatioIs1);
         if (res != OK) {
-            SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)",
-                    frameNumber, strerror(res), res);
+            SET_ERR("Failed to update camera %s's physical zoom ratio metadata for "
+                    "frame %d: %s(%d)", cameraId8.c_str(), frameNumber, strerror(-res), res);
             return;
         }
     }
@@ -3790,7 +3843,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
                 sendCaptureResult(metadata, request.resultExtras,
                     collectedPartialResult, frameNumber,
                     hasInputBufferInRequest, request.zslCapture && request.stillCapture,
-                    request.physicalMetadatas);
+                    request.cameraIdsWithZoom, request.physicalMetadatas);
             }
         }
 
@@ -4008,7 +4061,7 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg,
                 sendCaptureResult(r.pendingMetadata, r.resultExtras,
                     r.collectedPartialResult, msg.frame_number,
                     r.hasInputBuffer, r.zslCapture && r.stillCapture,
-                    r.physicalMetadatas);
+                    r.cameraIdsWithZoom, r.physicalMetadatas);
             }
             bool timestampIncreasing = !(r.zslCapture || r.hasInputBuffer);
             returnOutputBuffers(r.pendingOutputBuffers.array(),
@@ -5610,6 +5663,7 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
                 // request in a batch as new
                 !(batchedRequest && i > 0);
         if (newRequest) {
+            std::set<std::string> cameraIdsWithZoom;
             /**
              * HAL workaround:
              * Insert a dummy trigger ID if a trigger is set but no trigger ID is
@@ -5642,6 +5696,28 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
                             return INVALID_OPERATION;
                         }
                     }
+
+                    for (it = captureRequest->mSettingsList.begin();
+                            it != captureRequest->mSettingsList.end(); it++) {
+                        if (parent->mZoomRatioMappers.find(it->cameraId) ==
+                                parent->mZoomRatioMappers.end()) {
+                            continue;
+                        }
+
+                        camera_metadata_entry_t e = it->metadata.find(ANDROID_CONTROL_ZOOM_RATIO);
+                        if (e.count > 0 && e.data.f[0] != 1.0f) {
+                            cameraIdsWithZoom.insert(it->cameraId);
+                        }
+
+                        res = parent->mZoomRatioMappers[it->cameraId].updateCaptureRequest(
+                            &(it->metadata));
+                        if (res != OK) {
+                            SET_ERR("RequestThread: Unable to correct capture requests "
+                                    "for zoom ratio for request %d: %s (%d)",
+                                    halRequest->frame_number, strerror(-res), res);
+                            return INVALID_OPERATION;
+                        }
+                    }
                 }
             }
 
@@ -5652,6 +5728,7 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
             captureRequest->mSettingsList.begin()->metadata.sort();
             halRequest->settings = captureRequest->mSettingsList.begin()->metadata.getAndLock();
             mPrevRequest = captureRequest;
+            mPrevCameraIdsWithZoom = cameraIdsWithZoom;
             ALOGVV("%s: Request settings are NEW", __FUNCTION__);
 
             IF_ALOGV() {
@@ -5839,7 +5916,7 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
                 /*hasInput*/halRequest->input_buffer != NULL,
                 hasCallback,
                 calculateMaxExpectedDuration(halRequest->settings),
-                requestedPhysicalCameras, isStillCapture, isZslCapture,
+                requestedPhysicalCameras, isStillCapture, isZslCapture, mPrevCameraIdsWithZoom,
                 (mUseHalBufManager) ? uniqueSurfaceIdMap :
                                       SurfaceMap{});
         ALOGVV("%s: registered in flight requestId = %" PRId32 ", frameNumber = %" PRId64

services/camera/libcameraservice/device3/Camera3Device.h

@@ -46,6 +46,7 @@
 #include "device3/StatusTracker.h"
 #include "device3/Camera3BufferManager.h"
 #include "device3/DistortionMapper.h"
+#include "device3/ZoomRatioMapper.h"
 #include "utils/TagMonitor.h"
 #include "utils/LatencyHistogram.h"
 #include <camera_metadata_hidden.h>
@@ -442,6 +443,7 @@ class Camera3Device :
     sp<HalInterface> mInterface;
 
     CameraMetadata             mDeviceInfo;
+    bool                       mSupportNativeZoomRatio;
     std::unordered_map<std::string, CameraMetadata> mPhysicalDeviceInfoMap;
 
     CameraMetadata             mRequestTemplateCache[CAMERA3_TEMPLATE_COUNT];
@@ -980,6 +982,7 @@ class Camera3Device :
 
         sp<CaptureRequest> mPrevRequest;
         int32_t            mPrevTriggers;
+        std::set<std::string> mPrevCameraIdsWithZoom;
 
         uint32_t           mFrameNumber;
 
@@ -1078,6 +1081,9 @@ class Camera3Device :
         // Indicates a ZSL capture request
         bool zslCapture;
 
+        // Requested camera ids (both logical and physical) with zoomRatio != 1.0f
+        std::set<std::string> cameraIdsWithZoom;
+
         // What shared surfaces an output should go to
         SurfaceMap outputSurfaces;
 
@@ -1099,7 +1105,7 @@ class Camera3Device :
         InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput,
                 bool hasAppCallback, nsecs_t maxDuration,
                 const std::set<String8>& physicalCameraIdSet, bool isStillCapture,
-                bool isZslCapture,
+                bool isZslCapture, const std::set<std::string>& idsWithZoom,
                 const SurfaceMap& outSurfaces = SurfaceMap{}) :
                 shutterTimestamp(0),
                 sensorTimestamp(0),
@@ -1114,6 +1120,7 @@ class Camera3Device :
                 physicalCameraIds(physicalCameraIdSet),
                 stillCapture(isStillCapture),
                 zslCapture(isZslCapture),
+                cameraIdsWithZoom(idsWithZoom),
                 outputSurfaces(outSurfaces) {
         }
     };
@@ -1131,7 +1138,7 @@ class Camera3Device :
     status_t registerInFlight(uint32_t frameNumber,
             int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
             bool callback, nsecs_t maxExpectedDuration, std::set<String8>& physicalCameraIds,
-            bool isStillCapture, bool isZslCapture,
+            bool isStillCapture, bool isZslCapture, const std::set<std::string>& cameraIdsWithZoom,
             const SurfaceMap& outputSurfaces);
 
     /**
@@ -1256,6 +1263,7 @@ class Camera3Device :
             CaptureResultExtras &resultExtras,
             CameraMetadata &collectedPartialResult, uint32_t frameNumber,
             bool reprocess, bool zslStillCapture,
+            const std::set<std::string>& cameraIdsWithZoom,
             const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas);
 
     bool isLastFullResult(const InFlightRequest& inFlightRequest);
@@ -1288,6 +1296,11 @@ class Camera3Device :
     // logical camera and its physical subcameras.
     std::unordered_map<std::string, camera3::DistortionMapper> mDistortionMappers;
 
+    /**
+     * Zoom ratio mapper support
+     */
+    std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers;
+
     // Debug tracker for metadata tag value changes
     // - Enabled with the -m <taglist> option to dumpsys, such as
     //   dumpsys -m android.control.aeState,android.control.aeMode