hwc: Rotate RGB layer using MDP Rotator

1. Use MDP rotator to rotate RGB layers which are rendered by software
2. Restrict total number of rotator sessions to 4 for all displays
3. Disable RGB layer rotation for all MDP versions < MDP5

Change-Id: Ie93111082dff9a16b614e9768df5d8ce83ff9e25
diff --git a/libgralloc/gpu.cpp b/libgralloc/gpu.cpp
index 78e875a..b4f0e70 100644
--- a/libgralloc/gpu.cpp
+++ b/libgralloc/gpu.cpp
@@ -152,6 +152,10 @@
             flags |= private_handle_t::PRIV_FLAGS_TILE_RENDERED;
         }
 
+        if(usage & (GRALLOC_USAGE_SW_READ_MASK | GRALLOC_USAGE_SW_WRITE_MASK)) {
+            flags |= private_handle_t::PRIV_FLAGS_CPU_RENDERED;
+        }
+
         flags |= data.allocType;
         uint64_t eBaseAddr = (uint64_t)(eData.base) + eData.offset;
         private_handle_t *hnd = new private_handle_t(data.fd, size, flags,
diff --git a/libgralloc/gralloc_priv.h b/libgralloc/gralloc_priv.h
index 7a4fc15..a763c37 100644
--- a/libgralloc/gralloc_priv.h
+++ b/libgralloc/gralloc_priv.h
@@ -202,7 +202,9 @@
             PRIV_FLAGS_ITU_R_709          = 0x00800000,
             PRIV_FLAGS_SECURE_DISPLAY     = 0x01000000,
             // Buffer is rendered in Tile Format
-            PRIV_FLAGS_TILE_RENDERED      = 0x02000000
+            PRIV_FLAGS_TILE_RENDERED      = 0x02000000,
+            // Buffer rendered using CPU/SW renderer
+            PRIV_FLAGS_CPU_RENDERED       = 0x04000000
         };
 
         // file-descriptors
diff --git a/libhwcomposer/hwc.cpp b/libhwcomposer/hwc.cpp
index b2466d0..efa4302 100644
--- a/libhwcomposer/hwc.cpp
+++ b/libhwcomposer/hwc.cpp
@@ -143,11 +143,11 @@
                     hwc_layer_1_t const* layer = &list->hwLayers[layerIndex];
                     private_handle_t *hnd = (private_handle_t *)layer->handle;
 
-                    /* If a video layer requires rotation, set the DMA state
+                    /* If a layer requires rotation, set the DMA state
                      * to BLOCK_MODE */
 
-                    if (UNLIKELY(isYuvBuffer(hnd)) && canUseRotator(ctx, dpy) &&
-                        (layer->transform & HWC_TRANSFORM_ROT_90)) {
+                    if (canUseRotator(ctx, dpy) &&
+                        has90Transform(layer) && isRotationDoable(ctx, hnd)) {
                         if(not ctx->mOverlay->isDMAMultiplexingSupported()) {
                             if(ctx->mOverlay->isPipeTypeAttached(
                                              overlay::utils::OV_MDP_PIPE_DMA))
diff --git a/libhwcomposer/hwc_fbupdate.cpp b/libhwcomposer/hwc_fbupdate.cpp
index 59f6245..942b8b2 100644
--- a/libhwcomposer/hwc_fbupdate.cpp
+++ b/libhwcomposer/hwc_fbupdate.cpp
@@ -208,7 +208,7 @@
                                    transform, orient);
         //Store the displayFrame, will be used in getDisplayViewFrame
         ctx->dpyAttr[mDpy].mDstRect = displayFrame;
-        setMdpFlags(layer, mdpFlags, 0, transform);
+        setMdpFlags(ctx, layer, mdpFlags, 0, transform);
         // For External use rotator if there is a rotation value set
         ret = preRotateExtDisplay(ctx, layer, info,
                 sourceCrop, mdpFlags, rotFlags);
diff --git a/libhwcomposer/hwc_mdpcomp.cpp b/libhwcomposer/hwc_mdpcomp.cpp
index cf68a3a..010d533 100644
--- a/libhwcomposer/hwc_mdpcomp.cpp
+++ b/libhwcomposer/hwc_mdpcomp.cpp
@@ -320,7 +320,7 @@
 
 bool MDPComp::isSupportedForMDPComp(hwc_context_t *ctx, hwc_layer_1_t* layer) {
     private_handle_t *hnd = (private_handle_t *)layer->handle;
-    if((not isYuvBuffer(hnd) and has90Transform(layer)) or
+    if((has90Transform(layer) and (not isRotationDoable(ctx, hnd))) ||
         (not isValidDimension(ctx,layer))
         //More conditions here, SKIP, sRGB+Blend etc
         ) {
@@ -615,8 +615,9 @@
 
     for(int index = 0; index < numAppLayers; index++ ) {
         hwc_layer_1_t* layer = &list->hwLayers[index];
+        private_handle_t *hnd = (private_handle_t *)layer->handle;
         if ((mCachedFrame.hnd[index] != layer->handle) ||
-                isYuvBuffer((private_handle_t *)layer->handle)) {
+                isYuvBuffer(hnd)) {
             hwc_rect_t dst = layer->displayFrame;
             hwc_rect_t updatingRect = dst;
 
@@ -713,7 +714,7 @@
         hwc_layer_1_t* layer = &list->hwLayers[i];
         private_handle_t *hnd = (private_handle_t *)layer->handle;
 
-        if(isYuvBuffer(hnd) && has90Transform(layer)) {
+        if(has90Transform(layer) && isRotationDoable(ctx, hnd)) {
             if(!canUseRotator(ctx, mDpy)) {
                 ALOGD_IF(isDebug(), "%s: Can't use rotator for dpy %d",
                         __FUNCTION__, mDpy);
@@ -1157,7 +1158,7 @@
         return false;
     }
 
-    if(layer->transform & HWC_TRANSFORM_ROT_90 && !canUseRotator(ctx,mDpy)) {
+    if(has90Transform(layer) && !canUseRotator(ctx, mDpy)) {
         ALOGD_IF(isDebug(), "%s: no free DMA pipe",__FUNCTION__);
         return false;
     }
@@ -1420,7 +1421,7 @@
         hwc_display_contents_1_t* list) {
 
     //Capability checks
-    if(!resourceCheck()) {
+    if(!resourceCheck(ctx, list)) {
         ALOGD_IF(isDebug(), "%s: resource check failed", __FUNCTION__);
         return false;
     }
@@ -1494,12 +1495,31 @@
     return true;
 }
 
-bool MDPComp::resourceCheck() {
+bool MDPComp::resourceCheck(hwc_context_t* ctx,
+        hwc_display_contents_1_t* list) {
     const bool fbUsed = mCurrentFrame.fbCount;
     if(mCurrentFrame.mdpCount > sMaxPipesPerMixer - fbUsed) {
         ALOGD_IF(isDebug(), "%s: Exceeds MAX_PIPES_PER_MIXER",__FUNCTION__);
         return false;
     }
+    // Init rotCount to number of rotate sessions used by other displays
+    int rotCount = ctx->mRotMgr->getNumActiveSessions();
+    // Count the number of rotator sessions required for current display
+    for (int index = 0; index < mCurrentFrame.layerCount; index++) {
+        if(!mCurrentFrame.isFBComposed[index]) {
+            hwc_layer_1_t* layer = &list->hwLayers[index];
+            private_handle_t *hnd = (private_handle_t *)layer->handle;
+            if(has90Transform(layer) && isRotationDoable(ctx, hnd)) {
+                rotCount++;
+            }
+        }
+    }
+    // if number of layers to rotate exceeds max rotator sessions, bail out.
+    if(rotCount > RotMgr::MAX_ROT_SESS) {
+        ALOGD_IF(isDebug(), "%s: Exceeds max rotator sessions  %d",
+                                    __FUNCTION__, mDpy);
+        return false;
+    }
     return true;
 }
 
@@ -2255,14 +2275,14 @@
     }
 
     eMdpFlags mdpFlags = OV_MDP_BACKEND_COMPOSITION;
-    setMdpFlags(layer, mdpFlags, 0, transform);
+    setMdpFlags(ctx, layer, mdpFlags, 0, transform);
 
     if(lDest != OV_INVALID && rDest != OV_INVALID) {
         //Enable overfetch
         setMdpFlags(mdpFlags, OV_MDSS_MDP_DUAL_PIPE);
     }
 
-    if(isYuvBuffer(hnd) && (transform & HWC_TRANSFORM_ROT_90)) {
+    if(has90Transform(layer) && isRotationDoable(ctx, hnd)) {
         (*rot) = ctx->mRotMgr->getNext();
         if((*rot) == NULL) return -1;
         ctx->mLayerRotMap[mDpy]->add(layer, *rot);
diff --git a/libhwcomposer/hwc_mdpcomp.h b/libhwcomposer/hwc_mdpcomp.h
index 850e242..5f39902 100644
--- a/libhwcomposer/hwc_mdpcomp.h
+++ b/libhwcomposer/hwc_mdpcomp.h
@@ -228,7 +228,7 @@
             hwc_display_contents_1_t* list);
     void reset(hwc_context_t *ctx);
     bool isSupportedForMDPComp(hwc_context_t *ctx, hwc_layer_1_t* layer);
-    bool resourceCheck();
+    bool resourceCheck(hwc_context_t* ctx, hwc_display_contents_1_t* list);
     hwc_rect_t getUpdatingFBRect(hwc_context_t *ctx,
             hwc_display_contents_1_t* list);
     /* checks for conditions to enable partial udpate */
diff --git a/libhwcomposer/hwc_utils.cpp b/libhwcomposer/hwc_utils.cpp
index 671388b..9adfe74 100644
--- a/libhwcomposer/hwc_utils.cpp
+++ b/libhwcomposer/hwc_utils.cpp
@@ -955,6 +955,32 @@
         return false;
 }
 
+bool isRotatorSupportedFormat(private_handle_t *hnd) {
+    // Following rotator src formats are supported by mdp driver
+    // TODO: Add more formats in future, if mdp driver adds support
+    switch(hnd->format) {
+        case HAL_PIXEL_FORMAT_RGBA_8888:
+        case HAL_PIXEL_FORMAT_RGB_565:
+        case HAL_PIXEL_FORMAT_RGB_888:
+        case HAL_PIXEL_FORMAT_BGRA_8888:
+            return true;
+        default:
+            return false;
+    }
+    return false;
+}
+
+bool isRotationDoable(hwc_context_t *ctx, private_handle_t *hnd) {
+    // Rotate layers, if it is YUV type or rendered by CPU and not
+    // for the MDP versions below MDP5
+    if((isCPURendered(hnd) && isRotatorSupportedFormat(hnd) &&
+        !ctx->mMDP.version < qdutils::MDSS_V5)
+                   || isYuvBuffer(hnd)) {
+        return true;
+    }
+    return false;
+}
+
 // returns true if Action safe dimensions are set and target supports Actionsafe
 bool isActionSafePresent(hwc_context_t *ctx, int dpy) {
     // if external supports underscan, do nothing
@@ -1422,7 +1448,7 @@
     return ret;
 }
 
-void setMdpFlags(hwc_layer_1_t *layer,
+void setMdpFlags(hwc_context_t *ctx, hwc_layer_1_t *layer,
         ovutils::eMdpFlags &mdpFlags,
         int rotDownscale, int transform) {
     private_handle_t *hnd = (private_handle_t *)layer->handle;
@@ -1443,11 +1469,6 @@
             ovutils::setMdpFlags(mdpFlags,
                     ovutils::OV_MDP_DEINTERLACE);
         }
-        //Pre-rotation will be used using rotator.
-        if(transform & HWC_TRANSFORM_ROT_90) {
-            ovutils::setMdpFlags(mdpFlags,
-                    ovutils::OV_MDP_SOURCE_ROTATED_90);
-        }
     }
 
     if(isSecureDisplayBuffer(hnd)) {
@@ -1457,6 +1478,12 @@
         ovutils::setMdpFlags(mdpFlags,
                              ovutils::OV_MDP_SECURE_DISPLAY_OVERLAY_SESSION);
     }
+
+    //Pre-rotation will be used using rotator.
+    if(has90Transform(layer) && isRotationDoable(ctx, hnd)) {
+        ovutils::setMdpFlags(mdpFlags,
+                ovutils::OV_MDP_SOURCE_ROTATED_90);
+    }
     //No 90 component and no rot-downscale then flips done by MDP
     //If we use rot then it might as well do flips
     if(!(transform & HWC_TRANSFORM_ROT_90) && !rotDownscale) {
@@ -1649,14 +1676,15 @@
         }
     }
 
-    setMdpFlags(layer, mdpFlags, downscale, transform);
+    setMdpFlags(ctx, layer, mdpFlags, downscale, transform);
 
-    if(isYuvBuffer(hnd) && //if 90 component or downscale, use rot
-            ((transform & HWC_TRANSFORM_ROT_90) || downscale)) {
+    //if 90 component or downscale, use rot
+    if((has90Transform(layer) && isRotationDoable(ctx, hnd)) || downscale) {
         *rot = ctx->mRotMgr->getNext();
         if(*rot == NULL) return -1;
         ctx->mLayerRotMap[dpy]->add(layer, *rot);
-        if(!dpy)
+        // BWC is not tested for other formats So enable it only for YUV format
+        if(!dpy && isYuvBuffer(hnd))
             BwcPM::setBwc(crop, dst, transform, mdpFlags);
         //Configure rotator for pre-rotation
         if(configRotator(*rot, whf, crop, mdpFlags, orient, downscale) < 0) {
@@ -1746,7 +1774,7 @@
        ActionSafe, and extorientation features. */
     calcExtDisplayPosition(ctx, hnd, dpy, crop, dst, transform, orient);
 
-    setMdpFlags(layer, mdpFlagsL, 0, transform);
+    setMdpFlags(ctx, layer, mdpFlagsL, 0, transform);
 
     if(lDest != OV_INVALID && rDest != OV_INVALID) {
         //Enable overfetch
@@ -1760,7 +1788,7 @@
         whf.format = wb->getOutputFormat();
     }
 
-    if(isYuvBuffer(hnd) && (transform & HWC_TRANSFORM_ROT_90)) {
+    if(has90Transform(layer) && isRotationDoable(ctx, hnd)) {
         (*rot) = ctx->mRotMgr->getNext();
         if((*rot) == NULL) return -1;
         ctx->mLayerRotMap[dpy]->add(layer, *rot);
@@ -1888,14 +1916,15 @@
        ActionSafe, and extorientation features. */
     calcExtDisplayPosition(ctx, hnd, dpy, crop, dst, transform, orient);
 
-    setMdpFlags(layer, mdpFlagsL, 0, transform);
+    setMdpFlags(ctx, layer, mdpFlagsL, 0, transform);
     trimLayer(ctx, dpy, transform, crop, dst);
 
-    if(isYuvBuffer(hnd) && (transform & HWC_TRANSFORM_ROT_90)) {
+    if(has90Transform(layer) && isRotationDoable(ctx, hnd)) {
         (*rot) = ctx->mRotMgr->getNext();
         if((*rot) == NULL) return -1;
         ctx->mLayerRotMap[dpy]->add(layer, *rot);
-        if(!dpy)
+        // BWC is not tested for other formats So enable it only for YUV format
+        if(!dpy && isYuvBuffer(hnd))
             BwcPM::setBwc(crop, dst, transform, mdpFlagsL);
         //Configure rotator for pre-rotation
         if(configRotator(*rot, whf, crop, mdpFlagsL, orient, downscale) < 0) {
diff --git a/libhwcomposer/hwc_utils.h b/libhwcomposer/hwc_utils.h
index 06f7f9f..6b7d87f 100644
--- a/libhwcomposer/hwc_utils.h
+++ b/libhwcomposer/hwc_utils.h
@@ -235,6 +235,10 @@
                               hwc_rect_t& nwr);
 bool isSecuring(hwc_context_t* ctx, hwc_layer_1_t const* layer);
 bool isSecureModePolicy(int mdpVersion);
+// Returns true, if the input layer format is supported by rotator
+bool isRotatorSupportedFormat(private_handle_t *hnd);
+//Returns true, if the layer is YUV or the layer has been rendered by CPU
+bool isRotationDoable(hwc_context_t *ctx, private_handle_t *hnd);
 bool isExternalActive(hwc_context_t* ctx);
 bool isAlphaScaled(hwc_layer_1_t const* layer);
 bool needsScaling(hwc_layer_1_t const* layer);
@@ -315,7 +319,7 @@
         int fd);
 
 //Sets appropriate mdp flags for a layer.
-void setMdpFlags(hwc_layer_1_t *layer,
+void setMdpFlags(hwc_context_t *ctx, hwc_layer_1_t *layer,
         ovutils::eMdpFlags &mdpFlags,
         int rotDownscale, int transform);
 
@@ -400,6 +404,10 @@
     return (hnd && (private_handle_t::PRIV_FLAGS_TILE_RENDERED & hnd->flags));
 }
 
+static inline bool isCPURendered(const private_handle_t* hnd) {
+    return (hnd && (private_handle_t::PRIV_FLAGS_CPU_RENDERED & hnd->flags));
+}
+
 //Return true if buffer is marked locked
 static inline bool isBufferLocked(const private_handle_t* hnd) {
     return (hnd && (private_handle_t::PRIV_FLAGS_HWC_LOCK & hnd->flags));
@@ -577,7 +585,7 @@
     return  ctx->listStats[dpy].yuvCount;
 }
 
-static inline bool has90Transform(hwc_layer_1_t *layer) {
+static inline bool has90Transform(hwc_layer_1_t const* layer) {
     return ((layer->transform & HWC_TRANSFORM_ROT_90) &&
             !(layer->flags & HWC_COLOR_FILL));
 }
diff --git a/liboverlay/overlayRotator.h b/liboverlay/overlayRotator.h
index f7b44bd..e0f542c 100644
--- a/liboverlay/overlayRotator.h
+++ b/liboverlay/overlayRotator.h
@@ -210,6 +210,9 @@
 // Holder of rotator objects. Manages lifetimes
 class RotMgr {
 public:
+    //Virtually we can support as many rotator sessions as possible, However
+    // more number of rotator sessions leads to performance issues, so
+    // restricting the max rotator session to 4
     enum { MAX_ROT_SESS = 4 };
 
     ~RotMgr();