SurfaceFlinger: Limit to 4k Layers

Sets a limit of 4k Layers which SurfaceFlinger will allow to be in
existence at any given time. An attempt to create Layers in excess of
this limit will fail with NO_MEMORY.

Bug: 20674586
Change-Id: I2dfaf59643d826f982b2fa44e8a9ed643176d972
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index df4ac2e..715b92f 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -1994,18 +1994,25 @@
     engine.fillRegionWithColor(region, height, 0, 0, 0, 0);
 }
 
-void SurfaceFlinger::addClientLayer(const sp<Client>& client,
+status_t SurfaceFlinger::addClientLayer(const sp<Client>& client,
         const sp<IBinder>& handle,
         const sp<IGraphicBufferProducer>& gbc,
         const sp<Layer>& lbc)
 {
+    // add this layer to the current state list
+    {
+        Mutex::Autolock _l(mStateLock);
+        if (mCurrentState.layersSortedByZ.size() >= MAX_LAYERS) {
+            return NO_MEMORY;
+        }
+        mCurrentState.layersSortedByZ.add(lbc);
+        mGraphicBufferProducerList.add(IInterface::asBinder(gbc));
+    }
+
     // attach this layer to the client
     client->attachLayer(handle, lbc);
 
-    // add this layer to the current state list
-    Mutex::Autolock _l(mStateLock);
-    mCurrentState.layersSortedByZ.add(lbc);
-    mGraphicBufferProducerList.add(IInterface::asBinder(gbc));
+    return NO_ERROR;
 }
 
 status_t SurfaceFlinger::removeLayer(const sp<Layer>& layer) {
@@ -2262,10 +2269,16 @@
             break;
     }
 
-    if (result == NO_ERROR) {
-        addClientLayer(client, *handle, *gbp, layer);
-        setTransactionFlags(eTransactionNeeded);
+    if (result != NO_ERROR) {
+        return result;
     }
+
+    result = addClientLayer(client, *handle, *gbp, layer);
+    if (result != NO_ERROR) {
+        return result;
+    }
+
+    setTransactionFlags(eTransactionNeeded);
     return result;
 }