commit 80de6f350cb23e8a78ad05ae4f8b781ff6fc681a
author Ivan Lozano <ivanlozano@google.com> Mon Oct 30 11:24:08 2017 -0700
committer Ivan Lozano <ivanlozano@google.com> Wed Nov 01 12:49:38 2017 -0700
tree 37bebf611571498cfff07666fbafbafb8ff50155
parent 15b519ba6d498e6ed7fd4e6fac61bb0b3eb7324c

Fix surfaceflinger on integer sanitized builds.

Refactors unsigned integer overflows in do_compare to fix integer
overflow sanitized builds. Also disables sanitization in
Layer::makeTraversalList.

Test: Compiled and checked output from sanitizer.
Bug: 30969751
Change-Id: I1a6d73435495e8b3ba3edb2587f62e4a0f2662f3

services/surfaceflinger/Layer.cpp

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 56328c9..aa7635d 100755
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -71,7 +71,7 @@
     :   contentDirty(false),
         sequence(uint32_t(android_atomic_inc(&sSequence))),
         mFlinger(flinger),
-        mTextureName(-1U),
+        mTextureName(UINT32_MAX),
         mPremultipliedAlpha(true),
         mName("unnamed"),
         mFormat(PIXEL_FORMAT_NONE),
@@ -2700,6 +2700,7 @@
     return mDrawingState.z;
 }
 
+__attribute__((no_sanitize("unsigned-integer-overflow")))
 LayerVector Layer::makeTraversalList(LayerVector::StateSet stateSet) {
     LOG_ALWAYS_FATAL_IF(stateSet == LayerVector::StateSet::Invalid,
                         "makeTraversalList received invalid stateSet");

services/surfaceflinger/LayerVector.cpp

diff --git a/services/surfaceflinger/LayerVector.cpp b/services/surfaceflinger/LayerVector.cpp
index 2233e78..d0f8fbe 100644
--- a/services/surfaceflinger/LayerVector.cpp
+++ b/services/surfaceflinger/LayerVector.cpp
@@ -35,14 +35,17 @@
     uint32_t ls = l->getCurrentState().layerStack;
     uint32_t rs = r->getCurrentState().layerStack;
     if (ls != rs)
-        return ls - rs;
+        return (ls > rs) ? 1 : -1;
 
     uint32_t lz = l->getCurrentState().z;
     uint32_t rz = r->getCurrentState().z;
     if (lz != rz)
-        return lz - rz;
+        return (lz > rz) ? 1 : -1;
 
-    return l->sequence - r->sequence;
+    if (l->sequence == r->sequence)
+        return 0;
+
+    return (l->sequence > r->sequence) ? 1 : -1;
 }
 
 void LayerVector::traverseInZOrder(StateSet stateSet, const Visitor& visitor) const {