Add clamp to Layer and update Transform inverse.
Ensure that the adjusted transparent region is within
the display bounds by clamping after the transformation
is applied and clean up transform's inverse function.
Bug: 18452546
Change-Id: Ia473e483ee8374177bcb84d8192eb1f0e86f022a
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index c3e1a76..5e3dfab 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -332,8 +332,12 @@
activeCrop.intersect(hw->getViewport(), &activeCrop);
activeCrop = s.transform.inverse().transform(activeCrop);
- // paranoia: make sure the window-crop is constrained in the
- // window's bounds
+ // This needs to be here as transform.transform(Rect) computes the
+ // transformed rect and then takes the bounding box of the result before
+ // returning. This means
+ // transform.inverse().transform(transform.transform(Rect)) != Rect
+ // in which case we need to make sure the final rect is clipped to the
+ // display bounds.
activeCrop.intersect(Rect(s.active.w, s.active.h), &activeCrop);
// subtract the transparent region and snap to the bounds
@@ -432,6 +436,13 @@
activeCrop = s.transform.transform(activeCrop);
activeCrop.intersect(hw->getViewport(), &activeCrop);
activeCrop = s.transform.inverse().transform(activeCrop);
+ // This needs to be here as transform.transform(Rect) computes the
+ // transformed rect and then takes the bounding box of the result before
+ // returning. This means
+ // transform.inverse().transform(transform.transform(Rect)) != Rect
+ // in which case we need to make sure the final rect is clipped to the
+ // display bounds.
+ activeCrop.intersect(Rect(s.active.w, s.active.h), &activeCrop);
// mark regions outside the crop as transparent
activeTransparentRegion.orSelf(Rect(0, 0, s.active.w, activeCrop.top));
activeTransparentRegion.orSelf(Rect(0, activeCrop.bottom,