From eab7ea0e64a24fc254fa71f0f501c2dc8794ed72 Mon Sep 17 00:00:00 2001
From: Lucas Dupin <dupin@google.com>
Date: Wed, 3 Jun 2020 17:27:28 -0700
Subject: [PATCH] Fix issue where surface corner rounding is ignored

It's not correct to only check the diagonal of the transform matrix
for scaling. The scale will be distributed across four components
when the layer is rotated.

Test: manual
Test: atest LayerTypeAndRenderTypeTransaction
Fixes: 147415720
Change-Id: I140b373efd7fad705d0cd54aa6e86b4142e190e5
---
 services/surfaceflinger/Layer.cpp             |  4 +-
 ...LayerTypeAndRenderTypeTransaction_test.cpp | 45 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 96552770c4..dcc213f943 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -2122,7 +2122,9 @@ Layer::RoundedCornerState Layer::getRoundedCornerState() const {
             // but a transform matrix can define horizontal and vertical scales.
             // Let's take the average between both of them and pass into the shader, practically we
             // never do this type of transformation on windows anyway.
-            parentState.radius *= (t[0][0] + t[1][1]) / 2.0f;
+            auto scaleX = sqrtf(t[0][0] * t[0][0] + t[0][1] * t[0][1]);
+            auto scaleY = sqrtf(t[1][0] * t[1][0] + t[1][1] * t[1][1]);
+            parentState.radius *= (scaleX + scaleY) / 2.0f;
             return parentState;
         }
     }
diff --git a/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp b/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp
index d666d7e01a..7d4314f4df 100644
--- a/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp
+++ b/services/surfaceflinger/tests/LayerTypeAndRenderTypeTransaction_test.cpp
@@ -220,6 +220,51 @@ TEST_P(LayerTypeAndRenderTypeTransactionTest, SetCornerRadius) {
         shot->expectColor(Rect(size - testArea, 0, right, testArea), Color::BLACK);
         shot->expectColor(Rect(0, bottom - testArea, testArea, bottom), Color::BLACK);
         shot->expectColor(Rect(size - testArea, bottom - testArea, right, bottom), Color::BLACK);
+        // Solid center
+        shot->expectColor(Rect(size / 2 - testArea / 2, size / 2 - testArea / 2,
+                               size / 2 + testArea / 2, size / 2 + testArea / 2),
+                          Color::RED);
+    }
+}
+
+TEST_P(LayerTypeAndRenderTypeTransactionTest, SetCornerRadiusRotated) {
+    sp<SurfaceControl> parent;
+    sp<SurfaceControl> child;
+    const uint8_t size = 64;
+    const uint8_t testArea = 4;
+    const float cornerRadius = 20.0f;
+    ASSERT_NO_FATAL_FAILURE(parent = createLayer("parent", size, size));
+    ASSERT_NO_FATAL_FAILURE(fillLayerColor(parent, Color::RED, size, size));
+    ASSERT_NO_FATAL_FAILURE(child = createLayer("child", size, size));
+    ASSERT_NO_FATAL_FAILURE(fillLayerColor(child, Color::GREEN, size, size));
+
+    auto transaction = Transaction()
+                               .setCornerRadius(parent, cornerRadius)
+                               .setCrop_legacy(parent, Rect(0, 0, size, size))
+                               .reparent(child, parent->getHandle())
+                               .setPosition(child, 0, size)
+                               // Rotate by half PI
+                               .setMatrix(child, 0.0f, -1.0f, 1.0f, 0.0f);
+    if (mLayerType == ISurfaceComposerClient::eFXSurfaceBufferQueue) {
+        transaction.setCrop_legacy(parent, Rect(0, 0, size, size));
+    } else {
+        transaction.setFrame(parent, Rect(0, 0, size, size));
+    }
+    transaction.apply();
+
+    {
+        const uint8_t bottom = size - 1;
+        const uint8_t right = size - 1;
+        auto shot = getScreenCapture();
+        // Edges are transparent
+        shot->expectColor(Rect(0, 0, testArea, testArea), Color::BLACK);
+        shot->expectColor(Rect(size - testArea, 0, right, testArea), Color::BLACK);
+        shot->expectColor(Rect(0, bottom - testArea, testArea, bottom - testArea), Color::BLACK);
+        shot->expectColor(Rect(right - testArea, bottom - testArea, right, bottom), Color::BLACK);
+        // Solid center
+        shot->expectColor(Rect(size / 2 - testArea / 2, size / 2 - testArea / 2,
+                               size / 2 + testArea / 2, size / 2 + testArea / 2),
+                          Color::GREEN);
     }
 }