Mark effectively final classes as final

Fixes: 202745475
Change-Id: I44e8da0e6544a57de8f23ea8474eada3eb545619
diff --git a/src/main/java/com/android/tools/r8/graph/AppView.java b/src/main/java/com/android/tools/r8/graph/AppView.java
index 492a4ec..0e165f7 100644
--- a/src/main/java/com/android/tools/r8/graph/AppView.java
+++ b/src/main/java/com/android/tools/r8/graph/AppView.java
@@ -31,6 +31,7 @@
 import com.android.tools.r8.naming.SeedMapper;
 import com.android.tools.r8.optimize.argumentpropagation.ArgumentPropagator;
 import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.shaking.KeepClassInfo;
 import com.android.tools.r8.shaking.KeepInfoCollection;
 import com.android.tools.r8.shaking.KeepMethodInfo;
 import com.android.tools.r8.shaking.LibraryModeledPredicate;
@@ -537,6 +538,10 @@
     return keepInfo;
   }
 
+  public KeepClassInfo getKeepInfo(DexProgramClass clazz) {
+    return getKeepInfo().getClassInfo(clazz);
+  }
+
   public KeepMethodInfo getKeepInfo(ProgramMethod method) {
     return getKeepInfo().getMethodInfo(method);
   }
diff --git a/src/main/java/com/android/tools/r8/optimize/argumentpropagation/codescanner/VirtualRootMethodsAnalysis.java b/src/main/java/com/android/tools/r8/optimize/argumentpropagation/codescanner/VirtualRootMethodsAnalysis.java
index 6e90810..cd1e030 100644
--- a/src/main/java/com/android/tools/r8/optimize/argumentpropagation/codescanner/VirtualRootMethodsAnalysis.java
+++ b/src/main/java/com/android/tools/r8/optimize/argumentpropagation/codescanner/VirtualRootMethodsAnalysis.java
@@ -20,10 +20,18 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
 
+/**
+ * Computes the set of virtual methods for which we can use a monomorphic method state as well as
+ * the mapping from virtual methods to their representative root methods.
+ *
+ * <p>The analysis can be used to easily mark effectively final classes and methods as final, and
+ * therefore does this as a side effect.
+ */
 public class VirtualRootMethodsAnalysis extends DepthFirstTopDownClassHierarchyTraversal {
 
   static class VirtualRootMethod {
@@ -118,6 +126,16 @@
   }
 
   @Override
+  public void forEachSubClass(DexProgramClass clazz, Consumer<DexProgramClass> consumer) {
+    List<DexProgramClass> subclasses = immediateSubtypingInfo.getSubclasses(clazz);
+    if (subclasses.isEmpty()) {
+      promoteToFinalIfPossible(clazz);
+    } else {
+      subclasses.forEach(consumer);
+    }
+  }
+
+  @Override
   public void visit(DexProgramClass clazz) {
     Map<DexMethodSignature, VirtualRootMethod> state = computeVirtualRootMethodsState(clazz);
     virtualRootMethodsPerClass.put(clazz, state);
@@ -157,12 +175,7 @@
         rootCandidate -> {
           VirtualRootMethod virtualRootMethod =
               virtualRootMethodsForClass.remove(rootCandidate.getMethodSignature());
-          if (!clazz.isInterface()
-              && !rootCandidate.getAccessFlags().isAbstract()
-              && !virtualRootMethod.hasOverrides()
-              && appView.getKeepInfo(rootCandidate).isOptimizationAllowed(appView.options())) {
-            rootCandidate.getAccessFlags().promoteToFinal();
-          }
+          promoteToFinalIfPossible(rootCandidate, virtualRootMethod);
           if (!rootCandidate.isStructurallyEqualTo(virtualRootMethod.getRoot())) {
             return;
           }
@@ -193,4 +206,21 @@
           }
         });
   }
+
+  private void promoteToFinalIfPossible(DexProgramClass clazz) {
+    if (!clazz.isAbstract()
+        && !clazz.isInterface()
+        && appView.getKeepInfo(clazz).isOptimizationAllowed(appView.options())) {
+      clazz.getAccessFlags().promoteToFinal();
+    }
+  }
+
+  private void promoteToFinalIfPossible(ProgramMethod method, VirtualRootMethod virtualRootMethod) {
+    if (!method.getHolder().isInterface()
+        && !method.getAccessFlags().isAbstract()
+        && !virtualRootMethod.hasOverrides()
+        && appView.getKeepInfo(method).isOptimizationAllowed(appView.options())) {
+      method.getAccessFlags().promoteToFinal();
+    }
+  }
 }
diff --git a/src/test/java/com/android/tools/r8/optimize/finalize/FinalizeKeptClassTest.java b/src/test/java/com/android/tools/r8/optimize/finalize/FinalizeKeptClassTest.java
new file mode 100644
index 0000000..b20a64b
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/optimize/finalize/FinalizeKeptClassTest.java
@@ -0,0 +1,55 @@
+// Copyright (c) 2021, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.optimize.finalize;
+
+import static com.android.tools.r8.utils.codeinspector.Matchers.isFinal;
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.AllOf.allOf;
+
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.utils.codeinspector.ClassSubject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class FinalizeKeptClassTest extends TestBase {
+
+  @Parameter(0)
+  public TestParameters parameters;
+
+  @Parameters(name = "{0}")
+  public static TestParametersCollection parameters() {
+    return getTestParameters().withAllRuntimesAndApiLevels().build();
+  }
+
+  @Test
+  public void test() throws Exception {
+    testForR8(parameters.getBackend())
+        .addInnerClasses(getClass())
+        .addKeepMainRule(Main.class)
+        .setMinApi(parameters.getApiLevel())
+        .compile()
+        .inspect(
+            inspector -> {
+              ClassSubject mainClassSubject = inspector.clazz(Main.class);
+              assertThat(mainClassSubject, allOf(isPresent(), not(isFinal())));
+            })
+        .run(parameters.getRuntime(), Main.class)
+        .assertSuccessWithEmptyOutput();
+  }
+
+  // Should not become final.
+  static class Main {
+
+    public static void main(String[] args) {}
+  }
+}
diff --git a/src/test/java/com/android/tools/r8/optimize/finalize/FinalizeSubclassTest.java b/src/test/java/com/android/tools/r8/optimize/finalize/FinalizeSubclassTest.java
new file mode 100644
index 0000000..f71d961
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/optimize/finalize/FinalizeSubclassTest.java
@@ -0,0 +1,100 @@
+// Copyright (c) 2021, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.optimize.finalize;
+
+import static com.android.tools.r8.utils.codeinspector.Matchers.isFinal;
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.AllOf.allOf;
+
+import com.android.tools.r8.NeverClassInline;
+import com.android.tools.r8.NeverInline;
+import com.android.tools.r8.NoVerticalClassMerging;
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.utils.codeinspector.ClassSubject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class FinalizeSubclassTest extends TestBase {
+
+  @Parameter(0)
+  public TestParameters parameters;
+
+  @Parameters(name = "{0}")
+  public static TestParametersCollection parameters() {
+    return getTestParameters().withAllRuntimesAndApiLevels().build();
+  }
+
+  @Test
+  public void test() throws Exception {
+    testForR8(parameters.getBackend())
+        .addInnerClasses(getClass())
+        .addKeepMainRule(Main.class)
+        .enableInliningAnnotations()
+        .enableNeverClassInliningAnnotations()
+        .enableNoVerticalClassMergingAnnotations()
+        .setMinApi(parameters.getApiLevel())
+        .compile()
+        .inspect(
+            inspector -> {
+              ClassSubject aClassSubject = inspector.clazz(A.class);
+              assertThat(aClassSubject, allOf(isPresent(), not(isFinal())));
+
+              ClassSubject bClassSubject = inspector.clazz(B.class);
+              assertThat(bClassSubject, allOf(isPresent(), not(isFinal())));
+
+              ClassSubject cClassSubject = inspector.clazz(C.class);
+              assertThat(cClassSubject, allOf(isPresent(), isFinal()));
+            })
+        .run(parameters.getRuntime(), Main.class)
+        .assertSuccessWithOutputLines("A.m()", "B.m()", "C.m()");
+  }
+
+  static class Main {
+
+    public static void main(String[] args) {
+      new A().m();
+      new B().m();
+      new C().m();
+    }
+  }
+
+  @NeverClassInline
+  @NoVerticalClassMerging
+  static class A {
+
+    @NeverInline
+    void m() {
+      System.out.println("A.m()");
+    }
+  }
+
+  @NeverClassInline
+  @NoVerticalClassMerging
+  static class B extends A {
+
+    @NeverInline
+    void m() {
+      System.out.println("B.m()");
+    }
+  }
+
+  // Should become final.
+  @NeverClassInline
+  static class C extends B {
+
+    @NeverInline
+    void m() {
+      System.out.println("C.m()");
+    }
+  }
+}