Prevent merging vertically merged classes

Bug: 163311975
Change-Id: Ibabe9097c6253a112e92696c5da9002b6082710a
diff --git a/src/main/java/com/android/tools/r8/horizontalclassmerging/HorizontalClassMerger.java b/src/main/java/com/android/tools/r8/horizontalclassmerging/HorizontalClassMerger.java
index a5a9f0a6..1886a48 100644
--- a/src/main/java/com/android/tools/r8/horizontalclassmerging/HorizontalClassMerger.java
+++ b/src/main/java/com/android/tools/r8/horizontalclassmerging/HorizontalClassMerger.java
@@ -19,6 +19,7 @@
 import com.android.tools.r8.horizontalclassmerging.policies.NoStaticClassInitializer;
 import com.android.tools.r8.horizontalclassmerging.policies.NotEntryPoint;
 import com.android.tools.r8.horizontalclassmerging.policies.NotMatchedByNoHorizontalClassMerging;
+import com.android.tools.r8.horizontalclassmerging.policies.NotVerticallyMergedIntoSubtype;
 import com.android.tools.r8.horizontalclassmerging.policies.PreventChangingVisibility;
 import com.android.tools.r8.horizontalclassmerging.policies.PreventMergeIntoMainDex;
 import com.android.tools.r8.horizontalclassmerging.policies.RespectPackageBoundaries;
@@ -58,6 +59,7 @@
             new NoInnerClasses(),
             new NoStaticClassInitializer(),
             new NoKeepRules(appView),
+            new NotVerticallyMergedIntoSubtype(appView),
             new NoRuntimeTypeChecks(classMergingEnqueuerExtension),
             new NotEntryPoint(appView.dexItemFactory()),
             new PreventMergeIntoMainDex(appView, mainDexTracingResult),
diff --git a/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotVerticallyMergedIntoSubtype.java b/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotVerticallyMergedIntoSubtype.java
new file mode 100644
index 0000000..67afbc3
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotVerticallyMergedIntoSubtype.java
@@ -0,0 +1,26 @@
+// Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.horizontalclassmerging.policies;
+
+import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.horizontalclassmerging.SingleClassPolicy;
+import com.android.tools.r8.shaking.AppInfoWithLiveness;
+
+public class NotVerticallyMergedIntoSubtype extends SingleClassPolicy {
+  private final AppView<AppInfoWithLiveness> appView;
+
+  public NotVerticallyMergedIntoSubtype(AppView<AppInfoWithLiveness> appView) {
+    this.appView = appView;
+  }
+
+  @Override
+  public boolean canMerge(DexProgramClass program) {
+    if (appView.verticallyMergedClasses() == null) {
+      return true;
+    }
+    return !appView.verticallyMergedClasses().hasBeenMergedIntoSubtype(program.type);
+  }
+}
diff --git a/src/test/java/com/android/tools/r8/classmerging/horizontal/VerticallyMergedClassTest.java b/src/test/java/com/android/tools/r8/classmerging/horizontal/VerticallyMergedClassTest.java
new file mode 100644
index 0000000..e40ff51
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/classmerging/horizontal/VerticallyMergedClassTest.java
@@ -0,0 +1,77 @@
+// Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.classmerging.horizontal;
+
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import com.android.tools.r8.NeverClassInline;
+import com.android.tools.r8.NeverInline;
+import com.android.tools.r8.NoHorizontalClassMerging;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.classmerging.horizontal.ConstructorMergingOverlapTest.Main;
+import org.junit.Test;
+
+public class VerticallyMergedClassTest extends HorizontalClassMergingTestBase {
+  public VerticallyMergedClassTest(
+      TestParameters parameters, boolean enableHorizontalClassMerging) {
+    super(parameters, enableHorizontalClassMerging);
+  }
+
+  @Test
+  public void testR8() throws Exception {
+    testForR8(parameters.getBackend())
+        .addInnerClasses(getClass())
+        .addKeepMainRule(Main.class)
+        .addOptionsModification(
+            options -> options.enableHorizontalClassMerging = enableHorizontalClassMerging)
+        .enableNoHorizontalClassMergingAnnotations()
+        .enableInliningAnnotations()
+        .enableNeverClassInliningAnnotations()
+        .setMinApi(parameters.getApiLevel())
+        .run(parameters.getRuntime(), Main.class)
+        .assertSuccessWithOutputLines("b", "a", "c")
+        .inspect(
+            codeInspector -> {
+              assertThat(codeInspector.clazz(A.class), not(isPresent()));
+              assertThat(codeInspector.clazz(B.class), isPresent());
+              assertThat(codeInspector.clazz(C.class), isPresent());
+            });
+  }
+
+  public static class A {
+    public void print() {
+      System.out.println("a");
+    }
+  }
+
+  @NoHorizontalClassMerging
+  @NeverClassInline
+  public static class B extends A {
+    public B() {
+      System.out.println("b");
+    }
+  }
+
+  @NeverClassInline
+  public static class C {
+    public C() {
+      System.out.println("c");
+    }
+  }
+
+  public static class Main {
+    @NeverInline
+    static void printA(A a) {
+      a.print();
+    }
+
+    public static void main(String[] args) {
+      printA(new B());
+      new C();
+    }
+  }
+}