Add an desugaring option for invokevirtual to private method on self

This option allows rewriting of invokevirtual to private method
on self to happen during desugaring.

Bug: b/247759997
Change-Id: Ia4f3f1389650a98f640b9cc8b5896e5ff170ccd9
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/InvokeToPrivateRewriter.java b/src/main/java/com/android/tools/r8/ir/desugar/InvokeToPrivateRewriter.java
new file mode 100644
index 0000000..65f23dc
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/ir/desugar/InvokeToPrivateRewriter.java
@@ -0,0 +1,76 @@
+// Copyright (c) 2021, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.ir.desugar;
+
+import com.android.tools.r8.cf.code.CfInstruction;
+import com.android.tools.r8.cf.code.CfInvoke;
+import com.android.tools.r8.contexts.CompilationContext.MethodProcessingContext;
+import com.android.tools.r8.graph.DexEncodedMethod;
+import com.android.tools.r8.graph.DexItemFactory;
+import com.android.tools.r8.graph.DexMethod;
+import com.android.tools.r8.graph.ProgramMethod;
+import com.google.common.collect.ImmutableList;
+import java.util.Collection;
+import org.objectweb.asm.Opcodes;
+
+/**
+ * If an invoke-virtual targets a private method in the current class overriding will not apply (see
+ * JVM 11 spec on method selection 5.4.6. In previous jvm specs this was not explicitly stated, but
+ * derived from method resolution 5.4.3.3 and overriding 5.4.5).
+ *
+ * <p>An invoke-interface can in the same way target a private method.
+ *
+ * <p>For desugaring we use invoke-direct instead. We need to do this as the Android Runtime will
+ * not allow invoke-virtual of a private method.
+ */
+public class InvokeToPrivateRewriter implements CfInstructionDesugaring {
+
+  @Override
+  public Collection<CfInstruction> desugarInstruction(
+      CfInstruction instruction,
+      FreshLocalProvider freshLocalProvider,
+      LocalStackAllocator localStackAllocator,
+      CfInstructionDesugaringEventConsumer eventConsumer,
+      ProgramMethod context,
+      MethodProcessingContext methodProcessingContext,
+      CfInstructionDesugaringCollection desugaringCollection,
+      DexItemFactory dexItemFactory) {
+    if (!instruction.isInvokeVirtual() && !instruction.isInvokeInterface()) {
+      return null;
+    }
+    CfInvoke invoke = instruction.asInvoke();
+    DexMethod method = invoke.getMethod();
+    DexEncodedMethod privateMethod = privateMethodInvokedOnSelf(invoke, context);
+    if (privateMethod == null) {
+      return null;
+    }
+    return ImmutableList.of(new CfInvoke(Opcodes.INVOKESPECIAL, method, invoke.isInterface()));
+  }
+
+  @Override
+  public boolean needsDesugaring(CfInstruction instruction, ProgramMethod context) {
+    if (!instruction.isInvokeVirtual() && !instruction.isInvokeInterface()) {
+      return false;
+    }
+    return isInvokingPrivateMethodOnSelf(instruction.asInvoke(), context);
+  }
+
+  private DexEncodedMethod privateMethodInvokedOnSelf(CfInvoke invoke, ProgramMethod context) {
+    DexMethod method = invoke.getMethod();
+    if (method.getHolderType() != context.getHolderType()) {
+      return null;
+    }
+    DexEncodedMethod directTarget = context.getHolder().lookupDirectMethod(method);
+    if (directTarget != null && !directTarget.isStatic()) {
+      assert method.holder == directTarget.getHolderType();
+      return directTarget;
+    }
+    return null;
+  }
+
+  private boolean isInvokingPrivateMethodOnSelf(CfInvoke invoke, ProgramMethod context) {
+    return privateMethodInvokedOnSelf(invoke, context) != null;
+  }
+}
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/NonEmptyCfInstructionDesugaringCollection.java b/src/main/java/com/android/tools/r8/ir/desugar/NonEmptyCfInstructionDesugaringCollection.java
index 4ee8f70..aa8ca78 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/NonEmptyCfInstructionDesugaringCollection.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/NonEmptyCfInstructionDesugaringCollection.java
@@ -121,6 +121,9 @@
     desugarings.add(new LambdaInstructionDesugaring(appView));
     desugarings.add(new ConstantDynamicInstructionDesugaring(appView));
     desugarings.add(new InvokeSpecialToSelfDesugaring(appView));
+    if (appView.options().rewriteInvokeToPrivateInDesugar) {
+      desugarings.add(new InvokeToPrivateRewriter());
+    }
     desugarings.add(new StringConcatInstructionDesugaring(appView));
     desugarings.add(new BufferCovariantReturnTypeRewriter(appView));
     if (backportedMethodRewriter.hasBackports()) {
@@ -358,7 +361,8 @@
           //  identification is explicitly non-overlapping and remove the exceptions below.
           assert !alsoApplicable
                   || (appliedDesugaring instanceof InterfaceMethodRewriter
-                      && desugaring instanceof NestBasedAccessDesugaring)
+                      && (desugaring instanceof InvokeToPrivateRewriter
+                          || desugaring instanceof NestBasedAccessDesugaring))
                   || (appliedDesugaring instanceof TwrInstructionDesugaring
                       && desugaring instanceof InterfaceMethodRewriter)
               : "Desugaring of "
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 88a8244..3939dd5 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -422,6 +422,11 @@
   public boolean createSingletonsForStatelessLambdas =
       System.getProperty("com.android.tools.r8.createSingletonsForStatelessLambdas") != null;
 
+  // Flag to control the representation of stateless lambdas.
+  // See b/222081665 for context.
+  public boolean rewriteInvokeToPrivateInDesugar =
+      System.getProperty("com.android.tools.r8.rewriteInvokeToPrivateInDesugar") != null;
+
   // Flag to allow record annotations in DEX. See b/231930852 for context.
   public boolean emitRecordAnnotationsInDex =
       System.getProperty("com.android.tools.r8.emitRecordAnnotationsInDex") != null;
diff --git a/src/test/java/com/android/tools/r8/desugar/InterfaceInvokePrivateTest.java b/src/test/java/com/android/tools/r8/desugar/InterfaceInvokePrivateTest.java
index 9cb8615..4055ea0 100644
--- a/src/test/java/com/android/tools/r8/desugar/InterfaceInvokePrivateTest.java
+++ b/src/test/java/com/android/tools/r8/desugar/InterfaceInvokePrivateTest.java
@@ -12,6 +12,7 @@
 import com.android.tools.r8.TestRuntime.CfVm;
 import com.android.tools.r8.cf.CfVersion;
 import com.android.tools.r8.utils.AndroidApiLevel;
+import com.android.tools.r8.utils.BooleanUtils;
 import com.android.tools.r8.utils.StringUtils;
 import java.io.IOException;
 import org.junit.Test;
@@ -29,11 +30,16 @@
   @Parameter(1)
   public CfVersion inputCfVersion;
 
-  @Parameterized.Parameters(name = "{0}, Input CfVersion = {1}")
+  @Parameter(2)
+  public boolean rewriteInvokeToPrivateInDesugar;
+
+  @Parameterized.Parameters(
+      name = "{0}, Input CfVersion = {1}, rewriteInvokeToPrivateInDesugar = {2}")
   public static Iterable<?> data() {
     return buildParameters(
         getTestParameters().withCfRuntimes().withDexRuntimes().withAllApiLevelsAlsoForCf().build(),
-        CfVersion.rangeInclusive(CfVersion.V1_8, CfVersion.V15));
+        CfVersion.rangeInclusive(CfVersion.V1_8, CfVersion.V15),
+        BooleanUtils.values());
   }
 
   private static final String EXPECTED_OUTPUT = StringUtils.unixLines("Hello, world!", "21", "6");
@@ -47,6 +53,7 @@
   public void testReference() throws Exception {
     assumeTrue(parameters.getRuntime().isCf());
     assumeTrue(parameters.getApiLevel().isEqualTo(AndroidApiLevel.B));
+    assumeTrue(rewriteInvokeToPrivateInDesugar);
 
     testForJvm()
         .addProgramClassFileData(transformIToPrivate(inputCfVersion))
@@ -69,7 +76,9 @@
 
   @Test
   public void testDesugar() throws Exception {
-    testForDesugaring(parameters)
+    testForDesugaring(
+            parameters,
+            options -> options.rewriteInvokeToPrivateInDesugar = rewriteInvokeToPrivateInDesugar)
         .addProgramClassFileData(transformIToPrivate(inputCfVersion))
         .addProgramClasses(TestRunner.class)
         .run(parameters.getRuntime(), TestRunner.class)
@@ -91,7 +100,8 @@
                 parameters.getRuntime().isCf()
                     && parameters.getRuntime().asCf().isOlderThan(CfVm.JDK11)
                     && (DesugarTestConfiguration.isNotDesugared(c)
-                        || parameters.getApiLevel().isGreaterThanOrEqualTo(AndroidApiLevel.N)),
+                        || (parameters.getApiLevel().isGreaterThanOrEqualTo(AndroidApiLevel.N)
+                            && !rewriteInvokeToPrivateInDesugar)),
             r -> r.assertFailureWithErrorThatThrows(IncompatibleClassChangeError.class),
             // All other conditions succeed.
             r -> r.assertSuccessWithOutput(EXPECTED_OUTPUT));