Enum unboxing and lambda cf to cf override

Bug: 204174455
Change-Id: Ic9ff2beb772da66e333c1b865f827e408e2a5ca3
diff --git a/src/main/java/com/android/tools/r8/ir/conversion/LensCodeRewriterUtils.java b/src/main/java/com/android/tools/r8/ir/conversion/LensCodeRewriterUtils.java
index dd5b328..aa2e065 100644
--- a/src/main/java/com/android/tools/r8/ir/conversion/LensCodeRewriterUtils.java
+++ b/src/main/java/com/android/tools/r8/ir/conversion/LensCodeRewriterUtils.java
@@ -97,8 +97,8 @@
     if (!isLambdaMetaFactory) {
       return callSite.methodName;
     }
-    assert callSite.getBootstrapArgs().size() > 0;
-    assert callSite.getBootstrapArgs().get(0).isDexValueMethodType();
+    assert callSite.bootstrapArgs.size() > 0;
+    assert callSite.bootstrapArgs.get(0).isDexValueMethodType();
     // The targeted method may have been renamed, we need to update the name if that is the case.
     DexMethod method =
         LambdaDescriptor.getMainFunctionalInterfaceMethodReference(
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java b/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
index 887f18e..3f76080 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 // Represents the lambda descriptor inferred from calls site.
@@ -182,6 +183,17 @@
     return implHandle.asMethod().getName().startsWith(factory.javacLambdaMethodPrefix);
   }
 
+  public void forEachErasedAndEnforcedTypes(BiConsumer<DexType, DexType> consumer) {
+    consumer.accept(erasedProto.returnType, enforcedProto.returnType);
+    for (int i = 0; i < enforcedProto.getArity(); i++) {
+      consumer.accept(erasedProto.getParameter(i), enforcedProto.getParameter(i));
+    }
+  }
+
+  public Iterable<DexType> getReferencedBaseTypes(DexItemFactory dexItemFactory) {
+    return enforcedProto.getBaseTypes(dexItemFactory);
+  }
+
   /** Is a stateless lambda, i.e. lambda does not capture any values */
   final boolean isStateless() {
     return captures.isEmpty();
@@ -257,7 +269,7 @@
 
   public static DexMethod getMainFunctionalInterfaceMethodReference(
       DexCallSite callSite, DexItemFactory factory) {
-    DexProto proto = callSite.getBootstrapArgs().get(0).asDexValueMethodType().value;
+    DexProto proto = callSite.bootstrapArgs.get(0).asDexValueMethodType().value;
     DexProto lambdaFactoryProto = callSite.methodProto;
     DexType mainInterface = lambdaFactoryProto.returnType;
     DexString funcMethodName = callSite.methodName;
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
index 71d5f33..bb18678 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
@@ -32,7 +32,9 @@
 import com.android.tools.r8.graph.DexField;
 import com.android.tools.r8.graph.DexItemFactory;
 import com.android.tools.r8.graph.DexMethod;
+import com.android.tools.r8.graph.DexMethodHandle;
 import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.graph.DexProto;
 import com.android.tools.r8.graph.DexType;
 import com.android.tools.r8.graph.DirectMappedDexApplication;
 import com.android.tools.r8.graph.FieldResolutionResult;
@@ -61,6 +63,7 @@
 import com.android.tools.r8.ir.code.If;
 import com.android.tools.r8.ir.code.InstanceGet;
 import com.android.tools.r8.ir.code.Instruction;
+import com.android.tools.r8.ir.code.InvokeCustom;
 import com.android.tools.r8.ir.code.InvokeMethod;
 import com.android.tools.r8.ir.code.InvokeStatic;
 import com.android.tools.r8.ir.code.MemberType;
@@ -71,6 +74,7 @@
 import com.android.tools.r8.ir.conversion.IRConverter;
 import com.android.tools.r8.ir.conversion.MethodConversionOptions.MutableMethodConversionOptions;
 import com.android.tools.r8.ir.conversion.PostMethodProcessor;
+import com.android.tools.r8.ir.desugar.LambdaDescriptor;
 import com.android.tools.r8.ir.optimize.Inliner.Constraint;
 import com.android.tools.r8.ir.optimize.enums.EnumDataMap.EnumData;
 import com.android.tools.r8.ir.optimize.enums.EnumInstanceFieldData.EnumInstanceFieldKnownData;
@@ -118,6 +122,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
@@ -247,6 +252,9 @@
           case Opcodes.CHECK_CAST:
             analyzeCheckCast(instruction.asCheckCast(), eligibleEnums);
             break;
+          case Opcodes.INVOKE_CUSTOM:
+            analyzeInvokeCustom(instruction.asInvokeCustom(), eligibleEnums, code.context());
+            break;
           case INVOKE_STATIC:
             analyzeInvokeStatic(instruction.asInvokeStatic(), eligibleEnums, code.context());
             break;
@@ -282,6 +290,70 @@
     }
   }
 
+  private void markEnumEligible(DexType type, Set<DexType> eligibleEnums) {
+    DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(type);
+    if (enumClass != null) {
+      eligibleEnums.add(enumClass.getType());
+    }
+  }
+
+  private void invalidateEnum(DexType type) {
+    DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(type);
+    if (enumClass != null) {
+      markEnumAsUnboxable(Reason.INVALID_INVOKE_CUSTOM, enumClass);
+    }
+  }
+
+  private void analyzeInvokeCustom(
+      InvokeCustom invoke, Set<DexType> eligibleEnums, ProgramMethod context) {
+    invoke.getCallSite().methodProto.forEachType(t -> markEnumEligible(t, eligibleEnums));
+    LambdaDescriptor lambdaDescriptor =
+        LambdaDescriptor.tryInfer(invoke.getCallSite(), appView.appInfo(), context);
+    if (lambdaDescriptor == null) {
+      // Based on lambda we can see that enums cannot be unboxed if used in call site bootstrap
+      // arguments, since there might be expectations on overrides. Enums used directly in the
+      // method proto should be fine.
+      analyzeInvokeCustomParameters(invoke, this::invalidateEnum);
+      return;
+    }
+
+    analyzeInvokeCustomParameters(invoke, t -> markEnumEligible(t, eligibleEnums));
+
+    lambdaDescriptor.forEachErasedAndEnforcedTypes(
+        (erasedType, enforcedType) -> {
+          if (erasedType != enforcedType) {
+            invalidateEnum(erasedType);
+            invalidateEnum(enforcedType);
+          }
+        });
+  }
+
+  private void analyzeInvokeCustomParameters(InvokeCustom invoke, Consumer<DexType> nonHolder) {
+    invoke
+        .getCallSite()
+        .bootstrapArgs
+        .forEach(
+            bootstrapArgument -> {
+              if (bootstrapArgument.isDexValueMethodHandle()) {
+                DexMethodHandle methodHandle =
+                    bootstrapArgument.asDexValueMethodHandle().getValue();
+                if (methodHandle.isMethodHandle()) {
+                  DexMethod method = methodHandle.asMethod();
+                  invalidateEnum(method.getHolderType());
+                  method.getProto().forEachType(nonHolder);
+                } else {
+                  assert methodHandle.isFieldHandle();
+                  DexField field = methodHandle.asField();
+                  invalidateEnum(field.getHolderType());
+                  nonHolder.accept(field.type);
+                }
+              } else if (bootstrapArgument.isDexValueMethodType()) {
+                DexProto proto = bootstrapArgument.asDexValueMethodType().getValue();
+                proto.forEachType(nonHolder);
+              }
+            });
+  }
+
   private void analyzeFieldInstruction(FieldInstruction fieldInstruction, IRCode code) {
     DexField field = fieldInstruction.getField();
     DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(field.holder);
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java
index 429bdae..2eb5439 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java
@@ -24,6 +24,7 @@
   public static final Reason INVALID_INIT = new StringReason("INVALID_INIT");
   public static final Reason INVALID_CLINIT = new StringReason("INVALID_CLINIT");
   public static final Reason INVALID_INVOKE = new StringReason("INVALID_INVOKE");
+  public static final Reason INVALID_INVOKE_CUSTOM = new StringReason("INVALID_INVOKE_CUSTOM");
   public static final Reason INVALID_INVOKE_CLASSPATH =
       new StringReason("INVALID_INVOKE_CLASSPATH");
   public static final Reason INVALID_INVOKE_ON_ARRAY = new StringReason("INVALID_INVOKE_ON_ARRAY");
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumObjectTest.java b/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumObjectTest.java
new file mode 100644
index 0000000..284f8ac
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumObjectTest.java
@@ -0,0 +1,77 @@
+// Copyright (c) 2021, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.enumunboxing;
+
+import com.android.tools.r8.NeverClassInline;
+import com.android.tools.r8.NeverInline;
+import com.android.tools.r8.NoVerticalClassMerging;
+import com.android.tools.r8.TestParameters;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class LambdaEnumUnboxingEmptyEnumObjectTest extends EnumUnboxingTestBase {
+
+  private final TestParameters parameters;
+  private final boolean enumValueOptimization;
+  private final EnumKeepRules enumKeepRules;
+
+  @Parameters(name = "{0} valueOpt: {1} keep: {2}")
+  public static List<Object[]> data() {
+    return enumUnboxingTestParameters(getTestParameters().withAllRuntimesAndApiLevels().build());
+  }
+
+  public LambdaEnumUnboxingEmptyEnumObjectTest(
+      TestParameters parameters, boolean enumValueOptimization, EnumKeepRules enumKeepRules) {
+    this.parameters = parameters;
+    this.enumValueOptimization = enumValueOptimization;
+    this.enumKeepRules = enumKeepRules;
+  }
+
+  @Test
+  public void testEnumUnboxing() throws Exception {
+    testForR8(parameters.getBackend())
+        .addInnerClasses(getClass())
+        .addKeepMainRule(Main.class)
+        .addKeepRules(enumKeepRules.getKeepRules())
+        .enableNeverClassInliningAnnotations()
+        .noMinification()
+        .enableNoVerticalClassMergingAnnotations()
+        .enableInliningAnnotations()
+        .addOptionsModification(opt -> enableEnumOptions(opt, enumValueOptimization))
+        .addEnumUnboxingInspector(
+            inspector ->
+                inspector.assertUnboxedIf(
+                    enumKeepRules.isNone() && parameters.getBackend().isDex(), MyEnum.class))
+        .setMinApi(parameters.getApiLevel())
+        .compile()
+        .run(parameters.getRuntime(), Main.class)
+        .assertSuccessWithOutputLines("null");
+  }
+
+  @NeverClassInline
+  enum MyEnum {}
+
+  @NoVerticalClassMerging
+  interface ObjectConsumer<T> {
+
+    void accept(T o);
+  }
+
+  static class Main {
+
+    public static void main(String[] args) {
+      executeObject(e -> System.out.println(String.valueOf(e)));
+    }
+
+    @NeverInline
+    static void executeObject(ObjectConsumer<MyEnum> consumer) {
+      consumer.accept(null);
+    }
+  }
+}
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumTest.java b/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumTest.java
index 288ace4..c205bcf 100644
--- a/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumTest.java
+++ b/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumTest.java
@@ -45,7 +45,9 @@
         .enableInliningAnnotations()
         .addOptionsModification(opt -> enableEnumOptions(opt, enumValueOptimization))
         .addEnumUnboxingInspector(
-            inspector -> inspector.assertUnboxedIf(enumKeepRules.isNone(), MyEnum.class))
+            inspector ->
+                inspector.assertUnboxedIf(
+                    enumKeepRules.isNone() && parameters.getBackend().isDex(), MyEnum.class))
         .setMinApi(parameters.getApiLevel())
         .compile()
         .run(parameters.getRuntime(), Main.class)