Enum unboxing and lambda cf to cf override

Bug: 204174455
Change-Id: Ic9ff2beb772da66e333c1b865f827e408e2a5ca3
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java b/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
index 887f18e..74de135 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 // Represents the lambda descriptor inferred from calls site.
@@ -182,6 +183,17 @@
     return implHandle.asMethod().getName().startsWith(factory.javacLambdaMethodPrefix);
   }
 
+  public void forEachErasedAndEnforcedTypes(BiConsumer<DexType, DexType> consumer) {
+    consumer.accept(erasedProto.returnType, enforcedProto.returnType);
+    for (int i = 0; i < enforcedProto.getArity(); i++) {
+      consumer.accept(erasedProto.getParameter(i), enforcedProto.getParameter(i));
+    }
+  }
+
+  public Iterable<DexType> getReferencedBaseTypes(DexItemFactory dexItemFactory) {
+    return enforcedProto.getBaseTypes(dexItemFactory);
+  }
+
   /** Is a stateless lambda, i.e. lambda does not capture any values */
   final boolean isStateless() {
     return captures.isEmpty();
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
index 2e74bb2..11e631a 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
@@ -76,6 +76,7 @@
 import com.android.tools.r8.ir.conversion.MethodConversionOptions.MutableMethodConversionOptions;
 import com.android.tools.r8.ir.conversion.MethodProcessor;
 import com.android.tools.r8.ir.conversion.PostMethodProcessor;
+import com.android.tools.r8.ir.desugar.LambdaDescriptor;
 import com.android.tools.r8.ir.optimize.Inliner.Constraint;
 import com.android.tools.r8.ir.optimize.enums.EnumDataMap.EnumData;
 import com.android.tools.r8.ir.optimize.enums.EnumInstanceFieldData.EnumInstanceFieldKnownData;
@@ -265,7 +266,7 @@
             analyzeCheckCast(instruction.asCheckCast(), eligibleEnums);
             break;
           case Opcodes.INVOKE_CUSTOM:
-            analyzeInvokeCustom(instruction.asInvokeCustom(), eligibleEnums);
+            analyzeInvokeCustom(instruction.asInvokeCustom(), eligibleEnums, code.context());
             break;
           case INVOKE_STATIC:
             analyzeInvokeStatic(instruction.asInvokeStatic(), eligibleEnums, code.context());
@@ -303,15 +304,45 @@
     }
   }
 
-  private void analyzeInvokeCustom(InvokeCustom invoke, Set<DexType> eligibleEnums) {
-    Consumer<DexType> typeReferenceConsumer =
-        type -> {
-          DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(type);
-          if (enumClass != null) {
-            eligibleEnums.add(enumClass.getType());
+  private void markEnumEligible(DexType type, Set<DexType> eligibleEnums) {
+    DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(type);
+    if (enumClass != null) {
+      eligibleEnums.add(enumClass.getType());
+    }
+  }
+
+  private void invalidateEnum(DexType type) {
+    DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(type);
+    if (enumClass != null) {
+      markEnumAsUnboxable(Reason.INVALID_INVOKE_CUSTOM, enumClass);
+    }
+  }
+
+  private void analyzeInvokeCustom(
+      InvokeCustom invoke, Set<DexType> eligibleEnums, ProgramMethod context) {
+    invoke.getCallSite().getMethodProto().forEachType(t -> markEnumEligible(t, eligibleEnums));
+    LambdaDescriptor lambdaDescriptor =
+        LambdaDescriptor.tryInfer(invoke.getCallSite(), appView.appInfo(), context);
+    if (lambdaDescriptor == null) {
+      // Based on lambda we can see that enums cannot be unboxed if used in call site bootstrap
+      // arguments, since there might be expectations on overrides. Enums used directly in the
+      // method proto should be fine.
+      analyzeInvokeCustomParameters(invoke, this::invalidateEnum);
+      return;
+    }
+
+    analyzeInvokeCustomParameters(invoke, t -> markEnumEligible(t, eligibleEnums));
+
+    lambdaDescriptor.forEachErasedAndEnforcedTypes(
+        (erasedType, enforcedType) -> {
+          if (erasedType != enforcedType) {
+            invalidateEnum(erasedType);
+            invalidateEnum(enforcedType);
           }
-        };
-    invoke.getCallSite().getMethodProto().forEachType(typeReferenceConsumer);
+        });
+  }
+
+  private void analyzeInvokeCustomParameters(InvokeCustom invoke, Consumer<DexType> nonHolder) {
     invoke
         .getCallSite()
         .getBootstrapArgs()
@@ -322,30 +353,22 @@
                     bootstrapArgument.asDexValueMethodHandle().getValue();
                 if (methodHandle.isMethodHandle()) {
                   DexMethod method = methodHandle.asMethod();
-                  DexProgramClass enumClass =
-                      getEnumUnboxingCandidateOrNull(method.getHolderType());
-                  if (enumClass != null) {
-                    markEnumAsUnboxable(Reason.INVALID_INVOKE_CUSTOM, enumClass);
-                  } else {
-                    method.getProto().forEachType(typeReferenceConsumer);
-                  }
+                  invalidateEnum(method.getHolderType());
+                  method.getProto().forEachType(nonHolder);
                 } else {
                   assert methodHandle.isFieldHandle();
                   DexField field = methodHandle.asField();
-                  DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(field.getHolderType());
-                  if (enumClass != null) {
-                    markEnumAsUnboxable(Reason.INVALID_INVOKE_CUSTOM, enumClass);
-                  } else {
-                    typeReferenceConsumer.accept(field.getType());
-                  }
+                  invalidateEnum(field.getHolderType());
+                  nonHolder.accept(field.type);
                 }
               } else if (bootstrapArgument.isDexValueMethodType()) {
                 DexProto proto = bootstrapArgument.asDexValueMethodType().getValue();
-                proto.forEachType(typeReferenceConsumer);
+                proto.forEachType(nonHolder);
               }
             });
   }
 
+
   private void analyzeFieldInstruction(
       FieldInstruction fieldInstruction, Set<DexType> eligibleEnums, ProgramMethod context) {
     DexField field = fieldInstruction.getField();
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumObjectTest.java b/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumObjectTest.java
new file mode 100644
index 0000000..284f8ac
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/enumunboxing/LambdaEnumUnboxingEmptyEnumObjectTest.java
@@ -0,0 +1,77 @@
+// Copyright (c) 2021, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.enumunboxing;
+
+import com.android.tools.r8.NeverClassInline;
+import com.android.tools.r8.NeverInline;
+import com.android.tools.r8.NoVerticalClassMerging;
+import com.android.tools.r8.TestParameters;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class LambdaEnumUnboxingEmptyEnumObjectTest extends EnumUnboxingTestBase {
+
+  private final TestParameters parameters;
+  private final boolean enumValueOptimization;
+  private final EnumKeepRules enumKeepRules;
+
+  @Parameters(name = "{0} valueOpt: {1} keep: {2}")
+  public static List<Object[]> data() {
+    return enumUnboxingTestParameters(getTestParameters().withAllRuntimesAndApiLevels().build());
+  }
+
+  public LambdaEnumUnboxingEmptyEnumObjectTest(
+      TestParameters parameters, boolean enumValueOptimization, EnumKeepRules enumKeepRules) {
+    this.parameters = parameters;
+    this.enumValueOptimization = enumValueOptimization;
+    this.enumKeepRules = enumKeepRules;
+  }
+
+  @Test
+  public void testEnumUnboxing() throws Exception {
+    testForR8(parameters.getBackend())
+        .addInnerClasses(getClass())
+        .addKeepMainRule(Main.class)
+        .addKeepRules(enumKeepRules.getKeepRules())
+        .enableNeverClassInliningAnnotations()
+        .noMinification()
+        .enableNoVerticalClassMergingAnnotations()
+        .enableInliningAnnotations()
+        .addOptionsModification(opt -> enableEnumOptions(opt, enumValueOptimization))
+        .addEnumUnboxingInspector(
+            inspector ->
+                inspector.assertUnboxedIf(
+                    enumKeepRules.isNone() && parameters.getBackend().isDex(), MyEnum.class))
+        .setMinApi(parameters.getApiLevel())
+        .compile()
+        .run(parameters.getRuntime(), Main.class)
+        .assertSuccessWithOutputLines("null");
+  }
+
+  @NeverClassInline
+  enum MyEnum {}
+
+  @NoVerticalClassMerging
+  interface ObjectConsumer<T> {
+
+    void accept(T o);
+  }
+
+  static class Main {
+
+    public static void main(String[] args) {
+      executeObject(e -> System.out.println(String.valueOf(e)));
+    }
+
+    @NeverInline
+    static void executeObject(ObjectConsumer<MyEnum> consumer) {
+      consumer.accept(null);
+    }
+  }
+}