Fix inadequate enum unboxing reprocessing when generating class files

Change-Id: I38b99999124804a06f5935ebdeebb726f3bea85f

src/main/java/com/android/tools/r8/graph/DexCallSite.java

diff --git a/src/main/java/com/android/tools/r8/graph/DexCallSite.java b/src/main/java/com/android/tools/r8/graph/DexCallSite.java
index 9cb4704..3aa03da 100644
--- a/src/main/java/com/android/tools/r8/graph/DexCallSite.java
+++ b/src/main/java/com/android/tools/r8/graph/DexCallSite.java
@@ -102,6 +102,14 @@
     return application.getCallSite(name, desc, bootstrapMethod, bootstrapArgs);
   }
 
+  public List<DexValue> getBootstrapArgs() {
+    return bootstrapArgs;
+  }
+
+  public DexProto getMethodProto() {
+    return methodProto;
+  }
+
   @Override
   public DexCallSite self() {
     return this;

src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java

diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
index a5ed9fb..2e74bb2 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
@@ -33,7 +33,9 @@
 import com.android.tools.r8.graph.DexField;
 import com.android.tools.r8.graph.DexItemFactory;
 import com.android.tools.r8.graph.DexMethod;
+import com.android.tools.r8.graph.DexMethodHandle;
 import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.graph.DexProto;
 import com.android.tools.r8.graph.DexType;
 import com.android.tools.r8.graph.FieldResolutionResult;
 import com.android.tools.r8.graph.GraphLens;
@@ -61,6 +63,7 @@
 import com.android.tools.r8.ir.code.If;
 import com.android.tools.r8.ir.code.InstanceGet;
 import com.android.tools.r8.ir.code.Instruction;
+import com.android.tools.r8.ir.code.InvokeCustom;
 import com.android.tools.r8.ir.code.InvokeMethod;
 import com.android.tools.r8.ir.code.InvokeStatic;
 import com.android.tools.r8.ir.code.InvokeVirtual;
@@ -122,6 +125,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
@@ -217,6 +221,13 @@
   }
 
   private DexProgramClass getEnumUnboxingCandidateOrNull(DexType type) {
+    if (type.isArrayType()) {
+      return getEnumUnboxingCandidateOrNull(type.toBaseType(appView.dexItemFactory()));
+    }
+    if (type.isPrimitiveType() || type.isVoidType()) {
+      return null;
+    }
+    assert type.isClassType();
     return enumUnboxingCandidatesInfo.getCandidateClassOrNull(type);
   }
 
@@ -253,6 +264,9 @@
           case Opcodes.CHECK_CAST:
             analyzeCheckCast(instruction.asCheckCast(), eligibleEnums);
             break;
+          case Opcodes.INVOKE_CUSTOM:
+            analyzeInvokeCustom(instruction.asInvokeCustom(), eligibleEnums);
+            break;
           case INVOKE_STATIC:
             analyzeInvokeStatic(instruction.asInvokeStatic(), eligibleEnums, code.context());
             break;
@@ -289,6 +303,49 @@
     }
   }
 
+  private void analyzeInvokeCustom(InvokeCustom invoke, Set<DexType> eligibleEnums) {
+    Consumer<DexType> typeReferenceConsumer =
+        type -> {
+          DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(type);
+          if (enumClass != null) {
+            eligibleEnums.add(enumClass.getType());
+          }
+        };
+    invoke.getCallSite().getMethodProto().forEachType(typeReferenceConsumer);
+    invoke
+        .getCallSite()
+        .getBootstrapArgs()
+        .forEach(
+            bootstrapArgument -> {
+              if (bootstrapArgument.isDexValueMethodHandle()) {
+                DexMethodHandle methodHandle =
+                    bootstrapArgument.asDexValueMethodHandle().getValue();
+                if (methodHandle.isMethodHandle()) {
+                  DexMethod method = methodHandle.asMethod();
+                  DexProgramClass enumClass =
+                      getEnumUnboxingCandidateOrNull(method.getHolderType());
+                  if (enumClass != null) {
+                    markEnumAsUnboxable(Reason.INVALID_INVOKE_CUSTOM, enumClass);
+                  } else {
+                    method.getProto().forEachType(typeReferenceConsumer);
+                  }
+                } else {
+                  assert methodHandle.isFieldHandle();
+                  DexField field = methodHandle.asField();
+                  DexProgramClass enumClass = getEnumUnboxingCandidateOrNull(field.getHolderType());
+                  if (enumClass != null) {
+                    markEnumAsUnboxable(Reason.INVALID_INVOKE_CUSTOM, enumClass);
+                  } else {
+                    typeReferenceConsumer.accept(field.getType());
+                  }
+                }
+              } else if (bootstrapArgument.isDexValueMethodType()) {
+                DexProto proto = bootstrapArgument.asDexValueMethodType().getValue();
+                proto.forEachType(typeReferenceConsumer);
+              }
+            });
+  }
+
   private void analyzeFieldInstruction(
       FieldInstruction fieldInstruction, Set<DexType> eligibleEnums, ProgramMethod context) {
     DexField field = fieldInstruction.getField();

src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java

diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java
index 429bdae..bac0275 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/eligibility/Reason.java
@@ -26,6 +26,7 @@
   public static final Reason INVALID_INVOKE = new StringReason("INVALID_INVOKE");
   public static final Reason INVALID_INVOKE_CLASSPATH =
       new StringReason("INVALID_INVOKE_CLASSPATH");
+  public static final Reason INVALID_INVOKE_CUSTOM = new StringReason("INVALID_INVOKE_CUSTOM");
   public static final Reason INVALID_INVOKE_ON_ARRAY = new StringReason("INVALID_INVOKE_ON_ARRAY");
   public static final Reason IMPLICIT_UP_CAST_IN_RETURN =
       new StringReason("IMPLICIT_UP_CAST_IN_RETURN");