Allow unboxing enums with multidimensional arrays

Bug: 185182242
Change-Id: Ieac93b60bf0474c6f3f1ca1ecbb14c6db7c09465
diff --git a/src/main/java/com/android/tools/r8/graph/DexItemFactory.java b/src/main/java/com/android/tools/r8/graph/DexItemFactory.java
index 6d57a75..ceb543f 100644
--- a/src/main/java/com/android/tools/r8/graph/DexItemFactory.java
+++ b/src/main/java/com/android/tools/r8/graph/DexItemFactory.java
@@ -390,6 +390,8 @@
   public final DexType stringBuilderType = createStaticallyKnownType(stringBuilderDescriptor);
   public final DexType stringBufferType = createStaticallyKnownType(stringBufferDescriptor);
 
+  public final DexType javaLangReflectArrayType =
+      createStaticallyKnownType("Ljava/lang/reflect/Array;");
   public final DexType javaLangSystemType = createStaticallyKnownType(javaLangSystemDescriptor);
   public final DexType javaIoPrintStreamType = createStaticallyKnownType("Ljava/io/PrintStream;");
 
@@ -557,6 +559,8 @@
   public final ClassMethods classMethods = new ClassMethods();
   public final ConstructorMethods constructorMethods = new ConstructorMethods();
   public final EnumMembers enumMembers = new EnumMembers();
+  public final JavaLangReflectArrayMembers javaLangReflectArrayMembers =
+      new JavaLangReflectArrayMembers();
   public final JavaLangSystemMethods javaLangSystemMethods = new JavaLangSystemMethods();
   public final NullPointerExceptionMethods npeMethods = new NullPointerExceptionMethods();
   public final IllegalArgumentExceptionMethods illegalArgumentExceptionMethods =
@@ -1453,6 +1457,17 @@
     }
   }
 
+  public class JavaLangReflectArrayMembers {
+
+    public final DexMethod newInstanceMethodWithDimensions =
+        createMethod(
+            javaLangReflectArrayType,
+            createProto(objectType, classType, intArrayType),
+            "newInstance");
+
+    private JavaLangReflectArrayMembers() {}
+  }
+
   public class JavaLangSystemMethods {
     public final DexMethod identityHashCode;
 
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
index cc7c72d..dc34e88 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
@@ -285,6 +285,19 @@
   }
 
   private void analyzeCheckCast(CheckCast checkCast, Set<DexType> eligibleEnums) {
+    // Casts to enum array types are fine as long all enum array creations are valid and have valid
+    // usages. Since creations of enum arrays are rewritten to primitive int arrays, enum array
+    // casts will continue to work after rewriting to int[] casts. Casts that failed with
+    // ClassCastException: "T[] cannot be cast to MyEnum[]" will continue to fail, but with "T[]
+    // cannot be cast to int[]".
+    //
+    // Note that strictly speaking, the rewriting from MyEnum[] to int[] could change the semantics
+    // of code that would fail with "int[] cannot be cast to MyEnum[]" in the input. However, javac
+    // does not allow such code ("incompatible types"), so we should generally not see such code.
+    if (checkCast.getType().isArrayType()) {
+      return;
+    }
+
     // We are doing a type check, which typically means the in-value is of an upper
     // type and cannot be dealt with.
     // If the cast is on a dynamically typed object, the checkCast can be simply removed.
@@ -311,6 +324,9 @@
       ConstClass constClass, Set<DexType> eligibleEnums, ProgramMethod context) {
     // We are using the ConstClass of an enum, which typically means the enum cannot be unboxed.
     // We however allow unboxing if the ConstClass is used only:
+    // - as an argument to java.lang.reflect.Array#newInstance(java.lang.Class, int[]), to allow
+    //   unboxing of:
+    //    MyEnum[][] a = new MyEnum[x][y];
     // - as an argument to Enum#valueOf, to allow unboxing of:
     //    MyEnum a = Enum.valueOf(MyEnum.class, "A");
     // - as a receiver for a name method, to allow unboxing of:
@@ -335,11 +351,18 @@
       }
       if (user.isInvokeStatic()) {
         DexClassAndMethod singleTarget = user.asInvokeStatic().lookupSingleTarget(appView, context);
-        if (singleTarget != null && singleTarget.getReference() == factory.enumMembers.valueOf) {
-          // The name data is required for the correct mapping from the enum name to the ordinal in
-          // the valueOf utility method.
-          addRequiredNameData(enumType);
-          continue;
+        if (singleTarget != null) {
+          if (singleTarget.getReference() == factory.enumMembers.valueOf) {
+            // The name data is required for the correct mapping from the enum name to the ordinal
+            // in
+            // the valueOf utility method.
+            addRequiredNameData(enumType);
+            continue;
+          }
+          if (singleTarget.getReference()
+              == factory.javaLangReflectArrayMembers.newInstanceMethodWithDimensions) {
+            continue;
+          }
         }
       }
       markEnumAsUnboxable(Reason.CONST_CLASS, enumClass);
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/EnumUnboxingArrayTest.java b/src/test/java/com/android/tools/r8/enumunboxing/EnumUnboxingArrayTest.java
index 2d5ddc0..4776d5f 100644
--- a/src/test/java/com/android/tools/r8/enumunboxing/EnumUnboxingArrayTest.java
+++ b/src/test/java/com/android/tools/r8/enumunboxing/EnumUnboxingArrayTest.java
@@ -53,14 +53,12 @@
             .addOptionsModification(opt -> enableEnumOptions(opt, enumValueOptimization))
             .addEnumUnboxingInspector(
                 inspector ->
-                    inspector
-                        .assertUnboxed(
-                            EnumArrayNullRead.MyEnum.class,
-                            EnumArrayReadWrite.MyEnum.class,
-                            EnumArrayReadWriteNoEscape.MyEnum.class,
-                            EnumVarArgs.MyEnum.class)
-                        // TODO(b/185182242): Should always be unboxed.
-                        .assertNotUnboxed(Enum2DimArrayReadWrite.MyEnum.class))
+                    inspector.assertUnboxed(
+                        Enum2DimArrayReadWrite.MyEnum.class,
+                        EnumArrayNullRead.MyEnum.class,
+                        EnumArrayReadWrite.MyEnum.class,
+                        EnumArrayReadWriteNoEscape.MyEnum.class,
+                        EnumVarArgs.MyEnum.class))
             .setMinApi(parameters.getApiLevel())
             .compile();
     for (Class<?> main : TESTS) {