Unbox enum with subtypes and static members

Bug: b/271385332
Change-Id: Ibbcd19a4f9998d1e49b5b1f0389751053402ea5a
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumDataMap.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumDataMap.java
index aacdd3a..6672c87 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumDataMap.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumDataMap.java
@@ -55,6 +55,15 @@
     }
   }
 
+  public boolean isAssignableTo(DexType subtype, DexType superType) {
+    assert superType != null;
+    assert subtype != null;
+    if (superType == subtype) {
+      return true;
+    }
+    return superType == subEnumToSuperEnumMap.get(subtype);
+  }
+
   public DexType representativeType(DexType type) {
     return subEnumToSuperEnumMap.getOrDefault(type, type);
   }
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java
index 8b82922..e12cdc8 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java
@@ -20,7 +20,6 @@
 import com.android.tools.r8.shaking.KeepInfoCollection;
 import com.android.tools.r8.utils.InternalOptions;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import java.util.IdentityHashMap;
 import java.util.Map;
@@ -133,14 +132,6 @@
       }
       result = false;
     }
-    // TODO(b/271385332): Support subEnums with static members (JDK16+).
-    if (!clazz.staticFields().isEmpty()
-        || !Iterables.isEmpty(clazz.directMethods(DexEncodedMethod::isStatic))) {
-      if (!enumUnboxer.reportFailure(clazz.superType, Reason.SUBENUM_STATIC_MEMBER)) {
-        return false;
-      }
-      result = false;
-    }
     return result;
   }
 
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingTreeFixer.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingTreeFixer.java
index 7d7ce1c..92f6d70 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingTreeFixer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingTreeFixer.java
@@ -130,7 +130,7 @@
     PrunedItems.Builder prunedItemsBuilder = PrunedItems.builder();
 
     // We do this before so that we can still perform lookup of definitions.
-    fixupEnumClassInitializers(converter, executorService);
+    fixupSuperEnumClassInitializers(converter, executorService);
 
     // Fix all methods and fields using enums to unbox.
     // TODO(b/191617665): Parallelize this fixup.
@@ -267,9 +267,8 @@
     return checkNotNullToCheckNotZeroMapping;
   }
 
-
-  private void fixupEnumClassInitializers(IRConverter converter, ExecutorService executorService)
-      throws ExecutionException {
+  private void fixupSuperEnumClassInitializers(
+      IRConverter converter, ExecutorService executorService) throws ExecutionException {
     DexEncodedField ordinalField =
         appView
             .appInfo()
@@ -277,11 +276,11 @@
             .getResolvedField();
     ThreadUtils.processItems(
         unboxedEnumHierarchy.keySet(),
-        unboxedEnum -> fixupEnumClassInitializer(converter, unboxedEnum, ordinalField),
+        unboxedEnum -> fixupSuperEnumClassInitializer(converter, unboxedEnum, ordinalField),
         executorService);
   }
 
-  private void fixupEnumClassInitializer(
+  private void fixupSuperEnumClassInitializer(
       IRConverter converter, DexProgramClass unboxedEnum, DexEncodedField ordinalField) {
     if (!unboxedEnum.hasClassInitializer()) {
       assert unboxedEnum.staticFields().isEmpty();
@@ -321,7 +320,7 @@
           // LocalEnumUtility.class.desiredAssertionStatus() instead of
           // int.class.desiredAssertionStatus().
           ConstClass constClass = instruction.asConstClass();
-          if (constClass.getType() != unboxedEnum.getType()) {
+          if (!enumDataMap.isAssignableTo(constClass.getType(), unboxedEnum.getType())) {
             continue;
           }
 
@@ -353,7 +352,7 @@
         } else if (instruction.isNewInstance()) {
           NewInstance newInstance = instruction.asNewInstance();
           DexType rewrittenType = appView.graphLens().lookupType(newInstance.getType());
-          if (rewrittenType == unboxedEnum.getType()) {
+          if (enumDataMap.isAssignableTo(rewrittenType, unboxedEnum.getType())) {
             InvokeDirect constructorInvoke =
                 newInstance.getUniqueConstructorInvoke(appView.dexItemFactory());
             assert constructorInvoke != null;
@@ -432,16 +431,15 @@
             // the enum unboxing rewriter.
             instructionIterator.replaceCurrentInstruction(
                 new NewUnboxedEnumInstance(
-                    unboxedEnum.getType(),
+                    rewrittenType,
                     ordinal,
                     code.createValue(
-                        ClassTypeElement.create(
-                            unboxedEnum.getType(), definitelyNotNull(), appView))));
+                        ClassTypeElement.create(rewrittenType, definitelyNotNull(), appView))));
           }
         } else if (instruction.isStaticPut()) {
           StaticPut staticPut = instruction.asStaticPut();
           DexField rewrittenField = appView.graphLens().lookupField(staticPut.getField());
-          if (rewrittenField.getHolderType() != unboxedEnum.getType()) {
+          if (!enumDataMap.isAssignableTo(rewrittenField.getHolderType(), unboxedEnum.getType())) {
             continue;
           }
 
@@ -593,6 +591,11 @@
           subEnum,
           prunedItemsBuilder,
           method -> {
+            if (method.getDefinition().isClassInitializer()) {
+              assert method.getDefinition().getCode().isEmptyVoidMethod();
+              prunedItemsBuilder.addRemovedMethod(method.getReference());
+              return;
+            }
             if (!methodsWithOverride.contains(method.getReference())) {
               DexEncodedMethod newLocalUtilityMethod =
                   installLocalUtilityMethod(localUtilityClass, localUtilityMethods, method);
diff --git a/src/test/examplesJava17/enumStatic/EnumStaticMain.java b/src/test/examplesJava17/enumStatic/EnumStaticMain.java
new file mode 100644
index 0000000..6e79499
--- /dev/null
+++ b/src/test/examplesJava17/enumStatic/EnumStaticMain.java
@@ -0,0 +1,29 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package enumStatic;
+
+public class EnumStaticMain {
+
+  enum EnumStatic {
+    A,
+    B {
+      static int i = 17;
+
+      static void print() {
+        System.out.println("-" + i);
+      }
+
+      public void virtualPrint() {
+        print();
+      }
+    };
+
+    public void virtualPrint() {}
+  }
+
+  public static void main(String[] args) throws Throwable {
+    EnumStatic.B.virtualPrint();
+  }
+}
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/enummerging/StaticEnumMergingTest.java b/src/test/java/com/android/tools/r8/enumunboxing/enummerging/StaticEnumMergingTest.java
new file mode 100644
index 0000000..1c1b779
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/enumunboxing/enummerging/StaticEnumMergingTest.java
@@ -0,0 +1,58 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.enumunboxing.enummerging;
+
+import static com.android.tools.r8.utils.FileUtils.JAR_EXTENSION;
+
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.ToolHelper;
+import com.android.tools.r8.enumunboxing.EnumUnboxingTestBase;
+import com.android.tools.r8.utils.StringUtils;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class StaticEnumMergingTest extends EnumUnboxingTestBase {
+
+  private static final Path JDK17_JAR =
+      Paths.get(ToolHelper.TESTS_BUILD_DIR, "examplesJava17").resolve("enumStatic" + JAR_EXTENSION);
+  private static final String EXPECTED_RESULT = StringUtils.lines("-17");
+  private static final String MAIN = "enumStatic.EnumStaticMain";
+
+  private final TestParameters parameters;
+  private final boolean enumValueOptimization;
+  private final EnumKeepRules enumKeepRules;
+
+  @Parameters(name = "{0} valueOpt: {1} keep: {2}")
+  public static List<Object[]> data() {
+    return enumUnboxingTestParameters();
+  }
+
+  public StaticEnumMergingTest(
+      TestParameters parameters, boolean enumValueOptimization, EnumKeepRules enumKeepRules) {
+    this.parameters = parameters;
+    this.enumValueOptimization = enumValueOptimization;
+    this.enumKeepRules = enumKeepRules;
+  }
+
+  @Test
+  public void testEnumUnboxing() throws Exception {
+    testForR8(parameters.getBackend())
+        .addProgramFiles(JDK17_JAR)
+        .addKeepMainRule(MAIN)
+        .addKeepRules(enumKeepRules.getKeepRules())
+        .addOptionsModification(opt -> opt.testing.enableEnumWithSubtypesUnboxing = true)
+        .addOptionsModification(opt -> enableEnumOptions(opt, enumValueOptimization))
+        .addOptionsModification(opt -> opt.testing.enableEnumUnboxingDebugLogs = true)
+        .setMinApi(parameters)
+        .allowDiagnosticInfoMessages()
+        .run(parameters.getRuntime(), MAIN)
+        .assertSuccessWithOutput(EXPECTED_RESULT);
+  }
+}