Enum unboxing: pinned enums

- Do not unbox pinned enums

Bug: 147860220
Change-Id: I86ff66c5fe100d80d817ad37fea39b32238c52b5
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
index 5bcfac5..ee87965 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxer.java
@@ -472,6 +472,7 @@
 
   public enum Reason {
     ELIGIBLE,
+    PINNED,
     DOWN_CAST,
     SUBTYPES,
     INTERFACE,
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java
index 53cbcc0..3d18f6a 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/enums/EnumUnboxingCandidateAnalysis.java
@@ -7,10 +7,16 @@
 import com.android.tools.r8.graph.AppView;
 import com.android.tools.r8.graph.DexEncodedField;
 import com.android.tools.r8.graph.DexEncodedMethod;
+import com.android.tools.r8.graph.DexField;
 import com.android.tools.r8.graph.DexItemFactory;
+import com.android.tools.r8.graph.DexMember;
+import com.android.tools.r8.graph.DexMethod;
 import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.graph.DexProto;
+import com.android.tools.r8.graph.DexReference;
 import com.android.tools.r8.graph.DexType;
 import com.android.tools.r8.ir.optimize.enums.EnumUnboxer.Reason;
+import com.android.tools.r8.shaking.AppInfoWithLiveness;
 import com.google.common.collect.Sets;
 import java.util.Map;
 import java.util.Set;
@@ -18,24 +24,25 @@
 
 class EnumUnboxingCandidateAnalysis {
 
-  private final AppView<?> appView;
+  private final AppView<AppInfoWithLiveness> appView;
   private final EnumUnboxer enumUnboxer;
   private final DexItemFactory factory;
+  private Map<DexType, Set<DexEncodedMethod>> enumToUnboxCandidates = new ConcurrentHashMap<>();
 
-  EnumUnboxingCandidateAnalysis(AppView<?> appView, EnumUnboxer enumUnboxer) {
+  EnumUnboxingCandidateAnalysis(AppView<AppInfoWithLiveness> appView, EnumUnboxer enumUnboxer) {
     this.appView = appView;
     this.enumUnboxer = enumUnboxer;
     factory = appView.dexItemFactory();
   }
 
   Map<DexType, Set<DexEncodedMethod>> findCandidates() {
-    Map<DexType, Set<DexEncodedMethod>> enums = new ConcurrentHashMap<>();
     for (DexProgramClass clazz : appView.appInfo().classes()) {
       if (isEnumUnboxingCandidate(clazz)) {
-        enums.put(clazz.type, Sets.newConcurrentHashSet());
+        enumToUnboxCandidates.put(clazz.type, Sets.newConcurrentHashSet());
       }
     }
-    return enums;
+    removePinnedCandidates();
+    return enumToUnboxCandidates;
   }
 
   private boolean isEnumUnboxingCandidate(DexProgramClass clazz) {
@@ -113,4 +120,39 @@
     }
     return true;
   }
+
+  private void removePinnedCandidates() {
+    // A holder type, for field or method, should block enum unboxing only if the enum type is
+    // also kept. This is to allow the keep rule -keepclassmembers to be used on enums while
+    // enum unboxing can still be performed.
+    for (DexReference item : appView.appInfo().getPinnedItems()) {
+      if (item.isDexType()) {
+        removePinnedCandidate(item.asDexType());
+      } else if (item.isDexField()) {
+        DexField field = item.asDexField();
+        removePinnedIfNotHolder(field, field.type);
+      } else {
+        assert item.isDexMethod();
+        DexMethod method = item.asDexMethod();
+        DexProto proto = method.proto;
+        removePinnedIfNotHolder(method, proto.returnType);
+        for (DexType parameterType : proto.parameters.values) {
+          removePinnedIfNotHolder(method, parameterType);
+        }
+      }
+    }
+  }
+
+  private void removePinnedIfNotHolder(DexMember<?, ?> member, DexType type) {
+    if (type != member.holder) {
+      removePinnedCandidate(type);
+    }
+  }
+
+  private void removePinnedCandidate(DexType type) {
+    if (enumToUnboxCandidates.containsKey(type)) {
+      enumUnboxer.reportFailure(type, Reason.PINNED);
+      enumToUnboxCandidates.remove(type);
+    }
+  }
 }
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/PinnedEnumUnboxingTest.java b/src/test/java/com/android/tools/r8/enumunboxing/PinnedEnumUnboxingTest.java
new file mode 100644
index 0000000..3c34d67
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/enumunboxing/PinnedEnumUnboxingTest.java
@@ -0,0 +1,70 @@
+// Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.enumunboxing;
+
+import com.android.tools.r8.NeverClassInline;
+import com.android.tools.r8.R8TestRunResult;
+import com.android.tools.r8.TestParameters;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class PinnedEnumUnboxingTest extends EnumUnboxingTestBase {
+
+  private static final Class<?> ENUM_CLASS = MyEnum.class;
+
+  private final TestParameters parameters;
+  private final boolean enumValueOptimization;
+  private final boolean enumKeepRules;
+
+  @Parameters(name = "{0} valueOpt: {1} keep: {2}")
+  public static List<Object[]> data() {
+    return enumUnboxingTestParameters();
+  }
+
+  public PinnedEnumUnboxingTest(
+      TestParameters parameters, boolean enumValueOptimization, boolean enumKeepRules) {
+    this.parameters = parameters;
+    this.enumValueOptimization = enumValueOptimization;
+    this.enumKeepRules = enumKeepRules;
+  }
+
+  @Test
+  public void testEnumUnboxing() throws Exception {
+    Class<MainWithKeptEnum> classToTest = MainWithKeptEnum.class;
+    R8TestRunResult run =
+        testForR8(parameters.getBackend())
+            .addProgramClasses(classToTest, ENUM_CLASS)
+            .addKeepMainRule(classToTest)
+            .addKeepClassRules(MyEnum.class)
+            .addKeepRules(enumKeepRules ? KEEP_ENUM : "")
+            .enableNeverClassInliningAnnotations()
+            .addOptionsModification(opt -> enableEnumOptions(opt, enumValueOptimization))
+            .allowDiagnosticInfoMessages()
+            .setMinApi(parameters.getApiLevel())
+            .compile()
+            .inspectDiagnosticMessages(
+                m -> assertEnumIsBoxed(ENUM_CLASS, classToTest.getSimpleName(), m))
+            .run(parameters.getRuntime(), classToTest)
+            .assertSuccessWithOutputLines("0");
+  }
+
+  @NeverClassInline
+  enum MyEnum {
+    A,
+    B,
+    C
+  }
+
+  static class MainWithKeptEnum {
+
+    public static void main(String[] args) {
+      System.out.println(MyEnum.A.ordinal());
+    }
+  }
+}