Avoid merging dead enum lite maps with proto enum shrinking

Change-Id: I22e22a0c08724dd1f14e661f7a52bb3585185bab
diff --git a/src/main/java/com/android/tools/r8/R8.java b/src/main/java/com/android/tools/r8/R8.java
index 80045fa..b97eb7f 100644
--- a/src/main/java/com/android/tools/r8/R8.java
+++ b/src/main/java/com/android/tools/r8/R8.java
@@ -422,7 +422,7 @@
                   appViewWithLiveness, appViewWithLiveness.appInfo().computeSubtypingInfo())
               .run();
 
-          if (appView.options().protoShrinking().isProtoShrinkingEnabled()) {
+          if (appView.options().protoShrinking().isProtoEnumShrinkingEnabled()) {
             appView.protoShrinker().enumProtoShrinker.clearDeadEnumLiteMaps();
           }
 
@@ -782,7 +782,9 @@
         }
 
         if (appView.options().protoShrinking().isProtoShrinkingEnabled()) {
-          appView.protoShrinker().enumProtoShrinker.verifyDeadEnumLiteMapsAreDead();
+          if (appView.options().protoShrinking().isProtoEnumShrinkingEnabled()) {
+            appView.protoShrinker().enumProtoShrinker.verifyDeadEnumLiteMapsAreDead();
+          }
 
           IRConverter converter = new IRConverter(appView, timing, null, mainDexTracingResult);
 
diff --git a/src/main/java/com/android/tools/r8/graph/AppView.java b/src/main/java/com/android/tools/r8/graph/AppView.java
index 37f45c6..ef9b742 100644
--- a/src/main/java/com/android/tools/r8/graph/AppView.java
+++ b/src/main/java/com/android/tools/r8/graph/AppView.java
@@ -13,6 +13,7 @@
 import com.android.tools.r8.graph.classmerging.MergedClassesCollection;
 import com.android.tools.r8.graph.classmerging.VerticallyMergedClasses;
 import com.android.tools.r8.horizontalclassmerging.HorizontallyMergedClasses;
+import com.android.tools.r8.ir.analysis.proto.EnumLiteProtoShrinker;
 import com.android.tools.r8.ir.analysis.proto.GeneratedExtensionRegistryShrinker;
 import com.android.tools.r8.ir.analysis.proto.GeneratedMessageLiteBuilderShrinker;
 import com.android.tools.r8.ir.analysis.proto.GeneratedMessageLiteShrinker;
@@ -315,6 +316,13 @@
     return defaultValue;
   }
 
+  public <U> U withProtoEnumShrinker(Function<EnumLiteProtoShrinker, U> fn, U defaultValue) {
+    if (protoShrinker != null && options().protoShrinking().isProtoEnumShrinkingEnabled()) {
+      return fn.apply(protoShrinker.enumProtoShrinker);
+    }
+    return defaultValue;
+  }
+
   public <E extends Throwable> void withGeneratedExtensionRegistryShrinker(
       ThrowingConsumer<GeneratedExtensionRegistryShrinker, E> consumer) throws E {
     if (protoShrinker != null && protoShrinker.generatedExtensionRegistryShrinker != null) {
diff --git a/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotMatchedByNoHorizontalClassMerging.java b/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotMatchedByNoHorizontalClassMerging.java
index 2311f16..5b11ff5 100644
--- a/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotMatchedByNoHorizontalClassMerging.java
+++ b/src/main/java/com/android/tools/r8/horizontalclassmerging/policies/NotMatchedByNoHorizontalClassMerging.java
@@ -8,18 +8,26 @@
 import com.android.tools.r8.graph.DexProgramClass;
 import com.android.tools.r8.graph.DexType;
 import com.android.tools.r8.horizontalclassmerging.SingleClassPolicy;
+import com.android.tools.r8.ir.analysis.proto.EnumLiteProtoShrinker;
 import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import java.util.Collections;
 import java.util.Set;
 
 public class NotMatchedByNoHorizontalClassMerging extends SingleClassPolicy {
+
+  private final Set<DexType> deadEnumLiteMaps;
   private final Set<DexType> neverMergeClassHorizontally;
 
   public NotMatchedByNoHorizontalClassMerging(AppView<AppInfoWithLiveness> appView) {
+    deadEnumLiteMaps =
+        appView.withProtoEnumShrinker(
+            EnumLiteProtoShrinker::getDeadEnumLiteMaps, Collections.emptySet());
     neverMergeClassHorizontally = appView.appInfo().getNoHorizontalClassMergingSet();
   }
 
   @Override
   public boolean canMerge(DexProgramClass program) {
-    return !neverMergeClassHorizontally.contains(program.getReference());
+    return !deadEnumLiteMaps.contains(program.getType())
+        && !neverMergeClassHorizontally.contains(program.getType());
   }
 }
diff --git a/src/main/java/com/android/tools/r8/ir/analysis/proto/EnumLiteProtoShrinker.java b/src/main/java/com/android/tools/r8/ir/analysis/proto/EnumLiteProtoShrinker.java
index 055882b..0b04f9b 100644
--- a/src/main/java/com/android/tools/r8/ir/analysis/proto/EnumLiteProtoShrinker.java
+++ b/src/main/java/com/android/tools/r8/ir/analysis/proto/EnumLiteProtoShrinker.java
@@ -45,6 +45,10 @@
     this.references = references;
   }
 
+  public Set<DexType> getDeadEnumLiteMaps() {
+    return deadEnumLiteMaps;
+  }
+
   private DexField createInternalValueMapField(DexType holder) {
     return appView
         .dexItemFactory()
@@ -52,9 +56,7 @@
   }
 
   public void clearDeadEnumLiteMaps() {
-    if (!appView.options().protoShrinking().enableEnumLiteProtoShrinking) {
-      return;
-    }
+    assert appView.options().protoShrinking().isProtoEnumShrinkingEnabled();
     // The optimization only enables further enums to be unboxed, no point to run it if enum
     // unboxing is disabled.
     if (!appView.options().enableEnumUnboxing) {
@@ -70,23 +72,28 @@
 
   private void internalClearDeadEnumLiteMaps() {
     for (DexProgramClass clazz : appView.appInfo().classes()) {
-      if (clazz.interfaces.contains(references.enumLiteMapType)) {
-        DexProgramClass enumLite = computeCorrespondingEnumLite(clazz);
-        if (enumLite != null) {
-          DexEncodedField field = enumLite.lookupField(createInternalValueMapField(enumLite.type));
-          if (field != null) {
-            if (appView.appInfo().isStaticFieldWrittenOnlyInEnclosingStaticInitializer(field)
-                && !appView.appInfo().isFieldRead(field)) {
-              deadEnumLiteMaps.add(clazz.type);
-              // Clears the EnumLiteMap methods to avoid them being IR processed.
-              clazz.setVirtualMethods(DexEncodedMethod.EMPTY_ARRAY);
-            }
-          }
-        }
+      if (isDeadEnumLiteMap(clazz)) {
+        deadEnumLiteMaps.add(clazz.getType());
+        // Clears the EnumLiteMap methods to avoid them being IR processed.
+        clazz.setVirtualMethods(DexEncodedMethod.EMPTY_ARRAY);
       }
     }
   }
 
+  public boolean isDeadEnumLiteMap(DexProgramClass clazz) {
+    if (clazz.getInterfaces().contains(references.enumLiteMapType)) {
+      DexProgramClass enumLite = computeCorrespondingEnumLite(clazz);
+      if (enumLite != null) {
+        DexEncodedField field =
+            enumLite.lookupField(createInternalValueMapField(enumLite.getType()));
+        return field != null
+            && appView.appInfo().isStaticFieldWrittenOnlyInEnclosingStaticInitializer(field)
+            && !appView.appInfo().isFieldRead(field);
+      }
+    }
+    return false;
+  }
+
   /**
    * Each EnumLiteMap subclass has only two virtual methods findValueByNumber:
    *
diff --git a/src/main/java/com/android/tools/r8/ir/analysis/proto/ProtoShrinker.java b/src/main/java/com/android/tools/r8/ir/analysis/proto/ProtoShrinker.java
index 6cec099..2674265 100644
--- a/src/main/java/com/android/tools/r8/ir/analysis/proto/ProtoShrinker.java
+++ b/src/main/java/com/android/tools/r8/ir/analysis/proto/ProtoShrinker.java
@@ -42,7 +42,10 @@
         appView.options().protoShrinking().enableGeneratedMessageLiteBuilderShrinking
             ? new GeneratedMessageLiteBuilderShrinker(appView, references)
             : null;
-    this.enumProtoShrinker = new EnumLiteProtoShrinker(appView, references);
+    this.enumProtoShrinker =
+        appView.options().protoShrinking().isProtoEnumShrinkingEnabled()
+            ? new EnumLiteProtoShrinker(appView, references)
+            : null;
     this.references = references;
   }
 
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 48fb234..c74ed38 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -1237,6 +1237,10 @@
           || enableGeneratedMessageLiteBuilderShrinking
           || enableEnumLiteProtoShrinking;
     }
+
+    public boolean isProtoEnumShrinkingEnabled() {
+      return enableEnumLiteProtoShrinking;
+    }
   }
 
   public static class TestingOptions {