Adjust the rewriting of main dex rootsets

This fixes the following:

- uses rewriteReference instead of lookupReference
- allow for delayed rootset actions which is added by the enqueuer
- prevents adding to noObfuscation sets
- checks if rewritten type is a basetype

Change-Id: I0b1716b033c909575c71ba73b5ac184996f611ce
diff --git a/src/main/java/com/android/tools/r8/graph/GraphLens.java b/src/main/java/com/android/tools/r8/graph/GraphLens.java
index 2f6c224..50fcfbf 100644
--- a/src/main/java/com/android/tools/r8/graph/GraphLens.java
+++ b/src/main/java/com/android/tools/r8/graph/GraphLens.java
@@ -413,14 +413,7 @@
   }
 
   public DexReference lookupReference(DexReference reference) {
-    if (reference.isDexType()) {
-      return lookupType(reference.asDexType());
-    } else if (reference.isDexMethod()) {
-      return lookupMethod(reference.asDexMethod());
-    } else {
-      assert reference.isDexField();
-      return lookupField(reference.asDexField());
-    }
+    return reference.apply(this::lookupType, this::lookupField, this::lookupMethod);
   }
 
   // The method lookupMethod() maps a pair INVOKE=(method signature, invoke type) to a new pair
@@ -532,14 +525,9 @@
 
   @SuppressWarnings("unchecked")
   public <T extends DexReference> T rewriteReference(T reference) {
-    if (reference.isDexField()) {
-      return (T) getRenamedFieldSignature(reference.asDexField());
-    }
-    if (reference.isDexMethod()) {
-      return (T) getRenamedMethodSignature(reference.asDexMethod());
-    }
-    assert reference.isDexType();
-    return (T) lookupType(reference.asDexType());
+    return (T)
+        reference.apply(
+            this::lookupType, this::getRenamedFieldSignature, this::getRenamedMethodSignature);
   }
 
   public Set<DexReference> rewriteReferences(Set<DexReference> references) {
diff --git a/src/main/java/com/android/tools/r8/shaking/RootSetUtils.java b/src/main/java/com/android/tools/r8/shaking/RootSetUtils.java
index 245de17..3d51c1a 100644
--- a/src/main/java/com/android/tools/r8/shaking/RootSetUtils.java
+++ b/src/main/java/com/android/tools/r8/shaking/RootSetUtils.java
@@ -1791,8 +1791,10 @@
       MutableItemsWithRules rewrittenItemsWithRules = new MutableItemsWithRules();
       forEachReference(
           (reference, rules) ->
-              rewrittenItemsWithRules.addReferenceWithRules(
-                  graphLens.lookupReference(reference), rules));
+              rewriteAndApplyIfNotPrimitiveType(
+                  graphLens,
+                  reference,
+                  rewritten -> rewrittenItemsWithRules.addReferenceWithRules(rewritten, rules)));
       return rewrittenItemsWithRules;
     }
   }
@@ -2237,13 +2239,13 @@
     public MainDexRootSet build(ExecutorService executorService) throws ExecutionException {
       // Call the super builder to have if-tests calculated automatically.
       RootSet rootSet = super.build(executorService);
-      assert rootSet.delayedRootSetActionItems.isEmpty();
       return new MainDexRootSet(
           rootSet.noShrinking,
           rootSet.reasonAsked,
           rootSet.checkDiscarded,
           rootSet.dependentNoShrinking,
-          rootSet.ifRules);
+          rootSet.ifRules,
+          rootSet.delayedRootSetActionItems);
     }
   }
 
@@ -2254,7 +2256,8 @@
         ImmutableList<DexReference> reasonAsked,
         ImmutableList<DexReference> checkDiscarded,
         Map<DexReference, MutableItemsWithRules> dependentNoShrinking,
-        Set<ProguardIfRule> ifRules) {
+        Set<ProguardIfRule> ifRules,
+        List<DelayedRootSetActionItem> delayedRootSetActionItems) {
       super(
           noShrinking,
           new MutableItemsWithRules(),
@@ -2285,8 +2288,7 @@
           Collections.emptyMap(),
           Collections.emptySet(),
           ifRules,
-          // This list can be modified during tracing but will be cleared after each round.
-          new ArrayList<>());
+          delayedRootSetActionItems);
     }
 
     @Override
@@ -2309,6 +2311,11 @@
       return new MainDexRootSetBuilder(appView, subtypingInfo, rules);
     }
 
+    @Override
+    void shouldNotBeMinified(DexReference reference) {
+      // Do nothing.
+    }
+
     public MainDexRootSet rewrittenWithLens(GraphLens graphLens) {
       if (graphLens.isIdentityLens()) {
         return this;
@@ -2317,25 +2324,39 @@
       dependentNoShrinking.forEach(
           (reference, rules) -> {
             // Rewriting a reference can result in us having to merge items with rules.
-            MutableItemsWithRules rewrittenRules =
-                rewrittenDependent.computeIfAbsent(
-                    graphLens.lookupReference(reference),
-                    rewrittenRef -> new MutableItemsWithRules());
-            rewrittenRules.addAll(rules.rewrittenWithLens(graphLens));
+            rewriteAndApplyIfNotPrimitiveType(
+                graphLens,
+                reference,
+                rewritten -> {
+                  MutableItemsWithRules rewrittenRules =
+                      rewrittenDependent.computeIfAbsent(
+                          graphLens.lookupReference(reference),
+                          rewrittenRef -> new MutableItemsWithRules());
+                  rewrittenRules.addAll(rules.rewrittenWithLens(graphLens));
+                });
           });
+
       ImmutableList.Builder<DexReference> rewrittenCheckDiscarded = ImmutableList.builder();
-      checkDiscarded.forEach(ref -> rewrittenCheckDiscarded.add(graphLens.lookupReference(ref)));
+      checkDiscarded.forEach(
+          reference ->
+              rewriteAndApplyIfNotPrimitiveType(
+                  graphLens, reference, rewrittenCheckDiscarded::add));
       ImmutableList.Builder<DexReference> rewrittenReasonAsked = ImmutableList.builder();
-      reasonAsked.forEach(ref -> rewrittenReasonAsked.add(graphLens.lookupReference(ref)));
+      reasonAsked.forEach(
+          reference ->
+              rewriteAndApplyIfNotPrimitiveType(graphLens, reference, rewrittenReasonAsked::add));
       // TODO(b/164019179): If rules can now reference dead items. These should be pruned or
       //  rewritten
       ifRules.forEach(ProguardIfRule::canReferenceDeadTypes);
+      // All delayed root set actions should have been processed at this point.
+      assert delayedRootSetActionItems.isEmpty();
       return new MainDexRootSet(
           noShrinking.rewrittenWithLens(graphLens),
           rewrittenReasonAsked.build(),
           rewrittenCheckDiscarded.build(),
           rewrittenDependent,
-          ifRules);
+          ifRules,
+          delayedRootSetActionItems);
     }
 
     public MainDexRootSet withoutPrunedItems(PrunedItems prunedItems) {
@@ -2354,12 +2375,26 @@
       // TODO(b/164019179): If rules can now reference dead items. These should be pruned or
       //  rewritten
       ifRules.forEach(ProguardIfRule::canReferenceDeadTypes);
+      // All delayed root set actions should have been processed at this point.
+      assert delayedRootSetActionItems.isEmpty();
       return new MainDexRootSet(
           noShrinking.prune(prunedItems.getRemovedClasses()),
           reasonAsked,
           checkDiscarded,
           prunedDependent,
-          ifRules);
+          ifRules,
+          delayedRootSetActionItems);
     }
   }
+
+  private static void rewriteAndApplyIfNotPrimitiveType(
+      GraphLens graphLens, DexReference reference, Consumer<DexReference> rewrittenConsumer) {
+    DexReference rewrittenReference = graphLens.rewriteReference(reference);
+    // Enum unboxing can change a class type to int which leads to errors going forward since
+    // the root set should not have primitive types.
+    if (rewrittenReference.isDexType() && rewrittenReference.asDexType().isPrimitiveType()) {
+      return;
+    }
+    rewrittenConsumer.accept(rewrittenReference);
+  }
 }