Rename dangling types, i.e., types that are only referenced in signatures.

The VM does not actually care about the details of the type (as in, its
definition or subtyping relationship), it is simply a placeholder. If it
is from the set of program classes, we can rename it, as library code
cannot reference it.

Bug:
Change-Id: I44fa307a7f7c86793534403fdbcf7b3d88cb2b07
diff --git a/src/main/java/com/android/tools/r8/R8.java b/src/main/java/com/android/tools/r8/R8.java
index da90e82..a0a5e4b 100644
--- a/src/main/java/com/android/tools/r8/R8.java
+++ b/src/main/java/com/android/tools/r8/R8.java
@@ -270,9 +270,10 @@
           proguardSeedsData = bytes.toByteArray();
         }
         if (options.useTreeShaking) {
-          application = new TreePruner(application, appInfo.withLiveness(), options).run();
+          TreePruner pruner = new TreePruner(application, appInfo.withLiveness(), options);
+          application = pruner.run();
           // Recompute the subtyping information.
-          appInfo = appInfo.withLiveness().prunedCopyFrom(application);
+          appInfo = appInfo.withLiveness().prunedCopyFrom(application, pruner.getRemovedClasses());
           new AbstractMethodRemover(appInfo).run();
           new AnnotationRemover(appInfo.withLiveness(), options).run();
         }
@@ -295,11 +296,13 @@
         // Class merging requires inlining.
         if (!options.skipClassMerging && options.inlineAccessors) {
           timing.begin("ClassMerger");
-          graphLense = new SimpleClassMerger(application, appInfo.withLiveness(), graphLense,
-              timing).run();
+          SimpleClassMerger classMerger = new SimpleClassMerger(application,
+              appInfo.withLiveness(), graphLense, timing);
+          graphLense = classMerger.run();
           timing.end();
+          appInfo = appInfo.withLiveness()
+              .prunedCopyFrom(application, classMerger.getRemovedClasses());
         }
-        appInfo = appInfo.withLiveness().prunedCopyFrom(application);
         appInfo = appInfo.withLiveness().rewrittenWithLense(graphLense);
         // Collect switch maps and ordinals maps.
         new SwitchMapCollector(appInfo.withLiveness(), options).run();
@@ -333,8 +336,10 @@
           Enqueuer enqueuer = new Enqueuer(appInfo);
           appInfo = enqueuer.traceApplication(rootSet, timing);
           if (options.useTreeShaking) {
-            application = new TreePruner(application, appInfo.withLiveness(), options).run();
-            appInfo = appInfo.withLiveness().prunedCopyFrom(application);
+            TreePruner pruner = new TreePruner(application, appInfo.withLiveness(), options);
+            application = pruner.run();
+            appInfo = appInfo.withLiveness()
+                .prunedCopyFrom(application, pruner.getRemovedClasses());
             // Print reasons on the application after pruning, so that we reflect the actual result.
             ReasonPrinter reasonPrinter = enqueuer.getReasonPrinter(rootSet.reasonAsked);
             reasonPrinter.run(application);
diff --git a/src/main/java/com/android/tools/r8/naming/ClassNameMinifier.java b/src/main/java/com/android/tools/r8/naming/ClassNameMinifier.java
index c1e8515..7c4af5b 100644
--- a/src/main/java/com/android/tools/r8/naming/ClassNameMinifier.java
+++ b/src/main/java/com/android/tools/r8/naming/ClassNameMinifier.java
@@ -9,7 +9,10 @@
 
 import com.android.tools.r8.graph.DexAnnotation;
 import com.android.tools.r8.graph.DexClass;
+import com.android.tools.r8.graph.DexEncodedField;
+import com.android.tools.r8.graph.DexEncodedMethod;
 import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.graph.DexProto;
 import com.android.tools.r8.graph.DexString;
 import com.android.tools.r8.graph.DexType;
 import com.android.tools.r8.graph.DexValue;
@@ -78,7 +81,7 @@
     // Collect names we have to keep.
     timing.begin("reserve");
     for (DexClass clazz : classes) {
-      if (rootSet.noObfuscation.contains(clazz)) {
+      if (rootSet.noObfuscation.contains(clazz.type)) {
         assert !renaming.containsKey(clazz.type);
         registerClassAsUsed(clazz.type);
       }
@@ -94,6 +97,12 @@
     }
     timing.end();
 
+    timing.begin("rename-dangling-types");
+    for (DexClass clazz : classes) {
+      renameDanglingTypes(clazz);
+    }
+    timing.end();
+
     timing.begin("rename-generic");
     renameTypesInGenericSignatures();
     timing.end();
@@ -105,6 +114,35 @@
     return Collections.unmodifiableMap(renaming);
   }
 
+  private void renameDanglingTypes(DexClass clazz) {
+    clazz.forEachMethod(this::renameDanglingTypesInMethod);
+    clazz.forEachField(this::renameDanglingTypesInField);
+  }
+
+  private void renameDanglingTypesInField(DexEncodedField field) {
+    renameDanglingType(field.field.type);
+  }
+
+  private void renameDanglingTypesInMethod(DexEncodedMethod method) {
+    DexProto proto = method.method.proto;
+    renameDanglingType(proto.returnType);
+    for (DexType type : proto.parameters.values) {
+      renameDanglingType(type);
+    }
+  }
+
+  private void renameDanglingType(DexType type) {
+    if (appInfo.wasPruned(type)
+        && !renaming.containsKey(type)
+        && !rootSet.noObfuscation.contains(type)) {
+      // We have a type that is defined in the program source but is only used in a proto or
+      // return type. As we don't need the class, we can rename it to anything as long as it is
+      // unique.
+      assert appInfo.definitionFor(type) == null;
+      renaming.put(type, topLevelState.nextTypeName());
+    }
+  }
+
   private void renameTypesInGenericSignatures() {
     for (DexClass clazz : appInfo.classes()) {
       rewriteGenericSignatures(clazz.annotations.annotations,
diff --git a/src/main/java/com/android/tools/r8/shaking/Enqueuer.java b/src/main/java/com/android/tools/r8/shaking/Enqueuer.java
index 524207e..ff8adee 100644
--- a/src/main/java/com/android/tools/r8/shaking/Enqueuer.java
+++ b/src/main/java/com/android/tools/r8/shaking/Enqueuer.java
@@ -41,6 +41,7 @@
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -927,13 +928,13 @@
   SortedSet<DexField> collectFieldsRead() {
     return ImmutableSortedSet.copyOf(PresortedComparable::slowCompareTo,
         Sets.union(collectReachedFields(instanceFieldsRead, this::tryLookupInstanceField),
-        collectReachedFields(staticFieldsRead, this::tryLookupStaticField)));
+            collectReachedFields(staticFieldsRead, this::tryLookupStaticField)));
   }
 
   SortedSet<DexField> collectFieldsWritten() {
     return ImmutableSortedSet.copyOf(PresortedComparable::slowCompareTo,
         Sets.union(collectReachedFields(instanceFieldsWritten, this::tryLookupInstanceField),
-        collectReachedFields(staticFieldsWritten, this::tryLookupStaticField)));
+            collectReachedFields(staticFieldsWritten, this::tryLookupStaticField)));
   }
 
   private static class Action {
@@ -1099,6 +1100,10 @@
      * Map from the class of an extension to the state it produced.
      */
     public final Map<Class, Object> extensions;
+    /**
+     * A set of types that have been removed by the {@link TreePruner}.
+     */
+    public final Set<DexType> prunedTypes;
 
     private AppInfoWithLiveness(AppInfoWithSubtyping appInfo, Enqueuer enqueuer) {
       super(appInfo);
@@ -1124,11 +1129,13 @@
       this.assumedValues = enqueuer.rootSet.assumedValues;
       this.alwaysInline = enqueuer.rootSet.alwaysInline;
       this.extensions = enqueuer.extensionsState;
+      this.prunedTypes = Collections.emptySet();
       assert Sets.intersection(instanceFieldReads, staticFieldReads).size() == 0;
       assert Sets.intersection(instanceFieldWrites, staticFieldWrites).size() == 0;
     }
 
-    private AppInfoWithLiveness(AppInfoWithLiveness previous, DexApplication application) {
+    private AppInfoWithLiveness(AppInfoWithLiveness previous, DexApplication application,
+        Collection<DexType> removedClasses) {
       super(application);
       this.liveTypes = previous.liveTypes;
       this.instantiatedTypes = previous.instantiatedTypes;
@@ -1151,6 +1158,7 @@
       this.staticInvokes = previous.staticInvokes;
       this.extensions = previous.extensions;
       this.alwaysInline = previous.alwaysInline;
+      this.prunedTypes = mergeSets(previous.prunedTypes, removedClasses);
       assert Sets.intersection(instanceFieldReads, staticFieldReads).size() == 0;
       assert Sets.intersection(instanceFieldWrites, staticFieldWrites).size() == 0;
     }
@@ -1178,6 +1186,7 @@
       this.staticInvokes = rewriteItems(previous.staticInvokes, lense::lookupMethod);
       this.alwaysInline = previous.alwaysInline;
       this.extensions = previous.extensions;
+      this.prunedTypes = rewriteItems(previous.prunedTypes, lense::lookupType);
       assert Sets.intersection(instanceFieldReads, staticFieldReads).size() == 0;
       assert Sets.intersection(instanceFieldWrites, staticFieldWrites).size() == 0;
     }
@@ -1209,6 +1218,13 @@
       return builder.build();
     }
 
+    private static <T> Set<T> mergeSets(Collection<T> first, Collection<T> second) {
+      ImmutableSet.Builder<T> builder = ImmutableSet.builder();
+      builder.addAll(first);
+      builder.addAll(second);
+      return builder.build();
+    }
+
     @SuppressWarnings("unchecked")
     public <T> T getExtension(Class extension, T defaultValue) {
       if (extensions.containsKey(extension)) {
@@ -1237,14 +1253,23 @@
      * Returns a copy of this AppInfoWithLiveness where the set of classes is pruned using the
      * given DexApplication object.
      */
-    public AppInfoWithLiveness prunedCopyFrom(DexApplication application) {
-      return new AppInfoWithLiveness(this, application);
+    public AppInfoWithLiveness prunedCopyFrom(DexApplication application,
+        Collection<DexType> removedClasses) {
+      return new AppInfoWithLiveness(this, application, removedClasses);
     }
 
     public AppInfoWithLiveness rewrittenWithLense(GraphLense lense) {
       assert lense.isContextFree();
       return new AppInfoWithLiveness(this, lense);
     }
+
+    /**
+     * Returns true if the given type was part of the original program but has been removed during
+     * tree shaking.
+     */
+    public boolean wasPruned(DexType type) {
+      return prunedTypes.contains(type);
+    }
   }
 
   private static class SetWithReason<T> {
diff --git a/src/main/java/com/android/tools/r8/shaking/RootSetBuilder.java b/src/main/java/com/android/tools/r8/shaking/RootSetBuilder.java
index 0fa5051..5e7b42d 100644
--- a/src/main/java/com/android/tools/r8/shaking/RootSetBuilder.java
+++ b/src/main/java/com/android/tools/r8/shaking/RootSetBuilder.java
@@ -452,7 +452,7 @@
     dependentNoShrinking.computeIfAbsent(item, x -> new IdentityHashMap<>())
         .put(definition, context);
     // Unconditionally add to no-obfuscation, as that is only checked for surviving items.
-    noObfuscation.add(definition);
+    noObfuscation.add(type);
   }
 
   private void includeDescriptorClasses(DexItem item, ProguardKeepRule context) {
@@ -487,7 +487,11 @@
         noOptimization.add(item);
       }
       if (!modifiers.allowsObfuscation) {
-        noObfuscation.add(item);
+        if (item instanceof DexClass) {
+          noObfuscation.add(((DexClass) item).type);
+        } else {
+          noObfuscation.add(item);
+        }
       }
       if (modifiers.includeDescriptorClasses) {
         includeDescriptorClasses(item, keepRule);
@@ -520,38 +524,24 @@
     public final Map<DexItem, ProguardMemberRule> assumedValues;
     private final Map<DexItem, Map<DexItem, ProguardKeepRule>> dependentNoShrinking;
 
-    private boolean legalNoObfuscationItem(DexItem item) {
-      if (!(item instanceof DexProgramClass
-          || item instanceof DexLibraryClass
-          || item instanceof DexEncodedMethod
-          || item instanceof DexEncodedField)) {
-      }
-      assert item instanceof DexProgramClass
-          || item instanceof DexLibraryClass
-          || item instanceof DexEncodedMethod
-          || item instanceof DexEncodedField;
-      return true;
-    }
-
-    private boolean legalNoObfuscationItems(Set<DexItem> items) {
-      items.forEach(this::legalNoObfuscationItem);
-      return true;
-    }
-
-    private boolean legalDependentNoShrinkingItem(DexItem item) {
-      if (!(item instanceof DexType
-          || item instanceof DexEncodedMethod
-          || item instanceof DexEncodedField)) {
-      }
+    private boolean isTypeEncodedMethodOrEncodedField(DexItem item) {
       assert item instanceof DexType
           || item instanceof DexEncodedMethod
           || item instanceof DexEncodedField;
+      return item instanceof DexType
+          || item instanceof DexEncodedMethod
+          || item instanceof DexEncodedField;
+    }
+
+    private boolean legalNoObfuscationItems(Set<DexItem> items) {
+      assert items.stream().allMatch(this::isTypeEncodedMethodOrEncodedField);
       return true;
     }
 
     private boolean legalDependentNoShrinkingItems(
         Map<DexItem, Map<DexItem, ProguardKeepRule>> dependentNoShrinking) {
-      dependentNoShrinking.keySet().forEach(this::legalDependentNoShrinkingItem);
+      assert dependentNoShrinking.keySet().stream()
+          .allMatch(this::isTypeEncodedMethodOrEncodedField);
       return true;
     }
 
diff --git a/src/main/java/com/android/tools/r8/shaking/SimpleClassMerger.java b/src/main/java/com/android/tools/r8/shaking/SimpleClassMerger.java
index c04b29a..193ebdf 100644
--- a/src/main/java/com/android/tools/r8/shaking/SimpleClassMerger.java
+++ b/src/main/java/com/android/tools/r8/shaking/SimpleClassMerger.java
@@ -655,4 +655,8 @@
       return result;
     }
   }
+
+  public Collection<DexType> getRemovedClasses() {
+    return Collections.unmodifiableCollection(mergedClasses.keySet());
+  }
 }
diff --git a/src/main/java/com/android/tools/r8/shaking/TreePruner.java b/src/main/java/com/android/tools/r8/shaking/TreePruner.java
index 64294bd..153f604 100644
--- a/src/main/java/com/android/tools/r8/shaking/TreePruner.java
+++ b/src/main/java/com/android/tools/r8/shaking/TreePruner.java
@@ -8,13 +8,17 @@
 import com.android.tools.r8.graph.DexEncodedField;
 import com.android.tools.r8.graph.DexEncodedMethod;
 import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.graph.DexType;
 import com.android.tools.r8.graph.KeyedDexItem;
 import com.android.tools.r8.graph.PresortedComparable;
 import com.android.tools.r8.logging.Log;
 import com.android.tools.r8.shaking.Enqueuer.AppInfoWithLiveness;
 import com.android.tools.r8.utils.InternalOptions;
+import com.google.common.collect.Sets;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
@@ -23,7 +27,8 @@
   private DexApplication application;
   private final AppInfoWithLiveness appInfo;
   private final InternalOptions options;
-  private UsagePrinter usagePrinter;
+  private final UsagePrinter usagePrinter;
+  private final Set<DexType> prunedTypes = Sets.newIdentityHashSet();
 
   public TreePruner(
       DexApplication application, AppInfoWithLiveness appInfo, InternalOptions options) {
@@ -64,6 +69,7 @@
         if (Log.ENABLED) {
           Log.debug(getClass(), "Removing class: " + clazz);
         }
+        prunedTypes.add(clazz.type);
         usagePrinter.printUnusedClass(clazz);
       } else {
         newClasses.add(clazz);
@@ -190,4 +196,8 @@
     }
     return reachableFields.toArray(new DexEncodedField[reachableFields.size()]);
   }
+
+  public Collection<DexType> getRemovedClasses() {
+    return Collections.unmodifiableCollection(prunedTypes);
+  }
 }