diff --git a/src/main/java/com/android/tools/r8/ir/optimize/staticizer/ClassStaticizer.java b/src/main/java/com/android/tools/r8/ir/optimize/staticizer/ClassStaticizer.java
index 675e8f4..041cf09 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/staticizer/ClassStaticizer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/staticizer/ClassStaticizer.java
@@ -35,12 +35,12 @@
 import com.android.tools.r8.utils.ListUtils;
 import com.android.tools.r8.utils.SetUtils;
 import com.android.tools.r8.utils.collections.LongLivedProgramMethodSetBuilder;
-import com.android.tools.r8.utils.collections.ProgramMethodSetOrBuilder;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
@@ -69,7 +69,6 @@
     final AtomicInteger fieldWrites = new AtomicInteger();
     // Number of instances created.
     final AtomicInteger instancesCreated = new AtomicInteger();
-    ProgramMethodSetOrBuilder referencedFrom = new LongLivedProgramMethodSetBuilder();
     final AtomicReference<DexEncodedMethod> constructor = new AtomicReference<>();
     final AtomicReference<DexEncodedMethod> getter = new AtomicReference<>();
 
@@ -103,6 +102,9 @@
     }
   }
 
+  final Map<CandidateInfo, LongLivedProgramMethodSetBuilder> referencedFrom =
+      new IdentityHashMap<>();
+
   // The map storing all the potential candidates for staticizing.
   final ConcurrentHashMap<DexType, CandidateInfo> candidates = new ConcurrentHashMap<>();
 
@@ -289,7 +291,9 @@
             }
             // Ignore just read instruction.
           }
-          candidateInfo.referencedFrom.asLongLivedBuilder().add(method);
+          referencedFrom
+              .computeIfAbsent(candidateInfo, ignore -> new LongLivedProgramMethodSetBuilder())
+              .add(method);
         }
         continue;
       }
@@ -309,7 +313,9 @@
         // Check the field being read: make sure all usages are valid.
         CandidateInfo info = processStaticFieldRead(instruction.asStaticGet());
         if (info != null) {
-          info.referencedFrom.asLongLivedBuilder().add(method);
+          referencedFrom
+              .computeIfAbsent(info, ignore -> new LongLivedProgramMethodSetBuilder())
+              .add(method);
           // If the candidate is still valid, ignore all usages in further analysis.
           Value value = instruction.outValue();
           if (value != null) {
@@ -323,7 +329,9 @@
         // Check if it is a static singleton getter.
         CandidateInfo info = processInvokeStatic(instruction.asInvokeStatic());
         if (info != null) {
-          info.referencedFrom.asLongLivedBuilder().add(method);
+          referencedFrom
+              .computeIfAbsent(info, ignore -> new LongLivedProgramMethodSetBuilder())
+              .add(method);
           // If the candidate is still valid, ignore all usages in further analysis.
           Value value = instruction.outValue();
           if (value != null) {
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/staticizer/StaticizingProcessor.java b/src/main/java/com/android/tools/r8/ir/optimize/staticizer/StaticizingProcessor.java
index 9e1b0f8..1e7fb3c 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/staticizer/StaticizingProcessor.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/staticizer/StaticizingProcessor.java
@@ -94,10 +94,11 @@
   final void run(OptimizationFeedback feedback, ExecutorService executorService)
       throws ExecutionException {
     // Filter out candidates based on the information we collected while examining methods.
-    finalEligibilityCheck();
+    Map<CandidateInfo, ProgramMethodSet> materializedReferencedFromCollections =
+        finalEligibilityCheck();
 
     // Prepare interim data.
-    prepareCandidates();
+    prepareCandidates(materializedReferencedFromCollections);
 
     // Enqueue all host class initializers (only remove instantiations).
     ProgramMethodSet hostClassInitMethods = ProgramMethodSet.create();
@@ -147,11 +148,12 @@
     classStaticizer.candidates.clear();
   }
 
-  private void finalEligibilityCheck() {
+  private Map<CandidateInfo, ProgramMethodSet> finalEligibilityCheck() {
+    Map<CandidateInfo, ProgramMethodSet> materializedReferencedFromCollections =
+        new IdentityHashMap<>();
     Set<Phi> visited = Sets.newIdentityHashSet();
     Set<Phi> trivialPhis = Sets.newIdentityHashSet();
-    Iterator<Entry<DexType, CandidateInfo>> it =
-        classStaticizer.candidates.entrySet().iterator();
+    Iterator<Entry<DexType, CandidateInfo>> it = classStaticizer.candidates.entrySet().iterator();
     while (it.hasNext()) {
       Entry<DexType, CandidateInfo> entry = it.next();
       DexType candidateType = entry.getKey();
@@ -222,8 +224,13 @@
         continue;
       }
 
-      ProgramMethodSet referencedFrom = info.referencedFrom.asLongLivedBuilder().build(appView);
-      info.referencedFrom = referencedFrom;
+      ProgramMethodSet referencedFrom;
+      if (classStaticizer.referencedFrom.containsKey(info)) {
+        referencedFrom = classStaticizer.referencedFrom.remove(info).build(appView);
+        materializedReferencedFromCollections.put(info, referencedFrom);
+      } else {
+        referencedFrom = ProgramMethodSet.empty();
+      }
 
       // CHECK: references to field read usages are fixable.
       boolean fixableFieldReads = true;
@@ -275,9 +282,11 @@
         continue;
       }
     }
+    return materializedReferencedFromCollections;
   }
 
-  private void prepareCandidates() {
+  private void prepareCandidates(
+      Map<CandidateInfo, ProgramMethodSet> materializedReferencedFromCollections) {
     Set<DexEncodedMethod> removedInstanceMethods = Sets.newIdentityHashSet();
 
     for (CandidateInfo candidate : classStaticizer.candidates.values()) {
@@ -304,7 +313,8 @@
       if (getter != null) {
         singletonGetters.put(getter.method, candidate);
       }
-      ProgramMethodSet referencedFrom = candidate.referencedFrom.asSet();
+      ProgramMethodSet referencedFrom =
+          materializedReferencedFromCollections.getOrDefault(candidate, ProgramMethodSet.empty());
       assert validMethods(referencedFrom);
       referencingExtraMethods.addAll(referencedFrom);
     }
diff --git a/src/main/java/com/android/tools/r8/utils/collections/LongLivedProgramMethodSetBuilder.java b/src/main/java/com/android/tools/r8/utils/collections/LongLivedProgramMethodSetBuilder.java
index 269b24d..b76c14c 100644
--- a/src/main/java/com/android/tools/r8/utils/collections/LongLivedProgramMethodSetBuilder.java
+++ b/src/main/java/com/android/tools/r8/utils/collections/LongLivedProgramMethodSetBuilder.java
@@ -12,7 +12,7 @@
 import com.google.common.collect.Sets;
 import java.util.Set;
 
-public class LongLivedProgramMethodSetBuilder implements ProgramMethodSetOrBuilder {
+public class LongLivedProgramMethodSetBuilder {
 
   private Set<DexMethod> methods = Sets.newIdentityHashSet();
 
@@ -39,14 +39,4 @@
   public boolean isEmpty() {
     return methods.isEmpty();
   }
-
-  @Override
-  public boolean isLongLivedBuilder() {
-    return true;
-  }
-
-  @Override
-  public LongLivedProgramMethodSetBuilder asLongLivedBuilder() {
-    return this;
-  }
 }
diff --git a/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSet.java b/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSet.java
index d288177..1795899 100644
--- a/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSet.java
+++ b/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSet.java
@@ -18,7 +18,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Stream;
 
-public class ProgramMethodSet implements ProgramMethodSetOrBuilder, Iterable<ProgramMethod> {
+public class ProgramMethodSet implements Iterable<ProgramMethod> {
 
   private static final ProgramMethodSet EMPTY = new ProgramMethodSet(ImmutableMap.of());
 
@@ -68,11 +68,6 @@
     backing.putAll(methods.backing);
   }
 
-  @Override
-  public ProgramMethodSet asSet() {
-    return this;
-  }
-
   public boolean createAndAdd(DexProgramClass clazz, DexEncodedMethod definition) {
     return add(new ProgramMethod(clazz, definition));
   }
diff --git a/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSetOrBuilder.java b/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSetOrBuilder.java
deleted file mode 100644
index f7298be..0000000
--- a/src/main/java/com/android/tools/r8/utils/collections/ProgramMethodSetOrBuilder.java
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-package com.android.tools.r8.utils.collections;
-
-public interface ProgramMethodSetOrBuilder {
-
-  default boolean isLongLivedBuilder() {
-    return false;
-  }
-
-  default LongLivedProgramMethodSetBuilder asLongLivedBuilder() {
-    return null;
-  }
-
-  default ProgramMethodSet asSet() {
-    return null;
-  }
-}