diff --git a/src/main/java/com/android/tools/r8/dex/ApplicationReader.java b/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
index 315c217..3c239c8 100644
--- a/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
+++ b/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
@@ -228,7 +228,7 @@
           if (options.passthroughDexCode) {
             computedMinApiLevel = validateOrComputeMinApiLevel(computedMinApiLevel, dexReader);
           }
-          dexParsers.add(new DexParser(dexReader, classKind, options));
+          dexParsers.add(new DexParser(dexReader, classKind, itemFactory, options.reporter));
         }
         options.minApiLevel = computedMinApiLevel;
         for (DexParser dexParser : dexParsers) {
diff --git a/src/main/java/com/android/tools/r8/dex/DexParser.java b/src/main/java/com/android/tools/r8/dex/DexParser.java
index 3ef485f..1760ca4 100644
--- a/src/main/java/com/android/tools/r8/dex/DexParser.java
+++ b/src/main/java/com/android/tools/r8/dex/DexParser.java
@@ -8,6 +8,7 @@
 import static com.android.tools.r8.utils.EncodedValueUtils.parseSigned;
 import static com.android.tools.r8.utils.EncodedValueUtils.parseUnsigned;
 
+import com.android.tools.r8.DiagnosticsHandler;
 import com.android.tools.r8.ProgramResource.Kind;
 import com.android.tools.r8.code.Instruction;
 import com.android.tools.r8.code.InstructionFactory;
@@ -58,7 +59,6 @@
 import com.android.tools.r8.logging.Log;
 import com.android.tools.r8.origin.Origin;
 import com.android.tools.r8.origin.PathOrigin;
-import com.android.tools.r8.utils.InternalOptions;
 import com.android.tools.r8.utils.Pair;
 import com.google.common.io.ByteStreams;
 import it.unimi.dsi.fastutil.ints.Int2IntArrayMap;
@@ -85,7 +85,7 @@
   private final DexSection[] dexSections;
   private int[] stringIDs;
   private final ClassKind classKind;
-  private final InternalOptions options;
+  private final DiagnosticsHandler reporter;
 
   public static DexSection[] parseMapFrom(Path file) throws IOException {
     return parseMapFrom(Files.newInputStream(file), new PathOrigin(file));
@@ -96,7 +96,9 @@
   }
 
   private static DexSection[] parseMapFrom(DexReader dexReader) {
-    DexParser dexParser = new DexParser(dexReader, ClassKind.PROGRAM, new InternalOptions());
+    DexParser dexParser =
+        new DexParser(
+            dexReader, ClassKind.PROGRAM, new DexItemFactory(), new DiagnosticsHandler() {});
     return dexParser.dexSections;
   }
 
@@ -121,16 +123,17 @@
   // Factory to canonicalize certain dexitems.
   private final DexItemFactory dexItemFactory;
 
-  public DexParser(DexReader dexReader, ClassKind classKind, InternalOptions options) {
+  public DexParser(DexReader dexReader,
+      ClassKind classKind, DexItemFactory dexItemFactory, DiagnosticsHandler reporter) {
     assert dexReader.getOrigin() != null;
     this.origin = dexReader.getOrigin();
     this.dexReader = dexReader;
-    this.dexItemFactory = options.itemFactory;
+    this.dexItemFactory = dexItemFactory;
     dexReader.setByteOrder();
     dexSections = parseMap();
     parseStringIDs();
     this.classKind = classKind;
-    this.options = options;
+    this.reporter = reporter;
   }
 
   private void ensureCodesInited() {
@@ -423,17 +426,8 @@
       annotationOffsets[i] = dexReader.getUint();
     }
     DexAnnotation[] result = new DexAnnotation[size];
-    int actualSize = 0;
     for (int i = 0; i < size; i++) {
-      DexAnnotation dexAnnotation = annotationAt(annotationOffsets[i]);
-      if (retainAnnotation(dexAnnotation)) {
-        result[actualSize++] = dexAnnotation;
-      }
-    }
-    if (actualSize < size) {
-      DexAnnotation[] temp = new DexAnnotation[actualSize];
-      System.arraycopy(result, 0, temp, 0, actualSize);
-      result = temp;
+      result[i] = annotationAt(annotationOffsets[i]);
     }
     DexType dupType = DexAnnotationSet.findDuplicateEntryType(result);
     if (dupType != null) {
@@ -443,11 +437,6 @@
     return new DexAnnotationSet(result);
   }
 
-  private boolean retainAnnotation(DexAnnotation annotation) {
-    return annotation.visibility != DexAnnotation.VISIBILITY_BUILD
-        || DexAnnotation.retainCompileTimeAnnotation(annotation.annotation.type, options);
-  }
-
   private DexAnnotationSet annotationSetAt(int offset) {
     return (DexAnnotationSet) cacheAt(offset, this::parseAnnotationSet, DexAnnotationSet::empty);
   }
@@ -702,7 +691,7 @@
       }
 
       AttributesAndAnnotations attrs =
-          new AttributesAndAnnotations(type, annotationsDirectory.clazz, options.itemFactory);
+          new AttributesAndAnnotations(type, annotationsDirectory.clazz, dexItemFactory);
 
       DexClass clazz =
           classKind.create(
diff --git a/src/main/java/com/android/tools/r8/graph/DexAnnotation.java b/src/main/java/com/android/tools/r8/graph/DexAnnotation.java
index c3d914f..6fc1e1d 100644
--- a/src/main/java/com/android/tools/r8/graph/DexAnnotation.java
+++ b/src/main/java/com/android/tools/r8/graph/DexAnnotation.java
@@ -13,8 +13,6 @@
 import com.android.tools.r8.graph.DexValue.DexValueNull;
 import com.android.tools.r8.graph.DexValue.DexValueString;
 import com.android.tools.r8.graph.DexValue.DexValueType;
-import com.android.tools.r8.ir.desugar.CovariantReturnTypeAnnotationTransformer;
-import com.android.tools.r8.utils.InternalOptions;
 import com.android.tools.r8.utils.Pair;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -69,24 +67,6 @@
     mixedItems.add(this);
   }
 
-  public static boolean retainCompileTimeAnnotation(DexType annotation, InternalOptions options) {
-    if (options.readCompileTimeAnnotations) {
-      return true;
-    }
-    if (annotation == options.itemFactory.dalvikFastNativeAnnotation
-        || annotation == options.itemFactory.dalvikCriticalNativeAnnotation
-        || annotation == options.itemFactory.annotationSynthesizedClassMap) {
-      return true;
-    }
-    if (options.processCovariantReturnTypeAnnotations) {
-      // @CovariantReturnType annotations are processed by CovariantReturnTypeAnnotationTransformer,
-      // they thus need to be read here and will then be removed as part of the processing.
-      return CovariantReturnTypeAnnotationTransformer.isCovariantReturnTypeAnnotation(
-          annotation, options.itemFactory);
-    }
-    return false;
-  }
-
   public static DexAnnotation createEnclosingClassAnnotation(DexType enclosingClass,
       DexItemFactory factory) {
     return createSystemValueAnnotation(factory.annotationEnclosingClass, factory,
diff --git a/src/main/java/com/android/tools/r8/graph/JarClassFileReader.java b/src/main/java/com/android/tools/r8/graph/JarClassFileReader.java
index 87777e3..16e9d44 100644
--- a/src/main/java/com/android/tools/r8/graph/JarClassFileReader.java
+++ b/src/main/java/com/android/tools/r8/graph/JarClassFileReader.java
@@ -29,6 +29,7 @@
 import com.android.tools.r8.graph.DexValue.DexValueShort;
 import com.android.tools.r8.graph.DexValue.DexValueString;
 import com.android.tools.r8.graph.DexValue.DexValueType;
+import com.android.tools.r8.ir.desugar.CovariantReturnTypeAnnotationTransformer;
 import com.android.tools.r8.jar.CfApplicationWriter;
 import com.android.tools.r8.origin.Origin;
 import com.android.tools.r8.shaking.ProguardKeepAttributes;
@@ -167,9 +168,21 @@
 
   private static boolean retainCompileTimeAnnotation(
       String desc, JarApplicationReader application) {
-    return application.options.readCompileTimeAnnotations
-        || DexAnnotation.retainCompileTimeAnnotation(
-            application.getTypeFromDescriptor(desc), application.options);
+    if (application.options.readCompileTimeAnnotations) {
+      return true;
+    }
+    DexType type = application.getTypeFromDescriptor(desc);
+    if (type == application.options.itemFactory.dalvikFastNativeAnnotation
+        || type == application.options.itemFactory.dalvikCriticalNativeAnnotation) {
+      return true;
+    }
+    if (application.options.processCovariantReturnTypeAnnotations) {
+      // @CovariantReturnType annotations are processed by CovariantReturnTypeAnnotationTransformer,
+      // they thus need to be read here and will then be removed as part of the processing.
+      return CovariantReturnTypeAnnotationTransformer.isCovariantReturnTypeAnnotation(
+          type, application.options.itemFactory);
+    }
+    return false;
   }
 
   private static DexEncodedAnnotation createEncodedAnnotation(String desc,
diff --git a/src/test/java/com/android/tools/r8/annotations/RetentionPolicyTest.java b/src/test/java/com/android/tools/r8/annotations/RetentionPolicyTest.java
index 1509320..be166ec 100644
--- a/src/test/java/com/android/tools/r8/annotations/RetentionPolicyTest.java
+++ b/src/test/java/com/android/tools/r8/annotations/RetentionPolicyTest.java
@@ -9,8 +9,8 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-import com.android.tools.r8.D8TestCompileResult;
 import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestBuilder;
 import com.android.tools.r8.TestParameters;
 import com.android.tools.r8.utils.StringUtils;
 import com.android.tools.r8.utils.codeinspector.ClassSubject;
@@ -43,7 +43,7 @@
         .flatMap(
             parameters -> {
               if (parameters.isCfRuntime()) {
-                return Stream.of((Object) new Object[] {parameters, true});
+                return Stream.of((Object) new Object[] {parameters, false});
               }
               return Stream.of(new Object[] {parameters, true}, new Object[] {parameters, false});
             })
@@ -81,48 +81,26 @@
 
   @Test
   public void test() throws Exception {
-    if (parameters.isCfRuntime()) {
-      assertTrue(intermediate);
-      checkAnnotations(
-          testForJvm()
-              .addProgramClasses(CLASSES)
-              .run(parameters.getRuntime(), A.class)
-              .assertSuccessWithOutput(EXPECTED)
-              .inspector(),
-          intermediate);
-    } else {
-      D8TestCompileResult compile =
-          testForD8()
-              .setMinApi(parameters.getRuntime())
-              .setIntermediate(intermediate)
-              .addProgramClasses(CLASSES)
-              .compile();
-      checkAnnotations(
-          compile
-              .run(parameters.getRuntime(), A.class)
-              .assertSuccessWithOutput(EXPECTED)
-              .inspector(),
-          intermediate);
-      // If the first build was an intermediate, re-compile and check the final output.
-      if (intermediate) {
-        checkAnnotations(
-            testForD8()
-                .setMinApi(parameters.getRuntime())
-                .addProgramFiles(compile.writeToZip())
-                .run(parameters.getRuntime(), A.class)
-                .inspector(),
-            false);
-      }
-    }
-  }
+    TestBuilder<?, ?> testBuilder =
+        parameters.isCfRuntime()
+            ? testForJvm()
+            : testForD8().setMinApi(parameters.getRuntime()).setIntermediate(intermediate);
 
-  private static void checkAnnotations(CodeInspector inspector, boolean isClassRetained) {
+    CodeInspector inspector =
+        testBuilder
+            .addProgramClasses(CLASSES)
+            .run(parameters.getRuntime(), A.class)
+            .assertSuccessWithOutput(EXPECTED)
+            .inspector();
+
     ClassSubject clazz = inspector.clazz(A.class);
     assertThat(clazz, isPresent());
     // Source retained annotations are always gone, even in the CF inputs.
     assertFalse(clazz.annotation(SourceRetained.class.getName()).isPresent());
     // Class retained annotations are present in CF and in intermediate builds.
-    assertEquals(isClassRetained, clazz.annotation(ClassRetained.class.getName()).isPresent());
+    assertEquals(
+        parameters.isCfRuntime() || intermediate,
+        clazz.annotation(ClassRetained.class.getName()).isPresent());
     // Runtime retained annotations are present in all.
     assertTrue(clazz.annotation(RuntimeRetained.class.getName()).isPresent());
   }