Add support for reading DEX container format
Bug: b/249922554
Change-Id: I3357dd7a922f779d1f45750d95aa97dc9a3ad565
diff --git a/src/main/java/com/android/tools/r8/dex/ApplicationReader.java b/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
index c741ee9..eda00f9 100644
--- a/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
+++ b/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
@@ -49,6 +49,8 @@
import com.android.tools.r8.utils.StringDiagnostic;
import com.android.tools.r8.utils.ThreadUtils;
import com.android.tools.r8.utils.Timing;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -337,9 +339,17 @@
for (ProgramResource input : dexSources) {
DexReader dexReader = new DexReader(input);
if (options.passthroughDexCode) {
- computedMinApiLevel = validateOrComputeMinApiLevel(computedMinApiLevel, dexReader);
+ if (!options.testing.dexContainerExperiment) {
+ computedMinApiLevel = validateOrComputeMinApiLevel(computedMinApiLevel, dexReader);
+ } else {
+ assert dexReader.getDexVersion() == DexVersion.V41;
+ }
}
- dexParsers.add(new DexParser<>(dexReader, PROGRAM, options));
+ if (!options.testing.dexContainerExperiment) {
+ dexParsers.add(new DexParser<>(dexReader, PROGRAM, options));
+ } else {
+ addDexParsersForContainer(dexParsers, dexReader);
+ }
}
options.setMinApiLevel(computedMinApiLevel);
@@ -349,17 +359,56 @@
// Read the DexCode items and DexProgramClass items in parallel.
if (!options.skipReadingDexCode) {
ApplicationReaderMap applicationReaderMap = ApplicationReaderMap.getInstance(options);
- for (DexParser<DexProgramClass> dexParser : dexParsers) {
- futures.add(
- executorService.submit(
- () -> {
- dexParser.addClassDefsTo(
- classes::add, applicationReaderMap); // Depends on Methods, Code items etc.
- }));
+ if (!options.testing.dexContainerExperiment) {
+ for (DexParser<DexProgramClass> dexParser : dexParsers) {
+ futures.add(
+ executorService.submit(
+ () -> {
+ dexParser.addClassDefsTo(
+ classes::add,
+ applicationReaderMap); // Depends on Methods, Code items etc.
+ }));
+ }
+ } else {
+ // All Dex parsers use the same DEX reader, so don't process in parallel.
+ for (int i = 0; i < dexParsers.size(); i++) {
+ dexParsers.get(i).addClassDefsTo(classes::add, applicationReaderMap);
+ }
}
}
}
+ private void addDexParsersForContainer(
+ List<DexParser<DexProgramClass>> dexParsers, DexReader dexReader) {
+ // Find the start offsets of each dex section.
+ IntList offsets = new IntArrayList();
+ dexReader.setByteOrder();
+ int offset = 0;
+ while (offset < dexReader.end()) {
+ offsets.add(offset);
+ DexReader tmp = new DexReader(Origin.unknown(), dexReader.buffer.array(), offset);
+ assert tmp.getDexVersion() == DexVersion.V41;
+ assert dexReader.getUint(offset + Constants.HEADER_SIZE_OFFSET)
+ == Constants.TYPE_HEADER_ITEM_SIZE_V41;
+ assert dexReader.getUint(offset + Constants.HEADER_OFF_OFFSET) == offset;
+ int dataSize = dexReader.getUint(offset + Constants.DATA_SIZE_OFFSET);
+ int dataOffset = dexReader.getUint(offset + Constants.DATA_OFF_OFFSET);
+ int file_size = dexReader.getUint(offset + Constants.FILE_SIZE_OFFSET);
+ assert dataOffset == 0;
+ assert dataSize == 0;
+ offset += file_size;
+ }
+ assert offset == dexReader.end();
+ // Create a parser for the last section with string data.
+ DexParser<DexProgramClass> last =
+ new DexParser<>(dexReader, PROGRAM, options, offsets.getInt(offsets.size() - 1), null);
+ // Create a parsers for the remaining sections with reference to the string data.
+ for (int i = 0; i < offsets.size() - 1; i++) {
+ dexParsers.add(new DexParser<>(dexReader, PROGRAM, options, offsets.getInt(i), last));
+ }
+ dexParsers.add(last);
+ }
+
private boolean includeAnnotationClass(DexProgramClass clazz) {
if (!options.pruneNonVissibleAnnotationClasses) {
return true;
diff --git a/src/main/java/com/android/tools/r8/dex/DexParser.java b/src/main/java/com/android/tools/r8/dex/DexParser.java
index 0d9fac5..263acfc 100644
--- a/src/main/java/com/android/tools/r8/dex/DexParser.java
+++ b/src/main/java/com/android/tools/r8/dex/DexParser.java
@@ -95,21 +95,23 @@
private final int NO_INDEX = -1;
private final Origin origin;
private DexReader dexReader;
- private final DexSection[] dexSections;
+ private final List<DexSection> dexSections;
+ private final int offset;
private int[] stringIDs;
private final ClassKind<T> classKind;
private final InternalOptions options;
private Object2LongMap<String> checksums;
- public static DexSection[] parseMapFrom(Path file) throws IOException {
+ public static List<DexSection> parseMapFrom(Path file) throws IOException {
return parseMapFrom(Files.newInputStream(file), new PathOrigin(file));
}
- public static DexSection[] parseMapFrom(InputStream stream, Origin origin) throws IOException {
+ public static List<DexSection> parseMapFrom(InputStream stream, Origin origin)
+ throws IOException {
return parseMapFrom(new DexReader(origin, ByteStreams.toByteArray(stream)));
}
- private static DexSection[] parseMapFrom(DexReader dexReader) {
+ private static List<DexSection> parseMapFrom(DexReader dexReader) {
DexParser<DexProgramClass> dexParser =
new DexParser<>(dexReader, ClassKind.PROGRAM, new InternalOptions());
return dexParser.dexSections;
@@ -144,13 +146,27 @@
private final DexItemFactory dexItemFactory;
public DexParser(DexReader dexReader, ClassKind<T> classKind, InternalOptions options) {
+ this(dexReader, classKind, options, 0, null);
+ }
+
+ public DexParser(
+ DexReader dexReader,
+ ClassKind<T> classKind,
+ InternalOptions options,
+ int offset,
+ DexParser<T> parserWithStringIDs) {
assert dexReader.getOrigin() != null;
this.origin = dexReader.getOrigin();
this.dexReader = dexReader;
+ this.offset = offset;
this.dexItemFactory = options.itemFactory;
dexReader.setByteOrder();
dexSections = parseMap();
- parseStringIDs();
+ if (parserWithStringIDs == null) {
+ parseStringIDs();
+ } else {
+ stringIDs = parserWithStringIDs.stringIDs;
+ }
this.classKind = classKind;
this.options = options;
}
@@ -919,12 +935,12 @@
return new DexSection(type, 0, 0, 0);
}
- private DexSection[] parseMap() {
+ private List<DexSection> parseMap() {
// Read the dexSections information from the MAP.
- int mapOffset = dexReader.getUint(Constants.MAP_OFF_OFFSET);
+ int mapOffset = dexReader.getUint(offset + Constants.MAP_OFF_OFFSET);
dexReader.position(mapOffset);
int mapSize = dexReader.getUint();
- final DexSection[] result = new DexSection[mapSize];
+ final List<DexSection> result = new ArrayList<>(mapSize);
for (int i = 0; i < mapSize; i++) {
int type = dexReader.getUshort();
int unused = dexReader.getUshort();
@@ -943,12 +959,12 @@
+ dexReader.end(),
origin);
}
- result[i] = new DexSection(type, unused, size, offset);
+ result.add(new DexSection(type, unused, size, offset));
}
for (int i = 0; i < mapSize - 1; i++) {
- result[i].setEnd(result[i + 1].offset);
+ result.get(i).setEnd(result.get(i + 1).offset);
}
- result[mapSize - 1].setEnd(dexReader.end());
+ result.get(mapSize - 1).setEnd(dexReader.end());
return result;
}
diff --git a/src/main/java/com/android/tools/r8/dex/DexReader.java b/src/main/java/com/android/tools/r8/dex/DexReader.java
index 18c0beb..87829fe 100644
--- a/src/main/java/com/android/tools/r8/dex/DexReader.java
+++ b/src/main/java/com/android/tools/r8/dex/DexReader.java
@@ -25,7 +25,7 @@
public DexReader(ProgramResource resource) throws ResourceException, IOException {
super(resource);
- version = parseMagic(buffer);
+ version = parseMagic(buffer, 0);
}
/**
@@ -35,18 +35,23 @@
*/
DexReader(Origin origin, byte[] bytes) {
super(origin, bytes);
- version = parseMagic(buffer);
+ version = parseMagic(buffer, 0);
+ }
+
+ DexReader(Origin origin, byte[] bytes, int offset) {
+ super(origin, bytes);
+ version = parseMagic(buffer, offset);
}
// Parse the magic header and determine the dex file version.
- private DexVersion parseMagic(CompatByteBuffer buffer) {
+ private DexVersion parseMagic(CompatByteBuffer buffer, int offset) {
try {
buffer.get();
buffer.rewind();
} catch (BufferUnderflowException e) {
throw new CompilationError("Dex file is empty", origin);
}
- int index = 0;
+ int index = offset;
for (byte prefixByte : DEX_FILE_MAGIC_PREFIX) {
byte actualByte = buffer.get(index++);
if (actualByte != prefixByte) {
diff --git a/src/test/java/com/android/tools/r8/debuginfo/CanonicalizeWithInline.java b/src/test/java/com/android/tools/r8/debuginfo/CanonicalizeWithInline.java
index 455bfe4..0373c01 100644
--- a/src/test/java/com/android/tools/r8/debuginfo/CanonicalizeWithInline.java
+++ b/src/test/java/com/android/tools/r8/debuginfo/CanonicalizeWithInline.java
@@ -15,6 +15,7 @@
import com.android.tools.r8.utils.AndroidApiLevel;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -34,7 +35,7 @@
}
private int getNumberOfDebugInfos(Path file) throws IOException {
- DexSection[] dexSections = DexParser.parseMapFrom(file);
+ List<DexSection> dexSections = DexParser.parseMapFrom(file);
for (DexSection dexSection : dexSections) {
if (dexSection.type == Constants.TYPE_DEBUG_INFO_ITEM) {
return dexSection.length;
diff --git a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java
index 28a01e0..25b028a 100644
--- a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java
+++ b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java
@@ -33,6 +33,7 @@
import com.android.tools.r8.utils.BitUtils;
import com.android.tools.r8.utils.DescriptorUtils;
import com.android.tools.r8.utils.DexVersion;
+import com.android.tools.r8.utils.ListUtils;
import com.android.tools.r8.utils.ZipUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.io.ByteStreams;
@@ -106,7 +107,7 @@
}
@Test
- public void testD8Experiment() throws Exception {
+ public void testD8ExperimentSimpleMerge() throws Exception {
Path outputFromDexing =
testForD8(Backend.DEX)
.addProgramFiles(inputA)
@@ -116,10 +117,23 @@
.compile()
.writeToZip();
validateSingleContainerDex(outputFromDexing);
+
+ Path outputFromMerging =
+ testForD8(Backend.DEX)
+ .addProgramFiles(outputFromDexing)
+ .addOptionsModification(
+ options -> options.getTestingOptions().dexContainerExperiment = true)
+ .compile()
+ .writeToZip();
+ validateSingleContainerDex(outputFromMerging);
+
+ // Identical DEX after re-merging.
+ assertArrayEquals(
+ unzipContent(outputFromDexing).get(0), unzipContent(outputFromMerging).get(0));
}
@Test
- public void testD8Experiment2() throws Exception {
+ public void testD8ExperimentMoreMerge() throws Exception {
Path outputA =
testForD8(Backend.DEX)
.addProgramFiles(inputA)
@@ -139,6 +153,27 @@
.compile()
.writeToZip();
validateSingleContainerDex(outputB);
+
+ Path outputBoth =
+ testForD8(Backend.DEX)
+ .addProgramFiles(inputA, inputB)
+ .setMinApi(AndroidApiLevel.L)
+ .addOptionsModification(
+ options -> options.getTestingOptions().dexContainerExperiment = true)
+ .compile()
+ .writeToZip();
+ validateSingleContainerDex(outputBoth);
+
+ Path outputMerged =
+ testForD8(Backend.DEX)
+ .addProgramFiles(outputA, outputB)
+ .addOptionsModification(
+ options -> options.getTestingOptions().dexContainerExperiment = true)
+ .compile()
+ .writeToZip();
+ validateSingleContainerDex(outputMerged);
+
+ assertArrayEquals(unzipContent(outputBoth).get(0), unzipContent(outputMerged).get(0));
}
private void validateDex(Path output, int expectedDexes, DexVersion expectedVersion)
@@ -168,10 +203,14 @@
int dataSize = buffer.getInt(offset + DATA_SIZE_OFFSET);
int dataOffset = buffer.getInt(offset + DATA_OFF_OFFSET);
int file_size = buffer.getInt(offset + FILE_SIZE_OFFSET);
- if (!expectedVersion.isContainerDex()) {
+ if (expectedVersion.isContainerDex()) {
+ assertEquals(0, dataSize);
+ assertEquals(0, dataOffset);
+ } else {
assertEquals(file_size, dataOffset + dataSize);
}
offset += expectedVersion.isContainerDex() ? file_size : dataOffset + dataSize;
+ assertEquals(file_size, offset - ListUtils.last(sections));
}
assertEquals(buffer.capacity(), offset);
