Decrease peak memory usage for the common case.
Byte arrays created for files in input archives should not be reachable
from the outside again and therefore we can make them one-shot
resources that do not hold on to their byte arrays after you get
their stream.
Applications can be used multiple times. Therefore, this change
allows the rereading of the program archives which recreates
the content.
This makes IOExceptions happen at a slightly different time in the
pipeline, not sure that we care much.
R=zerny@google.com
Change-Id: Ia9b7eb78b0ea9814d37a3611180543796c31ec26
diff --git a/src/main/java/com/android/tools/r8/BaseOutput.java b/src/main/java/com/android/tools/r8/BaseOutput.java
index b1b9166..48e2309 100644
--- a/src/main/java/com/android/tools/r8/BaseOutput.java
+++ b/src/main/java/com/android/tools/r8/BaseOutput.java
@@ -43,7 +43,7 @@
* @return an immutable list of compiled DEX resources.
*/
public List<Resource> getDexResources() {
- return ImmutableList.copyOf(app.getDexProgramResources());
+ return ImmutableList.copyOf(app.getDexProgramResourcesForOutput());
}
/**
diff --git a/src/main/java/com/android/tools/r8/Resource.java b/src/main/java/com/android/tools/r8/Resource.java
index 3208e52..15b5dd1 100644
--- a/src/main/java/com/android/tools/r8/Resource.java
+++ b/src/main/java/com/android/tools/r8/Resource.java
@@ -18,7 +18,7 @@
DEX, CLASSFILE
}
- private Resource(Kind kind) {
+ protected Resource(Kind kind) {
this.kind = kind;
}
diff --git a/src/main/java/com/android/tools/r8/utils/AndroidApp.java b/src/main/java/com/android/tools/r8/utils/AndroidApp.java
index 19d5153..b328895 100644
--- a/src/main/java/com/android/tools/r8/utils/AndroidApp.java
+++ b/src/main/java/com/android/tools/r8/utils/AndroidApp.java
@@ -15,7 +15,6 @@
import com.google.common.io.Closer;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -25,19 +24,15 @@
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipException;
-import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
/**
@@ -52,6 +47,7 @@
private final ImmutableList<Resource> programResources;
private final ImmutableList<ClassFileResourceProvider> classpathResourceProviders;
private final ImmutableList<ClassFileResourceProvider> libraryResourceProviders;
+ private final ImmutableList<ProgramFileArchiveReader> programFileArchiveReaders;
private final Resource deadCode;
private final Resource proguardMap;
private final Resource proguardSeeds;
@@ -61,6 +57,7 @@
// See factory methods and AndroidApp.Builder below.
private AndroidApp(
ImmutableList<Resource> programResources,
+ ImmutableList<ProgramFileArchiveReader> programFileArchiveReaders,
ImmutableList<ClassFileResourceProvider> classpathResourceProviders,
ImmutableList<ClassFileResourceProvider> libraryResourceProviders,
Resource deadCode,
@@ -69,6 +66,7 @@
Resource packageDistribution,
Resource mainDexList) {
this.programResources = programResources;
+ this.programFileArchiveReaders = programFileArchiveReaders;
this.classpathResourceProviders = classpathResourceProviders;
this.libraryResourceProviders = libraryResourceProviders;
this.deadCode = deadCode;
@@ -142,13 +140,26 @@
}
/** Get input streams for all dex program resources. */
- public List<Resource> getDexProgramResources() {
+ public List<Resource> getDexProgramResources() throws IOException {
+ List<Resource> dexResources = filter(programResources, Resource.Kind.DEX);
+ for (ProgramFileArchiveReader reader : programFileArchiveReaders) {
+ dexResources.addAll(reader.getDexProgramResources());
+ }
+ return dexResources;
+ }
+
+ public List<Resource> getDexProgramResourcesForOutput() {
+ assert programFileArchiveReaders.isEmpty();
return filter(programResources, Resource.Kind.DEX);
}
/** Get input streams for all Java-bytecode program resources. */
- public List<Resource> getClassProgramResources() {
- return filter(programResources, Resource.Kind.CLASSFILE);
+ public List<Resource> getClassProgramResources() throws IOException {
+ List<Resource> classResources = filter(programResources, Resource.Kind.CLASSFILE);
+ for (ProgramFileArchiveReader reader : programFileArchiveReaders) {
+ classResources.addAll(reader.getClassProgramResources());
+ }
+ return classResources;
}
/** Get classpath resource providers. */
@@ -367,6 +378,7 @@
public static class Builder {
private final List<Resource> programResources = new ArrayList<>();
+ private final List<ProgramFileArchiveReader> programFileArchiveReaders = new ArrayList<>();
private final List<ClassFileResourceProvider> classpathResourceProviders = new ArrayList<>();
private final List<ClassFileResourceProvider> libraryResourceProviders = new ArrayList<>();
private Resource deadCode;
@@ -382,6 +394,7 @@
// See AndroidApp::builder(AndroidApp).
private Builder(AndroidApp app) {
programResources.addAll(app.programResources);
+ programFileArchiveReaders.addAll(app.programFileArchiveReaders);
classpathResourceProviders.addAll(app.classpathResourceProviders);
libraryResourceProviders.addAll(app.libraryResourceProviders);
deadCode = app.deadCode;
@@ -597,6 +610,7 @@
public AndroidApp build() {
return new AndroidApp(
ImmutableList.copyOf(programResources),
+ ImmutableList.copyOf(programFileArchiveReaders),
ImmutableList.copyOf(classpathResourceProviders),
ImmutableList.copyOf(libraryResourceProviders),
deadCode,
@@ -615,7 +629,7 @@
} else if (isClassFile(file)) {
programResources.add(Resource.fromFile(Resource.Kind.CLASSFILE, file));
} else if (isArchive(file)) {
- addProgramArchive(file);
+ programFileArchiveReaders.add(new ProgramFileArchiveReader(file));
} else {
throw new CompilationError("Unsupported source file type for file: " + file);
}
@@ -634,35 +648,5 @@
throw new CompilationError("Unsupported source file type for file: " + file);
}
}
-
- private void addProgramArchive(Path archive) throws IOException {
- assert isArchive(archive);
- boolean containsDexData = false;
- boolean containsClassData = false;
- try (ZipInputStream stream = new ZipInputStream(new FileInputStream(archive.toFile()))) {
- ZipEntry entry;
- while ((entry = stream.getNextEntry()) != null) {
- Path name = Paths.get(entry.getName());
- if (isDexFile(name)) {
- containsDexData = true;
- programResources.add(Resource.fromBytes(
- Resource.Kind.DEX, ByteStreams.toByteArray(stream)));
- } else if (isClassFile(name)) {
- containsClassData = true;
- String descriptor = PreloadedClassFileProvider.guessTypeDescriptor(name);
- programResources.add(Resource.fromBytes(Resource.Kind.CLASSFILE,
- ByteStreams.toByteArray(stream), Collections.singleton(descriptor)));
- }
- }
- } catch (ZipException e) {
- throw new CompilationError(
- "Zip error while reading '" + archive + "': " + e.getMessage(), e);
- }
- if (containsDexData && containsClassData) {
- throw new CompilationError(
- "Cannot create android app from an archive '" + archive
- + "' containing both DEX and Java-bytecode content");
- }
- }
}
}
diff --git a/src/main/java/com/android/tools/r8/utils/OneShotByteResource.java b/src/main/java/com/android/tools/r8/utils/OneShotByteResource.java
new file mode 100644
index 0000000..51aa312
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/utils/OneShotByteResource.java
@@ -0,0 +1,36 @@
+// Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.utils;
+
+import com.android.tools.r8.Resource;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+
+class OneShotByteResource extends Resource {
+
+ private byte[] bytes;
+ private final Set<String> classDescriptors;
+
+ OneShotByteResource(Kind kind, byte[] bytes, Set<String> classDescriptors) {
+ super(kind);
+ assert bytes != null;
+ this.bytes = bytes;
+ this.classDescriptors = classDescriptors;
+ }
+
+ @Override
+ public Set<String> getClassDescriptors() {
+ return classDescriptors;
+ }
+
+ @Override
+ public InputStream getStream() throws IOException {
+ assert bytes != null;
+ InputStream result = new ByteArrayInputStream(bytes);
+ bytes = null;
+ return result;
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/utils/ProgramFileArchiveReader.java b/src/main/java/com/android/tools/r8/utils/ProgramFileArchiveReader.java
new file mode 100644
index 0000000..814ffd1
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/utils/ProgramFileArchiveReader.java
@@ -0,0 +1,82 @@
+// Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.utils;
+
+import static com.android.tools.r8.utils.FileUtils.isArchive;
+import static com.android.tools.r8.utils.FileUtils.isClassFile;
+import static com.android.tools.r8.utils.FileUtils.isDexFile;
+
+import com.android.tools.r8.Resource;
+import com.android.tools.r8.errors.CompilationError;
+import com.google.common.io.ByteStreams;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipInputStream;
+
+class ProgramFileArchiveReader {
+
+ private final Path archive;
+ private List<Resource> dexResources = null;
+ private List<Resource> classResources = null;
+
+ ProgramFileArchiveReader(Path archive) {
+ this.archive = archive;
+ }
+
+ private void readArchive() throws IOException {
+ assert isArchive(archive);
+ dexResources = new ArrayList<>();
+ classResources = new ArrayList<>();
+ try (ZipInputStream stream = new ZipInputStream(new FileInputStream(archive.toFile()))) {
+ ZipEntry entry;
+ while ((entry = stream.getNextEntry()) != null) {
+ Path name = Paths.get(entry.getName());
+ if (isDexFile(name)) {
+ Resource resource =
+ new OneShotByteResource(Resource.Kind.DEX, ByteStreams.toByteArray(stream), null);
+ dexResources.add(resource);
+ } else if (isClassFile(name)) {
+ String descriptor = PreloadedClassFileProvider.guessTypeDescriptor(name);
+ Resource resource = new OneShotByteResource(Resource.Kind.CLASSFILE,
+ ByteStreams.toByteArray(stream), Collections.singleton(descriptor));
+ classResources.add(resource);
+ }
+ }
+ } catch (ZipException e) {
+ throw new CompilationError(
+ "Zip error while reading '" + archive + "': " + e.getMessage(), e);
+ }
+ if (!dexResources.isEmpty() && !classResources.isEmpty()) {
+ throw new CompilationError(
+ "Cannot create android app from an archive '" + archive
+ + "' containing both DEX and Java-bytecode content");
+ }
+ }
+
+ public Collection<Resource> getDexProgramResources() throws IOException {
+ if (dexResources == null) {
+ readArchive();
+ }
+ List<Resource> result = dexResources;
+ dexResources = null;
+ return result;
+ }
+
+ public Collection<Resource> getClassProgramResources() throws IOException {
+ if (classResources == null) {
+ readArchive();
+ }
+ List<Resource> result = classResources;
+ classResources = null;
+ return result;
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/utils/D8CommandTest.java b/src/test/java/com/android/tools/r8/utils/D8CommandTest.java
index ed146b72..443e257 100644
--- a/src/test/java/com/android/tools/r8/utils/D8CommandTest.java
+++ b/src/test/java/com/android/tools/r8/utils/D8CommandTest.java
@@ -46,7 +46,7 @@
verifyEmptyCommand(parse("\t", "\t"));
}
- private void verifyEmptyCommand(D8Command command) {
+ private void verifyEmptyCommand(D8Command command) throws IOException {
assertEquals(0, ToolHelper.getApp(command).getDexProgramResources().size());
assertEquals(0, ToolHelper.getApp(command).getClassProgramResources().size());
assertFalse(ToolHelper.getApp(command).hasMainDexList());
diff --git a/src/test/java/com/android/tools/r8/utils/R8CommandTest.java b/src/test/java/com/android/tools/r8/utils/R8CommandTest.java
index 27f0367..a1e57ac 100644
--- a/src/test/java/com/android/tools/r8/utils/R8CommandTest.java
+++ b/src/test/java/com/android/tools/r8/utils/R8CommandTest.java
@@ -46,7 +46,7 @@
verifyEmptyCommand(parse("\t", "\t"));
}
- private void verifyEmptyCommand(R8Command command) {
+ private void verifyEmptyCommand(R8Command command) throws IOException {
assertEquals(0, ToolHelper.getApp(command).getDexProgramResources().size());
assertEquals(0, ToolHelper.getApp(command).getClassProgramResources().size());
assertFalse(ToolHelper.getApp(command).hasMainDexList());
