Add support for splitting non-code resources in dex splitter.
This patch introduces a new option: --split-non-class-resources
which makes the dex splitter also copy over non-class resources
from the input to the feature jar it came from or to the base
if it cannot determine where a non-class resource came from.
Bug: 111082605
Change-Id: I82bd99b20ac4efa0e930b84bc4f13f2aad5cf38c
diff --git a/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java b/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
index b0a5a3c..1956e8e 100644
--- a/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
+++ b/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
@@ -10,19 +10,29 @@
import com.android.tools.r8.DexSplitterHelper;
import com.android.tools.r8.DiagnosticsHandler;
import com.android.tools.r8.Keep;
+import com.android.tools.r8.origin.PathOrigin;
import com.android.tools.r8.utils.AbortException;
import com.android.tools.r8.utils.DefaultDiagnosticsHandler;
+import com.android.tools.r8.utils.ExceptionDiagnostic;
import com.android.tools.r8.utils.ExceptionUtils;
import com.android.tools.r8.utils.FeatureClassMapping;
import com.android.tools.r8.utils.FeatureClassMapping.FeatureMappingException;
import com.android.tools.r8.utils.OptionsParsing;
import com.android.tools.r8.utils.OptionsParsing.ParseContext;
import com.android.tools.r8.utils.StringDiagnostic;
+import com.android.tools.r8.utils.ZipUtils;
import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Enumeration;
import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
@Keep
public final class DexSplitter {
@@ -61,7 +71,18 @@
}
return featureName;
}
+ }
+ private static class ZipFileOrigin extends PathOrigin {
+
+ public ZipFileOrigin(Path path) {
+ super(path);
+ }
+
+ @Override
+ public String part() {
+ return "splitting of file '" + super.part() + "'";
+ }
}
@Keep
@@ -73,6 +94,7 @@
private String output = DEFAULT_OUTPUT_DIR;
private String featureSplitMapping;
private String proguardMap;
+ private boolean splitNonClassResources = false;
public DiagnosticsHandler getDiagnosticsHandler() {
return diagnosticsHandler;
@@ -126,6 +148,10 @@
featureJars.add(new FeatureJar(jar, outputName));
}
+ public void setSplitNonClassResources(boolean value) {
+ splitNonClassResources = value;
+ }
+
public ImmutableList<String> getInputArchives() {
return ImmutableList.copyOf(inputArchives);
}
@@ -193,6 +219,11 @@
options.setFeatureSplitMapping(featureSplit);
continue;
}
+ Boolean b = OptionsParsing.tryParseBoolean(context, "--split-non-class-resources");
+ if (b != null) {
+ options.setSplitNonClassResources(b);
+ continue;
+ }
throw new RuntimeException(String.format("Unknown options: '%s'.", context.head()));
}
return options;
@@ -246,6 +277,39 @@
DexSplitterHelper.run(
builder.build(), featureClassMapping, options.getOutput(), options.getProguardMap());
+
+ if (options.splitNonClassResources) {
+ splitNonClassResources(options, featureClassMapping);
+ }
+ }
+
+ private static void splitNonClassResources(Options options,
+ FeatureClassMapping featureClassMapping) {
+ for (String s : options.inputArchives) {
+ try (ZipFile zipFile = new ZipFile(s, StandardCharsets.UTF_8)) {
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ String name = entry.getName();
+ if (!ZipUtils.isDexFile(name) && !ZipUtils.isClassFile(name)) {
+ String feature = featureClassMapping.featureForNonClass(name);
+ Path outputDir = Paths.get(options.getOutput()).resolve(feature);
+ try (InputStream stream = zipFile.getInputStream(entry)) {
+ Path outputFile = outputDir.resolve(name);
+ Path parent = outputFile.getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ Files.copy(stream, outputFile);
+ }
+ }
+ }
+ } catch (IOException e) {
+ options.getDiagnosticsHandler().error(
+ new ExceptionDiagnostic(e, new ZipFileOrigin(Paths.get(s))));
+ throw new AbortException();
+ }
+ }
}
public static void main(String[] args) {
diff --git a/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java b/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java
index 31dc439..b4e6ae2 100644
--- a/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java
+++ b/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java
@@ -9,12 +9,16 @@
import com.android.tools.r8.dexsplitter.DexSplitter.FeatureJar;
import com.android.tools.r8.origin.PathOrigin;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
/**
* Provides a mappings of classes to modules. The structure of the input file is as follows:
@@ -43,6 +47,7 @@
public final class FeatureClassMapping {
HashMap<String, String> parsedRules = new HashMap<>(); // Already parsed rules.
+ HashMap<String, String> parseNonClassRules = new HashMap<>();
boolean usesOnlyExactMappings = true;
HashSet<FeaturePredicate> mappings = new HashSet<>();
@@ -122,6 +127,19 @@
String javaType = DescriptorUtils.descriptorToJavaType(javaDescriptor);
mapping.addMapping(javaType, featureJar.getOutputName());
}
+ try (ZipFile zipfile = new ZipFile(jarPath.toString(), StandardCharsets.UTF_8)) {
+ Enumeration<? extends ZipEntry> entries = zipfile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ String name = entry.getName();
+ if (!ZipUtils.isClassFile(name)) {
+ mapping.addNonClassMapping(name, featureJar.getOutputName());
+ }
+ }
+ } catch (IOException e) {
+ reporter.error(new ExceptionDiagnostic(e, new JarFileOrigin(jarPath)));
+ throw new AbortException();
+ }
}
assert mapping.usesOnlyExactMappings;
return mapping;
@@ -135,6 +153,16 @@
addRule(clazz, feature, 0);
}
+ public void addNonClassMapping(String name, String feature) throws FeatureMappingException {
+ if (parseNonClassRules.containsKey(name)) {
+ throw new FeatureMappingException(
+ "Non-code files with the same name present in multiple feature splits. " +
+ "File '" + name + "' present in both '" + feature + "' and '" +
+ parseNonClassRules.get(name) + "'.");
+ }
+ parseNonClassRules.put(name, feature);
+ }
+
FeatureClassMapping(List<String> lines) throws FeatureMappingException {
for (int i = 0; i < lines.size(); i++) {
String line = lines.get(i);
@@ -161,6 +189,10 @@
}
}
+ public String featureForNonClass(String nonClass) {
+ return parseNonClassRules.getOrDefault(nonClass, baseName);
+ }
+
private void parseAndAdd(String line, int lineNumber) throws FeatureMappingException {
if (line.startsWith(COMMENT)) {
return; // Ignore comments
diff --git a/src/test/examplesAndroidN/dexsplitsample/TextFile.txt b/src/test/examplesAndroidN/dexsplitsample/TextFile.txt
new file mode 100644
index 0000000..0bad44c
--- /dev/null
+++ b/src/test/examplesAndroidN/dexsplitsample/TextFile.txt
@@ -0,0 +1,9 @@
+// Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore
+et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut
+aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
+culpa qui officia deserunt mollit anim id est laborum.
diff --git a/src/test/java/com/android/tools/r8/ToolHelper.java b/src/test/java/com/android/tools/r8/ToolHelper.java
index 2499b5f..44b491e 100644
--- a/src/test/java/com/android/tools/r8/ToolHelper.java
+++ b/src/test/java/com/android/tools/r8/ToolHelper.java
@@ -73,6 +73,7 @@
public static final String LIBS_DIR = BUILD_DIR + "libs/";
public static final String TESTS_DIR = "src/test/";
public static final String EXAMPLES_DIR = TESTS_DIR + "examples/";
+ public static final String EXAMPLES_ANDROID_N_DIR = TESTS_DIR + "examplesAndroidN/";
public static final String EXAMPLES_ANDROID_O_DIR = TESTS_DIR + "examplesAndroidO/";
public static final String EXAMPLES_ANDROID_P_DIR = TESTS_DIR + "examplesAndroidP/";
public static final String EXAMPLES_KOTLIN_DIR = TESTS_DIR + "examplesKotlin/";
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java
index f1c8742..bc8bd8c 100644
--- a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java
+++ b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java
@@ -32,12 +32,15 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.internal.runners.statements.ExpectException;
+import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
public class DexSplitterTests {
@@ -50,6 +53,8 @@
private static final String CLASS3_INNER_CLASS = CLASS_DIR + "/Class3$InnerClass.class";
private static final String CLASS4_CLASS = CLASS_DIR + "/Class4.class";
private static final String CLASS4_LAMBDA_INTERFACE = CLASS_DIR + "/Class4$LambdaInterface.class";
+ private static final String TEXT_FILE =
+ ToolHelper.EXAMPLES_ANDROID_N_DIR + "dexsplitsample/TextFile.txt";
@Rule public TemporaryFolder temp = ToolHelper.getTemporaryFolderForTest();
@@ -201,6 +206,19 @@
}
}
+ private void validateNonClassOutput(Path base, Path feature) throws IOException {
+ byte[] contents = Files.readAllBytes(Paths.get(TEXT_FILE));
+ byte[] contents2 = new byte[contents.length * 2];
+ System.arraycopy(contents, 0, contents2, 0, contents.length);
+ System.arraycopy(contents, 0, contents2, contents.length, contents.length);
+ Path baseTextFile = base.resolve("dexsplitsample/TextFile.txt");
+ Path featureTextFile = feature.resolve("dexsplitsample/TextFile2.txt");
+ assert Files.exists(baseTextFile);
+ assert Files.exists(featureTextFile);
+ assert Arrays.equals(Files.readAllBytes(baseTextFile), contents);
+ assert Arrays.equals(Files.readAllBytes(featureTextFile), contents2);
+ }
+
private Path createSplitSpec() throws FileNotFoundException, UnsupportedEncodingException {
Path splitSpec = temp.getRoot().toPath().resolve("split_spec");
try (PrintWriter out = new PrintWriter(splitSpec.toFile(), "UTF-8")) {
@@ -222,8 +240,7 @@
@Test
public void splitFilesFromJar()
- throws IOException, CompilationFailedException, FeatureMappingException, ResourceException,
- ExecutionException {
+ throws IOException, CompilationFailedException, FeatureMappingException {
splitFromJars(true, true);
splitFromJars(false, true);
splitFromJars(true, false);
@@ -304,7 +321,6 @@
validateUnobfuscatedOutput(base, feature);
}
-
@Test
public void splitFilesObfuscation()
throws CompilationFailedException, IOException, ExecutionException {
@@ -368,4 +384,83 @@
assertTrue(subject.isPresent());
assertTrue(subject.isRenamed());
}
+
+ @Test
+ public void splitNonClassFiles()
+ throws CompilationFailedException, IOException, FeatureMappingException {
+ Path inputZip = temp.getRoot().toPath().resolve("input-zip-with-non-class-files.jar");
+ ZipOutputStream inputZipStream = new ZipOutputStream(Files.newOutputStream(inputZip));
+ String name = "dexsplitsample/TextFile.txt";
+ inputZipStream.putNextEntry(new ZipEntry(name));
+ inputZipStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ inputZipStream.closeEntry();
+ name = "dexsplitsample/TextFile2.txt";
+ inputZipStream.putNextEntry(new ZipEntry(name));
+ inputZipStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ inputZipStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ inputZipStream.closeEntry();
+ inputZipStream.close();
+ Path output = temp.newFolder().toPath().resolve("output");
+ Files.createDirectory(output);
+ Path baseJar = temp.getRoot().toPath().resolve("base.jar");
+ Path featureJar = temp.getRoot().toPath().resolve("feature1.jar");
+ ZipOutputStream baseStream = new ZipOutputStream(Files.newOutputStream(baseJar));
+ name = "dexsplitsample/TextFile.txt";
+ baseStream.putNextEntry(new ZipEntry(name));
+ baseStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ baseStream.closeEntry();
+ baseStream.close();
+ ZipOutputStream featureStream = new ZipOutputStream(Files.newOutputStream(featureJar));
+ name = "dexsplitsample/TextFile2.txt";
+ featureStream.putNextEntry(new ZipEntry(name));
+ featureStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ featureStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ featureStream.closeEntry();
+ featureStream.close();
+ Options options = new Options();
+ options.addInputArchive(inputZip.toString());
+ options.setOutput(output.toString());
+ options.addFeatureJar(baseJar.toString());
+ options.addFeatureJar(featureJar.toString());
+ options.setSplitNonClassResources(true);
+ DexSplitter.run(options);
+ Path baseDir = output.resolve("base");
+ Path featureDir = output.resolve("feature1");
+ validateNonClassOutput(baseDir, featureDir);
+ }
+
+ @Test(expected = FeatureMappingException.class)
+ public void splitDuplicateNonClassFiles()
+ throws IOException, CompilationFailedException, FeatureMappingException {
+ Path inputZip = temp.getRoot().toPath().resolve("input-zip-with-non-class-files.jar");
+ ZipOutputStream inputZipStream = new ZipOutputStream(Files.newOutputStream(inputZip));
+ String name = "dexsplitsample/TextFile.txt";
+ inputZipStream.putNextEntry(new ZipEntry(name));
+ inputZipStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ inputZipStream.closeEntry();
+ inputZipStream.close();
+ Path output = temp.newFolder().toPath().resolve("output");
+ Files.createDirectory(output);
+ Path featureJar = temp.getRoot().toPath().resolve("feature1.jar");
+ Path feature2Jar = temp.getRoot().toPath().resolve("feature2.jar");
+ ZipOutputStream featureStream = new ZipOutputStream(Files.newOutputStream(feature2Jar));
+ name = "dexsplitsample/TextFile.txt";
+ featureStream.putNextEntry(new ZipEntry(name));
+ featureStream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ featureStream.closeEntry();
+ featureStream.close();
+ ZipOutputStream feature2Stream = new ZipOutputStream(Files.newOutputStream(featureJar));
+ name = "dexsplitsample/TextFile.txt";
+ feature2Stream.putNextEntry(new ZipEntry(name));
+ feature2Stream.write(Files.readAllBytes(Paths.get(TEXT_FILE)));
+ feature2Stream.closeEntry();
+ feature2Stream.close();
+ Options options = new Options();
+ options.addInputArchive(inputZip.toString());
+ options.setOutput(output.toString());
+ options.addFeatureJar(feature2Jar.toString());
+ options.addFeatureJar(featureJar.toString());
+ options.setSplitNonClassResources(true);
+ DexSplitter.run(options);
+ }
}
