Remove DexFileMerger
Bug: 155464736
Change-Id: I911b4096b3060a0a1efff5bf95e4685439a4d86a
diff --git a/src/main/java/com/android/tools/r8/SwissArmyKnife.java b/src/main/java/com/android/tools/r8/SwissArmyKnife.java
index dd7da82..f079277 100644
--- a/src/main/java/com/android/tools/r8/SwissArmyKnife.java
+++ b/src/main/java/com/android/tools/r8/SwissArmyKnife.java
@@ -5,7 +5,6 @@
import com.android.tools.r8.bisect.Bisect;
import com.android.tools.r8.compatproguard.CompatProguard;
-import com.android.tools.r8.dexfilemerger.DexFileMerger;
import com.android.tools.r8.dexsplitter.DexSplitter;
import com.android.tools.r8.relocator.RelocatorCommandLine;
import java.util.Arrays;
@@ -37,9 +36,6 @@
case "d8":
D8.main(shift(args));
break;
- case "dexfilemerger":
- DexFileMerger.main(shift(args));
- break;
case "dexsegments":
DexSegments.main(shift(args));
break;
diff --git a/src/main/java/com/android/tools/r8/dexfilemerger/DexFileMerger.java b/src/main/java/com/android/tools/r8/dexfilemerger/DexFileMerger.java
deleted file mode 100644
index 7cbf816..0000000
--- a/src/main/java/com/android/tools/r8/dexfilemerger/DexFileMerger.java
+++ /dev/null
@@ -1,403 +0,0 @@
-// Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-package com.android.tools.r8.dexfilemerger;
-
-import com.android.tools.r8.ByteDataView;
-import com.android.tools.r8.CompilationFailedException;
-import com.android.tools.r8.D8Command;
-import com.android.tools.r8.DexFileMergerHelper;
-import com.android.tools.r8.DexIndexedConsumer;
-import com.android.tools.r8.DiagnosticsHandler;
-import com.android.tools.r8.errors.Unreachable;
-import com.android.tools.r8.origin.Origin;
-import com.android.tools.r8.origin.PathOrigin;
-import com.android.tools.r8.utils.ExceptionDiagnostic;
-import com.android.tools.r8.utils.FileUtils;
-import com.android.tools.r8.utils.OptionsParsing;
-import com.android.tools.r8.utils.OptionsParsing.ParseContext;
-import com.android.tools.r8.utils.StringDiagnostic;
-import com.android.tools.r8.utils.ZipUtils;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-public class DexFileMerger {
- /** File name prefix of a {@code .dex} file automatically loaded in an archive. */
- private static final String DEX_PREFIX = "classes";
-
- private static final String DEFAULT_OUTPUT_ARCHIVE_FILENAME = "classes.dex.jar";
-
- private static final boolean PRINT_ARGS = false;
-
- /** Strategies for outputting multiple {@code .dex} files supported by {@link DexFileMerger}. */
- private enum MultidexStrategy {
- /** Create exactly one .dex file. The operation will fail if .dex limits are exceeded. */
- OFF,
- /** Create exactly one <prefixN>.dex file with N taken from the (single) input archive. */
- GIVEN_SHARD,
- /**
- * Assemble .dex files similar to {@link com.android.dx.command.dexer.Main dx}, with all but one
- * file as large as possible.
- */
- MINIMAL,
- /**
- * Allow some leeway and sometimes use additional .dex files to speed up processing. This option
- * exists to give flexibility but it often (or always) may be identical to {@link #MINIMAL}.
- */
- BEST_EFFORT;
-
- public boolean isMultidexAllowed() {
- switch (this) {
- case OFF:
- case GIVEN_SHARD:
- return false;
- case MINIMAL:
- case BEST_EFFORT:
- return true;
- }
- throw new AssertionError("Unknown: " + this);
- }
-
- public static MultidexStrategy parse(String value) {
- switch (value) {
- case "off":
- return OFF;
- case "given_shard":
- return GIVEN_SHARD;
- case "minimal":
- return MINIMAL;
- case "best_effort":
- return BEST_EFFORT;
- default:
- throw new RuntimeException(
- "Multidex argument must be either 'off', 'given_shard', 'minimal' or 'best_effort'.");
- }
- }
- }
-
- private static class Options {
- List<String> inputArchives = new ArrayList<>();
- String outputArchive = DEFAULT_OUTPUT_ARCHIVE_FILENAME;
- MultidexStrategy multidexMode = MultidexStrategy.OFF;
- String mainDexListFile = null;
- boolean minimalMainDex = false;
- boolean verbose = false;
- String dexPrefix = DEX_PREFIX;
- }
-
-
- private static Options parseArguments(String[] args) throws IOException {
- // We may have a single argument which is a parameter file path, prefixed with '@'.
- if (args.length == 1 && args[0].startsWith("@")) {
- // TODO(tamaskenez) Implement more sophisticated processing
- // which is aligned with Blaze's
- // com.google.devtools.common.options.ShellQuotedParamsFilePreProcessor
- Path paramsFile = Paths.get(args[0].substring(1));
- List<String> argsList = new ArrayList<>();
- for (String s : Files.readAllLines(paramsFile)) {
- s = s.trim();
- if (s.isEmpty()) {
- continue;
- }
- // Trim optional enclosing single quotes. Unescaping omitted for now.
- if (s.length() >= 2 && s.startsWith("'") && s.endsWith("'")) {
- s = s.substring(1, s.length() - 1);
- }
- argsList.add(s);
- }
- args = argsList.toArray(new String[argsList.size()]);
- }
-
- Options options = new Options();
- ParseContext context = new ParseContext(args);
- List<String> strings;
- String string;
- Boolean b;
- while (context.head() != null) {
- if (context.head().startsWith("@")) {
- throw new RuntimeException("A params file must be the only argument: " + context.head());
- }
- strings = OptionsParsing.tryParseMulti(context, "--input");
- if (strings != null) {
- options.inputArchives.addAll(strings);
- continue;
- }
- string = OptionsParsing.tryParseSingle(context, "--output", "-o");
- if (string != null) {
- options.outputArchive = string;
- continue;
- }
- string = OptionsParsing.tryParseSingle(context, "--multidex", null);
- if (string != null) {
- options.multidexMode = MultidexStrategy.parse(string);
- continue;
- }
- string = OptionsParsing.tryParseSingle(context, "--main-dex-list", null);
- if (string != null) {
- options.mainDexListFile = string;
- continue;
- }
- b = OptionsParsing.tryParseBoolean(context, "--minimal-main-dex");
- if (b != null) {
- options.minimalMainDex = b;
- continue;
- }
- b = OptionsParsing.tryParseBoolean(context, "--verbose");
- if (b != null) {
- options.verbose = b;
- continue;
- }
- string = OptionsParsing.tryParseSingle(context, "--max-bytes-wasted-per-file", null);
- if (string != null) {
- System.err.println("Warning: '--max-bytes-wasted-per-file' is ignored.");
- continue;
- }
- string = OptionsParsing.tryParseSingle(context, "--set-max-idx-number", null);
- if (string != null) {
- System.err.println("Warning: The '--set-max-idx-number' option is ignored.");
- continue;
- }
- b = OptionsParsing.tryParseBoolean(context, "--forceJumbo");
- if (b != null) {
- System.err.println(
- "Warning: '--forceJumbo' can be safely omitted. Strings will only use "
- + "jumbo-string indexing if necessary.");
- continue;
- }
- string = OptionsParsing.tryParseSingle(context, "--dex_prefix", null);
- if (string != null) {
- options.dexPrefix = string;
- continue;
- }
- throw new RuntimeException(String.format("Unknown options: '%s'.", context.head()));
- }
- return options;
- }
-
- /**
- * Implements a DexIndexedConsumer writing into a ZipStream with support for custom dex file name
- * prefix, reindexing a single dex output file to a nonzero index and reporting if any data has
- * been written.
- */
- private static class ArchiveConsumer implements DexIndexedConsumer {
- private final Path path;
- private final String prefix;
- private final Integer singleFixedFileIndex;
- private final Origin origin;
- private ZipOutputStream stream = null;
-
- private int highestIndexWritten = -1;
- private final Map<Integer, Runnable> writers = new TreeMap<>();
- private boolean hasWrittenSomething = false;
-
- /** If singleFixedFileIndex is not null then we expect only one output dex file */
- private ArchiveConsumer(Path path, String prefix, Integer singleFixedFileIndex) {
- this.path = path;
- this.prefix = prefix;
- this.singleFixedFileIndex = singleFixedFileIndex;
- this.origin = new PathOrigin(path);
- }
-
- private boolean hasWrittenSomething() {
- return hasWrittenSomething;
- }
-
- private String getDexFileName(int fileIndex) {
- if (singleFixedFileIndex != null) {
- fileIndex = singleFixedFileIndex;
- }
- return prefix + (fileIndex == 0 ? "" : (fileIndex + 1)) + FileUtils.DEX_EXTENSION;
- }
-
- @Override
- public synchronized void accept(
- int fileIndex, ByteDataView data, Set<String> descriptors, DiagnosticsHandler handler) {
- if (singleFixedFileIndex != null && fileIndex != 0) {
- handler.error(new StringDiagnostic("Result does not fit into a single dex file."));
- return;
- }
- // Make a copy of the actual bytes as they will possibly be accessed later by the runner.
- final byte[] bytes = data.copyByteData();
- writers.put(fileIndex, () -> writeEntry(fileIndex, bytes, descriptors, handler));
-
- while (writers.containsKey(highestIndexWritten + 1)) {
- ++highestIndexWritten;
- writers.get(highestIndexWritten).run();
- writers.remove(highestIndexWritten);
- }
- }
-
- /** Get or open the zip output stream. */
- private synchronized ZipOutputStream getStream(DiagnosticsHandler handler) {
- if (stream == null) {
- try {
- stream =
- new ZipOutputStream(
- Files.newOutputStream(
- path, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING));
- } catch (IOException e) {
- handler.error(new ExceptionDiagnostic(e, origin));
- }
- }
- return stream;
- }
-
- private void writeEntry(
- int fileIndex, byte[] data, Set<String> descriptors, DiagnosticsHandler handler) {
- try {
- ZipUtils.writeToZipStream(
- getStream(handler),
- getDexFileName(fileIndex),
- ByteDataView.of(data),
- ZipEntry.DEFLATED);
- hasWrittenSomething = true;
- } catch (IOException e) {
- handler.error(new ExceptionDiagnostic(e, origin));
- }
- }
-
- @Override
- public void finished(DiagnosticsHandler handler) {
- if (!writers.isEmpty()) {
- handler.error(
- new StringDiagnostic(
- "Failed to write zip, for a multidex output some of the classes.dex files were"
- + " not produced."));
- }
- try {
- if (stream != null) {
- stream.close();
- stream = null;
- }
- } catch (IOException e) {
- handler.error(new ExceptionDiagnostic(e, origin));
- }
- }
- }
-
- private static int parseFileIndexFromShardFilename(String inputArchive) {
- Pattern namingPattern = Pattern.compile("([0-9]+)\\..*");
- String name = new File(inputArchive).getName();
- Matcher matcher = namingPattern.matcher(name);
- if (!matcher.matches()) {
- throw new RuntimeException(
- String.format(
- "Expect input named <N>.xxx.zip for --multidex=given_shard but got %s.", name));
- }
- int shard = Integer.parseInt(matcher.group(1));
- if (shard <= 0) {
- throw new RuntimeException(
- String.format("Expect positive N in input named <N>.xxx.zip but got %d.", shard));
- }
- return shard;
- }
-
- public static void run(String[] args) throws CompilationFailedException, IOException {
- Options options = parseArguments(args);
-
- if (options.inputArchives.isEmpty()) {
- throw new RuntimeException("Need at least one --input");
- }
-
- if (options.mainDexListFile != null && options.inputArchives.size() != 1) {
- throw new RuntimeException(
- "--main-dex-list only supported with exactly one --input, use DexFileSplitter for more");
- }
-
- if (!options.multidexMode.isMultidexAllowed()) {
- if (options.mainDexListFile != null) {
- throw new RuntimeException(
- "--main-dex-list is only supported with multidex enabled, but mode is: "
- + options.multidexMode.toString());
- }
- if (options.minimalMainDex) {
- throw new RuntimeException(
- "--minimal-main-dex is only supported with multidex enabled, but mode is: "
- + options.multidexMode.toString());
- }
- }
-
- D8Command.Builder builder = D8Command.builder();
-
- Map<String, Integer> inputOrdering = new HashMap<>(options.inputArchives.size());
- int sequenceNumber = 0;
- for (String s : options.inputArchives) {
- builder.addProgramFiles(Paths.get(s));
- inputOrdering.put(s, sequenceNumber++);
- }
-
- // Determine enabling multidexing and file indexing.
- Integer singleFixedFileIndex = null;
- switch (options.multidexMode) {
- case OFF:
- singleFixedFileIndex = 0;
- break;
- case GIVEN_SHARD:
- if (options.inputArchives.size() != 1) {
- throw new RuntimeException("'--multidex=given_shard' requires exactly one --input.");
- }
- singleFixedFileIndex = parseFileIndexFromShardFilename(options.inputArchives.get(0)) - 1;
- break;
- case MINIMAL:
- case BEST_EFFORT:
- // Nothing to do.
- break;
- default:
- throw new Unreachable("Unexpected enum: " + options.multidexMode);
- }
-
- if (options.mainDexListFile != null) {
- builder.addMainDexListFiles(Paths.get(options.mainDexListFile));
- }
-
- ArchiveConsumer consumer =
- new ArchiveConsumer(
- Paths.get(options.outputArchive), options.dexPrefix, singleFixedFileIndex);
- builder.setProgramConsumer(consumer);
-
- DexFileMergerHelper.run(builder.build(), options.minimalMainDex, inputOrdering);
-
- // If input was empty we still need to write out an empty zip.
- if (!consumer.hasWrittenSomething()) {
- File f = new File(options.outputArchive);
- ZipOutputStream out = new ZipOutputStream(new FileOutputStream(f));
- out.close();
- }
- }
-
- public static void main(String[] args) {
- try {
- if (PRINT_ARGS) {
- printArgs(args);
- }
- run(args);
- } catch (CompilationFailedException | IOException e) {
- System.err.println("Merge failed: " + e.getMessage());
- System.exit(1);
- }
- }
-
- private static void printArgs(String[] args) {
- System.err.print("r8.DexFileMerger");
- for (String s : args) {
- System.err.printf(" %s", s);
- }
- System.err.println("");
- }
-}
diff --git a/src/main/keep.txt b/src/main/keep.txt
index 083584f..7791d4e 100644
--- a/src/main/keep.txt
+++ b/src/main/keep.txt
@@ -8,7 +8,6 @@
-keep public class com.android.tools.r8.D8 { public static void main(java.lang.String[]); }
-keep public class com.android.tools.r8.R8 { public static void main(java.lang.String[]); }
-keep public class com.android.tools.r8.ExtractMarker { public static void main(java.lang.String[]); }
--keep public class com.android.tools.r8.dexfilemerger.DexFileMerger { public static void main(java.lang.String[]); }
-keep public class com.android.tools.r8.dexsplitter.DexSplitter { public static void main(java.lang.String[]); }
-keep public class com.android.tools.r8.Version { public static java.lang.String getVersionString(); }
diff --git a/src/test/java/com/android/tools/r8/dexfilemerger/DexFileMergerTests.java b/src/test/java/com/android/tools/r8/dexfilemerger/DexFileMergerTests.java
deleted file mode 100644
index 91364d4..0000000
--- a/src/test/java/com/android/tools/r8/dexfilemerger/DexFileMergerTests.java
+++ /dev/null
@@ -1,149 +0,0 @@
-// Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-package com.android.tools.r8.dexfilemerger;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import com.android.tools.r8.CompilationFailedException;
-import com.android.tools.r8.D8;
-import com.android.tools.r8.D8Command;
-import com.android.tools.r8.DexFileMergerHelper;
-import com.android.tools.r8.ExtractMarker;
-import com.android.tools.r8.OutputMode;
-import com.android.tools.r8.ResourceException;
-import com.android.tools.r8.TestBase;
-import com.android.tools.r8.ToolHelper;
-import com.android.tools.r8.ToolHelper.ArtCommandBuilder;
-import com.android.tools.r8.dex.Constants;
-import com.android.tools.r8.dex.Marker;
-import com.android.tools.r8.maindexlist.MainDexListTests;
-import com.android.tools.r8.utils.AndroidApiLevel;
-import com.android.tools.r8.utils.AndroidApp;
-import com.google.common.collect.ImmutableList;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collection;
-import java.util.concurrent.ExecutionException;
-import org.junit.Test;
-
-public class DexFileMergerTests extends TestBase {
-
- private static final String CLASS_DIR = ToolHelper.EXAMPLES_BUILD_DIR + "classes/dexmergesample";
- private static final String CLASS1_CLASS = CLASS_DIR + "/Class1.class";
- private static final String CLASS2_CLASS = CLASS_DIR + "/Class2.class";
- private static final int MAX_METHOD_COUNT = Constants.U16BIT_MAX;
-
- private Path createMergerInputWithTwoClasses(OutputMode outputMode, boolean addMarker)
- throws CompilationFailedException, IOException {
- // Compile Class1 and Class2
- Path mergerInputZip = temp.newFolder().toPath().resolve("merger-input.zip");
- D8Command command =
- D8Command.builder()
- .setOutput(mergerInputZip, outputMode)
- .addProgramFiles(Paths.get(CLASS1_CLASS))
- .addProgramFiles(Paths.get(CLASS2_CLASS))
- .build();
-
- DexFileMergerHelper.runD8ForTesting(command, !addMarker);
-
- return mergerInputZip;
- }
-
- private void testMarker(boolean addMarkerToInput)
- throws CompilationFailedException, IOException, ResourceException, ExecutionException {
- Path mergerInputZip = createMergerInputWithTwoClasses(OutputMode.DexIndexed, addMarkerToInput);
- int expectedNumberOfMarkers = addMarkerToInput ? 1 : 0;
-
- Collection<Marker> inputMarkers = ExtractMarker.extractMarkerFromDexFile(mergerInputZip);
- assertEquals(expectedNumberOfMarkers, inputMarkers.size());
-
- // Test that the DexFileMerger preserves markers.
- Path mergerOutputZip = temp.getRoot().toPath().resolve("merger-out.zip");
- DexFileMerger.main(
- new String[] {
- "--input", mergerInputZip.toString(), "--output", mergerOutputZip.toString()
- });
- Collection<Marker> outputMarkers = ExtractMarker.extractMarkerFromDexFile(mergerOutputZip);
- assertEquals(expectedNumberOfMarkers, outputMarkers.size());
-
- // Test that D8 when used for merging preserves markers.
- D8.main(new String[] { mergerInputZip.toString(), "--output", mergerOutputZip.toString() });
- Collection<Marker> d8OutputMarkers = ExtractMarker.extractMarkerFromDexFile((mergerOutputZip));
- assertEquals(expectedNumberOfMarkers, d8OutputMarkers.size());
- }
-
- @Test
- public void testMarkerPreserved()
- throws CompilationFailedException, IOException, ResourceException, ExecutionException {
- testMarker(true);
- }
-
- @Test
- public void testMarkerNotAdded()
- throws CompilationFailedException, IOException, ResourceException, ExecutionException {
- testMarker(false);
- }
-
- @Test
- public void mergeTwoFiles() throws CompilationFailedException, IOException {
- Path mergerInputZip = createMergerInputWithTwoClasses(OutputMode.DexFilePerClassFile, false);
-
- Path mergerOutputZip = temp.getRoot().toPath().resolve("merger-out.zip");
- DexFileMerger.main(
- new String[] {
- "--input", mergerInputZip.toString(), "--output", mergerOutputZip.toString()
- });
-
- // Test by running methods of Class1 and Class2
- for (String className : new String[] {"Class1", "Class2"}) {
- ArtCommandBuilder builder = new ArtCommandBuilder();
- builder.appendClasspath(mergerOutputZip.toString());
- builder.setMainClass("dexmergesample." + className);
- String out = ToolHelper.runArt(builder);
- assertEquals(out, className + "\n");
- }
- }
-
- private void generateClassesAndTest(int extraMethodCount, int programResourcesSize)
- throws IOException, ExecutionException, CompilationFailedException {
- AndroidApp generatedApp =
- MainDexListTests.generateApplication(
- ImmutableList.of("A", "B"),
- AndroidApiLevel.N.getLevel(),
- MAX_METHOD_COUNT / 2 + 1 + extraMethodCount);
- Path appDir = temp.newFolder().toPath().resolve("merger-input.zip");
- assertEquals(programResourcesSize, generatedApp.getDexProgramResourcesForTesting().size());
- generatedApp.write(appDir, OutputMode.DexIndexed);
-
- Path outZip = temp.getRoot().toPath().resolve("out.zip");
- DexFileMerger.run(
- new String[] {
- "--input", appDir.toString(), "--output", outZip.toString(), "--multidex=off"
- });
- }
-
- @Test
- public void failIfTooBig() throws IOException, ExecutionException {
- // Generates an application with two classes, each with the number of methods just enough not to
- // fit into a single dex file.
- try {
- generateClassesAndTest(1, 2);
- fail("Expect to fail");
- } catch (CompilationFailedException e) {
- assertThat(e.getCause().getMessage(), containsString("does not fit into a single dex file"));
- }
- }
-
- @Test
- public void failIfTooBigControl()
- throws IOException, ExecutionException, CompilationFailedException {
- // Control test for failIfTooBig to make sure we don't fail with less methods.
- generateClassesAndTest(0, 1);
- }
-}
