Revert "Remove DexSplitter from codebase" This reverts commit 908f1f58db9b9aa30e228cbd2816e655774d62f4. Reason for revert: old code in google3 is relying on this, filed b/230692867 to get that removed Bug: 230692867 Bug: 155577610 Change-Id: I9c21e43fc8c7648681f92f9e68236f09afea9b01
diff --git a/src/main/java/com/android/tools/r8/DexSplitterHelper.java b/src/main/java/com/android/tools/r8/DexSplitterHelper.java new file mode 100644 index 0000000..a7ac9e3 --- /dev/null +++ b/src/main/java/com/android/tools/r8/DexSplitterHelper.java
@@ -0,0 +1,155 @@ +// Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +package com.android.tools.r8; + +import static com.android.tools.r8.utils.ExceptionUtils.unwrapExecutionException; + +import com.android.tools.r8.DexIndexedConsumer.DirectoryConsumer; +import com.android.tools.r8.dex.ApplicationReader; +import com.android.tools.r8.dex.ApplicationWriter; +import com.android.tools.r8.dex.Marker; +import com.android.tools.r8.graph.AppInfo; +import com.android.tools.r8.graph.AppView; +import com.android.tools.r8.graph.DexApplication; +import com.android.tools.r8.graph.DexProgramClass; +import com.android.tools.r8.graph.LazyLoadedDexApplication; +import com.android.tools.r8.naming.ClassNameMapper; +import com.android.tools.r8.naming.NamingLens; +import com.android.tools.r8.shaking.MainDexInfo; +import com.android.tools.r8.utils.ExceptionUtils; +import com.android.tools.r8.utils.FeatureClassMapping; +import com.android.tools.r8.utils.FeatureClassMapping.FeatureMappingException; +import com.android.tools.r8.utils.InternalOptions; +import com.android.tools.r8.utils.InternalOptions.DesugarState; +import com.android.tools.r8.utils.ThreadUtils; +import com.android.tools.r8.utils.Timing; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; + +@Keep +public final class DexSplitterHelper { + + public static void run( + D8Command command, FeatureClassMapping featureClassMapping, String output, String proguardMap) + throws CompilationFailedException { + ExecutorService executor = ThreadUtils.getExecutorService(ThreadUtils.NOT_SPECIFIED); + try { + ExceptionUtils.withCompilationHandler( + command.getReporter(), + () -> run(command, featureClassMapping, output, proguardMap, executor)); + } finally { + executor.shutdown(); + } + } + + public static void run( + D8Command command, + FeatureClassMapping featureClassMapping, + String output, + String proguardMap, + ExecutorService executor) + throws IOException { + InternalOptions options = command.getInternalOptions(); + options.desugarState = DesugarState.OFF; + options.enableMainDexListCheck = false; + options.ignoreMainDexMissingClasses = true; + options.minimalMainDex = false; + assert !options.isMinifying(); + options.inlinerOptions().enableInlining = false; + options.outline.enabled = false; + + try { + Timing timing = new Timing("DexSplitter"); + ApplicationReader applicationReader = + new ApplicationReader(command.getInputApp(), options, timing); + DexApplication app = applicationReader.read(executor); + MainDexInfo mainDexInfo = applicationReader.readMainDexClasses(app); + + List<Marker> markers = app.dexItemFactory.extractMarkers(); + + ClassNameMapper mapper = null; + if (proguardMap != null) { + mapper = ClassNameMapper.mapperFromFile(Paths.get(proguardMap)); + } + Map<String, LazyLoadedDexApplication.Builder> applications = + getDistribution(app, featureClassMapping, mapper); + for (Entry<String, LazyLoadedDexApplication.Builder> entry : applications.entrySet()) { + String feature = entry.getKey(); + timing.begin("Feature " + feature); + DexApplication featureApp = entry.getValue().build(); + assert !options.hasMethodsFilter(); + + // If this is the base, we add the main dex list. + AppInfo appInfo = + feature.equals(featureClassMapping.getBaseName()) + ? AppInfo.createInitialAppInfo(featureApp, mainDexInfo) + : AppInfo.createInitialAppInfo(featureApp); + AppView<AppInfo> appView = AppView.createForD8(appInfo); + + // Run d8 optimize to ensure jumbo strings are handled. + D8.optimize(appView, options, timing, executor); + + // We create a specific consumer for each split. + Path outputDir = Paths.get(output).resolve(entry.getKey()); + if (!Files.exists(outputDir)) { + Files.createDirectory(outputDir); + } + DexIndexedConsumer consumer = new DirectoryConsumer(outputDir); + + try { + new ApplicationWriter( + appView, + markers, + NamingLens.getIdentityLens(), + consumer) + .write(executor); + options.printWarnings(); + } finally { + consumer.finished(options.reporter); + } + timing.end(); + } + } catch (ExecutionException e) { + throw unwrapExecutionException(e); + } catch (FeatureMappingException e) { + options.reporter.error(e.getMessage()); + } finally { + options.signalFinishedToConsumers(); + } + } + + private static Map<String, LazyLoadedDexApplication.Builder> getDistribution( + DexApplication app, FeatureClassMapping featureClassMapping, ClassNameMapper mapper) + throws FeatureMappingException { + Map<String, LazyLoadedDexApplication.Builder> applications = new HashMap<>(); + for (DexProgramClass clazz : app.classes()) { + String clazzName = + mapper != null ? mapper.deobfuscateClassName(clazz.toString()) : clazz.toString(); + String feature = featureClassMapping.featureForClass(clazzName); + LazyLoadedDexApplication.Builder featureApplication = applications.get(feature); + if (featureApplication == null) { + featureApplication = DexApplication.builder(app.options, app.timing); + applications.put(feature, featureApplication); + } + featureApplication.addProgramClass(clazz); + } + return applications; + } + + public static void runD8ForTesting(D8Command command, boolean dontCreateMarkerInD8) + throws CompilationFailedException { + InternalOptions options = command.getInternalOptions(); + options.testing.dontCreateMarkerInD8 = dontCreateMarkerInD8; + D8.runForTesting(command.getInputApp(), options); + } +}
diff --git a/src/main/java/com/android/tools/r8/SwissArmyKnife.java b/src/main/java/com/android/tools/r8/SwissArmyKnife.java index 2e44e14..2d836ae 100644 --- a/src/main/java/com/android/tools/r8/SwissArmyKnife.java +++ b/src/main/java/com/android/tools/r8/SwissArmyKnife.java
@@ -6,6 +6,7 @@ import com.android.tools.r8.bisect.Bisect; import com.android.tools.r8.cf.CfVerifierTool; import com.android.tools.r8.compatproguard.CompatProguard; +import com.android.tools.r8.dexsplitter.DexSplitter; import com.android.tools.r8.relocator.RelocatorCommandLine; import com.android.tools.r8.tracereferences.TraceReferences; import java.util.Arrays; @@ -40,6 +41,9 @@ case "dexsegments": DexSegments.main(shift(args)); break; + case "dexsplitter": + DexSplitter.main(shift(args)); + break; case "disasm": Disassemble.main(shift(args)); break;
diff --git a/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java b/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java new file mode 100644 index 0000000..4ee416c --- /dev/null +++ b/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
@@ -0,0 +1,377 @@ +// Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +package com.android.tools.r8.dexsplitter; + +import com.android.tools.r8.CompilationFailedException; +import com.android.tools.r8.D8Command; +import com.android.tools.r8.DexIndexedConsumer; +import com.android.tools.r8.DexSplitterHelper; +import com.android.tools.r8.Diagnostic; +import com.android.tools.r8.DiagnosticsHandler; +import com.android.tools.r8.Keep; +import com.android.tools.r8.origin.PathOrigin; +import com.android.tools.r8.utils.AbortException; +import com.android.tools.r8.utils.ExceptionDiagnostic; +import com.android.tools.r8.utils.ExceptionUtils; +import com.android.tools.r8.utils.FeatureClassMapping; +import com.android.tools.r8.utils.FeatureClassMapping.FeatureMappingException; +import com.android.tools.r8.utils.OptionsParsing; +import com.android.tools.r8.utils.OptionsParsing.ParseContext; +import com.android.tools.r8.utils.StringDiagnostic; +import com.android.tools.r8.utils.ZipUtils; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +@Keep +public final class DexSplitter { + + private static final String DEFAULT_OUTPUT_DIR = "output"; + private static final String DEFAULT_BASE_NAME = "base"; + + private static final boolean PRINT_ARGS = false; + + public static class FeatureJar { + private String jar; + private String outputName; + + public FeatureJar(String jar, String outputName) { + this.jar = jar; + this.outputName = outputName; + } + + public FeatureJar(String jar) { + this(jar, featureNameFromJar(jar)); + } + + public String getJar() { + return jar; + } + + public String getOutputName() { + return outputName; + } + + private static String featureNameFromJar(String jar) { + Path jarPath = Paths.get(jar); + String featureName = jarPath.getFileName().toString(); + if (featureName.endsWith(".jar") || featureName.endsWith(".zip")) { + featureName = featureName.substring(0, featureName.length() - 4); + } + return featureName; + } + } + + private static class ZipFileOrigin extends PathOrigin { + + public ZipFileOrigin(Path path) { + super(path); + } + + @Override + public String part() { + return "splitting of file '" + super.part() + "'"; + } + } + + @Keep + public static final class Options { + private final DiagnosticsHandler diagnosticsHandler; + private List<String> inputArchives = new ArrayList<>(); + private List<FeatureJar> featureJars = new ArrayList<>(); + private List<String> baseJars = new ArrayList<>(); + private String baseOutputName = DEFAULT_BASE_NAME; + private String output = DEFAULT_OUTPUT_DIR; + private String featureSplitMapping; + private String proguardMap; + private String mainDexList; + private boolean splitNonClassResources = false; + + public Options() { + this(new DiagnosticsHandler() {}); + } + + public Options(DiagnosticsHandler diagnosticsHandler) { + this.diagnosticsHandler = diagnosticsHandler; + } + + public DiagnosticsHandler getDiagnosticsHandler() { + return diagnosticsHandler; + } + + public String getMainDexList() { + return mainDexList; + } + + public void setMainDexList(String mainDexList) { + this.mainDexList = mainDexList; + } + + public String getOutput() { + return output; + } + + public void setOutput(String output) { + this.output = output; + } + + public String getFeatureSplitMapping() { + return featureSplitMapping; + } + + public void setFeatureSplitMapping(String featureSplitMapping) { + this.featureSplitMapping = featureSplitMapping; + } + + public String getProguardMap() { + return proguardMap; + } + + public void setProguardMap(String proguardMap) { + this.proguardMap = proguardMap; + } + + public String getBaseOutputName() { + return baseOutputName; + } + + public void setBaseOutputName(String baseOutputName) { + this.baseOutputName = baseOutputName; + } + + public void addInputArchive(String inputArchive) { + inputArchives.add(inputArchive); + } + + public void addBaseJar(String baseJar) { + baseJars.add(baseJar); + } + + private void addFeatureJar(FeatureJar featureJar) { + featureJars.add(featureJar); + } + + public void addFeatureJar(String jar) { + featureJars.add(new FeatureJar(jar)); + } + + public void addFeatureJar(String jar, String outputName) { + featureJars.add(new FeatureJar(jar, outputName)); + } + + public void setSplitNonClassResources(boolean value) { + splitNonClassResources = value; + } + + public ImmutableList<String> getInputArchives() { + return ImmutableList.copyOf(inputArchives); + } + + ImmutableList<FeatureJar> getFeatureJars() { + return ImmutableList.copyOf(featureJars); + } + + ImmutableList<String> getBaseJars() { + return ImmutableList.copyOf(baseJars); + } + + // Shorthand error messages. + public Diagnostic error(String msg) { + StringDiagnostic error = new StringDiagnostic(msg); + diagnosticsHandler.error(error); + return error; + } + } + + /** + * Parse a feature jar argument and return the corresponding FeatureJar representation. + * Default to use the name of the jar file if the argument contains no ':', if the argument + * contains ':', then use the value after the ':' as the name. + * @param argument + */ + private static FeatureJar parseFeatureJarArgument(String argument) { + if (argument.contains(":")) { + String[] parts = argument.split(":"); + if (parts.length > 2) { + throw new RuntimeException("--feature-jar argument contains more than one :"); + } + return new FeatureJar(parts[0], parts[1]); + } + return new FeatureJar(argument); + } + + private static Options parseArguments(String[] args) { + Options options = new Options(); + ParseContext context = new ParseContext(args); + while (context.head() != null) { + List<String> inputs = OptionsParsing.tryParseMulti(context, "--input"); + if (inputs != null) { + inputs.forEach(options::addInputArchive); + continue; + } + List<String> featureJars = OptionsParsing.tryParseMulti(context, "--feature-jar"); + if (featureJars != null) { + featureJars.forEach((feature) -> options.addFeatureJar(parseFeatureJarArgument(feature))); + continue; + } + List<String> baseJars = OptionsParsing.tryParseMulti(context, "--base-jar"); + if (baseJars != null) { + baseJars.forEach(options::addBaseJar); + continue; + } + String output = OptionsParsing.tryParseSingle(context, "--output", "-o"); + if (output != null) { + options.setOutput(output); + continue; + } + + String mainDexList= OptionsParsing.tryParseSingle(context, "--main-dex-list", null); + if (mainDexList!= null) { + options.setMainDexList(mainDexList); + continue; + } + + String proguardMap = OptionsParsing.tryParseSingle(context, "--proguard-map", null); + if (proguardMap != null) { + options.setProguardMap(proguardMap); + continue; + } + String baseOutputName = OptionsParsing.tryParseSingle(context, "--base-output-name", null); + if (baseOutputName != null) { + options.setBaseOutputName(baseOutputName); + continue; + } + String featureSplit = OptionsParsing.tryParseSingle(context, "--feature-splits", null); + if (featureSplit != null) { + options.setFeatureSplitMapping(featureSplit); + continue; + } + Boolean b = OptionsParsing.tryParseBoolean(context, "--split-non-class-resources"); + if (b != null) { + options.setSplitNonClassResources(b); + continue; + } + throw new RuntimeException(String.format("Unknown options: '%s'.", context.head())); + } + return options; + } + + private static FeatureClassMapping createFeatureClassMapping(Options options) + throws FeatureMappingException { + if (options.getFeatureSplitMapping() != null) { + return FeatureClassMapping.fromSpecification( + Paths.get(options.getFeatureSplitMapping()), options.getDiagnosticsHandler()); + } + assert !options.getFeatureJars().isEmpty(); + return FeatureClassMapping.Internal.fromJarFiles(options.getFeatureJars(), + options.getBaseJars(), options.getBaseOutputName(), options.getDiagnosticsHandler()); + } + + private static void run(String[] args) + throws CompilationFailedException, FeatureMappingException { + Options options = parseArguments(args); + run(options); + } + + public static void run(Options options) + throws FeatureMappingException, CompilationFailedException { + Diagnostic error = null; + if (options.getInputArchives().isEmpty()) { + error = options.error("Need at least one --input"); + } + if (options.getFeatureSplitMapping() == null && options.getFeatureJars().isEmpty()) { + error = options.error("You must supply a feature split mapping or feature jars"); + } + if (options.getFeatureSplitMapping() != null && !options.getFeatureJars().isEmpty()) { + error = options.error("You can't supply both a feature split mapping and feature jars"); + } + if (error != null) { + throw new AbortException(error); + } + + D8Command.Builder builder = D8Command.builder(options.diagnosticsHandler); + + + for (String s : options.inputArchives) { + builder.addProgramFiles(Paths.get(s)); + } + // We set the actual consumer on the ApplicationWriter when we have calculated the distribution + // since we don't yet know the distribution. + builder.setProgramConsumer(DexIndexedConsumer.emptyConsumer()); + if (options.getMainDexList() != null) { + builder.addMainDexListFiles(Paths.get(options.getMainDexList())); + } + + FeatureClassMapping featureClassMapping = createFeatureClassMapping(options); + + DexSplitterHelper.run( + builder.build(), featureClassMapping, options.getOutput(), options.getProguardMap()); + + if (options.splitNonClassResources) { + splitNonClassResources(options, featureClassMapping); + } + } + + private static void splitNonClassResources(Options options, + FeatureClassMapping featureClassMapping) { + for (String s : options.inputArchives) { + try (ZipFile zipFile = new ZipFile(s, StandardCharsets.UTF_8)) { + Enumeration<? extends ZipEntry> entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + String name = entry.getName(); + if (!ZipUtils.isDexFile(name) && !ZipUtils.isClassFile(name)) { + String feature = featureClassMapping.featureForNonClass(name); + Path outputDir = Paths.get(options.getOutput()).resolve(feature); + try (InputStream stream = zipFile.getInputStream(entry)) { + Path outputFile = outputDir.resolve(name); + Path parent = outputFile.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Files.copy(stream, outputFile); + } + } + } + } catch (IOException e) { + ExceptionDiagnostic error = new ExceptionDiagnostic(e, new ZipFileOrigin(Paths.get(s))); + options.getDiagnosticsHandler().error(error); + throw new AbortException(error); + } + } + } + + public static void main(String[] args) { + if (PRINT_ARGS) { + printArgs(args); + } + ExceptionUtils.withMainProgramHandler( + () -> { + try { + run(args); + } catch (FeatureMappingException e) { + // TODO(ricow): Report feature mapping errors via the reporter. + throw new RuntimeException("Splitting failed: " + e.getMessage()); + } + }); + } + + private static void printArgs(String[] args) { + System.err.printf("r8.DexSplitter"); + for (String s : args) { + System.err.printf(" %s", s); + } + System.err.println(""); + } +}
diff --git a/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java b/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java new file mode 100644 index 0000000..00412d9 --- /dev/null +++ b/src/main/java/com/android/tools/r8/utils/FeatureClassMapping.java
@@ -0,0 +1,313 @@ +// Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +package com.android.tools.r8.utils; + +import com.android.tools.r8.ArchiveClassFileProvider; +import com.android.tools.r8.DiagnosticsHandler; +import com.android.tools.r8.Keep; +import com.android.tools.r8.dexsplitter.DexSplitter.FeatureJar; +import com.android.tools.r8.origin.PathOrigin; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Provides a mappings of classes to modules. The structure of the input file is as follows: + * packageOrClass:module + * + * <p>Lines with a # prefix are ignored. + * + * <p>We will do most specific matching, i.e., + * <pre> + * com.google.foobar.*:feature2 + * com.google.*:base + * </pre> + * will put everything in the com.google namespace into base, except classes in com.google.foobar + * that will go to feature2. Class based mappings takes precedence over packages (since they are + * more specific): + * <pre> + * com.google.A:feature2 + * com.google.*:base + * </pre> + * Puts A into feature2, and all other classes from com.google into base. + * + * <p>Note that this format does not allow specifying inter-module dependencies, this is simply a + * placement tool. + */ +@Keep +public final class FeatureClassMapping { + + Map<String, String> parsedRules = new HashMap<>(); // Already parsed rules. + Map<String, String> parseNonClassRules = new HashMap<>(); + boolean usesOnlyExactMappings = true; + + Set<FeaturePredicate> mappings = new HashSet<>(); + + Path mappingFile; + String baseName = DEFAULT_BASE_NAME; + + static final String DEFAULT_BASE_NAME = "base"; + + static final String COMMENT = "#"; + static final String SEPARATOR = ":"; + + public String getBaseName() { + return baseName; + } + + private static class SpecificationOrigin extends PathOrigin { + + public SpecificationOrigin(Path path) { + super(path); + } + + @Override + public String part() { + return "specification file '" + super.part() + "'"; + } + } + + private static class JarFileOrigin extends PathOrigin { + + public JarFileOrigin(Path path) { + super(path); + } + + @Override + public String part() { + return "jar file '" + super.part() + "'"; + } + } + + public static FeatureClassMapping fromSpecification(Path file) throws FeatureMappingException { + return fromSpecification(file, new DiagnosticsHandler() {}); + } + + public static FeatureClassMapping fromSpecification(Path file, DiagnosticsHandler reporter) + throws FeatureMappingException { + FeatureClassMapping mapping = new FeatureClassMapping(); + List<String> lines = null; + try { + lines = FileUtils.readAllLines(file); + } catch (IOException e) { + ExceptionDiagnostic error = new ExceptionDiagnostic(e, new SpecificationOrigin(file)); + reporter.error(error); + throw new AbortException(error); + } + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + mapping.parseAndAdd(line, i); + } + return mapping; + } + + public static class Internal { + private static List<String> getClassFileDescriptors(String jar, DiagnosticsHandler reporter) { + Path jarPath = Paths.get(jar); + try { + return new ArchiveClassFileProvider(jarPath).getClassDescriptors() + .stream() + .map(DescriptorUtils::descriptorToJavaType) + .collect(Collectors.toList()); + } catch (IOException e) { + ExceptionDiagnostic error = new ExceptionDiagnostic(e, new JarFileOrigin(jarPath)); + reporter.error(error); + throw new AbortException(error); + } + } + + private static List<String> getNonClassFiles(String jar, DiagnosticsHandler reporter) { + try (ZipFile zipfile = new ZipFile(jar, StandardCharsets.UTF_8)) { + return zipfile.stream() + .filter(entry -> !ZipUtils.isClassFile(entry.getName())) + .map(ZipEntry::getName) + .collect(Collectors.toList()); + } catch (IOException e) { + ExceptionDiagnostic error = new ExceptionDiagnostic(e, new JarFileOrigin(Paths.get(jar))); + reporter.error(error); + throw new AbortException(error); + } + } + + public static FeatureClassMapping fromJarFiles( + List<FeatureJar> featureJars, List<String> baseJars, String baseName, + DiagnosticsHandler reporter) + throws FeatureMappingException { + FeatureClassMapping mapping = new FeatureClassMapping(); + if (baseName != null) { + mapping.baseName = baseName; + } + for (FeatureJar featureJar : featureJars) { + for (String javaType : getClassFileDescriptors(featureJar.getJar(), reporter)) { + mapping.addMapping(javaType, featureJar.getOutputName()); + } + for (String nonClass : getNonClassFiles(featureJar.getJar(), reporter)) { + mapping.addNonClassMapping(nonClass, featureJar.getOutputName()); + } + } + for (String baseJar : baseJars) { + for (String javaType : getClassFileDescriptors(baseJar, reporter)) { + mapping.addBaseMapping(javaType); + } + for (String nonClass : getNonClassFiles(baseJar, reporter)) { + mapping.addBaseNonClassMapping(nonClass); + } + } + assert mapping.usesOnlyExactMappings; + return mapping; + } + + } + + private FeatureClassMapping() {} + + public void addBaseMapping(String clazz) throws FeatureMappingException { + addMapping(clazz, baseName); + } + + public void addBaseNonClassMapping(String name) { + addNonClassMapping(name, baseName); + } + + public void addMapping(String clazz, String feature) throws FeatureMappingException { + addRule(clazz, feature, 0); + } + + public void addNonClassMapping(String name, String feature) { + // If a non-class file is present in multiple features put the resource in the base. + parseNonClassRules.put(name, parseNonClassRules.containsKey(name) ? baseName : feature); + } + + FeatureClassMapping(List<String> lines) throws FeatureMappingException { + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + parseAndAdd(line, i); + } + } + + public String featureForClass(String clazz) { + if (usesOnlyExactMappings) { + return parsedRules.getOrDefault(clazz, baseName); + } else { + FeaturePredicate bestMatch = null; + for (FeaturePredicate mapping : mappings) { + if (mapping.match(clazz)) { + if (bestMatch == null || bestMatch.predicate.length() < mapping.predicate.length()) { + bestMatch = mapping; + } + } + } + if (bestMatch == null) { + return baseName; + } + return bestMatch.feature; + } + } + + public String featureForNonClass(String nonClass) { + return parseNonClassRules.getOrDefault(nonClass, baseName); + } + + private void parseAndAdd(String line, int lineNumber) throws FeatureMappingException { + if (line.startsWith(COMMENT)) { + return; // Ignore comments + } + if (line.isEmpty()) { + return; // Ignore blank lines + } + + if (!line.contains(SEPARATOR)) { + error("Mapping lines must contain a " + SEPARATOR, lineNumber); + } + String[] values = line.split(SEPARATOR); + if (values.length != 2) { + error("Mapping lines can only contain one " + SEPARATOR, lineNumber); + } + + String predicate = values[0]; + String feature = values[1]; + addRule(predicate, feature, lineNumber); + } + + private void addRule(String predicate, String feature, int lineNumber) + throws FeatureMappingException { + if (parsedRules.containsKey(predicate)) { + if (!parsedRules.get(predicate).equals(feature)) { + error("Redefinition of predicate " + predicate + "not allowed", lineNumber); + } + return; // Already have this rule. + } + parsedRules.put(predicate, feature); + FeaturePredicate featurePredicate = new FeaturePredicate(predicate, feature); + mappings.add(featurePredicate); + usesOnlyExactMappings &= featurePredicate.isExactmapping(); + } + + private void error(String error, int line) throws FeatureMappingException { + throw new FeatureMappingException( + "Invalid mappings specification: " + error + "\n in file " + mappingFile + ":" + line); + } + + @Keep + public static class FeatureMappingException extends Exception { + FeatureMappingException(String message) { + super(message); + } + } + + /** A feature predicate can either be a wildcard or class predicate. */ + private static class FeaturePredicate { + private static Pattern identifier = Pattern.compile("[A-Za-z_\\-][A-Za-z0-9_$\\-]*"); + final String predicate; + final String feature; + final boolean isCatchAll; + // False implies class predicate. + final boolean isWildcard; + + FeaturePredicate(String predicate, String feature) throws FeatureMappingException { + isWildcard = predicate.endsWith(".*"); + isCatchAll = predicate.equals("*"); + if (isCatchAll) { + this.predicate = ""; + } else if (isWildcard) { + String packageName = predicate.substring(0, predicate.length() - 2); + if (!DescriptorUtils.isValidJavaType(packageName)) { + throw new FeatureMappingException(packageName + " is not a valid identifier"); + } + // Prefix of a fully-qualified class name, including a terminating dot. + this.predicate = predicate.substring(0, predicate.length() - 1); + } else { + if (!DescriptorUtils.isValidJavaType(predicate)) { + throw new FeatureMappingException(predicate + " is not a valid identifier"); + } + this.predicate = predicate; + } + this.feature = feature; + } + + boolean match(String className) { + if (isCatchAll) { + return true; + } else if (isWildcard) { + return className.startsWith(predicate); + } else { + return className.equals(predicate); + } + } + + boolean isExactmapping() { + return !isWildcard && !isCatchAll; + } + } +}
diff --git a/src/main/keep.txt b/src/main/keep.txt index deb8e6d..e376924 100644 --- a/src/main/keep.txt +++ b/src/main/keep.txt
@@ -12,6 +12,7 @@ -keep public class com.android.tools.r8.D8 { public static void main(java.lang.String[]); } -keep public class com.android.tools.r8.R8 { public static void main(java.lang.String[]); } -keep public class com.android.tools.r8.ExtractMarker { public static void main(java.lang.String[]); } +-keep public class com.android.tools.r8.dexsplitter.DexSplitter { public static void main(java.lang.String[]); } -keep public class com.android.tools.r8.Version { public static final java.lang.String LABEL; } -keep public class com.android.tools.r8.Version { public static java.lang.String getVersionString(); }
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterFieldTypeStrengtheningTest.java b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterFieldTypeStrengtheningTest.java index 71e7df5..45a3be4 100644 --- a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterFieldTypeStrengtheningTest.java +++ b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterFieldTypeStrengtheningTest.java
@@ -47,6 +47,28 @@ } @Test + public void testPropagationFromFeature() throws Exception { + ThrowingConsumer<R8TestCompileResult, Exception> ensureGetFromFeatureGone = + r8TestCompileResult -> { + // Ensure that getFromFeature from FeatureClass is inlined into the run method. + ClassSubject clazz = r8TestCompileResult.inspector().clazz(FeatureClass.class); + assertThat(clazz.uniqueMethodWithName("getFromFeature"), not(isPresent())); + }; + ProcessResult processResult = + testDexSplitter( + parameters, + ImmutableSet.of(BaseSuperClass.class), + ImmutableSet.of(FeatureClass.class), + FeatureClass.class, + EXPECTED, + ensureGetFromFeatureGone, + TestShrinkerBuilder::noMinification); + // We expect art to fail on this with the dex splitter, see b/122902374 + assertNotEquals(processResult.exitCode, 0); + assertTrue(processResult.stderr.contains("NoClassDefFoundError")); + } + + @Test public void testOnR8Splitter() throws IOException, CompilationFailedException { assumeTrue(parameters.isDexRuntime()); ProcessResult processResult =
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterInlineRegression.java b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterInlineRegression.java index 53af2b6..96220f8 100644 --- a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterInlineRegression.java +++ b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterInlineRegression.java
@@ -48,6 +48,31 @@ } @Test + public void testInliningFromFeature() throws Exception { + ThrowingConsumer<R8TestCompileResult, Exception> ensureGetFromFeatureGone = + r8TestCompileResult -> { + // Ensure that getFromFeature from FeatureClass is inlined into the run method. + ClassSubject clazz = r8TestCompileResult.inspector().clazz(FeatureClass.class); + assertThat(clazz.uniqueMethodWithName("getFromFeature"), not(isPresent())); + }; + Consumer<R8FullTestBuilder> configurator = + r8FullTestBuilder -> + r8FullTestBuilder.enableNoVerticalClassMergingAnnotations().noMinification(); + ProcessResult processResult = + testDexSplitter( + parameters, + ImmutableSet.of(BaseSuperClass.class), + ImmutableSet.of(FeatureClass.class), + FeatureClass.class, + EXPECTED, + ensureGetFromFeatureGone, + configurator); + // We expect art to fail on this with the dex splitter, see b/122902374 + assertNotEquals(processResult.exitCode, 0); + assertTrue(processResult.stderr.contains("NoClassDefFoundError")); + } + + @Test public void testOnR8Splitter() throws IOException, CompilationFailedException { assumeTrue(parameters.isDexRuntime()); ThrowableConsumer<R8FullTestBuilder> configurator =
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMemberValuePropagationRegression.java b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMemberValuePropagationRegression.java index a2e1b1d..38d19da 100644 --- a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMemberValuePropagationRegression.java +++ b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMemberValuePropagationRegression.java
@@ -46,6 +46,28 @@ } @Test + public void testPropagationFromFeature() throws Exception { + ThrowingConsumer<R8TestCompileResult, Exception> ensureGetFromFeatureGone = + r8TestCompileResult -> { + // Ensure that getFromFeature from FeatureClass is inlined into the run method. + ClassSubject clazz = r8TestCompileResult.inspector().clazz(FeatureClass.class); + assertThat(clazz.uniqueMethodWithName("getFromFeature"), not(isPresent())); + }; + ProcessResult processResult = + testDexSplitter( + parameters, + ImmutableSet.of(BaseSuperClass.class), + ImmutableSet.of(FeatureClass.class, FeatureEnum.class), + FeatureClass.class, + EXPECTED, + ensureGetFromFeatureGone, + builder -> builder.enableInliningAnnotations().noMinification()); + // We expect art to fail on this with the dex splitter, see b/122902374 + assertNotEquals(processResult.exitCode, 0); + assertTrue(processResult.stderr.contains("NoClassDefFoundError")); + } + + @Test public void testOnR8Splitter() throws IOException, CompilationFailedException { assumeTrue(parameters.isDexRuntime()); ProcessResult processResult =
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMergeRegression.java b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMergeRegression.java index a7d137f..71ddab8 100644 --- a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMergeRegression.java +++ b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterMergeRegression.java
@@ -48,6 +48,43 @@ } @Test + public void testInliningFromFeature() throws Exception { + // Static merging is based on sorting order, we assert that we merged to the feature. + ThrowingConsumer<R8TestCompileResult, Exception> ensureMergingToFeature = + r8TestCompileResult -> { + ClassSubject clazz = r8TestCompileResult.inspector().clazz(AFeatureWithStatic.class); + assertEquals(2, clazz.allMethods().size()); + assertThat(clazz.uniqueMethodWithName("getBase42"), isPresent()); + assertThat(clazz.uniqueMethodWithName("getFoobar"), isPresent()); + }; + Consumer<R8FullTestBuilder> configurator = + r8FullTestBuilder -> + r8FullTestBuilder + .addOptionsModification( + options -> + options.testing.horizontalClassMergingTarget = + (appView, candidates, target) -> candidates.iterator().next()) + .addHorizontallyMergedClassesInspector( + inspector -> + inspector.assertMergedInto(BaseWithStatic.class, AFeatureWithStatic.class)) + .enableNoVerticalClassMergingAnnotations() + .enableInliningAnnotations() + .noMinification(); + ProcessResult processResult = + testDexSplitter( + parameters, + ImmutableSet.of(BaseClass.class, BaseWithStatic.class), + ImmutableSet.of(FeatureClass.class, AFeatureWithStatic.class), + FeatureClass.class, + EXPECTED, + ensureMergingToFeature, + configurator); + // We expect art to fail on this with the dex splitter. + assertNotEquals(processResult.exitCode, 0); + assertTrue(processResult.stderr.contains("NoClassDefFoundError")); + } + + @Test public void testOnR8Splitter() throws IOException, CompilationFailedException { assumeTrue(parameters.isDexRuntime()); ThrowableConsumer<R8FullTestBuilder> configurator =
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java new file mode 100644 index 0000000..e90e0ab --- /dev/null +++ b/src/test/java/com/android/tools/r8/dexsplitter/DexSplitterTests.java
@@ -0,0 +1,484 @@ +// Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +package com.android.tools.r8.dexsplitter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.android.tools.r8.CompilationFailedException; +import com.android.tools.r8.D8Command; +import com.android.tools.r8.DexSplitterHelper; +import com.android.tools.r8.ExtractMarker; +import com.android.tools.r8.OutputMode; +import com.android.tools.r8.R8; +import com.android.tools.r8.R8Command; +import com.android.tools.r8.ResourceException; +import com.android.tools.r8.ToolHelper; +import com.android.tools.r8.ToolHelper.ArtCommandBuilder; +import com.android.tools.r8.dex.Marker; +import com.android.tools.r8.dexsplitter.DexSplitter.Options; +import com.android.tools.r8.origin.Origin; +import com.android.tools.r8.utils.FeatureClassMapping.FeatureMappingException; +import com.android.tools.r8.utils.StringUtils; +import com.android.tools.r8.utils.codeinspector.ClassSubject; +import com.android.tools.r8.utils.codeinspector.CodeInspector; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class DexSplitterTests { + + private static final String CLASS_DIR = + ToolHelper.EXAMPLES_ANDROID_N_BUILD_DIR + "classes/dexsplitsample"; + private static final String CLASS1_CLASS = CLASS_DIR + "/Class1.class"; + private static final String CLASS2_CLASS = CLASS_DIR + "/Class2.class"; + private static final String CLASS3_CLASS = CLASS_DIR + "/Class3.class"; + private static final String CLASS3_INNER_CLASS = CLASS_DIR + "/Class3$InnerClass.class"; + private static final String CLASS3_SYNTHETIC_CLASS = CLASS_DIR + "/Class3$1.class"; + private static final String CLASS4_CLASS = CLASS_DIR + "/Class4.class"; + private static final String CLASS4_LAMBDA_INTERFACE = CLASS_DIR + "/Class4$LambdaInterface.class"; + private static final String TEXT_FILE = + ToolHelper.EXAMPLES_ANDROID_N_DIR + "dexsplitsample/TextFile.txt"; + + + @Rule public TemporaryFolder temp = ToolHelper.getTemporaryFolderForTest(); + + private Path createInput(boolean dontCreateMarkerInD8) + throws IOException, CompilationFailedException { + // Initial normal compile to create dex files. + Path inputZip = temp.newFolder().toPath().resolve("input.zip"); + D8Command command = + D8Command.builder() + .setOutput(inputZip, OutputMode.DexIndexed) + .addProgramFiles(Paths.get(CLASS1_CLASS)) + .addProgramFiles(Paths.get(CLASS2_CLASS)) + .addProgramFiles(Paths.get(CLASS3_CLASS)) + .addProgramFiles(Paths.get(CLASS3_INNER_CLASS)) + .addProgramFiles(Paths.get(CLASS3_SYNTHETIC_CLASS)) + .addProgramFiles(Paths.get(CLASS4_CLASS)) + .addProgramFiles(Paths.get(CLASS4_LAMBDA_INTERFACE)) + .build(); + + DexSplitterHelper.runD8ForTesting(command, dontCreateMarkerInD8); + + return inputZip; + } + + private void testMarker(boolean addMarkerToInput) + throws CompilationFailedException, IOException, ResourceException, ExecutionException { + Path inputZip = createInput(!addMarkerToInput); + + Path output = temp.newFolder().toPath().resolve("output"); + Files.createDirectory(output); + Path splitSpec = createSplitSpec(); + + DexSplitter.main( + new String[] { + "--input", inputZip.toString(), + "--output", output.toString(), + "--feature-splits", splitSpec.toString() + }); + + Path base = output.resolve("base").resolve("classes.dex"); + Path feature = output.resolve("feature1").resolve("classes.dex"); + + for (Path path : new Path[] {inputZip, base, feature}) { + Collection<Marker> markers = ExtractMarker.extractMarkerFromDexFile(path); + assertEquals(addMarkerToInput ? 1 : 0, markers.size()); + } + } + + @Test + public void testMarkerPreserved() + throws CompilationFailedException, IOException, ResourceException, ExecutionException { + testMarker(true); + } + + @Test + public void testMarkerNotAdded() + throws CompilationFailedException, IOException, ResourceException, ExecutionException { + testMarker(false); + } + + /** + * To test the file splitting we have 3 classes that we distribute like this: Class1 -> base + * Class2 -> feature1 Class3 -> feature1 + * + * <p>Class1 and Class2 works independently of each other, but Class3 extends Class1, and + * therefore can't run without the base being loaded. + */ + @Test + public void splitFilesNoObfuscation() + throws CompilationFailedException, IOException, FeatureMappingException { + noObfuscation(false); + noObfuscation(true); + } + + private void noObfuscation(boolean useOptions) + throws IOException, CompilationFailedException, FeatureMappingException { + Path inputZip = createInput(false); + Path output = temp.newFolder().toPath().resolve("output"); + Files.createDirectory(output); + Path splitSpec = createSplitSpec(); + + if (useOptions) { + Options options = new Options(); + options.addInputArchive(inputZip.toString()); + options.setFeatureSplitMapping(splitSpec.toString()); + options.setOutput(output.toString()); + DexSplitter.run(options); + } else { + DexSplitter.main( + new String[] { + "--input", inputZip.toString(), + "--output", output.toString(), + "--feature-splits", splitSpec.toString() + }); + } + + Path base = output.resolve("base").resolve("classes.dex"); + Path feature = output.resolve("feature1").resolve("classes.dex"); + validateUnobfuscatedOutput(base, feature); + } + + private void validateUnobfuscatedOutput(Path base, Path feature) throws IOException { + // Both classes should still work if we give all dex files to the system. + for (String className : new String[] {"Class1", "Class2", "Class3"}) { + ArtCommandBuilder builder = new ArtCommandBuilder(); + builder.appendClasspath(base.toString()); + builder.appendClasspath(feature.toString()); + builder.setMainClass("dexsplitsample." + className); + String out = ToolHelper.runArt(builder); + assertEquals(out, className + "\n"); + } + // Individual classes should also work from the individual files. + String className = "Class1"; + ArtCommandBuilder builder = new ArtCommandBuilder(); + builder.appendClasspath(base.toString()); + builder.setMainClass("dexsplitsample." + className); + String out = ToolHelper.runArt(builder); + assertEquals(out, className + "\n"); + + className = "Class2"; + builder = new ArtCommandBuilder(); + builder.appendClasspath(feature.toString()); + builder.setMainClass("dexsplitsample." + className); + out = ToolHelper.runArt(builder); + assertEquals(out, className + "\n"); + + className = "Class3"; + builder = new ArtCommandBuilder(); + builder.appendClasspath(feature.toString()); + builder.setMainClass("dexsplitsample." + className); + try { + ToolHelper.runArt(builder); + assertFalse(true); + } catch (AssertionError assertionError) { + // We expect this to throw since base is not in the path and Class3 depends on Class1 + } + + className = "Class4"; + builder = new ArtCommandBuilder(); + builder.appendClasspath(feature.toString()); + builder.setMainClass("dexsplitsample." + className); + try { + ToolHelper.runArt(builder); + assertFalse(true); + } catch (AssertionError assertionError) { + // We expect this to throw since base is not in the path and Class4 includes a lambda that + // would have been pushed to base. + } + } + + private Path createSplitSpec() throws FileNotFoundException, UnsupportedEncodingException { + Path splitSpec = temp.getRoot().toPath().resolve("split_spec"); + try (PrintWriter out = new PrintWriter(splitSpec.toFile(), "UTF-8")) { + out.write( + "dexsplitsample.Class1:base\n" + + "dexsplitsample.Class2:feature1\n" + + "dexsplitsample.Class3:feature1\n" + + "dexsplitsample.Class4:feature1"); + } + return splitSpec; + } + + private List<String> getProguardConf() { + return ImmutableList.of( + "-keep class dexsplitsample.Class3 {", + " public static void main(java.lang.String[]);", + "}"); + } + + @Test + public void splitFilesFromJar() + throws IOException, CompilationFailedException, FeatureMappingException { + for (boolean useOptions : new boolean[]{false, true}) { + for (boolean explicitBase: new boolean[]{false, true}) { + for (boolean renameBase: new boolean[]{false, true}) { + splitFromJars(useOptions, explicitBase, renameBase); + } + } + } + } + + private void splitFromJars(boolean useOptions, boolean explicitBase, boolean renameBase) + throws IOException, CompilationFailedException, FeatureMappingException { + Path inputZip = createInput(false); + Path output = temp.newFolder().toPath().resolve("output"); + Files.createDirectory(output); + Path baseJar = temp.getRoot().toPath().resolve("base.jar"); + Path featureJar = temp.getRoot().toPath().resolve("feature1.jar"); + ZipOutputStream baseStream = new ZipOutputStream(Files.newOutputStream(baseJar)); + String name = "dexsplitsample/Class1.class"; + baseStream.putNextEntry(new ZipEntry(name)); + baseStream.write(Files.readAllBytes(Paths.get(CLASS1_CLASS))); + baseStream.closeEntry(); + baseStream.close(); + + ZipOutputStream featureStream = new ZipOutputStream(Files.newOutputStream(featureJar)); + name = "dexsplitsample/Class2.class"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(Files.readAllBytes(Paths.get(CLASS2_CLASS))); + featureStream.closeEntry(); + name = "dexsplitsample/Class3.class"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(Files.readAllBytes(Paths.get(CLASS3_CLASS))); + featureStream.closeEntry(); + name = "dexsplitsample/Class3$InnerClass.class"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(Files.readAllBytes(Paths.get(CLASS3_INNER_CLASS))); + featureStream.closeEntry(); + name = "dexsplitsample/Class3$1.class"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(Files.readAllBytes(Paths.get(CLASS3_SYNTHETIC_CLASS))); + featureStream.closeEntry(); + name = "dexsplitsample/Class4"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(Files.readAllBytes(Paths.get(CLASS4_CLASS))); + featureStream.closeEntry(); + name = "dexsplitsample/Class4$LambdaInterface"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(Files.readAllBytes(Paths.get(CLASS4_LAMBDA_INTERFACE))); + featureStream.closeEntry(); + featureStream.close(); + // Make sure that we can pass in a name for the output. + String specificOutputName = "renamed"; + if (useOptions) { + Options options = new Options(); + options.addInputArchive(inputZip.toString()); + options.setOutput(output.toString()); + if (explicitBase) { + options.addBaseJar(baseJar.toString()); + } else if (renameBase){ + // Ensure that we can rename base (if people called a feature base) + options.setBaseOutputName("base_renamed"); + } + options.addFeatureJar(featureJar.toString(), specificOutputName); + DexSplitter.run(options); + } else { + List<String> args = Lists.newArrayList( + "--input", + inputZip.toString(), + "--output", + output.toString(), + "--feature-jar", + featureJar.toString().concat(":").concat(specificOutputName)); + if (explicitBase) { + args.add("--base-jar"); + args.add(baseJar.toString()); + } else if (renameBase) { + args.add("--base-output-name"); + args.add("base_renamed"); + } + + DexSplitter.main(args.toArray(StringUtils.EMPTY_ARRAY)); + } + String baseOutputName = explicitBase || !renameBase ? "base" : "base_renamed"; + Path base = output.resolve(baseOutputName).resolve("classes.dex"); + Path feature = output.resolve(specificOutputName).resolve("classes.dex");; + validateUnobfuscatedOutput(base, feature); + } + + @Test + public void splitFilesObfuscation() + throws CompilationFailedException, IOException, ExecutionException { + // Initial normal compile to create dex files. + Path inputDex = temp.newFolder().toPath().resolve("input.zip"); + Path proguardMap = temp.getRoot().toPath().resolve("proguard.map"); + + R8.run( + R8Command.builder() + .setOutput(inputDex, OutputMode.DexIndexed) + .addProgramFiles(Paths.get(CLASS1_CLASS)) + .addProgramFiles(Paths.get(CLASS2_CLASS)) + .addProgramFiles(Paths.get(CLASS3_CLASS)) + .addProgramFiles(Paths.get(CLASS3_INNER_CLASS)) + .addProgramFiles(Paths.get(CLASS3_SYNTHETIC_CLASS)) + .addProgramFiles(Paths.get(CLASS4_CLASS)) + .addProgramFiles(Paths.get(CLASS4_LAMBDA_INTERFACE)) + .addLibraryFiles(ToolHelper.getDefaultAndroidJar()) + .setProguardMapOutputPath(proguardMap) + .addProguardConfiguration(getProguardConf(), Origin.unknown()) + .build()); + + Path outputDex = temp.newFolder().toPath().resolve("output"); + Files.createDirectory(outputDex); + Path splitSpec = createSplitSpec(); + + DexSplitter.main( + new String[] { + "--input", inputDex.toString(), + "--output", outputDex.toString(), + "--feature-splits", splitSpec.toString(), + "--proguard-map", proguardMap.toString() + }); + + Path base = outputDex.resolve("base").resolve("classes.dex"); + Path feature = outputDex.resolve("feature1").resolve("classes.dex"); + String class3 = "dexsplitsample.Class3"; + // We should still be able to run the Class3 which we kept, it has a call to the obfuscated + // class1 which is in base. + ArtCommandBuilder builder = new ArtCommandBuilder(); + builder.appendClasspath(base.toString()); + builder.appendClasspath(feature.toString()); + builder.setMainClass(class3); + String out = ToolHelper.runArt(builder); + assertEquals(out, "Class3\n"); + + // Class1 should not be in the feature, it should still be in base. + builder = new ArtCommandBuilder(); + builder.appendClasspath(feature.toString()); + builder.setMainClass(class3); + try { + ToolHelper.runArt(builder); + assertFalse(true); + } catch (AssertionError assertionError) { + // We expect this to throw since base is not in the path and Class3 depends on Class1. + } + + // Ensure that the Class1 is actually in the correct split. Note that Class2 would have been + // shaken away. + CodeInspector inspector = new CodeInspector(base, proguardMap); + ClassSubject subject = inspector.clazz("dexsplitsample.Class1"); + assertTrue(subject.isPresent()); + assertTrue(subject.isRenamed()); + } + + @Test + public void splitNonClassFiles() + throws CompilationFailedException, IOException, FeatureMappingException { + Path inputZip = temp.getRoot().toPath().resolve("input-zip-with-non-class-files.jar"); + ZipOutputStream inputZipStream = new ZipOutputStream(Files.newOutputStream(inputZip)); + String name = "dexsplitsample/TextFile.txt"; + inputZipStream.putNextEntry(new ZipEntry(name)); + byte[] fileBytes = Files.readAllBytes(Paths.get(TEXT_FILE)); + inputZipStream.write(fileBytes); + inputZipStream.closeEntry(); + name = "dexsplitsample/TextFile2.txt"; + inputZipStream.putNextEntry(new ZipEntry(name)); + inputZipStream.write(fileBytes); + inputZipStream.write(fileBytes); + inputZipStream.closeEntry(); + inputZipStream.close(); + Path output = temp.newFolder().toPath().resolve("output"); + Files.createDirectory(output); + Path baseJar = temp.getRoot().toPath().resolve("base.jar"); + Path featureJar = temp.getRoot().toPath().resolve("feature1.jar"); + ZipOutputStream baseStream = new ZipOutputStream(Files.newOutputStream(baseJar)); + name = "dexsplitsample/TextFile.txt"; + baseStream.putNextEntry(new ZipEntry(name)); + baseStream.write(fileBytes); + baseStream.closeEntry(); + baseStream.close(); + ZipOutputStream featureStream = new ZipOutputStream(Files.newOutputStream(featureJar)); + name = "dexsplitsample/TextFile2.txt"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(fileBytes); + featureStream.write(fileBytes); + featureStream.closeEntry(); + featureStream.close(); + Options options = new Options(); + options.addInputArchive(inputZip.toString()); + options.setOutput(output.toString()); + options.addFeatureJar(baseJar.toString()); + options.addFeatureJar(featureJar.toString()); + options.setSplitNonClassResources(true); + DexSplitter.run(options); + Path baseDir = output.resolve("base"); + Path featureDir = output.resolve("feature1"); + byte[] contents = fileBytes; + byte[] contents2 = new byte[contents.length * 2]; + System.arraycopy(contents, 0, contents2, 0, contents.length); + System.arraycopy(contents, 0, contents2, contents.length, contents.length); + Path baseTextFile = baseDir.resolve("dexsplitsample/TextFile.txt"); + Path featureTextFile = featureDir.resolve("dexsplitsample/TextFile2.txt"); + assert Files.exists(baseTextFile); + assert Files.exists(featureTextFile); + assert Arrays.equals(Files.readAllBytes(baseTextFile), contents); + assert Arrays.equals(Files.readAllBytes(featureTextFile), contents2); + } + + @Test + public void splitDuplicateNonClassFiles() + throws IOException, CompilationFailedException, FeatureMappingException { + Path inputZip = temp.getRoot().toPath().resolve("input-zip-with-non-class-files.jar"); + ZipOutputStream inputZipStream = new ZipOutputStream(Files.newOutputStream(inputZip)); + String name = "dexsplitsample/TextFile.txt"; + inputZipStream.putNextEntry(new ZipEntry(name)); + byte[] fileBytes = Files.readAllBytes(Paths.get(TEXT_FILE)); + inputZipStream.write(fileBytes); + inputZipStream.closeEntry(); + inputZipStream.close(); + Path output = temp.newFolder().toPath().resolve("output"); + Files.createDirectory(output); + Path featureJar = temp.getRoot().toPath().resolve("feature1.jar"); + Path feature2Jar = temp.getRoot().toPath().resolve("feature2.jar"); + ZipOutputStream featureStream = new ZipOutputStream(Files.newOutputStream(featureJar)); + name = "dexsplitsample/TextFile.txt"; + featureStream.putNextEntry(new ZipEntry(name)); + featureStream.write(fileBytes); + featureStream.closeEntry(); + featureStream.close(); + ZipOutputStream feature2Stream = new ZipOutputStream(Files.newOutputStream(feature2Jar)); + name = "dexsplitsample/TextFile.txt"; + feature2Stream.putNextEntry(new ZipEntry(name)); + feature2Stream.write(fileBytes); + feature2Stream.closeEntry(); + feature2Stream.close(); + Options options = new Options(); + options.addInputArchive(inputZip.toString()); + options.setOutput(output.toString()); + options.addFeatureJar(feature2Jar.toString()); + options.addFeatureJar(featureJar.toString()); + options.setSplitNonClassResources(true); + DexSplitter.run(options); + Path baseDir = output.resolve("base"); + Path feature1Dir = output.resolve("feature1"); + Path feature2Dir = output.resolve("feature2"); + Path baseTextFile = baseDir.resolve("dexsplitsample/TextFile.txt"); + Path feature1TextFile = feature1Dir.resolve("dexsplitsample/TextFile2.txt"); + Path feature2TextFile = feature2Dir.resolve("dexsplitsample/TextFile2.txt"); + assert !Files.exists(feature1TextFile); + assert !Files.exists(feature2TextFile); + assert Files.exists(baseTextFile); + assert Arrays.equals(Files.readAllBytes(baseTextFile), fileBytes); + } +}
diff --git a/src/test/java/com/android/tools/r8/dexsplitter/SplitterTestBase.java b/src/test/java/com/android/tools/r8/dexsplitter/SplitterTestBase.java index bd7929d..05d6d3d 100644 --- a/src/test/java/com/android/tools/r8/dexsplitter/SplitterTestBase.java +++ b/src/test/java/com/android/tools/r8/dexsplitter/SplitterTestBase.java
@@ -17,9 +17,12 @@ import com.android.tools.r8.ToolHelper; import com.android.tools.r8.ToolHelper.ArtCommandBuilder; import com.android.tools.r8.ToolHelper.ProcessResult; +import com.android.tools.r8.dexsplitter.DexSplitter.Options; import com.android.tools.r8.utils.ArchiveResourceProvider; import com.android.tools.r8.utils.Pair; +import com.android.tools.r8.utils.ThrowingConsumer; import com.android.tools.r8.utils.ZipUtils; +import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; import dalvik.system.PathClassLoader; import java.io.IOException; @@ -28,7 +31,9 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Set; +import java.util.function.Consumer; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -150,6 +155,96 @@ toRun, baseOutput, r8TestCompileResult.getFeature(0), parameters.getRuntime()); } + // Compile the passed in classes plus RunInterface and SplitRunner using R8, then split + // based on the base/feature sets. toRun must implement the BaseRunInterface + <E extends Throwable> ProcessResult testDexSplitter( + TestParameters parameters, + Set<Class<?>> baseClasses, + Set<Class<?>> featureClasses, + Class<?> toRun, + String expectedOutput, + ThrowingConsumer<R8TestCompileResult, E> compileResultConsumer, + Consumer<R8FullTestBuilder> r8TestConfigurator) + throws Exception, E { + List<Class<?>> baseClassesWithRunner = + ImmutableList.<Class<?>>builder() + .add(RunInterface.class, SplitRunner.class) + .addAll(baseClasses) + .build(); + + Path baseJar = jarTestClasses(baseClassesWithRunner); + Path featureJar = jarTestClasses(featureClasses); + + Path featureOnly = + testForR8(parameters.getBackend()) + .addProgramClasses(featureClasses) + .addClasspathClasses(baseClasses) + .addClasspathClasses(RunInterface.class) + .addKeepAllClassesRule() + .addInliningAnnotations() + .setMinApi(parameters.getApiLevel()) + .compile() + .writeToZip(); + if (parameters.isDexRuntime()) { + // With D8 this should just work. We compile all of the base classes, then run with the + // feature loaded at runtime. Since there is no inlining/class merging we don't + // have any issues. + testForD8() + .addProgramClasses(SplitRunner.class, RunInterface.class) + .addProgramClasses(baseClasses) + .setMinApi(parameters.getApiLevel()) + .compile() + .run( + parameters.getRuntime(), + SplitRunner.class, + toRun.getName(), + featureOnly.toAbsolutePath().toString()) + .assertSuccessWithOutput(expectedOutput); + } + + R8FullTestBuilder builder = testForR8(parameters.getBackend()); + if (parameters.isCfRuntime()) { + // Compiling to jar we need to support the same way of loading code at runtime as + // android supports. + builder + .addProgramClasses(PathClassLoader.class) + .addKeepClassAndMembersRules(PathClassLoader.class); + } + + R8FullTestBuilder r8FullTestBuilder = + builder + .setMinApi(parameters.getApiLevel()) + .addProgramClasses(SplitRunner.class, RunInterface.class) + .addProgramClasses(baseClasses) + .addProgramClasses(featureClasses) + .addKeepMainRule(SplitRunner.class) + .addKeepClassRules(toRun); + r8TestConfigurator.accept(r8FullTestBuilder); + R8TestCompileResult r8TestCompileResult = r8FullTestBuilder.compile(); + compileResultConsumer.accept(r8TestCompileResult); + Path fullFiles = r8TestCompileResult.writeToZip(); + + // Ensure that we can run the program as a unit (i.e., without splitting) + r8TestCompileResult + .run(parameters.getRuntime(), SplitRunner.class, toRun.getName()) + .assertSuccessWithOutput(expectedOutput); + + Path splitterOutput = temp.newFolder().toPath(); + Path splitterBaseDexFile = splitterOutput.resolve("base").resolve("classes.dex"); + Path splitterFeatureDexFile = splitterOutput.resolve("feature").resolve("classes.dex"); + + Options options = new Options(); + options.setOutput(splitterOutput.toString()); + options.addBaseJar(baseJar.toString()); + options.addFeatureJar(featureJar.toString(), "feature"); + + options.addInputArchive(fullFiles.toString()); + DexSplitter.run(options); + + return runFeatureOnArt( + toRun, splitterBaseDexFile, splitterFeatureDexFile, parameters.getRuntime()); + } + ProcessResult runFeatureOnArt( Class toRun, Path splitterBaseDexFile, Path splitterFeatureDexFile, TestRuntime runtime) throws IOException {
diff --git a/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java b/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java index 75b7dda..dd40ecf 100644 --- a/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java +++ b/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java
@@ -25,6 +25,8 @@ import com.android.tools.r8.ToolHelper; import com.android.tools.r8.dex.ApplicationWriter; import com.android.tools.r8.dex.Constants; +import com.android.tools.r8.dexsplitter.DexSplitter; +import com.android.tools.r8.dexsplitter.DexSplitter.Options; import com.android.tools.r8.errors.CompilationError; import com.android.tools.r8.errors.DexFileOverflowDiagnostic; import com.android.tools.r8.errors.Unreachable; @@ -253,6 +255,49 @@ } @Test + public void everyThirdClassInMainWithDexSplitter() throws Throwable { + List<String> featureMappings = new ArrayList<>(); + List<String> inFeatureMapping = new ArrayList<>(); + + ImmutableList.Builder<String> mainDexBuilder = ImmutableList.builder(); + for (int i = 0; i < MANY_CLASSES.size(); i++) { + String clazz = MANY_CLASSES.get(i); + // Write the first 2 classes into the split. + if (i < 10) { + featureMappings.add(clazz + ":feature1"); + inFeatureMapping.add(clazz); + } + if (i % 3 == 0) { + mainDexBuilder.add(clazz); + } + } + Path featureSplitMapping = temp.getRoot().toPath().resolve("splitmapping"); + Path mainDexFile = temp.getRoot().toPath().resolve("maindex"); + FileUtils.writeTextFile(featureSplitMapping, featureMappings); + List<String> mainDexList = mainDexBuilder.build(); + FileUtils.writeTextFile(mainDexFile, ListUtils.map(mainDexList, MainDexListTests::typeToEntry)); + Path output = temp.getRoot().toPath().resolve("split_output"); + Files.createDirectories(output); + TestDiagnosticsHandler diagnosticsHandler = new TestDiagnosticsHandler(); + Options options = new Options(diagnosticsHandler); + options.addInputArchive(getManyClassesMultiDexAppPath().toString()); + options.setFeatureSplitMapping(featureSplitMapping.toString()); + options.setOutput(output.toString()); + options.setMainDexList(mainDexFile.toString()); + DexSplitter.run(options); + assertEquals(0, diagnosticsHandler.numberOfErrorsAndWarnings()); + Path baseDir = output.resolve("base"); + CodeInspector inspector = + new CodeInspector( + AndroidApp.builder().addProgramFiles(baseDir.resolve("classes.dex")).build()); + for (String clazz : mainDexList) { + if (!inspector.clazz(clazz).isPresent() && !inFeatureMapping.contains(clazz)) { + failedToFindClassInExpectedFile(baseDir, clazz); + } + } + } + + @Test public void singleClassInMainDex() throws Throwable { ImmutableList<String> mainDex = ImmutableList.of(MANY_CLASSES.get(0)); verifyMainDexContains(mainDex, getManyClassesSingleDexAppPath(), true);
diff --git a/src/test/java/com/android/tools/r8/utils/FeatureClassMappingTest.java b/src/test/java/com/android/tools/r8/utils/FeatureClassMappingTest.java new file mode 100644 index 0000000..f41be5c --- /dev/null +++ b/src/test/java/com/android/tools/r8/utils/FeatureClassMappingTest.java
@@ -0,0 +1,132 @@ +// Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +package com.android.tools.r8.utils; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; + +import com.android.tools.r8.utils.FeatureClassMapping.FeatureMappingException; +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.junit.Test; + +public class FeatureClassMappingTest { + + @Test + public void testSimpleParse() throws Exception { + + List<String> lines = + ImmutableList.of( + "# Comment, don't care about contents: even more ::::", + "com.google.base:base", + "", // Empty lines allowed + "com.google.feature1:feature1", + "com.google.feature1:feature1", // Multiple definitions of the same predicate allowed. + "com.google$:feature1", + "_com.google:feature21", + "com.google.*:feature32"); + FeatureClassMapping mapping = new FeatureClassMapping(lines); + } + + private void ensureThrowsMappingException(List<String> lines) { + try { + new FeatureClassMapping(lines); + assertFalse(true); + } catch (FeatureMappingException e) { + // Expected + } + } + + private void ensureThrowsMappingException(String string) { + ensureThrowsMappingException(ImmutableList.of(string)); + } + + @Test + public void testLookup() throws Exception { + List<String> lines = + ImmutableList.of( + "com.google.Base:base", + "", + "com.google.Feature1:feature1", + "com.google.Feature1:feature1", // Multiple definitions of the same predicate allowed. + "com.google.different.*:feature1", + "_com.Google:feature21", + "com.google.bas42.*:feature42"); + FeatureClassMapping mapping = new FeatureClassMapping(lines); + assertEquals(mapping.featureForClass("com.google.Feature1"), "feature1"); + assertEquals(mapping.featureForClass("com.google.different.Feature1"), "feature1"); + assertEquals(mapping.featureForClass("com.google.different.Foobar"), "feature1"); + assertEquals(mapping.featureForClass("com.google.Base"), "base"); + assertEquals(mapping.featureForClass("com.google.bas42.foo.bar.bar.Foo"), "feature42"); + assertEquals(mapping.featureForClass("com.google.bas42.f$o$o$.bar43.bar.Foo"), "feature42"); + assertEquals(mapping.featureForClass("_com.Google"), "feature21"); + } + + @Test + public void testCatchAllWildcards() throws Exception { + testBaseWildcard(true); + testBaseWildcard(false); + testNonBaseCatchAll(); + } + + private void testNonBaseCatchAll() throws FeatureMappingException { + List<String> lines = + ImmutableList.of( + "com.google.Feature1:feature1", + "*:nonbase", + "com.strange.*:feature2"); + FeatureClassMapping mapping = new FeatureClassMapping(lines); + assertEquals(mapping.featureForClass("com.google.Feature1"), "feature1"); + assertEquals(mapping.featureForClass("com.google.different.Feature1"), "nonbase"); + assertEquals(mapping.featureForClass("com.strange.different.Feature1"), "feature2"); + assertEquals(mapping.featureForClass("Feature1"), "nonbase"); + assertEquals(mapping.featureForClass("a.b.z.A"), "nonbase"); + } + + private void testBaseWildcard(boolean explicitBase) throws FeatureMappingException { + List<String> lines = + ImmutableList.of( + "com.google.Feature1:feature1", + explicitBase ? "*:base" : "", + "com.strange.*:feature2"); + FeatureClassMapping mapping = new FeatureClassMapping(lines); + assertEquals(mapping.featureForClass("com.google.Feature1"), "feature1"); + assertEquals(mapping.featureForClass("com.google.different.Feature1"), "base"); + assertEquals(mapping.featureForClass("com.strange.different.Feature1"), "feature2"); + assertEquals(mapping.featureForClass("com.stranger.Clazz"), "base"); + assertEquals(mapping.featureForClass("Feature1"), "base"); + assertEquals(mapping.featureForClass("a.b.z.A"), "base"); + } + + @Test + public void testWrongLines() throws Exception { + // No colon. + ensureThrowsMappingException("foo"); + ensureThrowsMappingException("com.google.base"); + // Two colons. + ensureThrowsMappingException(ImmutableList.of("a:b:c")); + + // Empty identifier. + ensureThrowsMappingException("com..google:feature1"); + + // Ambiguous redefinition + ensureThrowsMappingException( + ImmutableList.of("com.google.foo:feature1", "com.google.foo:feature2")); + ensureThrowsMappingException( + ImmutableList.of("com.google.foo.*:feature1", "com.google.foo.*:feature2")); + } + + @Test + public void testUsesOnlyExactMappings() throws Exception { + List<String> lines = + ImmutableList.of( + "com.pkg1.Clazz:feature1", + "com.pkg2.Clazz:feature2"); + FeatureClassMapping mapping = new FeatureClassMapping(lines); + + assertEquals(mapping.featureForClass("com.pkg1.Clazz"), "feature1"); + assertEquals(mapping.featureForClass("com.pkg2.Clazz"), "feature2"); + assertEquals(mapping.featureForClass("com.pkg1.Other"), mapping.baseName); + } +}
diff --git a/tools/dexsplitter.py b/tools/dexsplitter.py new file mode 100755 index 0000000..a97d089 --- /dev/null +++ b/tools/dexsplitter.py
@@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import sys +import toolhelper + +if __name__ == '__main__': + sys.exit(toolhelper.run('dexsplitter', sys.argv[1:]))