Merge "Revert "Relax the dead condition of field reads.""
diff --git a/.gitignore b/.gitignore
index 14e24ec..c04d9dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,6 +63,8 @@
third_party/kotlin
third_party/nest/*
!third_party/nest/*.sha1
+third_party/opensource_apps
+third_party/opensource_apps.tar.gz
third_party/photos/*
!third_party/photos/*.sha1
third_party/proguard/*
diff --git a/build.gradle b/build.gradle
index f129fb5..4ce2e18 100644
--- a/build.gradle
+++ b/build.gradle
@@ -10,7 +10,6 @@
ext {
androidSupportVersion = '25.4.0'
asmVersion = '6.2.1'
- autoValueVersion = '1.5'
espressoVersion = '3.0.0'
fastutilVersion = '7.2.0'
guavaVersion = '23.0'
@@ -66,7 +65,6 @@
dependencies {
classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.0.13"
- classpath "net.ltgt.gradle:gradle-apt-plugin:0.12"
classpath "com.gradle:build-scan-plugin:1.14"
}
}
@@ -81,7 +79,6 @@
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'net.ltgt.errorprone-base'
-apply plugin: "net.ltgt.apt"
if (project.hasProperty('with_code_coverage')) {
apply plugin: 'jacoco'
@@ -257,7 +254,6 @@
examplesCompile "com.google.guava:guava:$guavaVersion"
examplesCompile "junit:junit:$junitVersion"
examplesCompile "org.mockito:mockito-core:$mockitoVersion"
- examplesCompileOnly "com.google.auto.value:auto-value:$autoValueVersion"
supportLibs "com.android.support:support-v4:$androidSupportVersion"
supportLibs "junit:junit:$junitVersion"
supportLibs "com.android.support.test.espresso:espresso-core:$espressoVersion"
@@ -266,7 +262,6 @@
debugTestResourcesKotlinCompileOnly "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
examplesKotlinCompileOnly "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
kotlinR8TestResourcesCompileOnly "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
- apt "com.google.auto.value:auto-value:$autoValueVersion"
}
def r8LibPath = "$buildDir/libs/r8lib.jar"
diff --git a/src/main/java/com/android/tools/r8/DexSplitterHelper.java b/src/main/java/com/android/tools/r8/DexSplitterHelper.java
index f78912f..27c564a 100644
--- a/src/main/java/com/android/tools/r8/DexSplitterHelper.java
+++ b/src/main/java/com/android/tools/r8/DexSplitterHelper.java
@@ -58,6 +58,7 @@
InternalOptions options = command.getInternalOptions();
options.enableDesugaring = false;
options.enableMainDexListCheck = false;
+ options.ignoreMainDexMissingClasses = true;
options.minimalMainDex = false;
options.enableMinification = false;
options.enableInlining = false;
diff --git a/src/main/java/com/android/tools/r8/dex/VirtualFile.java b/src/main/java/com/android/tools/r8/dex/VirtualFile.java
index 0e62775..0d12915 100644
--- a/src/main/java/com/android/tools/r8/dex/VirtualFile.java
+++ b/src/main/java/com/android/tools/r8/dex/VirtualFile.java
@@ -307,11 +307,13 @@
mainDexFile.addClass(programClass);
classes.remove(programClass);
} else {
- options.reporter.warning(
- new StringDiagnostic(
- "Application does not contain `"
- + type.toSourceString()
- + "` as referenced in main-dex-list."));
+ if (!options.ignoreMainDexMissingClasses) {
+ options.reporter.warning(
+ new StringDiagnostic(
+ "Application does not contain `"
+ + type.toSourceString()
+ + "` as referenced in main-dex-list."));
+ }
}
mainDexFile.commitTransaction();
}
diff --git a/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java b/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
index 9062ae7..4b89d9e 100644
--- a/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
+++ b/src/main/java/com/android/tools/r8/dexsplitter/DexSplitter.java
@@ -86,7 +86,7 @@
@Keep
public static final class Options {
- private final DiagnosticsHandler diagnosticsHandler = new DiagnosticsHandler() {};
+ private final DiagnosticsHandler diagnosticsHandler;
private List<String> inputArchives = new ArrayList<>();
private List<FeatureJar> featureJars = new ArrayList<>();
private List<String> baseJars = new ArrayList<>();
@@ -97,6 +97,14 @@
private String mainDexList;
private boolean splitNonClassResources = false;
+ public Options() {
+ this(new DiagnosticsHandler() {});
+ }
+
+ public Options(DiagnosticsHandler diagnosticsHandler) {
+ this.diagnosticsHandler = diagnosticsHandler;
+ }
+
public DiagnosticsHandler getDiagnosticsHandler() {
return diagnosticsHandler;
}
@@ -293,7 +301,9 @@
throw new AbortException();
}
- D8Command.Builder builder = D8Command.builder();
+ D8Command.Builder builder = D8Command.builder(options.diagnosticsHandler);
+
+
for (String s : options.inputArchives) {
builder.addProgramFiles(Paths.get(s));
}
diff --git a/src/main/java/com/android/tools/r8/graph/DexByteCodeWriter.java b/src/main/java/com/android/tools/r8/graph/DexByteCodeWriter.java
index 21900a01..42ebde9 100644
--- a/src/main/java/com/android/tools/r8/graph/DexByteCodeWriter.java
+++ b/src/main/java/com/android/tools/r8/graph/DexByteCodeWriter.java
@@ -62,7 +62,8 @@
private void write(ThrowingFunction<DexClass, PrintStream, IOException> outputStreamProvider,
Consumer<PrintStream> closer)
throws IOException {
- for (DexProgramClass clazz : application.classes()) {
+ Iterable<DexProgramClass> classes = application.classesWithDeterministicOrder();
+ for (DexProgramClass clazz : classes) {
if (anyMethodMatches(clazz)) {
PrintStream ps = outputStreamProvider.apply(clazz);
try {
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index a35100e..a5854b6 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -147,6 +147,10 @@
// Throw exception if there is a warning about invalid debug info.
public boolean invalidDebugInfoFatal = false;
+ // When dexsplitting we ignore main dex classes missing in the application. These will be
+ // fused together by play store when shipped for pre-L devices.
+ public boolean ignoreMainDexMissingClasses = false;
+
// Hidden marker for classes.dex
private boolean hasMarker = false;
private Marker marker;
diff --git a/src/test/examples/autovalue/SimpleAutoValue.java b/src/test/examples/autovalue/SimpleAutoValue.java
deleted file mode 100644
index 2c0b2c6..0000000
--- a/src/test/examples/autovalue/SimpleAutoValue.java
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-package autovalue;
-
-import com.google.auto.value.AutoValue;
-import javax.annotation.Nullable;
-
-public class SimpleAutoValue {
-
- @AutoValue
- static abstract class Pair {
-
- Pair() {
- // Intentionally left empty.
- }
-
- abstract int getOne();
-
- @Nullable
- abstract String getOther();
-
- abstract String getRequiredOther();
-
- static Builder builder() {
- return new AutoValue_SimpleAutoValue_Pair.Builder();
- }
-
- @AutoValue.Builder
- abstract static class Builder {
-
- abstract Builder setOne(int value);
-
- abstract Builder setOther(String value);
-
- abstract Builder setRequiredOther(String value);
-
- abstract Pair build();
- }
- }
-
- public static void main(String... args) {
- Pair.Builder builder = Pair.builder();
- builder.setOne(42);
- builder.setRequiredOther("123");
- System.out.println(builder.build());
- builder = Pair.builder();
- try {
- builder.build();
- } catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
-}
diff --git a/src/test/java/com/android/tools/r8/D8TestBuilder.java b/src/test/java/com/android/tools/r8/D8TestBuilder.java
index 37e4dec..c807a5d 100644
--- a/src/test/java/com/android/tools/r8/D8TestBuilder.java
+++ b/src/test/java/com/android/tools/r8/D8TestBuilder.java
@@ -87,8 +87,8 @@
return self();
}
- public D8TestBuilder setIntermediate(boolean b) {
- builder.setIntermediate(true);
+ public D8TestBuilder setIntermediate(boolean intermediate) {
+ builder.setIntermediate(intermediate);
return self();
}
}
diff --git a/src/test/java/com/android/tools/r8/TestDiagnosticMessagesImpl.java b/src/test/java/com/android/tools/r8/TestDiagnosticMessagesImpl.java
index 15c03dc..7ad544b 100644
--- a/src/test/java/com/android/tools/r8/TestDiagnosticMessagesImpl.java
+++ b/src/test/java/com/android/tools/r8/TestDiagnosticMessagesImpl.java
@@ -8,6 +8,7 @@
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.fail;
+import com.android.tools.r8.utils.ListUtils;
import java.util.ArrayList;
import java.util.List;
import org.hamcrest.Matcher;
@@ -44,25 +45,34 @@
return errors;
}
+ private void assertEmpty(String type, List<Diagnostic> messages) {
+ assertEquals(
+ "Expected no "
+ + type
+ + " messages, got:\n"
+ + String.join("\n", ListUtils.map(messages, m -> m.getDiagnosticMessage())),
+ 0,
+ messages.size());
+ }
public TestDiagnosticMessages assertNoMessages() {
- assertEquals(0, getInfos().size());
- assertEquals(0, getWarnings().size());
- assertEquals(0, getErrors().size());
+ assertEmpty("info", getInfos());
+ assertEmpty("warning", getWarnings());
+ assertEmpty("error", getErrors());
return this;
}
public TestDiagnosticMessages assertOnlyInfos() {
assertNotEquals(0, getInfos().size());
- assertEquals(0, getWarnings().size());
- assertEquals(0, getErrors().size());
+ assertEmpty("warning", getWarnings());
+ assertEmpty("error", getErrors());
return this;
}
public TestDiagnosticMessages assertOnlyWarnings() {
- assertEquals(0, getInfos().size());
+ assertEmpty("info", getInfos());
assertNotEquals(0, getWarnings().size());
- assertEquals(0, getErrors().size());
+ assertEmpty("error", getErrors());
return this;
}
diff --git a/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithInvokeInterfaceTest.java b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithInvokeInterfaceTest.java
new file mode 100644
index 0000000..e9b1b8c
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithInvokeInterfaceTest.java
@@ -0,0 +1,25 @@
+// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.desugar;
+
+public class DefaultLambdaWithInvokeInterfaceTest {
+
+ public interface I {
+ int run();
+ }
+
+ public interface J {
+ default String stateless() {
+ return "hest";
+ }
+
+ default I stateful() {
+ return () -> stateless().length();
+ }
+ }
+
+ public static void main(String[] args) {
+ System.out.println(new J() {}.stateful().run());
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithInvokeInterfaceTestRunner.java b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithInvokeInterfaceTestRunner.java
new file mode 100644
index 0000000..732612d
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithInvokeInterfaceTestRunner.java
@@ -0,0 +1,35 @@
+// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.desugar;
+
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.utils.AndroidApiLevel;
+import com.android.tools.r8.utils.StringUtils;
+import org.junit.Test;
+
+public class DefaultLambdaWithInvokeInterfaceTestRunner extends TestBase {
+
+ final Class<?> CLASS = DefaultLambdaWithInvokeInterfaceTest.class;
+ final String EXPECTED = StringUtils.lines("4");
+
+ @Test
+ public void testJvm() throws Exception {
+ testForJvm().addTestClasspath().run(CLASS).assertSuccessWithOutput(EXPECTED);
+ }
+
+ @Test
+ public void test() throws Exception {
+ testForD8()
+ .addProgramClassesAndInnerClasses(CLASS)
+ .setMinApi(AndroidApiLevel.K)
+ .compile()
+ // TODO(b/123506120): Add .assertNoMessages()
+ .run(CLASS)
+ .assertSuccessWithOutput(EXPECTED)
+ .inspect(inspector -> assertThat(inspector.clazz(CLASS), isPresent()));
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithSelfReferenceTest.java b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithSelfReferenceTest.java
new file mode 100644
index 0000000..eb4e75b
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithSelfReferenceTest.java
@@ -0,0 +1,23 @@
+// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.desugar;
+
+public class DefaultLambdaWithSelfReferenceTest {
+
+ interface I {
+ String foo();
+
+ default I stateless() {
+ return () -> "stateless";
+ }
+
+ default I stateful() {
+ return () -> "stateful(" + stateless().foo() + ")";
+ }
+ }
+
+ public static void main(String[] args) {
+ System.out.println(((I) () -> "foo").stateful().foo());
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithSelfReferenceTestRunner.java b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithSelfReferenceTestRunner.java
new file mode 100644
index 0000000..bc193c8
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/DefaultLambdaWithSelfReferenceTestRunner.java
@@ -0,0 +1,94 @@
+// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.desugar;
+
+import static org.junit.Assert.assertEquals;
+
+import com.android.tools.r8.Disassemble;
+import com.android.tools.r8.Disassemble.DisassembleCommand;
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.ToolHelper;
+import com.android.tools.r8.utils.AndroidApiLevel;
+import com.android.tools.r8.utils.StringUtils;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import org.junit.Test;
+
+public class DefaultLambdaWithSelfReferenceTestRunner extends TestBase {
+
+ final Class<?> CLASS = DefaultLambdaWithSelfReferenceTest.class;
+ final String EXPECTED = StringUtils.lines("stateful(stateless)");
+
+ @Test
+ public void testJvm() throws Exception {
+ testForJvm().addTestClasspath().run(CLASS).assertSuccessWithOutput(EXPECTED);
+ }
+
+ @Test
+ public void test() throws Exception {
+ Path out1 = temp.newFolder().toPath().resolve("out1.zip");
+ testForD8()
+ .addProgramClassesAndInnerClasses(CLASS)
+ .setMinApi(AndroidApiLevel.K)
+ .compile()
+ // TODO(b/123506120): Add .assertNoMessages()
+ .writeToZip(out1)
+ .run(CLASS)
+ .assertSuccessWithOutput(EXPECTED);
+
+ Path outPerClassDir = temp.newFolder().toPath();
+ Collection<Path> innerClasses =
+ ToolHelper.getClassFilesForInnerClasses(Collections.singleton(CLASS));
+
+ int i = 0;
+ List<Path> outs = new ArrayList<>();
+ {
+ Path mainOut = outPerClassDir.resolve("class" + i++ + ".zip");
+ outs.add(mainOut);
+ testForD8()
+ .addProgramClasses(CLASS)
+ .addClasspathFiles(ToolHelper.getClassPathForTests())
+ .setIntermediate(true)
+ .setMinApi(AndroidApiLevel.K)
+ .compile()
+ // TODO(b/123506120): Add .assertNoMessages()
+ .writeToZip(mainOut);
+ }
+ for (Path innerClass : innerClasses) {
+ Path out = outPerClassDir.resolve("class" + i++ + ".zip");
+ outs.add(out);
+ testForD8()
+ .addProgramFiles(innerClass)
+ .addClasspathFiles(ToolHelper.getClassPathForTests())
+ .setIntermediate(true)
+ .setMinApi(AndroidApiLevel.K)
+ .compile()
+ // TODO(b/123506120): Add .assertNoMessages()
+ .writeToZip(out);
+ }
+
+ Path out2 = temp.newFolder().toPath().resolve("out2.zip");
+ testForD8()
+ .addProgramFiles(outs)
+ .compile()
+ // TODO(b/123506120): Add .assertNoMessages()
+ .writeToZip(out2)
+ .run(CLASS)
+ .assertSuccessWithOutput(EXPECTED);
+
+ Path dissasemble1 = temp.newFolder().toPath().resolve("disassemble1.txt");
+ Path dissasemble2 = temp.newFolder().toPath().resolve("disassemble2.txt");
+ Disassemble.disassemble(
+ DisassembleCommand.builder().addProgramFiles(out1).setOutputPath(dissasemble1).build());
+ Disassemble.disassemble(
+ DisassembleCommand.builder().addProgramFiles(out2).setOutputPath(dissasemble2).build());
+ String content1 = StringUtils.join(Files.readAllLines(dissasemble1), "\n");
+ String content2 = StringUtils.join(Files.readAllLines(dissasemble2), "\n");
+ assertEquals(content1, content2);
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java b/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java
index ff948aa..5103951 100644
--- a/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java
+++ b/src/test/java/com/android/tools/r8/maindexlist/MainDexListTests.java
@@ -226,14 +226,15 @@
@Test
public void everyThirdClassInMainWithDexSplitter() throws Throwable {
List<String> featureMappings = new ArrayList<>();
+ List<String> inFeatureMapping = new ArrayList<>();
ImmutableList.Builder<String> mainDexBuilder = ImmutableList.builder();
for (int i = 0; i < MANY_CLASSES.size(); i++) {
String clazz = MANY_CLASSES.get(i);
// Write the first 2 classes into the split.
- if (i < 2) {
+ if (i < 10) {
featureMappings.add(clazz + ":feature1");
- continue;
+ inFeatureMapping.add(clazz);
}
if (i % 3 == 0) {
mainDexBuilder.add(clazz);
@@ -246,19 +247,20 @@
FileUtils.writeTextFile(mainDexFile, ListUtils.map(mainDexList, MainDexListTests::typeToEntry));
Path output = temp.getRoot().toPath().resolve("split_output");
Files.createDirectories(output);
-
- Options options = new Options();
+ TestDiagnosticsHandler diagnosticsHandler = new TestDiagnosticsHandler();
+ Options options = new Options(diagnosticsHandler);
options.addInputArchive(getManyClassesMultiDexAppPath().toString());
options.setFeatureSplitMapping(featureSplitMapping.toString());
options.setOutput(output.toString());
options.setMainDexList(mainDexFile.toString());
DexSplitter.run(options);
+ assertEquals(0, diagnosticsHandler.numberOfErrorsAndWarnings());
Path baseDir = output.resolve("base");
CodeInspector inspector =
new CodeInspector(
AndroidApp.builder().addProgramFiles(baseDir.resolve("classes.dex")).build());
for (String clazz : mainDexList) {
- if (!inspector.clazz(clazz).isPresent()) {
+ if (!inspector.clazz(clazz).isPresent() && !inFeatureMapping.contains(clazz)) {
failedToFindClassInExpectedFile(baseDir, clazz);
}
}
@@ -923,10 +925,20 @@
private class TestDiagnosticsHandler implements DiagnosticsHandler {
public List<Diagnostic> errors = new ArrayList<>();
+ public List<Diagnostic> warnings = new ArrayList<>();
+
+ public int numberOfErrorsAndWarnings() {
+ return errors.size() + warnings.size();
+ }
@Override
public void error(Diagnostic error) {
errors.add(error);
}
+
+ @Override
+ public void warning(Diagnostic warning) {
+ warnings.add(warning);
+ }
}
}
diff --git a/src/test/java/com/android/tools/r8/shaking/desugar/interfacemethods/BridgeInliningTest.java b/src/test/java/com/android/tools/r8/shaking/desugar/interfacemethods/BridgeInliningTest.java
index 5038729..e191921 100644
--- a/src/test/java/com/android/tools/r8/shaking/desugar/interfacemethods/BridgeInliningTest.java
+++ b/src/test/java/com/android/tools/r8/shaking/desugar/interfacemethods/BridgeInliningTest.java
@@ -4,7 +4,6 @@
package com.android.tools.r8.shaking.desugar.interfacemethods;
import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -60,11 +59,9 @@
MethodSubject m = c.uniqueMethodWithName("m");
assertThat(m, isPresent());
assertTrue(m.getMethod().hasCode());
- // TODO(b/123778921): why are I and C not merged without merge annotations?
+ // TODO(b/124017330): Verify that I$-CC.m() has been inlined into C.m().
//assertTrue(
// m.iterateInstructions(i -> i.isConstString("I::m", JumboStringMode.ALLOW)).hasNext());
-
- // TODO(b/123778921): No companion class is in the output.
//codeInspector.forAllClasses(classSubject -> {
// assertFalse(classSubject.getOriginalDescriptor().contains("$-CC"));
//});
diff --git a/third_party/opensource_apps.tar.gz.sha1 b/third_party/opensource_apps.tar.gz.sha1
new file mode 100644
index 0000000..9fb2bb2
--- /dev/null
+++ b/third_party/opensource_apps.tar.gz.sha1
@@ -0,0 +1 @@
+64b0689bce8f6789320b34da4e91c6dbcc1296e9
\ No newline at end of file
diff --git a/tools/apk_utils.py b/tools/apk_utils.py
index 5a6aa94..af24ec1 100644
--- a/tools/apk_utils.py
+++ b/tools/apk_utils.py
@@ -23,7 +23,8 @@
]
utils.RunCmd(cmd, quiet=quiet)
-def sign_with_apksigner(build_tools_dir, unsigned_apk, signed_apk, keystore, password):
+def sign_with_apksigner(
+ build_tools_dir, unsigned_apk, signed_apk, keystore, password, quiet=False):
cmd = [
os.path.join(build_tools_dir, 'apksigner'),
'sign',
@@ -34,5 +35,4 @@
'--out', signed_apk,
unsigned_apk
]
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
+ utils.RunCmd(cmd, quiet=quiet)
diff --git a/tools/as_utils.py b/tools/as_utils.py
index 5f224a6..7a6528a 100644
--- a/tools/as_utils.py
+++ b/tools/as_utils.py
@@ -127,7 +127,7 @@
if '-printconfiguration' not in line:
f.write(line)
# Check that there is a line-break at the end of the file or insert one.
- if lines[-1].strip():
+ if len(lines) and lines[-1].strip():
f.write('\n')
f.write('-printconfiguration {}\n'.format(destination))
@@ -159,7 +159,7 @@
Move(src, dst, quiet=quiet)
def MoveFile(src, dst, quiet=False):
- assert os.path.isfile(src)
+ assert os.path.isfile(src), "Expected a file to be present at " + src
Move(src, dst, quiet=quiet)
def MoveProfileReportTo(dest_dir, build_stdout, quiet=False):
diff --git a/tools/download_all_benchmark_dependencies.py b/tools/download_all_benchmark_dependencies.py
new file mode 100755
index 0000000..43ec0ab
--- /dev/null
+++ b/tools/download_all_benchmark_dependencies.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# Utility script to make it easier to update what golem builds.
+
+import gradle
+import sys
+import utils
+
+BUILD_TARGETS = ['downloadDeps', 'downloadAndroidCts', 'downloadDx']
+
+def Main():
+ gradle.RunGradle(BUILD_TARGETS)
+ # Download opensource_apps and place in build.
+ utils.DownloadFromX20(utils.OPENSOURCE_APPS_SHA_FILE)
+
+
+if __name__ == '__main__':
+ sys.exit(Main())
diff --git a/tools/run_on_as_app.py b/tools/run_on_as_app.py
index 3a9bf8c..55c8857 100755
--- a/tools/run_on_as_app.py
+++ b/tools/run_on_as_app.py
@@ -5,6 +5,7 @@
import apk_masseur
import apk_utils
+import golem
import gradle
import os
import optparse
@@ -17,15 +18,20 @@
import as_utils
-SHRINKERS = ['r8', 'r8-minified', 'r8full', 'r8full-minified', 'proguard']
-WORKING_DIR = utils.BUILD
+SHRINKERS = ['r8', 'r8-full', 'r8-nolib', 'r8-nolib-full', 'pg']
+WORKING_DIR = os.path.join(utils.BUILD, 'opensource_apps')
-if 'R8_BENCHMARK_DIR' in os.environ and os.path.isdir(os.environ['R8_BENCHMARK_DIR']):
+if ('R8_BENCHMARK_DIR' in os.environ
+ and os.path.isdir(os.environ['R8_BENCHMARK_DIR'])):
WORKING_DIR = os.environ['R8_BENCHMARK_DIR']
+# For running on Golem all APPS are bundled as an x20-dependency and then copied
+# to WORKING_DIR. To make it easier to update the app-bundle, remove the folder
+# WORKING_DIR and then run run_on_as_app.py --download-only.
APPS = {
# 'app-name': {
# 'git_repo': ...
+ # 'revision': ...,
# 'app_module': ... (default app)
# 'archives_base_name': ... (default same as app_module)
# 'flavor': ... (default no flavor)
@@ -34,6 +40,7 @@
'AnExplorer': {
'app_id': 'dev.dworks.apps.anexplorer.pro',
'git_repo': 'https://github.com/christofferqa/AnExplorer',
+ 'revision': '365927477b8eab4052a1882d5e358057ae3dee4d',
'flavor': 'googleMobilePro',
'signed-apk-name': 'AnExplorer-googleMobileProRelease-4.0.3.apk',
'min_sdk': 17
@@ -41,6 +48,7 @@
'AntennaPod': {
'app_id': 'de.danoeh.antennapod',
'git_repo': 'https://github.com/christofferqa/AntennaPod.git',
+ 'revision': '77e94f4783a16abe9cc5b78dc2d2b2b1867d8c06',
'flavor': 'play',
'min_sdk': 14,
'compile_sdk': 26
@@ -48,78 +56,105 @@
'apps-android-wikipedia': {
'app_id': 'org.wikipedia',
'git_repo': 'https://github.com/christofferqa/apps-android-wikipedia',
+ 'revision': '686e8aa5682af8e6a905054b935dd2daa57e63ee',
'flavor': 'prod',
- 'signed-apk-name': 'app-prod-universal-release.apk'
+ 'signed-apk-name': 'app-prod-universal-release.apk',
},
'chanu': {
- 'app_id': 'com.chanapps.four.activity',
- 'git_repo': 'https://github.com/mkj-gram/chanu.git',
+ 'app_id': 'com.chanapps.four.activity',
+ 'git_repo': 'https://github.com/mkj-gram/chanu.git',
+ 'revision': '04ade1e9c33d707f0850d5eb9d6fa5e8af814a26',
},
'friendlyeats-android': {
'app_id': 'com.google.firebase.example.fireeats',
- 'git_repo': 'https://github.com/christofferqa/friendlyeats-android.git'
+ 'git_repo': 'https://github.com/christofferqa/friendlyeats-android.git',
+ 'revision': '10091fa0ec37da12e66286559ad1b6098976b07b',
+ },
+ 'Instabug-Android': {
+ 'app_id': 'com.example.instabug',
+ 'git_repo': 'https://github.com/christofferqa/Instabug-Android.git',
+ 'revision': 'b8df78c96630a6537fbc07787b4990afc030cc0f'
},
'KISS': {
'app_id': 'fr.neamar.kiss',
'git_repo': 'https://github.com/christofferqa/KISS',
+ 'revision': '093da9ee0512e67192f62951c45a07a616fc3224',
},
'materialistic': {
'app_id': 'io.github.hidroh.materialistic',
'git_repo': 'https://github.com/christofferqa/materialistic',
+ 'revision': '2b2b2ee25ce9e672d5aab1dc90a354af1522b1d9',
},
'Minimal-Todo': {
'app_id': 'com.avjindersinghsekhon.minimaltodo',
'git_repo': 'https://github.com/christofferqa/Minimal-Todo',
+ 'revision': '9d8c73746762cd376b718858ec1e8783ca07ba7c',
},
'NewPipe': {
'app_id': 'org.schabi.newpipe',
'git_repo': 'https://github.com/christofferqa/NewPipe',
+ 'revision': 'ed543099c7823be00f15d9340f94bdb7cb37d1e6',
},
'rover-android': {
- 'app_id': 'io.rover.app.debug',
- 'app_module': 'debug-app',
- 'git_repo': 'https://github.com/mkj-gram/rover-android.git',
+ 'app_id': 'io.rover.app.debug',
+ 'app_module': 'debug-app',
+ 'git_repo': 'https://github.com/mkj-gram/rover-android.git',
+ 'revision': 'd2e876e597b3af7eab406e38a0e08327a38bd942',
},
'Signal-Android': {
- 'app_id': 'org.thoughtcrime.securesms',
- 'app_module': '',
- 'flavor': 'play',
- 'git_repo': 'https://github.com/mkj-gram/Signal-Android.git',
- 'releaseTarget': 'assemblePlayRelease',
- 'signed-apk-name': 'Signal-play-release-4.32.7.apk',
+ 'app_id': 'org.thoughtcrime.securesms',
+ 'app_module': '',
+ 'flavor': 'play',
+ 'git_repo': 'https://github.com/mkj-gram/Signal-Android.git',
+ 'revision': '85e1a10993e5e9ffe923f0798b26cbc44068ba31',
+ 'releaseTarget': 'assemblePlayRelease',
+ 'signed-apk-name': 'Signal-play-release-4.32.7.apk',
},
'Simple-Calendar': {
'app_id': 'com.simplemobiletools.calendar.pro',
'git_repo': 'https://github.com/christofferqa/Simple-Calendar',
+ 'revision': '82dad8c203eea5a0f0ddb513506d8f1de986ef2b',
'signed-apk-name': 'calendar-release.apk'
},
+ 'sqldelight': {
+ 'app_id': 'com.example.sqldelight.hockey',
+ 'git_repo': 'https://github.com/christofferqa/sqldelight.git',
+ 'revision': '2e67a1126b6df05e4119d1e3a432fde51d76cdc8',
+ 'app_module': 'sample/android',
+ 'archives_base_name': 'android',
+ 'min_sdk': 14,
+ 'compile_sdk': 28,
+ },
'tachiyomi': {
'app_id': 'eu.kanade.tachiyomi',
'git_repo': 'https://github.com/sgjesse/tachiyomi.git',
+ 'revision': 'b15d2fe16864645055af6a745a62cc5566629798',
'flavor': 'standard',
'releaseTarget': 'app:assembleRelease',
'min_sdk': 16
},
'tivi': {
'app_id': 'app.tivi',
- # Forked from https://github.com/chrisbanes/tivi.git removing
- # signingConfigs.
'git_repo': 'https://github.com/sgjesse/tivi.git',
- # TODO(123047413): Fails with R8.
- 'skip': True,
+ 'revision': '7d7f591d6f39d7caeb88dd13bf476c0c06accdfb',
+ 'min_sdk': 23,
+ 'compile_sdk': 28,
},
'Tusky': {
- 'app_id': 'com.keylesspalace.tusky',
- 'git_repo': 'https://github.com/mkj-gram/Tusky.git',
- 'flavor': 'blue'
+ 'app_id': 'com.keylesspalace.tusky',
+ 'git_repo': 'https://github.com/mkj-gram/Tusky.git',
+ 'revision': 'b794f3ab90388add98461ffe70edb65c39351c33',
+ 'flavor': 'blue'
},
'Vungle-Android-SDK': {
- 'app_id': 'com.publisher.vungle.sample',
- 'git_repo': 'https://github.com/mkj-gram/Vungle-Android-SDK.git',
+ 'app_id': 'com.publisher.vungle.sample',
+ 'git_repo': 'https://github.com/mkj-gram/Vungle-Android-SDK.git',
+ 'revision': '3e231396ea7ce97b2655e03607497c75730e45f6',
},
# This does not build yet.
'muzei': {
'git_repo': 'https://github.com/sgjesse/muzei.git',
+ 'revision': 'bed2a5f79c6e08b0a21e3e3f9242232d0848ef74',
'app_module': 'main',
'archives_base_name': 'muzei',
'skip': True,
@@ -151,17 +186,21 @@
return '~~R8' in subprocess.check_output(cmd).strip()
def IsMinifiedR8(shrinker):
- return shrinker == 'r8-minified' or shrinker == 'r8full-minified'
+ return 'nolib' not in shrinker
def IsTrackedByGit(file):
return subprocess.check_output(['git', 'ls-files', file]).strip() != ''
-def GitClone(git_url):
- return subprocess.check_output(['git', 'clone', git_url]).strip()
-
-def GitPull():
- # Use --no-edit to accept the auto-generated merge message, if any.
- return subprocess.call(['git', 'pull', '--no-edit']) == 0
+def GitClone(git_url, revision, checkout_dir, options):
+ result = subprocess.check_output(
+ ['git', 'clone', git_url, checkout_dir]).strip()
+ head_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir)
+ if revision == head_rev:
+ return result
+ warn('Target revision is not head in {}.'.format(checkout_dir))
+ with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet):
+ subprocess.check_output(['git', 'reset', '--hard', revision])
+ return result
def GitCheckout(file):
return subprocess.check_output(['git', 'checkout', file]).strip()
@@ -220,26 +259,23 @@
return True
def GetResultsForApp(app, config, options, temp_dir):
- git_repo = config['git_repo']
-
# Checkout and build in the build directory.
checkout_dir = os.path.join(WORKING_DIR, app)
result = {}
- if not os.path.exists(checkout_dir):
+ if not os.path.exists(checkout_dir) and not options.golem:
with utils.ChangedWorkingDirectory(WORKING_DIR, quiet=options.quiet):
- GitClone(git_repo)
- elif options.pull:
- with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet):
- # Checkout build.gradle to avoid merge conflicts.
- if IsTrackedByGit('build.gradle'):
- GitCheckout('build.gradle')
+ GitClone(config['git_repo'], config['revision'], checkout_dir, options)
- if not GitPull():
- result['status'] = 'failed'
- result['error_message'] = 'Unable to pull from remote'
- return result
+ checkout_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir)
+ if config['revision'] != checkout_rev:
+ msg = 'Checkout is not target revision for {} in {}.'.format(
+ app, checkout_dir)
+ if options.ignore_versions:
+ warn(msg)
+ else:
+ raise Exception(msg)
result['status'] = 'success'
@@ -267,7 +303,7 @@
BuildAppWithShrinker(app, config, shrinker, checkout_dir, out_dir,
temp_dir, options)
dex_size = ComputeSizeOfDexFilesInApk(apk_dest)
- result['apk_dest'] = apk_dest,
+ result['apk_dest'] = apk_dest
result['build_status'] = 'success'
result['dex_size'] = dex_size
result['profile_dest_dir'] = profile_dest_dir
@@ -357,11 +393,8 @@
shrinker,
' for recompilation' if keepRuleSynthesisForRecompilation else ''))
- # Add/remove 'r8.jar' from top-level build.gradle.
- if options.disable_tot:
- as_utils.remove_r8_dependency(checkout_dir)
- else:
- as_utils.add_r8_dependency(checkout_dir, temp_dir, IsMinifiedR8(shrinker))
+ # Add 'r8.jar' from top-level build.gradle.
+ as_utils.add_r8_dependency(checkout_dir, temp_dir, IsMinifiedR8(shrinker))
app_module = config.get('app_module', 'app')
archives_base_name = config.get('archives_base_name', app_module)
@@ -382,13 +415,13 @@
releaseTarget = config.get('releaseTarget')
if not releaseTarget:
- releaseTarget = app_module + ':' + 'assemble' + (
+ releaseTarget = app_module.replace('/', ':') + ':' + 'assemble' + (
flavor.capitalize() if flavor else '') + 'Release'
# Value for property android.enableR8.
enableR8 = 'r8' in shrinker
# Value for property android.enableR8.fullMode.
- enableR8FullMode = shrinker == 'r8full' or shrinker == 'r8full-minified'
+ enableR8FullMode = shrinker == 'r8-full' or shrinker == 'r8-nolib-full'
# Build gradlew command line.
cmd = ['./gradlew', '--no-daemon', 'clean', releaseTarget,
'--profile', '--stacktrace',
@@ -419,14 +452,13 @@
if options.sign_apks and not os.path.isfile(signed_apk):
assert os.path.isfile(unsigned_apk)
if options.sign_apks:
- keystore = 'app.keystore'
- keystore_password = 'android'
apk_utils.sign_with_apksigner(
utils.ANDROID_BUILD_TOOLS,
unsigned_apk,
signed_apk,
- keystore,
- keystore_password)
+ options.keystore,
+ options.keystore_password,
+ quiet=options.quiet)
if os.path.isfile(signed_apk):
apk_dest = os.path.join(out_dir, signed_apk_name)
@@ -507,10 +539,28 @@
def LogResultsForApps(result_per_shrinker_per_app, options):
print('')
for app, result_per_shrinker in sorted(
- result_per_shrinker_per_app.iteritems()):
+ result_per_shrinker_per_app.iteritems(), key=lambda s: s[0].lower()):
LogResultsForApp(app, result_per_shrinker, options)
def LogResultsForApp(app, result_per_shrinker, options):
+ if options.print_dexsegments:
+ LogSegmentsForApp(app, result_per_shrinker, options)
+ else:
+ LogComparisonResultsForApp(app, result_per_shrinker, options)
+
+def LogSegmentsForApp(app, result_per_shrinker, options):
+ for shrinker in SHRINKERS:
+ if shrinker not in result_per_shrinker:
+ continue
+ result = result_per_shrinker[shrinker];
+ benchmark_name = '{}-{}'.format(options.print_dexsegments, app)
+ utils.print_dexsegments(benchmark_name, [result.get('apk_dest')])
+ duration = sum(result.get('profile').values())
+ print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration * 1000))
+ print('%s-Total(CodeSize): %s' % (benchmark_name, result.get('dex_size')))
+
+
+def LogComparisonResultsForApp(app, result_per_shrinker, options):
print(app + ':')
if result_per_shrinker.get('status', 'success') != 'success':
@@ -518,7 +568,7 @@
print(' skipped ({})'.format(error_message))
return
- proguard_result = result_per_shrinker.get('proguard', {})
+ proguard_result = result_per_shrinker.get('pg', {})
proguard_dex_size = float(proguard_result.get('dex_size', -1))
proguard_duration = sum(proguard_result.get('profile', {}).values())
@@ -587,6 +637,27 @@
result.add_option('--app',
help='What app to run on',
choices=APPS.keys())
+ result.add_option('--download-only', '--download_only',
+ help='Whether to download apps without any compilation',
+ default=False,
+ action='store_true')
+ result.add_option('--golem',
+ help='Running on golem, do not download',
+ default=False,
+ action='store_true')
+ result.add_option('--gradle-flags', '--gradle_flags',
+ help='Flags to pass in to gradle')
+ result.add_option('--ignore-versions', '--ignore_versions',
+ help='Allow checked-out app to differ in revision from '
+ 'pinned',
+ default=False,
+ action='store_true')
+ result.add_option('--keystore',
+ help='Path to app.keystore',
+ default='app.keystore')
+ result.add_option('--keystore-password', '--keystore_password',
+ help='Password for app.keystore',
+ default='android')
result.add_option('--monkey',
help='Whether to install and run app(s) with monkey',
default=False,
@@ -595,10 +666,23 @@
help='Number of events that the monkey should trigger',
default=250,
type=int)
- result.add_option('--pull',
- help='Whether to pull the latest version of each app',
+ result.add_option('--no-build', '--no_build',
+ help='Run without building ToT first (only when using ToT)',
default=False,
action='store_true')
+ result.add_option('--quiet',
+ help='Disable verbose logging',
+ default=False,
+ action='store_true')
+ result.add_option('--print-dexsegments',
+ metavar='BENCHMARKNAME',
+ help='Print the sizes of individual dex segments as ' +
+ '\'<BENCHMARKNAME>-<APP>-<segment>(CodeSize): '
+ '<bytes>\'')
+ result.add_option('--r8-compilation-steps', '--r8_compilation_steps',
+ help='Number of times R8 should be run on each app',
+ default=2,
+ type=int)
result.add_option('--sign-apks', '--sign_apks',
help='Whether the APKs should be signed',
default=False,
@@ -606,57 +690,50 @@
result.add_option('--shrinker',
help='The shrinkers to use (by default, all are run)',
action='append')
- result.add_option('--r8-compilation-steps', '--r8_compilation_steps',
- help='Number of times R8 should be run on each app',
- default=2,
- type=int)
- result.add_option('--disable-tot', '--disable_tot',
- help='Whether to disable the use of the ToT version of R8',
- default=False,
- action='store_true')
- result.add_option('--no-build', '--no_build',
- help='Run without building ToT first (only when using ToT)',
- default=False,
- action='store_true')
- result.add_option('--gradle-flags', '--gradle_flags',
- help='Flags to pass in to gradle')
- result.add_option('--quiet',
- help='Disable verbose logging',
- default=False,
- action='store_true')
(options, args) = result.parse_args(argv)
- if options.disable_tot:
- # r8.jar is required for recompiling the generated APK
- options.r8_compilation_steps = 1
if options.shrinker:
for shrinker in options.shrinker:
assert shrinker in SHRINKERS
return (options, args)
+def download_apps(options):
+ # Download apps and place in build
+ with utils.ChangedWorkingDirectory(WORKING_DIR):
+ for app, config in APPS.iteritems():
+ app_dir = os.path.join(WORKING_DIR, app)
+ if not os.path.exists(app_dir):
+ GitClone(config['git_repo'], config['revision'], app_dir, options)
+
+
def main(argv):
global SHRINKERS
(options, args) = ParseOptions(argv)
+ if options.golem:
+ golem.link_third_party()
+ if os.path.exists(WORKING_DIR):
+ shutil.rmtree(WORKING_DIR)
+ shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
+
+ if not os.path.exists(WORKING_DIR):
+ os.makedirs(WORKING_DIR)
+
+ if options.download_only:
+ download_apps(options)
+ return
+
with utils.TempDir() as temp_dir:
- if options.disable_tot:
- # Cannot run r8 lib without adding r8lib.jar as an dependency
- SHRINKERS = [
- shrinker for shrinker in SHRINKERS
- if 'minified' not in shrinker]
- else:
- if not options.no_build:
- gradle.RunGradle(['r8', 'r8lib'])
+ if not options.no_build or options.golem:
+ gradle.RunGradle(['r8', 'r8lib'])
- assert os.path.isfile(utils.R8_JAR), (
- 'Cannot build from ToT without r8.jar')
- assert os.path.isfile(utils.R8LIB_JAR), (
- 'Cannot build from ToT without r8lib.jar')
+ assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar'
+ assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
- # Make a copy of r8.jar and r8lib.jar such that they stay the same for
- # the entire execution of this script.
- shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
- shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
+ # Make a copy of r8.jar and r8lib.jar such that they stay the same for
+ # the entire execution of this script.
+ shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
+ shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = {}
@@ -664,7 +741,7 @@
result_per_shrinker_per_app[options.app] = GetResultsForApp(
options.app, APPS.get(options.app), options, temp_dir)
else:
- for app, config in sorted(APPS.iteritems()):
+ for app, config in sorted(APPS.iteritems(), key=lambda s: s[0].lower()):
if not config.get('skip', False):
result_per_shrinker_per_app[app] = GetResultsForApp(
app, config, options, temp_dir)
diff --git a/tools/utils.py b/tools/utils.py
index aff96a3..08faec6 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -62,6 +62,10 @@
CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments')
PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar')
PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar')
+OPENSOURCE_APPS_SHA_FILE = os.path.join(
+ REPO_ROOT,
+ 'third_party/opensource_apps.tar.gz.sha1')
+OPENSOURCE_APPS_FOLDER = os.path.join(REPO_ROOT, 'third_party/opensource_apps/')
# Common environment setup.
@@ -184,9 +188,12 @@
return 'origin/master' in remotes
def get_HEAD_sha1():
+ return get_HEAD_sha1_for_checkout(REPO_ROOT)
+
+def get_HEAD_sha1_for_checkout(checkout):
cmd = ['git', 'rev-parse', 'HEAD']
PrintCmd(cmd)
- with ChangedWorkingDirectory(REPO_ROOT):
+ with ChangedWorkingDirectory(checkout):
return subprocess.check_output(cmd).strip()
def makedirs_if_needed(path):
