Infrastructure for test based benchmarks.
Bug: 210397080
Change-Id: I6938fc23a7578f100de0003656df3104646e66d9
diff --git a/build.gradle b/build.gradle
index 46548e4..eac0f5c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -264,14 +264,23 @@
main11Implementation group: 'org.ow2.asm', name: 'asm-util', version: asmVersion
examplesTestNGRunnerCompile group: 'org.testng', name: 'testng', version: testngVersion
+
testCompile sourceSets.examples.output
testCompile "junit:junit:$junitVersion"
+ testCompile "com.google.guava:guava:$guavaVersion"
testCompile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
testCompile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
testCompile group: 'org.smali', name: 'smali', version: smaliVersion
testCompile files('third_party/jasmin/jasmin-2.4.jar')
testCompile files('third_party/jdwp-tests/apache-harmony-jdwp-tests-host.jar')
testCompile files('third_party/ddmlib/ddmlib.jar')
+ testCompile group: 'org.ow2.asm', name: 'asm', version: asmVersion
+ testCompile group: 'org.ow2.asm', name: 'asm-commons', version: asmVersion
+ testCompile group: 'org.ow2.asm', name: 'asm-tree', version: asmVersion
+ testCompile group: 'org.ow2.asm', name: 'asm-analysis', version: asmVersion
+ testCompile group: 'org.ow2.asm', name: 'asm-util', version: asmVersion
+ testCompile group: 'it.unimi.dsi', name: 'fastutil', version: fastutilVersion
+
jctfCommonCompile "junit:junit:$junitVersion"
jctfTestsCompile "junit:junit:$junitVersion"
jctfTestsCompile sourceSets.jctfCommon.output
diff --git a/src/test/java/com/android/tools/r8/D8TestBuilder.java b/src/test/java/com/android/tools/r8/D8TestBuilder.java
index 1c74df5..cf27d33 100644
--- a/src/test/java/com/android/tools/r8/D8TestBuilder.java
+++ b/src/test/java/com/android/tools/r8/D8TestBuilder.java
@@ -5,6 +5,7 @@
import com.android.tools.r8.D8Command.Builder;
import com.android.tools.r8.TestBase.Backend;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.desugar.desugaredlibrary.DesugaredLibraryTestBase.KeepRuleConsumer;
import com.android.tools.r8.origin.Origin;
import com.android.tools.r8.utils.AndroidApiLevel;
@@ -64,10 +65,13 @@
@Override
D8TestCompileResult internalCompile(
- Builder builder, Consumer<InternalOptions> optionsConsumer, Supplier<AndroidApp> app)
+ Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ Supplier<AndroidApp> app,
+ BenchmarkResults benchmarkResults)
throws CompilationFailedException {
libraryDesugaringTestConfiguration.configure(builder);
- ToolHelper.runD8(builder, optionsConsumer);
+ ToolHelper.runAndBenchmarkD8(builder, optionsConsumer, benchmarkResults);
return new D8TestCompileResult(
getState(),
app.get(),
diff --git a/src/test/java/com/android/tools/r8/DXTestBuilder.java b/src/test/java/com/android/tools/r8/DXTestBuilder.java
index 436f1e7..fcef696 100644
--- a/src/test/java/com/android/tools/r8/DXTestBuilder.java
+++ b/src/test/java/com/android/tools/r8/DXTestBuilder.java
@@ -7,6 +7,7 @@
import com.android.tools.r8.D8Command.Builder;
import com.android.tools.r8.TestBase.Backend;
import com.android.tools.r8.ToolHelper.ProcessResult;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.errors.Unimplemented;
import com.android.tools.r8.utils.AndroidApp;
import com.android.tools.r8.utils.InternalOptions;
@@ -44,8 +45,12 @@
@Override
DXTestCompileResult internalCompile(
- Builder builder, Consumer<InternalOptions> optionsConsumer, Supplier<AndroidApp> app)
+ Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ Supplier<AndroidApp> app,
+ BenchmarkResults benchmarkResults)
throws CompilationFailedException {
+ assert benchmarkResults == null;
assert !libraryDesugaringTestConfiguration.isEnabled();
try {
Path dxOutputFolder = getState().getNewTempFolder();
diff --git a/src/test/java/com/android/tools/r8/ExternalR8TestBuilder.java b/src/test/java/com/android/tools/r8/ExternalR8TestBuilder.java
index 6d34346..2808f05 100644
--- a/src/test/java/com/android/tools/r8/ExternalR8TestBuilder.java
+++ b/src/test/java/com/android/tools/r8/ExternalR8TestBuilder.java
@@ -10,6 +10,7 @@
import com.android.tools.r8.R8Command.Builder;
import com.android.tools.r8.TestBase.Backend;
import com.android.tools.r8.ToolHelper.ProcessResult;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.errors.Unimplemented;
import com.android.tools.r8.utils.AndroidApp;
import com.android.tools.r8.utils.DescriptorUtils;
@@ -110,8 +111,12 @@
@Override
ExternalR8TestCompileResult internalCompile(
- Builder builder, Consumer<InternalOptions> optionsConsumer, Supplier<AndroidApp> app)
+ Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ Supplier<AndroidApp> app,
+ BenchmarkResults benchmarkResults)
throws CompilationFailedException {
+ assert benchmarkResults == null;
assert !libraryDesugaringTestConfiguration.isEnabled();
try {
Path outputFolder = getState().getNewTempFolder();
diff --git a/src/test/java/com/android/tools/r8/ProguardTestBuilder.java b/src/test/java/com/android/tools/r8/ProguardTestBuilder.java
index 8d5bdea..66e5fc8 100644
--- a/src/test/java/com/android/tools/r8/ProguardTestBuilder.java
+++ b/src/test/java/com/android/tools/r8/ProguardTestBuilder.java
@@ -6,6 +6,7 @@
import com.android.tools.r8.R8Command.Builder;
import com.android.tools.r8.TestBase.Backend;
import com.android.tools.r8.ToolHelper.ProcessResult;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.debug.DebugTestConfig;
import com.android.tools.r8.errors.Unimplemented;
import com.android.tools.r8.utils.AndroidApp;
@@ -64,8 +65,12 @@
@Override
ProguardTestCompileResult internalCompile(
- Builder builder, Consumer<InternalOptions> optionsConsumer, Supplier<AndroidApp> app)
+ Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ Supplier<AndroidApp> app,
+ BenchmarkResults benchmarkResults)
throws CompilationFailedException {
+ assert benchmarkResults == null;
assert !libraryDesugaringTestConfiguration.isEnabled();
try {
Path proguardOutputFolder = getState().getNewTempFolder();
diff --git a/src/test/java/com/android/tools/r8/R8TestBuilder.java b/src/test/java/com/android/tools/r8/R8TestBuilder.java
index 090ebb5..b3a595a 100644
--- a/src/test/java/com/android/tools/r8/R8TestBuilder.java
+++ b/src/test/java/com/android/tools/r8/R8TestBuilder.java
@@ -9,6 +9,7 @@
import com.android.tools.r8.R8Command.Builder;
import com.android.tools.r8.TestBase.Backend;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.desugar.desugaredlibrary.DesugaredLibraryTestBase.KeepRuleConsumer;
import com.android.tools.r8.dexsplitter.SplitterTestBase.RunInterface;
import com.android.tools.r8.dexsplitter.SplitterTestBase.SplitRunner;
@@ -71,7 +72,10 @@
@Override
R8TestCompileResult internalCompile(
- Builder builder, Consumer<InternalOptions> optionsConsumer, Supplier<AndroidApp> app)
+ Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ Supplier<AndroidApp> app,
+ BenchmarkResults benchmarkResults)
throws CompilationFailedException {
if (!keepRules.isEmpty()) {
builder.addProguardConfiguration(keepRules, Origin.unknown());
@@ -118,10 +122,11 @@
ToolHelper.addSyntheticProguardRulesConsumerForTesting(
builder, rules -> box.syntheticProguardRules = rules);
libraryDesugaringTestConfiguration.configure(builder);
- ToolHelper.runR8WithoutResult(
+ ToolHelper.runAndBenchmarkR8WithoutResult(
builder.build(),
optionsConsumer.andThen(
- options -> box.proguardConfiguration = options.getProguardConfiguration()));
+ options -> box.proguardConfiguration = options.getProguardConfiguration()),
+ benchmarkResults);
R8TestCompileResult compileResult =
new R8TestCompileResult(
getState(),
diff --git a/src/test/java/com/android/tools/r8/TestCompileResult.java b/src/test/java/com/android/tools/r8/TestCompileResult.java
index 5b6996d..efe353d 100644
--- a/src/test/java/com/android/tools/r8/TestCompileResult.java
+++ b/src/test/java/com/android/tools/r8/TestCompileResult.java
@@ -18,6 +18,7 @@
import com.android.tools.r8.ToolHelper.ArtCommandBuilder;
import com.android.tools.r8.ToolHelper.DexVm;
import com.android.tools.r8.ToolHelper.ProcessResult;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.debug.CfDebugTestConfig;
import com.android.tools.r8.debug.DebugTestConfig;
import com.android.tools.r8.debug.DexDebugTestConfig;
@@ -25,11 +26,13 @@
import com.android.tools.r8.origin.Origin;
import com.android.tools.r8.utils.AndroidApiLevel;
import com.android.tools.r8.utils.AndroidApp;
+import com.android.tools.r8.utils.Box;
import com.android.tools.r8.utils.DescriptorUtils;
import com.android.tools.r8.utils.FileUtils;
import com.android.tools.r8.utils.InternalOptions;
import com.android.tools.r8.utils.ThrowingConsumer;
import com.android.tools.r8.utils.TriFunction;
+import com.android.tools.r8.utils.ZipUtils;
import com.android.tools.r8.utils.codeinspector.ClassSubject;
import com.android.tools.r8.utils.codeinspector.CodeInspector;
import com.google.common.collect.ImmutableList;
@@ -658,4 +661,19 @@
app.writeToZip(jarFile, OutputMode.DexIndexed);
return new Dex2OatTestRunResult(app, runtime, ToolHelper.runDex2OatRaw(jarFile, oatFile, vm));
}
+
+ public CR benchmarkCodeSize(BenchmarkResults results) throws IOException {
+ Path out = writeToZip();
+ Box<Long> size = new Box<>(0L);
+ ZipUtils.iter(
+ out,
+ (entry, stream) -> {
+ if ((getBackend().isDex() && entry.getName().endsWith(".dex"))
+ || getBackend().isCf() && entry.getName().endsWith(".class")) {
+ size.set(size.get() + entry.getSize());
+ }
+ });
+ results.addCodeSizeResult(size.get());
+ return self();
+ }
}
diff --git a/src/test/java/com/android/tools/r8/TestCompilerBuilder.java b/src/test/java/com/android/tools/r8/TestCompilerBuilder.java
index a66a4b2..be94525 100644
--- a/src/test/java/com/android/tools/r8/TestCompilerBuilder.java
+++ b/src/test/java/com/android/tools/r8/TestCompilerBuilder.java
@@ -9,6 +9,7 @@
import static org.junit.Assert.assertTrue;
import com.android.tools.r8.TestBase.Backend;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.debug.DebugTestConfig;
import com.android.tools.r8.desugar.desugaredlibrary.DesugaredLibraryTestBase.KeepRuleConsumer;
import com.android.tools.r8.optimize.argumentpropagation.ArgumentPropagatorEventConsumer;
@@ -110,7 +111,10 @@
}
abstract CR internalCompile(
- B builder, Consumer<InternalOptions> optionsConsumer, Supplier<AndroidApp> app)
+ B builder,
+ Consumer<InternalOptions> optionsConsumer,
+ Supplier<AndroidApp> app,
+ BenchmarkResults benchmarkResults)
throws CompilationFailedException;
public T addArgumentPropagatorCodeScannerResultInspector(
@@ -183,7 +187,19 @@
dexItemFactory, verticallyMergedClasses))));
}
+ public CR benchmarkCompile(BenchmarkResults results) throws CompilationFailedException {
+ if (System.getProperty("com.android.tools.r8.printtimes") != null) {
+ allowStdoutMessages();
+ }
+ return internalCompileAndBenchmark(results);
+ }
+
public CR compile() throws CompilationFailedException {
+ return internalCompileAndBenchmark(null);
+ }
+
+ private CR internalCompileAndBenchmark(BenchmarkResults benchmark)
+ throws CompilationFailedException {
AndroidAppConsumers sink = new AndroidAppConsumers();
builder.setProgramConsumer(sink.wrapProgramConsumer(programConsumer));
if (mainDexClassesCollector != null || mainDexListConsumer != null) {
@@ -239,7 +255,7 @@
() -> new AssertionError("Unexpected print to stderr"))));
}
cr =
- internalCompile(builder, optionsConsumer, Suppliers.memoize(sink::build))
+ internalCompile(builder, optionsConsumer, Suppliers.memoize(sink::build), benchmark)
.addRunClasspathFiles(additionalRunClassPath);
if (isAndroidBuildVersionAdded) {
cr.setSystemProperty(AndroidBuildVersion.PROPERTY, "" + builder.getMinApiLevel());
diff --git a/src/test/java/com/android/tools/r8/ToolHelper.java b/src/test/java/com/android/tools/r8/ToolHelper.java
index 57fabd1..1534edb 100644
--- a/src/test/java/com/android/tools/r8/ToolHelper.java
+++ b/src/test/java/com/android/tools/r8/ToolHelper.java
@@ -14,6 +14,7 @@
import com.android.tools.r8.TestBase.Backend;
import com.android.tools.r8.TestRuntime.CfRuntime;
import com.android.tools.r8.ToolHelper.DexVm.Kind;
+import com.android.tools.r8.benchmarks.BenchmarkResults;
import com.android.tools.r8.desugar.desugaredlibrary.jdk11.DesugaredLibraryJDK11Undesugarer;
import com.android.tools.r8.dex.ApplicationReader;
import com.android.tools.r8.errors.Unreachable;
@@ -1252,9 +1253,25 @@
public static void runR8WithoutResult(
R8Command command, Consumer<InternalOptions> optionsConsumer)
throws CompilationFailedException {
+ runAndBenchmarkR8WithoutResult(command, optionsConsumer, null);
+ }
+
+ public static void runAndBenchmarkR8WithoutResult(
+ R8Command command,
+ Consumer<InternalOptions> optionsConsumer,
+ BenchmarkResults benchmarkResults)
+ throws CompilationFailedException {
InternalOptions internalOptions = command.getInternalOptions();
optionsConsumer.accept(internalOptions);
+ long start = 0;
+ if (benchmarkResults != null) {
+ start = System.nanoTime();
+ }
R8.runForTesting(command.getInputApp(), internalOptions);
+ if (benchmarkResults != null) {
+ long end = System.nanoTime();
+ benchmarkResults.addRuntimeRawResult(end - start);
+ }
}
public static AndroidApp runR8WithFullResult(
@@ -1324,6 +1341,14 @@
public static AndroidApp runD8(
D8Command.Builder builder, Consumer<InternalOptions> optionsConsumer)
throws CompilationFailedException {
+ return runAndBenchmarkD8(builder, optionsConsumer, null);
+ }
+
+ public static AndroidApp runAndBenchmarkD8(
+ D8Command.Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ BenchmarkResults benchmarkResults)
+ throws CompilationFailedException {
AndroidAppConsumers compatSink = new AndroidAppConsumers(builder);
D8Command command = builder.build();
InternalOptions options = command.getInternalOptions();
@@ -1331,7 +1356,15 @@
ExceptionUtils.withD8CompilationHandler(
options.reporter, () -> optionsConsumer.accept(options));
}
+ long start = 0;
+ if (benchmarkResults != null) {
+ start = System.nanoTime();
+ }
D8.runForTesting(command.getInputApp(), options);
+ if (benchmarkResults != null) {
+ long end = System.nanoTime();
+ benchmarkResults.addRuntimeRawResult(end - start);
+ }
return compatSink.build();
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java
new file mode 100644
index 0000000..3753d7f
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java
@@ -0,0 +1,41 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public abstract class BenchmarkBase extends TestBase {
+
+ // Benchmarks must be configured with the "none" runtime as each config defines a singleton
+ // benchmark in golem.
+ public static List<Object[]> parametersFromConfigs(Iterable<BenchmarkConfig> configs) {
+ return buildParameters(configs, getTestParameters().withNoneRuntime().build());
+ }
+
+ private final BenchmarkConfig config;
+
+ protected BenchmarkBase(BenchmarkConfig config, TestParameters parameters) {
+ this.config = config;
+ parameters.assertNoneRuntime();
+ }
+
+ protected BenchmarkConfig getConfig() {
+ return config;
+ }
+
+ @Test
+ public void testBenchmarks() throws Exception {
+ config.run(temp);
+ }
+
+ public static BenchmarkRunner runner(BenchmarkConfig config) {
+ return BenchmarkRunner.runner(config);
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java
new file mode 100644
index 0000000..f75aabe
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java
@@ -0,0 +1,41 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.android.tools.r8.benchmarks.helloworld.HelloWorldBenchmark;
+import com.android.tools.r8.errors.Unreachable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class BenchmarkCollection {
+
+ // Actual list of all configured benchmarks.
+ private final Map<BenchmarkIdentifier, BenchmarkConfig> benchmarks = new HashMap<>();
+
+ private void addBenchmark(BenchmarkConfig benchmark) {
+ BenchmarkIdentifier id = benchmark.getIdentifier();
+ if (benchmarks.containsKey(id)) {
+ throw new Unreachable("Duplicate definition of benchmark with name and target: " + id);
+ }
+ benchmarks.put(id, benchmark);
+ }
+
+ public BenchmarkConfig getBenchmark(BenchmarkIdentifier benchmark) {
+ return benchmarks.get(benchmark);
+ }
+
+ public static BenchmarkCollection computeCollection() {
+ BenchmarkCollection collection = new BenchmarkCollection();
+ // Every benchmark that should be active on golem must be setup in this method.
+ HelloWorldBenchmark.configs().forEach(collection::addBenchmark);
+ return collection;
+ }
+
+ /** Compute and print the golem configuration. */
+ public static void main(String[] args) throws IOException {
+ new BenchmarkCollectionPrinter(System.out)
+ .printGolemConfig(computeCollection().benchmarks.values());
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java
new file mode 100644
index 0000000..af25a57
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java
@@ -0,0 +1,187 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import static com.android.tools.r8.utils.StringUtils.join;
+
+import com.android.tools.r8.ToolHelper;
+import com.android.tools.r8.ToolHelper.ProcessResult;
+import com.android.tools.r8.errors.Unreachable;
+import com.android.tools.r8.utils.StringUtils.BraceType;
+import com.google.common.base.Strings;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+
+public class BenchmarkCollectionPrinter {
+
+ private interface RunnableIO {
+ void run() throws IOException;
+ }
+
+ private static final PrintStream QUIET =
+ new PrintStream(
+ new OutputStream() {
+ @Override
+ public void write(int b) {
+ // ignore
+ }
+ });
+
+ // Internal state for printing the benchmark config for golem.
+ private int currentIndent = 0;
+ private final PrintStream out;
+
+ public BenchmarkCollectionPrinter(PrintStream out) {
+ this.out = out;
+ }
+
+ private void indentScope(RunnableIO fn) throws IOException {
+ indentScope(2, fn);
+ }
+
+ private void indentScope(int spaces, RunnableIO fn) throws IOException {
+ currentIndent += spaces;
+ fn.run();
+ currentIndent -= spaces;
+ }
+
+ private static String quote(String str) {
+ return "\"" + str + "\"";
+ }
+
+ private void print(String string) {
+ printIndented(string, currentIndent);
+ }
+
+ private void printIndented(String string, int indent) {
+ out.print(Strings.repeat(" ", indent));
+ out.println(string);
+ }
+
+ public void printGolemConfig(Collection<BenchmarkConfig> benchmarks) throws IOException {
+ Path jdkHome = getJdkHome();
+ ArrayList<BenchmarkConfig> sortedBenchmarks = new ArrayList<>(benchmarks);
+ sortedBenchmarks.sort(
+ Comparator.comparing(BenchmarkConfig::getIdentifier)
+ .thenComparing(BenchmarkConfig::getTarget));
+ print(
+ "// AUTOGENERATED FILE from"
+ + " src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java");
+ print("part of r8_config;");
+ print("");
+ print("createTestBenchmarks() {");
+ indentScope(
+ () -> {
+ print("final cpus = [\"Lenovo M90\"];");
+ addGolemResource("openjdk", Paths.get(jdkHome + ".tar.gz"));
+ for (BenchmarkConfig benchmark : sortedBenchmarks) {
+ printBenchmarkBlock(benchmark);
+ }
+ });
+ print("}");
+ }
+
+ private void printBenchmarkBlock(BenchmarkConfig benchmark) throws IOException {
+ print("{");
+ indentScope(
+ () -> {
+ boolean addWarmupBenchmark = benchmark.hasTimeWarmupRuns();
+ String suite = benchmark.getSuite().getDartName();
+ print("final name = " + quote(benchmark.getName()) + ";");
+ print("final targets = [" + quote(benchmark.getTarget().getGolemName()) + "];");
+ print(
+ "final metrics = "
+ + join(
+ ", ", benchmark.getMetrics(), BenchmarkMetric::getDartType, BraceType.SQUARE)
+ + ";");
+ if (addWarmupBenchmark) {
+ print("final benchmark = new GroupBenchmark(name + \"Group\", metrics);");
+ } else {
+ print("final benchmark = new StandardBenchmark(name, metrics);");
+ }
+ print("final options = benchmark.addTargets(noImplementation, targets);");
+ print("options.cpus = cpus;");
+ print("options.isScript = true;");
+ print("options.fromRevision = " + benchmark.getFromRevision() + ";");
+ print(
+ "options.mainFile = "
+ + quote(
+ "tools/run_benchmark.py --golem"
+ + " --target "
+ + benchmark.getTarget().getIdentifierName()
+ + " --benchmark "
+ + benchmark.getName()));
+ print("options.resources.add(openjdk);");
+ if (addWarmupBenchmark) {
+ print("final warmupName = " + quote(benchmark.getWarmupName()) + ";");
+ print("benchmark.addBenchmark(warmupName, [Metric.RunTimeRaw]);");
+ print("benchmark.addBenchmark(name, metrics);");
+ print(suite + ".addBenchmark(warmupName);");
+ print(suite + ".addBenchmark(name);");
+ } else {
+ print(suite + ".addBenchmark(name);");
+ }
+ });
+ print("}");
+ }
+
+ private void addGolemResource(String name, Path tarball) throws IOException {
+ Path shaFile = Paths.get(tarball.toString() + ".sha1");
+ downloadDependency(shaFile);
+ String sha256 = computeSha256(tarball);
+ String shaFileContent = getShaFileContent(shaFile);
+ print("final " + name + " = BenchmarkResource(" + quote("") + ",");
+ indentScope(
+ 4,
+ () -> {
+ print("type: BenchmarkResourceType.Storage,");
+ print("uri: " + quote("gs://r8-deps/" + shaFileContent));
+ // Make dart formatter happy.
+ if (currentIndent > 2) {
+ print("hash: ");
+ indentScope(4, () -> print(quote(sha256)));
+ } else {
+ print("hash: " + quote(sha256));
+ }
+ print("extract: " + quote("gz") + ");");
+ });
+ }
+
+ private static Path getJdkHome() throws IOException {
+ ProcessBuilder builder = new ProcessBuilder("python", "tools/jdk.py");
+ ProcessResult result = ToolHelper.runProcess(builder, QUIET);
+ if (result.exitCode != 0) {
+ throw new Unreachable("Unexpected failure to determine jdk home: " + result);
+ }
+ return Paths.get(result.stdout.trim());
+ }
+
+ private static String computeSha256(Path path) throws IOException {
+ Hasher hasher = Hashing.sha256().newHasher();
+ return hasher.putBytes(Files.readAllBytes(path)).hash().toString();
+ }
+
+ private static String getShaFileContent(Path path) throws IOException {
+ return String.join("\n", Files.readAllLines(path)).trim();
+ }
+
+ private static void downloadDependency(Path path) throws IOException {
+ ProcessBuilder builder =
+ new ProcessBuilder(
+ "download_from_google_storage", "-n", "-b", "r8-deps", "-u", "-s", path.toString());
+ ProcessResult result = ToolHelper.runProcess(builder, QUIET);
+ if (result.exitCode != 0) {
+ throw new Unreachable("Unable to download dependency '" + path + "'\n" + result);
+ }
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java
new file mode 100644
index 0000000..09d7a6e
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java
@@ -0,0 +1,168 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.android.tools.r8.errors.Unreachable;
+import com.google.common.collect.ImmutableSet;
+import java.util.HashSet;
+import java.util.Set;
+import org.junit.rules.TemporaryFolder;
+
+public class BenchmarkConfig {
+
+ public static class Builder {
+
+ private String name = null;
+ private BenchmarkMethod method = null;
+ private BenchmarkTarget target = null;
+ private Set<BenchmarkMetric> metrics = new HashSet<>();
+ private BenchmarkSuite suite = BenchmarkSuite.getDefault();
+ private int fromRevision = -1;
+
+ private boolean timeWarmupRuns = false;
+
+ private Builder() {}
+
+ public BenchmarkConfig build() {
+ if (name == null) {
+ throw new Unreachable("Benchmark name must be set");
+ }
+ if (method == null) {
+ throw new Unreachable("Benchmark method must be set");
+ }
+ if (target == null) {
+ throw new Unreachable("Benchmark target must be set");
+ }
+ if (metrics.isEmpty()) {
+ throw new Unreachable("Benchmark must have at least one metric to measure");
+ }
+ if (suite == null) {
+ throw new Unreachable("Benchmark must have a suite");
+ }
+ if (fromRevision < 0) {
+ throw new Unreachable("Benchmark must specify from which golem revision it is valid");
+ }
+ if (timeWarmupRuns && !metrics.contains(BenchmarkMetric.RunTimeRaw)) {
+ throw new Unreachable("Benchmark with warmup time must measure RunTimeRaw");
+ }
+ return new BenchmarkConfig(
+ name, method, target, ImmutableSet.copyOf(metrics), suite, fromRevision, timeWarmupRuns);
+ }
+
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder setTarget(BenchmarkTarget target) {
+ this.target = target;
+ return this;
+ }
+
+ public Builder setMethod(BenchmarkMethod method) {
+ this.method = method;
+ return this;
+ }
+
+ public Builder measureRunTimeRaw() {
+ metrics.add(BenchmarkMetric.RunTimeRaw);
+ return this;
+ }
+
+ public Builder measureCodeSize() {
+ metrics.add(BenchmarkMetric.CodeSize);
+ return this;
+ }
+
+ public Builder setSuite(BenchmarkSuite suite) {
+ this.suite = suite;
+ return this;
+ }
+
+ public Builder setFromRevision(int fromRevision) {
+ this.fromRevision = fromRevision;
+ return this;
+ }
+
+ public Builder timeWarmupRuns() {
+ this.timeWarmupRuns = true;
+ return this;
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private final BenchmarkIdentifier id;
+ private final BenchmarkMethod method;
+ private final ImmutableSet<BenchmarkMetric> metrics;
+ private final BenchmarkSuite suite;
+ private final int fromRevision;
+ private final boolean timeWarmupRuns;
+
+ private BenchmarkConfig(
+ String name,
+ BenchmarkMethod benchmarkMethod,
+ BenchmarkTarget target,
+ ImmutableSet<BenchmarkMetric> metrics,
+ BenchmarkSuite suite,
+ int fromRevision,
+ boolean timeWarmupRuns) {
+ this.id = new BenchmarkIdentifier(name, target);
+ this.method = benchmarkMethod;
+ this.metrics = metrics;
+ this.suite = suite;
+ this.fromRevision = fromRevision;
+ this.timeWarmupRuns = timeWarmupRuns;
+ }
+
+ public BenchmarkIdentifier getIdentifier() {
+ return id;
+ }
+
+ public String getName() {
+ return id.getName();
+ }
+
+ public String getWarmupName() {
+ if (!timeWarmupRuns) {
+ throw new Unreachable("Invalid attempt at getting warmup benchmark name");
+ }
+ return getName() + "Warmup";
+ }
+
+ public BenchmarkTarget getTarget() {
+ return id.getTarget();
+ }
+
+ public Set<BenchmarkMetric> getMetrics() {
+ return metrics;
+ }
+
+ public boolean hasMetric(BenchmarkMetric metric) {
+ return metrics.contains(metric);
+ }
+
+ public BenchmarkSuite getSuite() {
+ return suite;
+ }
+
+ public int getFromRevision() {
+ return fromRevision;
+ }
+
+ public boolean hasTimeWarmupRuns() {
+ return timeWarmupRuns;
+ }
+
+ public void run(TemporaryFolder temp) throws Exception {
+ method.run(this, temp);
+ }
+
+ @Override
+ public String toString() {
+ return id.getName() + "/" + id.getTarget().getIdentifierName();
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkIdentifier.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkIdentifier.java
new file mode 100644
index 0000000..f9ac65e
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkIdentifier.java
@@ -0,0 +1,59 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.android.tools.r8.utils.structural.Equatable;
+import com.android.tools.r8.utils.structural.Ordered;
+import java.util.Comparator;
+import java.util.Objects;
+
+public class BenchmarkIdentifier implements Ordered<BenchmarkIdentifier> {
+
+ private final String name;
+ private final BenchmarkTarget target;
+
+ public static BenchmarkIdentifier parse(String benchmarkName, String targetIdentifier) {
+ for (BenchmarkTarget target : BenchmarkTarget.values()) {
+ if (target.getIdentifierName().equals(targetIdentifier)) {
+ return new BenchmarkIdentifier(benchmarkName, target);
+ }
+ }
+ return null;
+ }
+
+ public BenchmarkIdentifier(String name, BenchmarkTarget target) {
+ this.name = name;
+ this.target = target;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public BenchmarkTarget getTarget() {
+ return target;
+ }
+
+ @Override
+ public int compareTo(BenchmarkIdentifier other) {
+ return Comparator.comparing(BenchmarkIdentifier::getName)
+ .thenComparing(BenchmarkIdentifier::getTarget)
+ .compare(this, other);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return Equatable.equalsImpl(this, o);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, target);
+ }
+
+ @Override
+ public String toString() {
+ return "BenchmarkIdentifier{" + "name='" + name + '\'' + ", target=" + target + '}';
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMainEntryRunner.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMainEntryRunner.java
new file mode 100644
index 0000000..3e5a86a
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMainEntryRunner.java
@@ -0,0 +1,30 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import org.junit.rules.TemporaryFolder;
+
+public class BenchmarkMainEntryRunner {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ throw new RuntimeException("Invalid arguments. Expected exactly one benchmark and target");
+ }
+ String benchmarkName = args[0];
+ String targetIdentifier = args[1];
+ BenchmarkIdentifier identifier = BenchmarkIdentifier.parse(benchmarkName, targetIdentifier);
+ if (identifier == null) {
+ throw new RuntimeException("Invalid identifier identifier: " + benchmarkName);
+ }
+ BenchmarkCollection collection = BenchmarkCollection.computeCollection();
+ BenchmarkConfig config = collection.getBenchmark(identifier);
+ if (config == null) {
+ throw new RuntimeException("Unknown identifier: " + identifier);
+ }
+ TemporaryFolder temp = new TemporaryFolder();
+ temp.create();
+ config.run(temp);
+ temp.delete();
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMethod.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMethod.java
new file mode 100644
index 0000000..8c7e372
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMethod.java
@@ -0,0 +1,12 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import org.junit.rules.TemporaryFolder;
+
+@FunctionalInterface
+public interface BenchmarkMethod {
+
+ void run(BenchmarkConfig config, TemporaryFolder temp) throws Exception;
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMetric.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMetric.java
new file mode 100644
index 0000000..9483499
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkMetric.java
@@ -0,0 +1,13 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+public enum BenchmarkMetric {
+ RunTimeRaw,
+ CodeSize;
+
+ public String getDartType() {
+ return "Metric." + name();
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
new file mode 100644
index 0000000..cf55b86
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
@@ -0,0 +1,97 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.android.tools.r8.benchmarks.BenchmarkRunner.ResultMode;
+import com.android.tools.r8.errors.Unreachable;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+
+public class BenchmarkResults {
+
+ private final boolean isWarmupResults;
+ private final LongList runtimeRawResults = new LongArrayList();
+ private final LongList codeSizeResults = new LongArrayList();
+
+ public static BenchmarkResults create() {
+ return new BenchmarkResults(false);
+ }
+
+ public static BenchmarkResults createForWarmup() {
+ return new BenchmarkResults(true);
+ }
+
+ private BenchmarkResults(boolean isWarmupResults) {
+ this.isWarmupResults = isWarmupResults;
+ }
+
+ private String getName(BenchmarkConfig config) {
+ return isWarmupResults ? config.getWarmupName() : config.getName();
+ }
+
+ public void addRuntimeRawResult(long result) {
+ runtimeRawResults.add(result);
+ }
+
+ public void addCodeSizeResult(long result) {
+ codeSizeResults.add(result);
+ }
+
+ private static void verifyMetric(BenchmarkMetric metric, boolean expected, boolean actual) {
+ if (expected != actual) {
+ throw new Unreachable(
+ "Mismatched config and result for "
+ + metric.name()
+ + ". Expected by config: "
+ + expected
+ + ", but has result: "
+ + actual);
+ }
+ }
+
+ private void verifyConfigAndResults(BenchmarkConfig config) {
+ verifyMetric(
+ BenchmarkMetric.RunTimeRaw,
+ config.getMetrics().contains(BenchmarkMetric.RunTimeRaw),
+ !runtimeRawResults.isEmpty());
+ verifyMetric(
+ BenchmarkMetric.CodeSize,
+ config.getMetrics().contains(BenchmarkMetric.CodeSize),
+ !codeSizeResults.isEmpty());
+ }
+
+ public static String prettyTime(long nanoTime) {
+ return "" + (nanoTime / 1000000) + "ms";
+ }
+
+ private void printRunTimeRaw(BenchmarkConfig config, long duration) {
+ System.out.println(getName(config) + "(RunTimeRaw): " + prettyTime(duration));
+ }
+
+ private void printCodeSize(BenchmarkConfig config, long bytes) {
+ System.out.println(getName(config) + "(CodeSize): " + bytes);
+ }
+
+ public void printResults(ResultMode mode, BenchmarkConfig config) {
+ verifyConfigAndResults(config);
+ if (config.hasMetric(BenchmarkMetric.RunTimeRaw)) {
+ long sum = runtimeRawResults.stream().mapToLong(l -> l).sum();
+ if (mode == ResultMode.SUM) {
+ printRunTimeRaw(config, sum);
+ } else if (mode == ResultMode.AVERAGE) {
+ printRunTimeRaw(config, sum / runtimeRawResults.size());
+ }
+ }
+ if (!isWarmupResults && config.hasMetric(BenchmarkMetric.CodeSize)) {
+ long size = codeSizeResults.getLong(0);
+ for (int i = 1; i < codeSizeResults.size(); i++) {
+ if (size != codeSizeResults.getLong(i)) {
+ throw new Unreachable(
+ "Unexpected code size difference: " + size + " and " + codeSizeResults.getLong(i));
+ }
+ }
+ printCodeSize(config, size);
+ }
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
new file mode 100644
index 0000000..f0cb0b8
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
@@ -0,0 +1,92 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+public class BenchmarkRunner {
+
+ public interface BenchmarkRunnerFunction {
+ void run(BenchmarkResults results) throws Exception;
+ }
+
+ public enum ResultMode {
+ AVERAGE,
+ SUM;
+
+ @Override
+ public String toString() {
+ return name().toLowerCase();
+ }
+ }
+
+ private final BenchmarkConfig config;
+ private int warmups = 0;
+ private int iterations = 1;
+ private ResultMode resultMode = ResultMode.AVERAGE;
+
+ private BenchmarkRunner(BenchmarkConfig config) {
+ this.config = config;
+ }
+
+ public static BenchmarkRunner runner(BenchmarkConfig config) {
+ return new BenchmarkRunner(config);
+ }
+
+ public BenchmarkRunner setWarmupIterations(int iterations) {
+ this.warmups = iterations;
+ return this;
+ }
+
+ public BenchmarkRunner setBenchmarkIterations(int iterations) {
+ this.iterations = iterations;
+ return this;
+ }
+
+ public BenchmarkRunner reportResultAverage() {
+ resultMode = ResultMode.AVERAGE;
+ return this;
+ }
+
+ public BenchmarkRunner reportResultSum() {
+ resultMode = ResultMode.SUM;
+ return this;
+ }
+
+ public void run(BenchmarkRunnerFunction fn) throws Exception {
+ long warmupTotalTime = 0;
+ BenchmarkResults warmupResults = BenchmarkResults.createForWarmup();
+ if (warmups > 0) {
+ long start = System.nanoTime();
+ for (int i = 0; i < warmups; i++) {
+ fn.run(warmupResults);
+ }
+ warmupTotalTime = System.nanoTime() - start;
+ }
+ BenchmarkResults results = BenchmarkResults.create();
+ long start = System.nanoTime();
+ for (int i = 0; i < iterations; i++) {
+ fn.run(results);
+ }
+ long benchmarkTotalTime = System.nanoTime() - start;
+ System.out.println(
+ "Benchmark results for "
+ + config.getName()
+ + " on target "
+ + config.getTarget().getIdentifierName());
+ if (warmups > 0) {
+ printMetaInfo("warmup", warmups, warmupTotalTime);
+ if (config.hasTimeWarmupRuns()) {
+ warmupResults.printResults(resultMode, config);
+ }
+ }
+ printMetaInfo("benchmark", iterations, benchmarkTotalTime);
+ results.printResults(resultMode, config);
+ System.out.println();
+ }
+
+ private void printMetaInfo(String kind, int iterations, long totalTime) {
+ System.out.println(" " + kind + " reporting mode: " + resultMode);
+ System.out.println(" " + kind + " iterations: " + iterations);
+ System.out.println(" " + kind + " total time: " + BenchmarkResults.prettyTime(totalTime));
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkSuite.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkSuite.java
new file mode 100644
index 0000000..1068f72
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkSuite.java
@@ -0,0 +1,36 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+/** Enumeration of the benchmark suites on Golem. */
+public enum BenchmarkSuite {
+ R8_BENCHMARKS("R8Benchmarks", "suite"),
+ D8_BENCHMARKS("D8KeyBenchmarks", "d8KeySuite"),
+ D8_INCREMENTAL_BENCHMARKS("D8IncrementalBenchmarks", "incrementalSuite"),
+ OPENSOURCE_BENCHMARKS("OpenSourceAppDumps", "dumpsSuite"),
+ MEMORY_BENCHMARKS("R8MemoryBenchmarks", "r8MemorySuite"),
+ RETRACE_BENCHMARKS("R8RetraceBenchmarks", "r8RetraceSuite");
+
+ private final String golemName;
+ private final String dartName;
+
+ public static BenchmarkSuite getDefault() {
+ return R8_BENCHMARKS;
+ }
+
+ BenchmarkSuite(String golemName, String dartName) {
+ this.golemName = golemName;
+ this.dartName = dartName;
+ }
+
+ /** The name as shown in golems listings. */
+ public String getGolemName() {
+ return golemName;
+ }
+
+ /** The variable name used for the suite in the benchmarks.dart script. */
+ public String getDartName() {
+ return dartName;
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkTarget.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkTarget.java
new file mode 100644
index 0000000..cda7f6e
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkTarget.java
@@ -0,0 +1,28 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+public enum BenchmarkTarget {
+ // Possible dashboard targets on golem.
+ D8("d8", "D8"),
+ R8_COMPAT("r8-compat", "R8"),
+ R8_NON_COMPAT("r8", "R8-full"),
+ R8_FORCE_OPT("r8-force", "R8-full-minify-optimize-shrink");
+
+ private final String idName;
+ private final String golemName;
+
+ BenchmarkTarget(String idName, String golemName) {
+ this.idName = idName;
+ this.golemName = golemName;
+ }
+
+ public String getGolemName() {
+ return golemName;
+ }
+
+ public String getIdentifierName() {
+ return idName;
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java b/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java
new file mode 100644
index 0000000..b91e5b1
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java
@@ -0,0 +1,136 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks.helloworld;
+
+import com.android.tools.r8.TestBuilder;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.benchmarks.BenchmarkBase;
+import com.android.tools.r8.benchmarks.BenchmarkConfig;
+import com.android.tools.r8.benchmarks.BenchmarkMethod;
+import com.android.tools.r8.benchmarks.BenchmarkTarget;
+import com.android.tools.r8.utils.AndroidApiLevel;
+import com.android.tools.r8.utils.BooleanUtils;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import java.util.List;
+import java.util.function.Function;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/** Example of setting up a benchmark based on the testing infrastructure. */
+@RunWith(Parameterized.class)
+public class HelloWorldBenchmark extends BenchmarkBase {
+
+ @Parameters(name = "{0}")
+ public static List<Object[]> data() {
+ return parametersFromConfigs(configs());
+ }
+
+ public HelloWorldBenchmark(BenchmarkConfig config, TestParameters parameters) {
+ super(config, parameters);
+ }
+
+ /** Static method to add benchmarks to the benchmark collection. */
+ public static List<BenchmarkConfig> configs() {
+ Builder<BenchmarkConfig> benchmarks = ImmutableList.builder();
+ makeBenchmark(BenchmarkTarget.D8, HelloWorldBenchmark::benchmarkD8, benchmarks);
+ makeBenchmark(BenchmarkTarget.R8_NON_COMPAT, HelloWorldBenchmark::benchmarkR8, benchmarks);
+ return benchmarks.build();
+ }
+
+ // Options/parameter setup to define variants of the benchmark above.
+ private static class Options {
+ final BenchmarkTarget target;
+ final Backend backend;
+ final boolean includeLibrary;
+ final AndroidApiLevel minApi = AndroidApiLevel.B;
+
+ public Options(BenchmarkTarget target, Backend backend, boolean includeLibrary) {
+ this.target = target;
+ this.backend = backend;
+ this.includeLibrary = includeLibrary;
+ }
+
+ public String getName() {
+ // The name include each non-target option for the variants to ensure unique benchmarks.
+ String backendString = backend.isCf() ? "Cf" : "Dex";
+ String libraryString = includeLibrary ? "" : "NoLib";
+ return "HelloWorld" + backendString + libraryString;
+ }
+ }
+
+ private static void makeBenchmark(
+ BenchmarkTarget target,
+ Function<Options, BenchmarkMethod> method,
+ ImmutableList.Builder<BenchmarkConfig> benchmarks) {
+ for (boolean includeLibrary : BooleanUtils.values()) {
+ for (Backend backend : Backend.values()) {
+ Options options = new Options(target, backend, includeLibrary);
+ benchmarks.add(
+ BenchmarkConfig.builder()
+ // The benchmark is required to have a unique combination of name and target.
+ .setName(options.getName())
+ .setTarget(target)
+ // The benchmark is required to have at least one metric.
+ .measureRunTimeRaw()
+ .measureCodeSize()
+ // The benchmark is required to have a runner method which defines the actual
+ // execution.
+ .setMethod(method.apply(options))
+ // The benchmark is required to set a "golem from revision".
+ // Find this value by looking at the current revision on golem.
+ .setFromRevision(11900)
+ // The benchmark can optionally time the warmup. This is not needed to use a warmup
+ // in the actual run, only to include it as its own benchmark entry on golem.
+ .timeWarmupRuns()
+ .build());
+ }
+ }
+ }
+
+ public static BenchmarkMethod benchmarkD8(Options options) {
+ return (config, temp) ->
+ runner(config)
+ .setWarmupIterations(1)
+ .setBenchmarkIterations(100)
+ .reportResultSum()
+ .run(
+ results ->
+ testForD8(temp, options.backend)
+ .setMinApi(options.minApi)
+ .applyIf(!options.includeLibrary, TestBuilder::addLibraryFiles)
+ .addProgramClasses(TestClass.class)
+ // Compile and emit RunTimeRaw measure.
+ .benchmarkCompile(results)
+ // Measure the output size.
+ .benchmarkCodeSize(results));
+ }
+
+ public static BenchmarkMethod benchmarkR8(Options options) {
+ return (config, temp) ->
+ runner(config)
+ .setWarmupIterations(1)
+ .setBenchmarkIterations(4)
+ .reportResultSum()
+ .run(
+ results ->
+ testForR8(temp, options.backend)
+ .applyIf(!options.includeLibrary, b -> b.addLibraryFiles().addDontWarn("*"))
+ .setMinApi(options.minApi)
+ .addProgramClasses(TestClass.class)
+ .addKeepMainRule(TestClass.class)
+ // Compile and emit RunTimeRaw measure.
+ .benchmarkCompile(results)
+ // Measure the output size.
+ .benchmarkCodeSize(results));
+ }
+
+ static class TestClass {
+
+ public static void main(String[] args) {
+ System.out.println("Hello world!");
+ }
+ }
+}
diff --git a/tools/as_utils.py b/tools/as_utils.py
index 43153c6..4087137 100644
--- a/tools/as_utils.py
+++ b/tools/as_utils.py
@@ -3,11 +3,11 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
-import utils
-
-from distutils.version import LooseVersion
import os
import shutil
+from distutils.version import LooseVersion
+
+import utils
if utils.is_python3():
from html.parser import HTMLParser
@@ -174,7 +174,7 @@
shutil.rmtree(dst)
elif os.path.isfile(dst):
os.remove(dst)
- os.rename(src, dst)
+ shutil.move(src, dst)
def MoveDir(src, dst, quiet=False):
assert os.path.isdir(src)
diff --git a/tools/golem_build.py b/tools/golem_build.py
index 24a1906..c691293 100755
--- a/tools/golem_build.py
+++ b/tools/golem_build.py
@@ -5,15 +5,33 @@
# Utility script to make it easier to update what golem builds.
-import gradle
import sys
+import gradle
+import retrace_benchmark
+import run_benchmark
+import run_on_app_dump
+
GRADLE_ARGS = ['--no-daemon', '-Pno_internal']
-BUILD_TARGETS = ['R8', 'D8', 'R8Lib', 'buildExampleJars',
- 'downloadAndroidCts', 'downloadDx']
+
+LEGACY_BUILD_TARGETS = [
+ 'R8',
+ 'D8',
+ 'buildExampleJars',
+ 'downloadAndroidCts',
+ 'downloadDx']
+
+def lower(items):
+ return [ item.lower() for item in items ]
def Main():
- gradle.RunGradle(GRADLE_ARGS + BUILD_TARGETS)
+ targets = set()
+ targets.update(lower(LEGACY_BUILD_TARGETS))
+ targets.update(lower(retrace_benchmark.GOLEM_BUILD_TARGETS))
+ targets.update(lower(run_benchmark.GOLEM_BUILD_TARGETS))
+ targets.update(lower(run_on_app_dump.GOLEM_BUILD_TARGETS))
+ cmd = GRADLE_ARGS + [target for target in targets]
+ gradle.RunGradle(cmd)
if __name__ == '__main__':
sys.exit(Main())
diff --git a/tools/retrace_benchmark.py b/tools/retrace_benchmark.py
index e29bb86..42ed792 100755
--- a/tools/retrace_benchmark.py
+++ b/tools/retrace_benchmark.py
@@ -14,6 +14,7 @@
import proguard
import utils
+GOLEM_BUILD_TARGETS = ['R8Lib']
RETRACERS = ['r8', 'proguard', 'remapper']
def parse_arguments(argv):
diff --git a/tools/run_benchmark.py b/tools/run_benchmark.py
new file mode 100755
index 0000000..2ad5200
--- /dev/null
+++ b/tools/run_benchmark.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import argparse
+import os
+import subprocess
+import sys
+
+import gradle
+import jdk
+import utils
+
+NONLIB_BUILD_TARGET = 'R8WithRelocatedDeps'
+NONLIB_TEST_BUILD_TARGETS = [utils.R8_TESTS_TARGET, utils.R8_TESTS_DEPS_TARGET]
+
+R8LIB_BUILD_TARGET = utils.R8LIB
+R8LIB_TEST_BUILD_TARGETS = [utils.R8LIB_TESTS_TARGET, utils.R8LIB_TESTS_DEPS_TARGET]
+
+# The r8lib target is always the golem target.
+GOLEM_BUILD_TARGETS = [R8LIB_BUILD_TARGET] + R8LIB_TEST_BUILD_TARGETS
+
+def get_jdk_home(options, benchmark):
+ if options.golem:
+ return os.path.join('benchmarks', benchmark, 'linux')
+ return None
+
+def parse_options(argv):
+ result = argparse.ArgumentParser(description = 'Run test-based benchmarks.')
+ result.add_argument('--golem',
+ help='Indicate this as a run on golem',
+ default=False,
+ action='store_true')
+ result.add_argument('--benchmark',
+ help='The test benchmark to run',
+ required=True)
+ result.add_argument('--target',
+ help='The test target to run',
+ required=True,
+ # These should 1:1 with BenchmarkTarget.java
+ choices=['d8', 'r8', 'r8-force', 'r8-compat'])
+ result.add_argument('--nolib', '--no-lib', '--no-r8lib',
+ help='Run the non-lib R8 build (default false)',
+ default=False,
+ action='store_true')
+ result.add_argument('--no-build', '--no_build',
+ help='Run without building first (default false)',
+ default=False,
+ action='store_true')
+ result.add_argument('--enable-assertions', '--enable_assertions', '-ea',
+ help='Enable assertions when running',
+ default=False,
+ action='store_true')
+ result.add_argument('--print-times',
+ help='Print timing information from r8',
+ default=False,
+ action='store_true')
+ result.add_argument('--temp',
+ help='A directory to use for temporaries and outputs.',
+ default=None)
+ return result.parse_known_args(argv)
+
+def main(argv):
+ (options, args) = parse_options(argv)
+
+ if options.golem:
+ options.no_build = True
+ if options.nolib:
+ print("Error: golem should always run r8lib")
+ return 1
+
+ if options.nolib:
+ buildTargets = [NONLIB_BUILD_TARGET] + NONLIB_TEST_BUILD_TARGETS
+ r8jar = utils.R8_WITH_RELOCATED_DEPS_JAR
+ testjars = [utils.R8_TESTS_DEPS_JAR, utils.R8_TESTS_JAR]
+ else:
+ buildTargets = GOLEM_BUILD_TARGETS
+ r8jar = utils.R8LIB_JAR
+ testjars = [utils.R8LIB_TESTS_DEPS_JAR, utils.R8LIB_TESTS_JAR]
+
+ if not options.no_build:
+ gradle.RunGradle(buildTargets + ['-Pno_internal'])
+
+ return run(options, r8jar, testjars)
+
+def run(options, r8jar, testjars):
+ jdkhome = get_jdk_home(options, options.benchmark)
+ cmd = [jdk.GetJavaExecutable(jdkhome)]
+ if options.enable_assertions:
+ cmd.append('-ea')
+ if options.print_times:
+ cmd.append('-Dcom.android.tools.r8.printtimes=1')
+ cmd.extend(['-cp', ':'.join([r8jar] + testjars)])
+ cmd.extend([
+ 'com.android.tools.r8.benchmarks.BenchmarkMainEntryRunner',
+ options.benchmark,
+ options.target,
+ ])
+ return subprocess.check_call(cmd)
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/run_on_app_dump.py b/tools/run_on_app_dump.py
index c55a6f7..6035752 100755
--- a/tools/run_on_app_dump.py
+++ b/tools/run_on_app_dump.py
@@ -3,24 +3,25 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
+import argparse
+import hashlib
+import os
+import shutil
+import sys
+import time
+import zipfile
+from datetime import datetime
+
import adb
import apk_masseur
import as_utils
import compiledump
import gradle
-import hashlib
import jdk
-import argparse
-import os
-import shutil
-import sys
-import time
import update_prebuilds_in_android
import utils
-import zipfile
-from datetime import datetime
-
+GOLEM_BUILD_TARGETS = ['R8Lib', 'R8Retrace']
SHRINKERS = ['r8', 'r8-full', 'r8-nolib', 'r8-nolib-full']
class AttrDict(dict):
@@ -1060,9 +1061,7 @@
print_indented('final targetsFull = ["R8-full-minify-optimize-shrink"];', 2)
# Avoid calculating this for every app
jdk_gz = jdk.GetJdkHome() + '.tar.gz'
- download_sha(jdk_gz + '.sha1', False, quiet=True)
- jdk_sha256 = get_sha256(jdk_gz)
- add_golem_resource(2, jdk_gz, 'openjdk', sha256=jdk_sha256)
+ add_golem_resource(2, jdk_gz, 'openjdk')
for app in options.apps:
if app.folder and not app.internal:
indentation = 2;
@@ -1171,7 +1170,7 @@
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version == 'main':
- if not (options.no_build or options.golem):
+ if not options.no_build:
gradle.RunGradle(['R8Retrace', 'r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
diff --git a/tools/utils.py b/tools/utils.py
index 314fcd2..eb7849b 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -45,10 +45,15 @@
R8RETRACE_NO_DEPS = 'R8RetraceNoDeps'
R8_SRC = 'sourceJar'
LIBRARY_DESUGAR_CONVERSIONS = 'buildLibraryDesugarConversions'
+R8_TESTS_TARGET = 'TestJar'
+R8_TESTS_DEPS_TARGET = 'RepackageTestDeps'
+R8LIB_TESTS_TARGET = 'configureTestForR8Lib'
+R8LIB_TESTS_DEPS_TARGET = R8_TESTS_DEPS_TARGET
ALL_DEPS_JAR = os.path.join(LIBS, 'deps_all.jar')
D8_JAR = os.path.join(LIBS, 'd8.jar')
R8_JAR = os.path.join(LIBS, 'r8.jar')
+R8_WITH_RELOCATED_DEPS_JAR = os.path.join(LIBS, 'r8_with_relocated_deps.jar')
R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar')
R8LIB_MAP = os.path.join(LIBS, 'r8lib.jar.map')
R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')
@@ -56,6 +61,10 @@
R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar')
R8RETRACE_JAR = os.path.join(LIBS, 'r8retrace.jar')
R8RETRACE_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8retrace-exclude-deps.jar')
+R8_TESTS_JAR = os.path.join(LIBS, 'r8tests.jar')
+R8LIB_TESTS_JAR = os.path.join(LIBS, 'r8libtestdeps-cf.jar')
+R8_TESTS_DEPS_JAR = os.path.join(LIBS, 'test_deps_all.jar')
+R8LIB_TESTS_DEPS_JAR = R8_TESTS_DEPS_JAR
MAVEN_ZIP = os.path.join(LIBS, 'r8.zip')
MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip')
LIBRARY_DESUGAR_CONVERSIONS_ZIP = os.path.join(LIBS, 'library_desugar_conversions.zip')
