Support for writing benchmark result to json and overriding iterations
Change-Id: I2dc1482a36d3479f0fba7ee65ef850a93e1b9b83
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java
index 1cb50ba..917c268 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkBase.java
@@ -40,7 +40,7 @@
config.run(new BenchmarkEnvironment(config, temp, false));
}
- public static BenchmarkRunner runner(BenchmarkConfig config) {
- return BenchmarkRunner.runner(config);
+ public static BenchmarkRunner runner(BenchmarkEnvironment environment) {
+ return BenchmarkRunner.runner(environment);
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkEnvironment.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkEnvironment.java
index fe36c52..267ecc1 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkEnvironment.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkEnvironment.java
@@ -36,4 +36,20 @@
public Path getGolemDependencyRoot() {
return Paths.get("benchmarks", config.getDependencyDirectoryName());
}
+
+ public boolean hasBenchmarkIterationsOverride() {
+ return System.getProperty("BENCHMARK_ITERATIONS") != null;
+ }
+
+ public int getBenchmarkIterationsOverride() {
+ return Integer.parseInt(System.getProperty("BENCHMARK_ITERATIONS"));
+ }
+
+ public boolean hasOutputPath() {
+ return System.getProperty("BENCHMARK_OUTPUT") != null;
+ }
+
+ public Path getOutputPath() {
+ return Paths.get(System.getProperty("BENCHMARK_OUTPUT"));
+ }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
index 14b10f4..702374b 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
@@ -3,6 +3,8 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
+import com.android.tools.r8.errors.Unimplemented;
+import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -54,4 +56,9 @@
singleResults.printResults(mode);
}
}
+
+ @Override
+ public void writeResults(PrintStream out) {
+ throw new Unimplemented();
+ }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
index e0bb9f193..d44f1c0 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
@@ -3,8 +3,11 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
+import java.io.PrintStream;
import java.util.Set;
public class BenchmarkResultsSingle implements BenchmarkResults {
@@ -19,6 +22,18 @@
this.metrics = metrics;
}
+ public String getName() {
+ return name;
+ }
+
+ public LongList getCodeSizeResults() {
+ return codeSizeResults;
+ }
+
+ public LongList getRuntimeResults() {
+ return runtimeResults;
+ }
+
@Override
public void addRuntimeResult(long result) {
verifyMetric(BenchmarkMetric.RunTimeRaw, metrics.contains(BenchmarkMetric.RunTimeRaw), true);
@@ -93,4 +108,13 @@
printCodeSize(size);
}
}
+
+ @Override
+ public void writeResults(PrintStream out) {
+ Gson gson =
+ new GsonBuilder()
+ .registerTypeAdapter(BenchmarkResultsSingle.class, new BenchmarkResultsSingleAdapter())
+ .create();
+ out.print(gson.toJson(this));
+ }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java
new file mode 100644
index 0000000..55936d7
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java
@@ -0,0 +1,34 @@
+// Copyright (c) 2024, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.android.tools.r8.utils.ListUtils;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import java.lang.reflect.Type;
+
+public class BenchmarkResultsSingleAdapter implements JsonSerializer<BenchmarkResultsSingle> {
+
+ @Override
+ public JsonElement serialize(
+ BenchmarkResultsSingle result, Type type, JsonSerializationContext jsonSerializationContext) {
+ JsonArray resultsArray = new JsonArray();
+ ListUtils.forEachWithIndex(
+ result.getCodeSizeResults(),
+ (codeSizeResult, iteration) -> {
+ JsonObject resultObject = new JsonObject();
+ resultObject.addProperty("code_size", codeSizeResult);
+ resultObject.addProperty("runtime", result.getRuntimeResults().getLong(iteration));
+ resultsArray.add(resultObject);
+ });
+
+ JsonObject benchmarkObject = new JsonObject();
+ benchmarkObject.addProperty("benchmark_name", result.getName());
+ benchmarkObject.add("results", resultsArray);
+ return benchmarkObject;
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
index 2ac97a6..c7319f7 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
@@ -3,8 +3,10 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
+import com.android.tools.r8.errors.Unimplemented;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
+import java.io.PrintStream;
public class BenchmarkResultsWarmup implements BenchmarkResults {
@@ -61,4 +63,9 @@
BenchmarkResults.prettyMetric(
name, BenchmarkMetric.StartupTime, BenchmarkResults.prettyTime(result)));
}
+
+ @Override
+ public void writeResults(PrintStream out) {
+ throw new Unimplemented();
+ }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
index 2fed6dc..72d7adc 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
@@ -4,6 +4,9 @@
package com.android.tools.r8.benchmarks;
import com.android.tools.r8.benchmarks.BenchmarkResults.ResultMode;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
public class BenchmarkRunner {
@@ -11,17 +14,17 @@
void run(BenchmarkResults results) throws Exception;
}
- private final BenchmarkConfig config;
+ private final BenchmarkEnvironment environment;
private int warmups = 0;
private int iterations = 1;
private ResultMode resultMode = BenchmarkResults.ResultMode.AVERAGE;
- private BenchmarkRunner(BenchmarkConfig config) {
- this.config = config;
+ private BenchmarkRunner(BenchmarkEnvironment environment) {
+ this.environment = environment;
}
- public static BenchmarkRunner runner(BenchmarkConfig config) {
- return new BenchmarkRunner(config);
+ public static BenchmarkRunner runner(BenchmarkEnvironment environment) {
+ return new BenchmarkRunner(environment);
}
public BenchmarkRunner setWarmupIterations(int iterations) {
@@ -29,6 +32,12 @@
return this;
}
+ public int getBenchmarkIterations() {
+ return environment.hasBenchmarkIterationsOverride()
+ ? environment.getBenchmarkIterationsOverride()
+ : iterations;
+ }
+
public BenchmarkRunner setBenchmarkIterations(int iterations) {
this.iterations = iterations;
return this;
@@ -46,6 +55,7 @@
public void run(BenchmarkRunnerFunction fn) throws Exception {
long warmupTotalTime = 0;
+ BenchmarkConfig config = environment.getConfig();
BenchmarkResults warmupResults = new BenchmarkResultsWarmup(config.getName());
if (warmups > 0) {
long start = System.nanoTime();
@@ -59,7 +69,7 @@
? new BenchmarkResultsSingle(config.getName(), config.getMetrics())
: new BenchmarkResultsCollection(config.getSubBenchmarks());
long start = System.nanoTime();
- for (int i = 0; i < iterations; i++) {
+ for (int i = 0; i < getBenchmarkIterations(); i++) {
fn.run(results);
}
long benchmarkTotalTime = System.nanoTime() - start;
@@ -74,8 +84,11 @@
warmupResults.printResults(resultMode);
}
}
- printMetaInfo("benchmark", iterations, benchmarkTotalTime);
+ printMetaInfo("benchmark", getBenchmarkIterations(), benchmarkTotalTime);
results.printResults(resultMode);
+ if (environment.hasOutputPath()) {
+ writeResults(results);
+ }
System.out.println();
}
@@ -84,4 +97,11 @@
System.out.println(" " + kind + " iterations: " + iterations);
System.out.println(" " + kind + " total time: " + BenchmarkResults.prettyTime(totalTime));
}
+
+ private void writeResults(BenchmarkResults results) throws IOException {
+ try (PrintStream printStream =
+ new PrintStream(Files.newOutputStream(environment.getOutputPath()))) {
+ results.writeResults(printStream);
+ }
+ }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java b/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java
index 7e2a677..ac4f229 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java
@@ -203,7 +203,7 @@
private static BenchmarkMethod internalRunR8(
AppDumpBenchmarkBuilder builder, boolean enableResourceShrinking) {
return environment ->
- BenchmarkBase.runner(environment.getConfig())
+ BenchmarkBase.runner(environment)
.setWarmupIterations(1)
.run(
results -> {
@@ -244,7 +244,7 @@
private static BenchmarkMethod runBatchD8(AppDumpBenchmarkBuilder builder) {
return environment ->
- BenchmarkBase.runner(environment.getConfig())
+ BenchmarkBase.runner(environment)
.setWarmupIterations(1)
.run(
results -> {
@@ -262,7 +262,7 @@
private static BenchmarkMethod runIncrementalD8(AppDumpBenchmarkBuilder builder) {
return environment ->
- BenchmarkBase.runner(environment.getConfig())
+ BenchmarkBase.runner(environment)
.setWarmupIterations(1)
.reportResultSum()
.run(
diff --git a/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/L8Benchmark.java b/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/L8Benchmark.java
index 3f12fc8..adfa5f8 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/L8Benchmark.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/L8Benchmark.java
@@ -65,7 +65,7 @@
ImmutableSet.of(ANDROID_JAR.getRoot(environment).resolve("android.jar")),
LibraryDesugaringSpecification.JDK11_DESCRIPTOR,
"");
- runner(environment.getConfig())
+ runner(environment)
.setWarmupIterations(1)
.setBenchmarkIterations(10)
.reportResultSum()
diff --git a/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/LegacyDesugaredLibraryBenchmark.java b/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/LegacyDesugaredLibraryBenchmark.java
index 6b5e4f9..2d50a3d 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/LegacyDesugaredLibraryBenchmark.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/desugaredlib/LegacyDesugaredLibraryBenchmark.java
@@ -54,7 +54,7 @@
}
public static void run(BenchmarkEnvironment environment) throws Exception {
- runner(environment.getConfig())
+ runner(environment)
.setWarmupIterations(1)
.setBenchmarkIterations(10)
.reportResultSum()
diff --git a/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java b/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java
index cb43a1d..4c8c31c 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/helloworld/HelloWorldBenchmark.java
@@ -101,7 +101,7 @@
public static BenchmarkMethod benchmarkD8(Options options) {
return environment ->
- runner(environment.getConfig())
+ runner(environment)
.setWarmupIterations(1)
.setBenchmarkIterations(100)
.reportResultSum()
@@ -119,7 +119,7 @@
public static BenchmarkMethod benchmarkR8(Options options) {
return environment ->
- runner(environment.getConfig())
+ runner(environment)
.setWarmupIterations(1)
.setBenchmarkIterations(4)
.reportResultSum()
diff --git a/src/test/java/com/android/tools/r8/benchmarks/retrace/RetraceStackTraceBenchmark.java b/src/test/java/com/android/tools/r8/benchmarks/retrace/RetraceStackTraceBenchmark.java
index 28e1e83..c3cd9d0 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/retrace/RetraceStackTraceBenchmark.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/retrace/RetraceStackTraceBenchmark.java
@@ -59,7 +59,7 @@
public static BenchmarkMethod benchmarkRetrace() {
return environment ->
- runner(environment.getConfig())
+ runner(environment)
.setWarmupIterations(1)
.setBenchmarkIterations(4)
.reportResultSum()
diff --git a/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java b/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
index 3e52ba7..e19cd88 100644
--- a/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
+++ b/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
@@ -4,6 +4,7 @@
package com.android.tools.r8.benchmarks;
import com.android.tools.r8.utils.StringUtils;
+import java.io.PrintStream;
public interface BenchmarkResults {
// Append a runtime result. This may be summed or averaged depending on the benchmark set up.
@@ -21,6 +22,8 @@
void printResults(ResultMode resultMode);
+ void writeResults(PrintStream out);
+
static String prettyTime(long nanoTime) {
return "" + (nanoTime / 1000000) + " ms";
}
diff --git a/tools/run_benchmark.py b/tools/run_benchmark.py
index 3962d59..8948df7 100755
--- a/tools/run_benchmark.py
+++ b/tools/run_benchmark.py
@@ -63,6 +63,12 @@
help='Enable assertions when running',
default=False,
action='store_true')
+ result.add_argument('--iterations',
+ '-i',
+ help='Number of iterations to run',
+ type=int)
+ result.add_argument('--output',
+ help='Output path where to write the result')
result.add_argument('--print-times',
help='Print timing information from r8',
default=False,
@@ -81,8 +87,12 @@
def main(argv, temp):
(options, args) = parse_options(argv)
+ if options.output:
+ options.output = os.path.abspath(options.output)
+
if options.temp:
temp = options.temp
+ os.makedirs(temp, exist_ok=True)
if options.golem:
options.no_build = True
@@ -139,6 +149,10 @@
f'-DTEST_DATA_LOCATION={utils.REPO_ROOT}/d8_r8/test_modules/tests_java_8/build/classes/java/test',
f'-DTESTBASE_DATA_LOCATION={utils.REPO_ROOT}/d8_r8/test_modules/testbase/build/classes/java/main',
])
+ if options.iterations:
+ cmd.append(f'-DBENCHMARK_ITERATIONS={options.iterations}')
+ if options.output:
+ cmd.append(f'-DBENCHMARK_OUTPUT={options.output}')
cmd.extend(['-cp', ':'.join([r8jar] + testjars)])
cmd.extend([
'com.android.tools.r8.benchmarks.BenchmarkMainEntryRunner',
