Include warmup runtime in benchmark results
Change-Id: I3beb97b403aa14cd36785808350962b872387d91
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsAdapterBase.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsAdapterBase.java
new file mode 100644
index 0000000..ddc7b2d
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsAdapterBase.java
@@ -0,0 +1,30 @@
+// Copyright (c) 2024, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.google.gson.JsonObject;
+import java.util.Collection;
+import java.util.function.IntToLongFunction;
+
+public abstract class BenchmarkResultsAdapterBase {
+
+ void addPropertyIfValueDifferentFromRepresentative(
+ JsonObject resultObject,
+ String propertyName,
+ int iteration,
+ Collection<?> results,
+ IntToLongFunction getter) {
+ if (results.isEmpty()) {
+ return;
+ }
+ long result = getter.applyAsLong(iteration);
+ if (iteration != 0) {
+ long representativeResult = getter.applyAsLong(0);
+ if (result == representativeResult) {
+ return;
+ }
+ }
+ resultObject.addProperty(propertyName, result);
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
index 7d006fc..9ca4f81 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
@@ -86,11 +86,13 @@
}
@Override
- public void writeResults(Path path) throws IOException {
+ public void writeResults(Path path, BenchmarkResults warmupResults) throws IOException {
for (Entry<String, BenchmarkResultsSingle> entry : results.entrySet()) {
String name = entry.getKey();
BenchmarkResultsSingle result = entry.getValue();
- result.writeResults(path.resolve(name));
+ BenchmarkResults warmupSubResults =
+ warmupResults != null ? warmupResults.getSubResults(name) : null;
+ result.writeResults(path.resolve(name), warmupSubResults);
}
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
index e029bcd..22fbf4a 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
@@ -10,6 +10,7 @@
import com.android.tools.r8.dex.DexSection;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
import it.unimi.dsi.fastutil.ints.Int2ReferenceMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
@@ -268,14 +269,20 @@
}
@Override
- public void writeResults(Path path) throws IOException {
+ public void writeResults(Path path, BenchmarkResults warmupResults) throws IOException {
try (PrintStream out = new PrintStream(Files.newOutputStream(path))) {
Gson gson =
new GsonBuilder()
.registerTypeAdapter(
BenchmarkResultsSingle.class, new BenchmarkResultsSingleAdapter())
+ .registerTypeAdapter(
+ BenchmarkResultsWarmup.class, new BenchmarkResultsWarmupAdapter())
.create();
- out.print(gson.toJson(this));
+ JsonObject json = (JsonObject) gson.toJsonTree(this);
+ if (warmupResults != null) {
+ json.add("warmup", gson.toJsonTree(warmupResults));
+ }
+ out.print(json);
}
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java
index 90a257c..7e62cfc 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingleAdapter.java
@@ -11,10 +11,9 @@
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import java.lang.reflect.Type;
-import java.util.Collection;
-import java.util.function.IntToLongFunction;
-public class BenchmarkResultsSingleAdapter implements JsonSerializer<BenchmarkResultsSingle> {
+public class BenchmarkResultsSingleAdapter extends BenchmarkResultsAdapterBase
+ implements JsonSerializer<BenchmarkResultsSingle> {
@Override
public JsonElement serialize(
@@ -43,7 +42,7 @@
for (int section : DexSection.getConstants()) {
String sectionName = DexSection.typeName(section);
String sectionNameUnderscore =
- CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, sectionName);
+ CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, sectionName);
addPropertyIfValueDifferentFromRepresentative(
resultObject,
"dex_" + sectionNameUnderscore + "_size",
@@ -71,23 +70,4 @@
benchmarkObject.add("results", resultsArray);
return benchmarkObject;
}
-
- private void addPropertyIfValueDifferentFromRepresentative(
- JsonObject resultObject,
- String propertyName,
- int iteration,
- Collection<?> results,
- IntToLongFunction getter) {
- if (results.isEmpty()) {
- return;
- }
- long result = getter.applyAsLong(iteration);
- if (iteration != 0) {
- long representativeResult = getter.applyAsLong(0);
- if (result == representativeResult) {
- return;
- }
- }
- resultObject.addProperty(propertyName, result);
- }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
index 16084c1..12e4b9a 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
@@ -12,6 +12,9 @@
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
public class BenchmarkResultsWarmup implements BenchmarkResults {
@@ -20,10 +23,19 @@
private long codeSizeResult = -1;
private long resourceSizeResult = -1;
+ private final Map<String, BenchmarkResultsWarmup> results = new HashMap<>();
+
public BenchmarkResultsWarmup(String name) {
this.name = name;
}
+ public BenchmarkResultsWarmup(String name, Map<String, Set<BenchmarkMetric>> benchmarks) {
+ this.name = name;
+ benchmarks.forEach(
+ (benchmarkName, metrics) ->
+ results.put(benchmarkName, new BenchmarkResultsWarmup(benchmarkName)));
+ }
+
@Override
public void addRuntimeResult(long result) {
runtimeResults.add(result);
@@ -84,10 +96,13 @@
addRuntimeResult(averageRuntimeResult);
}
+ public LongList getRuntimeResults() {
+ return runtimeResults;
+ }
+
@Override
public BenchmarkResults getSubResults(String name) {
- // When running warmups all results are amended to the single warmup result.
- return this;
+ return results.get(name);
}
@Override
@@ -95,6 +110,10 @@
return false;
}
+ public int size() {
+ return runtimeResults.size();
+ }
+
@Override
public void printResults(ResultMode mode, boolean failOnCodeSizeDifferences) {
if (runtimeResults.isEmpty()) {
@@ -108,7 +127,7 @@
}
@Override
- public void writeResults(Path path) {
+ public void writeResults(Path path, BenchmarkResults warmupResults) {
throw new Unimplemented();
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmupAdapter.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmupAdapter.java
new file mode 100644
index 0000000..19cffef
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmupAdapter.java
@@ -0,0 +1,32 @@
+// Copyright (c) 2024, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import java.lang.reflect.Type;
+
+public class BenchmarkResultsWarmupAdapter extends BenchmarkResultsAdapterBase
+ implements JsonSerializer<BenchmarkResultsWarmup> {
+
+ @Override
+ public JsonElement serialize(
+ BenchmarkResultsWarmup result, Type type, JsonSerializationContext jsonSerializationContext) {
+ JsonArray resultsArray = new JsonArray();
+ for (int iteration = 0; iteration < result.size(); iteration++) {
+ JsonObject resultObject = new JsonObject();
+ addPropertyIfValueDifferentFromRepresentative(
+ resultObject,
+ "runtime",
+ iteration,
+ result.getRuntimeResults(),
+ i -> result.getRuntimeResults().getLong(i));
+ resultsArray.add(resultObject);
+ }
+ return resultsArray;
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
index 3f1db19..85ae724 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
@@ -53,8 +53,12 @@
public void run(BenchmarkRunnerFunction fn) throws Exception {
long warmupTotalTime = 0;
BenchmarkConfig config = environment.getConfig();
- BenchmarkResults warmupResults = new BenchmarkResultsWarmup(config.getName());
+ BenchmarkResultsWarmup warmupResults = null;
if (warmups > 0) {
+ warmupResults =
+ config.isSingleBenchmark()
+ ? new BenchmarkResultsWarmup(config.getName())
+ : new BenchmarkResultsWarmup(config.getName(), config.getSubBenchmarks());
long start = System.nanoTime();
for (int i = 0; i < warmups; i++) {
fn.run(warmupResults);
@@ -84,7 +88,7 @@
printMetaInfo("benchmark", getBenchmarkIterations(), benchmarkTotalTime);
results.printResults(resultMode, environment.failOnCodeSizeDifferences());
if (environment.hasOutputPath()) {
- results.writeResults(environment.getOutputPath());
+ results.writeResults(environment.getOutputPath(), warmupResults);
}
System.out.println();
}
diff --git a/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java b/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
index 42fb589..92cfd22 100644
--- a/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
+++ b/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
@@ -40,7 +40,7 @@
void printResults(ResultMode resultMode, boolean failOnCodeSizeDifferences);
- void writeResults(Path path) throws IOException;
+ void writeResults(Path path, BenchmarkResults warmupResults) throws IOException;
static String prettyTime(long nanoTime) {
return "" + (nanoTime / 1000000) + " ms";
