Verify consistency of shared benchmark properties.
Change-Id: Iee6e1920ac23a7cf22f30c62753503ea6e0f97c6
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java
index f75aabe..da0c335 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java
@@ -3,27 +3,40 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
+import static java.util.Collections.emptyList;
+
import com.android.tools.r8.benchmarks.helloworld.HelloWorldBenchmark;
-import com.android.tools.r8.errors.Unreachable;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
public class BenchmarkCollection {
// Actual list of all configured benchmarks.
- private final Map<BenchmarkIdentifier, BenchmarkConfig> benchmarks = new HashMap<>();
+ private final Map<String, List<BenchmarkConfig>> benchmarks = new HashMap<>();
private void addBenchmark(BenchmarkConfig benchmark) {
- BenchmarkIdentifier id = benchmark.getIdentifier();
- if (benchmarks.containsKey(id)) {
- throw new Unreachable("Duplicate definition of benchmark with name and target: " + id);
+ List<BenchmarkConfig> variants =
+ benchmarks.computeIfAbsent(benchmark.getName(), k -> new ArrayList<>());
+ for (BenchmarkConfig variant : variants) {
+ BenchmarkConfig.checkBenchmarkConsistency(benchmark, variant);
}
- benchmarks.put(id, benchmark);
+ variants.add(benchmark);
}
- public BenchmarkConfig getBenchmark(BenchmarkIdentifier benchmark) {
- return benchmarks.get(benchmark);
+ public BenchmarkConfig getBenchmark(BenchmarkIdentifier identifier) {
+ assert identifier != null;
+ List<BenchmarkConfig> configs = benchmarks.getOrDefault(identifier.getName(), emptyList());
+ for (BenchmarkConfig config : configs) {
+ if (identifier.equals(config.getIdentifier())) {
+ return config;
+ }
+ }
+ return null;
}
public static BenchmarkCollection computeCollection() {
@@ -36,6 +49,9 @@
/** Compute and print the golem configuration. */
public static void main(String[] args) throws IOException {
new BenchmarkCollectionPrinter(System.out)
- .printGolemConfig(computeCollection().benchmarks.values());
+ .printGolemConfig(
+ computeCollection().benchmarks.values().stream()
+ .flatMap(Collection::stream)
+ .collect(Collectors.toList()));
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java
index e3ede15..b69b6ee 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollectionPrinter.java
@@ -3,11 +3,9 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
-import static com.android.tools.r8.utils.ListUtils.map;
-
import com.android.tools.r8.ToolHelper;
import com.android.tools.r8.ToolHelper.ProcessResult;
-import com.android.tools.r8.errors.Unreachable;
+import com.android.tools.r8.utils.ListUtils;
import com.android.tools.r8.utils.StringUtils;
import com.android.tools.r8.utils.StringUtils.BraceType;
import com.google.common.base.Strings;
@@ -88,9 +86,9 @@
b -> nameToTargets.computeIfAbsent(b.getName(), k -> new ArrayList<>()).add(b));
List<String> sortedNames = new ArrayList<>(nameToTargets.keySet());
sortedNames.sort(String::compareTo);
- print(
- "// AUTOGENERATED FILE from"
- + " src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java");
+ print("// AUTOGENERATED FILE generated with");
+ print("// src/test/java/com/android/tools/r8/benchmarks/BenchmarkCollection.java");
+ print("// in the R8 repository.");
print("");
printSemi("part of r8_config");
print("");
@@ -108,21 +106,27 @@
print("}");
}
- private void printBenchmarkBlock(String benchmarkName, List<BenchmarkConfig> benchmarkTargets)
+ private void printBenchmarkBlock(String benchmarkName, List<BenchmarkConfig> benchmarkVariants)
throws IOException {
+ // Common properties that must be consistent among all the benchmark variants.
+ String suite = BenchmarkConfig.getCommonSuite(benchmarkVariants).getDartName();
+ boolean hasWarmup = BenchmarkConfig.getCommonTimeWarmupRuns(benchmarkVariants);
+ List<String> metrics =
+ new ArrayList<>(
+ ListUtils.map(
+ BenchmarkConfig.getCommonMetrics(benchmarkVariants), BenchmarkMetric::getDartType));
+ metrics.sort(String::compareTo);
printSemi("final name = " + quote(benchmarkName));
- printSemi("final group = new GroupBenchmark(name + \"Group\", [])");
- // NOTE: It appears these must be consistent for each target now?
- boolean hasWarmup = false;
- String suite = null;
- List<String> metrics = null;
- for (BenchmarkConfig benchmark : benchmarkTargets) {
- if (metrics == null) {
- // TODO: Verify equal on other runs.
- hasWarmup = benchmark.hasTimeWarmupRuns();
- suite = benchmark.getSuite().getDartName();
- metrics = map(benchmark.getMetrics(), BenchmarkMetric::getDartType);
- }
+ printSemi("final metrics = " + StringUtils.join(", ", metrics, BraceType.SQUARE));
+ printSemi("final group = new GroupBenchmark(name + \"Group\", metrics)");
+ printSemi("group.addBenchmark(name, metrics)");
+ printSemi(suite + ".addBenchmark(name)");
+ if (hasWarmup) {
+ printSemi("final warmupName = name + \"Warmup\"");
+ printSemi("group.addBenchmark(warmupName, [Metric.RunTimeRaw])");
+ printSemi(suite + ".addBenchmark(warmupName)");
+ }
+ for (BenchmarkConfig benchmark : benchmarkVariants) {
scopeBraces(
() -> {
printSemi("final target = " + quote(benchmark.getTarget().getGolemName()));
@@ -144,21 +148,6 @@
printSemi("options.resources.add(openjdk)");
});
}
-
- List<String> finalMetrics = metrics;
- String finalSuite = suite;
- boolean finalHasWarmup = hasWarmup;
- scopeBraces(
- () -> {
- printSemi("final metrics = " + StringUtils.join(", ", finalMetrics, BraceType.SQUARE));
- printSemi("group.addBenchmark(name, metrics)");
- printSemi(finalSuite + ".addBenchmark(name)");
- if (finalHasWarmup) {
- printSemi("final warmupName = name + \"Warmup\"");
- printSemi("group.addBenchmark(warmupName, [Metric.RunTimeRaw])");
- printSemi(finalSuite + ".addBenchmark(warmupName)");
- }
- });
}
private void addGolemResource(String name, Path tarball) throws IOException {
@@ -187,7 +176,7 @@
ProcessBuilder builder = new ProcessBuilder("python", "tools/jdk.py");
ProcessResult result = ToolHelper.runProcess(builder, QUIET);
if (result.exitCode != 0) {
- throw new Unreachable("Unexpected failure to determine jdk home: " + result);
+ throw new BenchmarkConfigError("Unexpected failure to determine jdk home: " + result);
}
return Paths.get(result.stdout.trim());
}
@@ -207,7 +196,7 @@
"download_from_google_storage", "-n", "-b", "r8-deps", "-u", "-s", path.toString());
ProcessResult result = ToolHelper.runProcess(builder, QUIET);
if (result.exitCode != 0) {
- throw new Unreachable("Unable to download dependency '" + path + "'\n" + result);
+ throw new BenchmarkConfigError("Unable to download dependency '" + path + "'\n" + result);
}
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java
index 09d7a6e..c64dbd2 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java
@@ -6,11 +6,53 @@
import com.android.tools.r8.errors.Unreachable;
import com.google.common.collect.ImmutableSet;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import org.junit.rules.TemporaryFolder;
public class BenchmarkConfig {
+ public static void checkBenchmarkConsistency(BenchmarkConfig benchmark, BenchmarkConfig other) {
+ if (benchmark.getTarget().equals(other.getTarget())) {
+ throw new BenchmarkConfigError("Duplicate benchmark name and target: " + benchmark);
+ }
+ if (!benchmark.getMetrics().equals(other.getMetrics())) {
+ throw new BenchmarkConfigError(
+ "Inconsistent metrics for benchmarks: " + benchmark + " and " + other);
+ }
+ if (!benchmark.getSuite().equals(other.getSuite())) {
+ throw new BenchmarkConfigError(
+ "Inconsistent suite for benchmarks: " + benchmark + " and " + other);
+ }
+ if (benchmark.hasTimeWarmupRuns() != other.hasTimeWarmupRuns()) {
+ throw new BenchmarkConfigError(
+ "Inconsistent time-warmup for benchmarks: " + benchmark + " and " + other);
+ }
+ }
+
+ public static Set<BenchmarkMetric> getCommonMetrics(List<BenchmarkConfig> variants) {
+ return getConsistentRepresentative(variants).getMetrics();
+ }
+
+ public static BenchmarkSuite getCommonSuite(List<BenchmarkConfig> variants) {
+ return getConsistentRepresentative(variants).getSuite();
+ }
+
+ public static boolean getCommonTimeWarmupRuns(List<BenchmarkConfig> variants) {
+ return getConsistentRepresentative(variants).hasTimeWarmupRuns();
+ }
+
+ private static BenchmarkConfig getConsistentRepresentative(List<BenchmarkConfig> variants) {
+ if (variants.isEmpty()) {
+ throw new BenchmarkConfigError("Unexpected attempt to check consistency of empty collection");
+ }
+ BenchmarkConfig representative = variants.get(0);
+ for (int i = 1; i < variants.size(); i++) {
+ checkBenchmarkConsistency(representative, variants.get(i));
+ }
+ return representative;
+ }
+
public static class Builder {
private String name = null;
@@ -128,7 +170,7 @@
public String getWarmupName() {
if (!timeWarmupRuns) {
- throw new Unreachable("Invalid attempt at getting warmup benchmark name");
+ throw new BenchmarkConfigError("Invalid attempt at getting warmup benchmark name");
}
return getName() + "Warmup";
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfigError.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfigError.java
new file mode 100644
index 0000000..9ed6e31
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfigError.java
@@ -0,0 +1,11 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.benchmarks;
+
+public class BenchmarkConfigError extends RuntimeException {
+
+ public BenchmarkConfigError(String message) {
+ super(message);
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
index cf55b86..f436c26 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
@@ -4,7 +4,6 @@
package com.android.tools.r8.benchmarks;
import com.android.tools.r8.benchmarks.BenchmarkRunner.ResultMode;
-import com.android.tools.r8.errors.Unreachable;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
@@ -40,7 +39,7 @@
private static void verifyMetric(BenchmarkMetric metric, boolean expected, boolean actual) {
if (expected != actual) {
- throw new Unreachable(
+ throw new BenchmarkConfigError(
"Mismatched config and result for "
+ metric.name()
+ ". Expected by config: "
@@ -87,7 +86,7 @@
long size = codeSizeResults.getLong(0);
for (int i = 1; i < codeSizeResults.size(); i++) {
if (size != codeSizeResults.getLong(i)) {
- throw new Unreachable(
+ throw new RuntimeException(
"Unexpected code size difference: " + size + " and " + codeSizeResults.getLong(i));
}
}
