Add NowInAndroid D8 incremental benchmark
This adds support for having benchmarks with sub benchmarks write the results to disk and integrates this with perf.py.
Change-Id: I487e40eafb2d6725486d0adfa2f39677be689dad
diff --git a/src/main/java/com/android/tools/r8/utils/ListUtils.java b/src/main/java/com/android/tools/r8/utils/ListUtils.java
index a183511..4a0c15e 100644
--- a/src/main/java/com/android/tools/r8/utils/ListUtils.java
+++ b/src/main/java/com/android/tools/r8/utils/ListUtils.java
@@ -18,6 +18,7 @@
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Function;
+import java.util.function.IntFunction;
import java.util.function.Predicate;
public class ListUtils {
@@ -247,6 +248,14 @@
return list;
}
+ public static <T> ArrayList<T> newInitializedArrayList(int size, IntFunction<T> fn) {
+ ArrayList<T> list = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ list.add(fn.apply(i));
+ }
+ return list;
+ }
+
public static <T> ImmutableList<T> newImmutableList(ForEachable<T> forEachable) {
ImmutableList.Builder<T> builder = ImmutableList.builder();
forEachable.forEach(builder::add);
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
index dc841ba..7d006fc 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsCollection.java
@@ -6,11 +6,13 @@
import com.android.tools.r8.DexSegments.SegmentInfo;
import com.android.tools.r8.errors.Unimplemented;
import it.unimi.dsi.fastutil.ints.Int2ReferenceMap;
-import java.io.PrintStream;
+import java.io.IOException;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
public class BenchmarkResultsCollection implements BenchmarkResults {
@@ -58,6 +60,11 @@
throw error();
}
+ @Override
+ public void doAverage() {
+ throw new Unimplemented();
+ }
+
private BenchmarkConfigError error() {
throw new BenchmarkConfigError(
"Unexpected attempt to add a result to a the root of a benchmark with sub-benchmarks");
@@ -79,7 +86,11 @@
}
@Override
- public void writeResults(PrintStream out) {
- throw new Unimplemented();
+ public void writeResults(Path path) throws IOException {
+ for (Entry<String, BenchmarkResultsSingle> entry : results.entrySet()) {
+ String name = entry.getKey();
+ BenchmarkResultsSingle result = entry.getValue();
+ result.writeResults(path.resolve(name));
+ }
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
index 6a6140a..e029bcd 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsSingle.java
@@ -3,6 +3,9 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import com.android.tools.r8.DexSegments.SegmentInfo;
import com.android.tools.r8.dex.DexSection;
import com.google.gson.Gson;
@@ -10,7 +13,10 @@
import it.unimi.dsi.fastutil.ints.Int2ReferenceMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
+import java.io.IOException;
import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -116,6 +122,21 @@
}
@Override
+ public void doAverage() {
+ assertFalse(runtimeResults.isEmpty());
+ long averageRuntimeResult =
+ Math.round(runtimeResults.stream().mapToLong(Long::longValue).average().orElse(0));
+ runtimeResults.clear();
+ addRuntimeResult(averageRuntimeResult);
+
+ assertTrue(codeSizeResults.isEmpty());
+ assertTrue(instructionCodeSizeResults.isEmpty());
+ assertTrue(composableInstructionCodeSizeResults.isEmpty());
+ assertTrue(dex2OatSizeResult.isEmpty());
+ assertTrue(dexSegmentsSizeResults.isEmpty());
+ }
+
+ @Override
public BenchmarkResults getSubResults(String name) {
throw new BenchmarkConfigError(
"Unexpected attempt to get sub-results for benchmark without sub-benchmarks");
@@ -247,11 +268,14 @@
}
@Override
- public void writeResults(PrintStream out) {
- Gson gson =
- new GsonBuilder()
- .registerTypeAdapter(BenchmarkResultsSingle.class, new BenchmarkResultsSingleAdapter())
- .create();
- out.print(gson.toJson(this));
+ public void writeResults(Path path) throws IOException {
+ try (PrintStream out = new PrintStream(Files.newOutputStream(path))) {
+ Gson gson =
+ new GsonBuilder()
+ .registerTypeAdapter(
+ BenchmarkResultsSingle.class, new BenchmarkResultsSingleAdapter())
+ .create();
+ out.print(gson.toJson(this));
+ }
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
index 4f16cc4..16084c1 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkResultsWarmup.java
@@ -3,13 +3,15 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;
+import static org.junit.Assert.assertFalse;
+
import com.android.tools.r8.DexSegments.SegmentInfo;
import com.android.tools.r8.errors.Unimplemented;
import com.android.tools.r8.errors.Unreachable;
import it.unimi.dsi.fastutil.ints.Int2ReferenceMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
-import java.io.PrintStream;
+import java.nio.file.Path;
public class BenchmarkResultsWarmup implements BenchmarkResults {
@@ -74,6 +76,15 @@
}
@Override
+ public void doAverage() {
+ assertFalse(runtimeResults.isEmpty());
+ long averageRuntimeResult =
+ Math.round(runtimeResults.stream().mapToLong(Long::longValue).average().orElse(0));
+ runtimeResults.clear();
+ addRuntimeResult(averageRuntimeResult);
+ }
+
+ @Override
public BenchmarkResults getSubResults(String name) {
// When running warmups all results are amended to the single warmup result.
return this;
@@ -97,7 +108,7 @@
}
@Override
- public void writeResults(PrintStream out) {
+ public void writeResults(Path path) {
throw new Unimplemented();
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
index 78441e3..3f1db19 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/BenchmarkRunner.java
@@ -4,9 +4,6 @@
package com.android.tools.r8.benchmarks;
import com.android.tools.r8.benchmarks.BenchmarkResults.ResultMode;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.nio.file.Files;
public class BenchmarkRunner {
@@ -87,7 +84,7 @@
printMetaInfo("benchmark", getBenchmarkIterations(), benchmarkTotalTime);
results.printResults(resultMode, environment.failOnCodeSizeDifferences());
if (environment.hasOutputPath()) {
- writeResults(results);
+ results.writeResults(environment.getOutputPath());
}
System.out.println();
}
@@ -97,11 +94,4 @@
System.out.println(" " + kind + " iterations: " + iterations);
System.out.println(" " + kind + " total time: " + BenchmarkResults.prettyTime(totalTime));
}
-
- private void writeResults(BenchmarkResults results) throws IOException {
- try (PrintStream printStream =
- new PrintStream(Files.newOutputStream(environment.getOutputPath()))) {
- results.writeResults(printStream);
- }
- }
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java b/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java
index 3f7d657..8383488 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/appdumps/AppDumpBenchmarkBuilder.java
@@ -161,8 +161,7 @@
.setMethod(runIncrementalD8(this))
.setFromRevision(fromRevision)
.addDependency(dumpDependency)
- .addSubBenchmark(nameForLibraryPart(), BenchmarkMetric.RunTimeRaw)
- .addSubBenchmark(nameForProgramPart(), BenchmarkMetric.RunTimeRaw)
+ .addSubBenchmark(nameForDexPart(), BenchmarkMetric.RunTimeRaw)
.addSubBenchmark(nameForMergePart(), BenchmarkMetric.RunTimeRaw)
.setTimeout(10, TimeUnit.MINUTES)
.build();
@@ -187,16 +186,8 @@
return name + "Code";
}
- private String nameForComposableCodePart() {
- return name + "ComposableCode";
- }
-
- private String nameForLibraryPart() {
- return name + "Library";
- }
-
- private String nameForProgramPart() {
- return name + "Program";
+ private String nameForDexPart() {
+ return name + "Dex";
}
private String nameForResourcePart() {
@@ -348,38 +339,45 @@
results -> {
CompilerDump dump = builder.getExtractedDump(environment);
DumpOptions dumpProperties = dump.getBuildProperties();
+
+ int numShards = 1;
PackageSplitResources resources =
PackageSplitResources.create(
- environment.getTemp(), dump.getProgramArchive(), builder.programPackages);
- if (resources.getPackageFiles().isEmpty()) {
- throw new RuntimeException("Unexpected empty set of program package files");
+ environment.getTemp(),
+ dump.getProgramArchive(),
+ builder.programPackages,
+ numShards);
+
+ // Compile all files to a single DEX file.
+ List<List<Path>> compiledShards = new ArrayList<>();
+ for (List<Path> shard : resources.getShards()) {
+ List<Path> compiledShard = new ArrayList<>(shard.size());
+ for (Path programFile : shard) {
+ TestBase.testForD8(environment.getTemp())
+ .addProgramFiles(programFile)
+ .addClasspathFiles(dump.getProgramArchive())
+ .addLibraryFiles(dump.getLibraryArchive())
+ .applyIf(
+ builder.enableLibraryDesugaring, b -> addDesugaredLibrary(b, dump))
+ .debug()
+ .setIntermediate(true)
+ .setMinApi(dumpProperties.getMinApi())
+ .benchmarkCompile(results.getSubResults(builder.nameForDexPart()))
+ .writeToZip(compiledShard::add);
+ }
+ compiledShards.add(compiledShard);
}
+ results.getSubResults(builder.nameForDexPart()).doAverage();
- TestBase.testForD8(environment.getTemp(), Backend.DEX)
- .addProgramFiles(resources.getOtherFiles())
- .addLibraryFiles(dump.getLibraryArchive())
- .setMinApi(dumpProperties.getMinApi())
- .benchmarkCompile(results.getSubResults(builder.nameForLibraryPart()));
-
- List<Path> programOutputs = new ArrayList<>();
- for (Path programFile : resources.getPackageFiles()) {
- programOutputs.add(
- TestBase.testForD8(environment.getTemp(), Backend.DEX)
- .addProgramFiles(programFile)
- .addClasspathFiles(dump.getProgramArchive())
- .addLibraryFiles(dump.getLibraryArchive())
- .setMinApi(dumpProperties.getMinApi())
- .apply(b -> addDesugaredLibrary(b, dump))
- .setIntermediate(true)
- .benchmarkCompile(results.getSubResults(builder.nameForProgramPart()))
- .writeToZip());
+ // Merge each compiled shard.
+ for (List<Path> compiledShard : compiledShards) {
+ TestBase.testForD8(environment.getTemp())
+ .addProgramFiles(compiledShard)
+ .addLibraryFiles(dump.getLibraryArchive())
+ .debug()
+ .setMinApi(dumpProperties.getMinApi())
+ .benchmarkCompile(results.getSubResults(builder.nameForMergePart()));
}
-
- TestBase.testForD8(environment.getTemp(), Backend.DEX)
- .addProgramFiles(programOutputs)
- .addLibraryFiles(dump.getLibraryArchive())
- .setMinApi(dumpProperties.getMinApi())
- .benchmarkCompile(results.getSubResults(builder.nameForMergePart()));
});
}
}
diff --git a/src/test/java/com/android/tools/r8/benchmarks/appdumps/NowInAndroidBenchmarks.java b/src/test/java/com/android/tools/r8/benchmarks/appdumps/NowInAndroidBenchmarks.java
index e635966..e0ce0b3 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/appdumps/NowInAndroidBenchmarks.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/appdumps/NowInAndroidBenchmarks.java
@@ -44,6 +44,12 @@
.setFromRevision(16017)
.buildBatchD8(),
AppDumpBenchmarkBuilder.builder()
+ .setName("NowInAndroidAppIncremental")
+ .setDumpDependencyPath(dump)
+ .setFromRevision(16017)
+ .addProgramPackages("com/google/samples/apps/nowinandroid")
+ .buildIncrementalD8(),
+ AppDumpBenchmarkBuilder.builder()
.setName("NowInAndroidApp")
.setDumpDependencyPath(dump)
.setFromRevision(16017)
diff --git a/src/test/java/com/android/tools/r8/benchmarks/appdumps/PackageSplitResources.java b/src/test/java/com/android/tools/r8/benchmarks/appdumps/PackageSplitResources.java
index b0566ac..d6dc468 100644
--- a/src/test/java/com/android/tools/r8/benchmarks/appdumps/PackageSplitResources.java
+++ b/src/test/java/com/android/tools/r8/benchmarks/appdumps/PackageSplitResources.java
@@ -4,6 +4,7 @@
package com.android.tools.r8.benchmarks.appdumps;
import com.android.tools.r8.utils.FileUtils;
+import com.android.tools.r8.utils.ListUtils;
import com.android.tools.r8.utils.ZipUtils;
import java.io.File;
import java.io.IOException;
@@ -15,34 +16,33 @@
public class PackageSplitResources {
- private final List<Path> packageFiles;
- private final List<Path> otherFiles;
+ private final List<List<Path>> shards;
- public PackageSplitResources(List<Path> packageFiles, List<Path> otherFiles) {
- this.packageFiles = packageFiles;
- this.otherFiles = otherFiles;
+ public PackageSplitResources(List<List<Path>> shards) {
+ this.shards = shards;
}
public static PackageSplitResources create(
- TemporaryFolder temp, Path archive, List<String> packagePrefixes) throws IOException {
+ TemporaryFolder temp, Path archive, List<String> packagePrefixes, int numShards)
+ throws IOException {
Path unzipDir = temp.newFolder().toPath();
ZipUtils.unzip(archive, unzipDir);
- List<Path> packageFiles = new ArrayList<>();
- List<Path> otherFiles = new ArrayList<>();
+ List<List<Path>> shards = ListUtils.newInitializedArrayList(numShards, i -> new ArrayList<>());
Files.walk(unzipDir)
.forEachOrdered(
file -> {
if (FileUtils.isClassFile(file)) {
Path relative = unzipDir.relativize(file);
if (isInPackagePrefixes(relative, packagePrefixes)) {
- packageFiles.add(file);
- } else {
- otherFiles.add(file);
+ String packageDir =
+ relative.getParent() != null ? relative.getParent().toString() : "";
+ int shard = Math.abs(packageDir.hashCode() % numShards);
+ shards.get(shard).add(file);
}
}
});
- return new PackageSplitResources(packageFiles, otherFiles);
+ return new PackageSplitResources(shards);
}
private static boolean isInPackagePrefixes(Path file, List<String> programPackages) {
@@ -58,11 +58,7 @@
return false;
}
- public List<Path> getPackageFiles() {
- return packageFiles;
- }
-
- public List<Path> getOtherFiles() {
- return otherFiles;
+ public List<List<Path>> getShards() {
+ return shards;
}
}
diff --git a/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java b/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
index 6f8d5b6..42fb589 100644
--- a/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
+++ b/src/test/testbase/java/com/android/tools/r8/benchmarks/BenchmarkResults.java
@@ -6,7 +6,8 @@
import com.android.tools.r8.DexSegments.SegmentInfo;
import com.android.tools.r8.utils.StringUtils;
import it.unimi.dsi.fastutil.ints.Int2ReferenceMap;
-import java.io.PrintStream;
+import java.io.IOException;
+import java.nio.file.Path;
public interface BenchmarkResults {
@@ -27,6 +28,8 @@
// Append a resource size result. This is always assumed to be identical if called multiple times.
void addResourceSizeResult(long result);
+ void doAverage();
+
// Get the results collection for a "sub-benchmark" when defining a group of benchmarks.
// This will throw if called on a benchmark without sub-benchmarks.
BenchmarkResults getSubResults(String name);
@@ -37,7 +40,7 @@
void printResults(ResultMode resultMode, boolean failOnCodeSizeDifferences);
- void writeResults(PrintStream out);
+ void writeResults(Path path) throws IOException;
static String prettyTime(long nanoTime) {
return "" + (nanoTime / 1000000) + " ms";
diff --git a/tools/perf.py b/tools/perf.py
index fec3449..ebf6d49 100755
--- a/tools/perf.py
+++ b/tools/perf.py
@@ -55,6 +55,12 @@
'NowInAndroidAppNoJ$': {
'targets': ['d8']
},
+ 'NowInAndroidAppIncremental': {
+ 'targets': ['d8'],
+ 'subBenchmarks': {
+ 'd8': ['Dex', 'Merge']
+ }
+ },
'OwlApp': {
'targets': ['r8-full']
},
@@ -177,6 +183,17 @@
return f'gs://{bucket}/{filename}'
+def ArchiveBenchmarkResult(benchmark, target, benchmark_result_json_files,
+ options, temp):
+ result_file = os.path.join(temp, 'result_file')
+ with open(result_file, 'w') as f:
+ json.dump(MergeBenchmarkResultJsonFiles(benchmark_result_json_files), f)
+ ArchiveOutputFile(result_file,
+ GetArtifactLocation(benchmark, target, options.version,
+ 'result.json'),
+ outdir=options.outdir)
+
+
def ArchiveOutputFile(file, dest, bucket=BUCKET, header=None, outdir=None):
if outdir:
dest_in_outdir = os.path.join(outdir, dest)
@@ -205,10 +222,15 @@
r8jar = compiledump.download_distribution(options.version,
download_options, temp)
for benchmark in options.benchmarks:
+ benchmark_info = BENCHMARKS[benchmark]
targets = [options.target
- ] if options.target else BENCHMARKS[benchmark]['targets']
+ ] if options.target else benchmark_info['targets']
for target in targets:
+ sub_benchmarks = benchmark_info.get('subBenchmarks', {})
+ sub_benchmarks_for_target = sub_benchmarks.get(target, [])
+
if options.skip_if_output_exists:
+ assert len(sub_benchmarks_for_target) == 0, 'Unimplemented'
if options.outdir:
raise NotImplementedError
output = GetGSLocation(
@@ -219,23 +241,41 @@
continue
# Run benchmark.
- benchmark_result_json_files = []
+ if sub_benchmarks_for_target:
+ benchmark_result_json_files = {}
+ for sub_benchmark in sub_benchmarks_for_target:
+ benchmark_result_json_files[sub_benchmark] = []
+ else:
+ benchmark_result_json_files = []
failed = False
for i in range(options.iterations):
utils.Print(
f'Benchmarking {benchmark} ({i+1}/{options.iterations})',
quiet=options.quiet)
- benchhmark_result_file = os.path.join(
- temp, f'result_file_{i}')
+ if sub_benchmarks_for_target:
+ benchmark_result_file = os.path.join(
+ temp, f'result_{i}')
+ os.makedirs(benchmark_result_file)
+ else:
+ benchmark_result_file = os.path.join(
+ temp, f'result_file_{i}')
iteration_cmd = GetRunCmd(benchmark, target, options, [
'--iterations',
str(options.iterations_inner), '--output',
- benchhmark_result_file, '--no-build'
+ benchmark_result_file, '--no-build'
])
try:
subprocess.check_call(iteration_cmd)
- benchmark_result_json_files.append(
- benchhmark_result_file)
+ if sub_benchmarks_for_target:
+ for sub_benchmark in sub_benchmarks_for_target:
+ sub_benchmark_result_file = os.path.join(
+ benchmark_result_file, sub_benchmark)
+ benchmark_result_json_files[
+ sub_benchmark].append(
+ sub_benchmark_result_file)
+ else:
+ benchmark_result_json_files.append(
+ benchmark_result_file)
except subprocess.CalledProcessError as e:
failed = True
any_failed = True
@@ -245,16 +285,16 @@
continue
# Merge results and write output.
- result_file = os.path.join(temp, 'result_file')
- with open(result_file, 'w') as f:
- json.dump(
- MergeBenchmarkResultJsonFiles(
- benchmark_result_json_files), f)
- ArchiveOutputFile(result_file,
- GetArtifactLocation(benchmark, target,
- options.version,
- 'result.json'),
- outdir=options.outdir)
+ if sub_benchmarks_for_target:
+ for sub_benchmark in sub_benchmarks:
+ ArchiveBenchmarkResult(
+ benchmark + sub_benchmark, target,
+ benchmark_result_json_files[sub_benchmark], options,
+ temp)
+ else:
+ ArchiveBenchmarkResult(benchmark, target,
+ benchmark_result_json_files, options,
+ temp)
# Write metadata.
if utils.is_bot():
diff --git a/tools/upload_benchmark_data_to_google_storage.py b/tools/upload_benchmark_data_to_google_storage.py
index f979010..8c47e8a 100755
--- a/tools/upload_benchmark_data_to_google_storage.py
+++ b/tools/upload_benchmark_data_to_google_storage.py
@@ -47,6 +47,19 @@
target, benchmarks):
if not target in benchmark_info['targets']:
return
+ sub_benchmarks = benchmark_info.get('subBenchmarks', {})
+ sub_benchmarks_for_target = sub_benchmarks.get(target, [])
+ if sub_benchmarks_for_target:
+ for sub_benchmark in sub_benchmarks_for_target:
+ RecordSingleBenchmarkResult(commit, benchmark + sub_benchmark,
+ local_bucket, target, benchmarks)
+ else:
+ RecordSingleBenchmarkResult(commit, benchmark, local_bucket, target,
+ benchmarks)
+
+
+def RecordSingleBenchmarkResult(commit, benchmark, local_bucket, target,
+ benchmarks):
filename = perf.GetArtifactLocation(benchmark, target, commit.hash(),
'result.json')
benchmark_data = ParseJsonFromCloudStorage(filename, local_bucket)
