Add retrace benchmark
Change-Id: Ie1c7f2cb1df5621486fd8b59fcccc5ad12053941
diff --git a/tools/perf.py b/tools/perf.py
index 409b61f..95f8af6 100755
--- a/tools/perf.py
+++ b/tools/perf.py
@@ -15,11 +15,44 @@
if utils.is_bot():
import upload_benchmark_data_to_google_storage
-BENCHMARKS = [
- 'ChromeApp', 'CraneApp', 'JetLaggedApp', 'JetNewsApp', 'JetCasterApp',
- 'JetChatApp', 'JetSnackApp', 'NowInAndroidApp', 'OwlApp', 'ReplyApp',
- 'TiviApp'
-]
+BENCHMARKS = {
+ 'ChromeApp': {
+ 'targets': ['r8-full']
+ },
+ 'CraneApp': {
+ 'targets': ['r8-full']
+ },
+ 'JetLaggedApp': {
+ 'targets': ['r8-full']
+ },
+ 'JetNewsApp': {
+ 'targets': ['r8-full']
+ },
+ 'JetCasterApp': {
+ 'targets': ['r8-full']
+ },
+ 'JetChatApp': {
+ 'targets': ['r8-full']
+ },
+ 'JetSnackApp': {
+ 'targets': ['r8-full']
+ },
+ 'NowInAndroidApp': {
+ 'targets': ['r8-full']
+ },
+ 'OwlApp': {
+ 'targets': ['r8-full']
+ },
+ 'R8': {
+ 'targets': ['retrace']
+ },
+ 'ReplyApp': {
+ 'targets': ['r8-full']
+ },
+ 'TiviApp': {
+ 'targets': ['r8-full']
+ },
+}
BUCKET = "r8-perf-results"
SAMPLE_BENCHMARK_RESULT_JSON = {
'benchmark_name': '<benchmark_name>',
@@ -54,10 +87,10 @@
help='Skip if output exists.',
action='store_true',
default=False)
- result.add_argument('--target',
- help='Specific target to run on.',
- default='r8-full',
- choices=['d8', 'r8-full', 'r8-force', 'r8-compat'])
+ result.add_argument(
+ '--target',
+ help='Specific target to run on.',
+ choices=['d8', 'r8-full', 'r8-force', 'r8-compat', 'retrace'])
result.add_argument('--verbose',
help='To enable verbose logging.',
action='store_true',
@@ -67,7 +100,7 @@
help='Use R8 hash for the run (default local build)',
default=None)
options, args = result.parse_known_args()
- options.benchmarks = options.benchmark or BENCHMARKS
+ options.benchmarks = options.benchmark or BENCHMARKS.keys()
options.quiet = not options.verbose
del options.benchmark
return options, args
@@ -75,13 +108,14 @@
def Build(options):
utils.Print('Building', quiet=options.quiet)
- build_cmd = GetRunCmd('N/A', options, ['--iterations', '0'])
+ target = options.target or 'r8-full'
+ build_cmd = GetRunCmd('N/A', target, options, ['--iterations', '0'])
subprocess.check_call(build_cmd)
-def GetRunCmd(benchmark, options, args):
+def GetRunCmd(benchmark, target, options, args):
base_cmd = [
- 'tools/run_benchmark.py', '--benchmark', benchmark, '--target', options.target
+ 'tools/run_benchmark.py', '--benchmark', benchmark, '--target', target
]
if options.verbose:
base_cmd.append('--verbose')
@@ -156,61 +190,69 @@
r8jar = compiledump.download_distribution(options.version,
download_options, temp)
for benchmark in options.benchmarks:
- if options.skip_if_output_exists:
- if options.outdir:
- raise NotImplementedError
- output = GetGSLocation(
- GetArtifactLocation(benchmark, options.target, options.version,
- 'result.json'))
- if utils.cloud_storage_exists(output):
- print(f'Skipping run, {output} already exists.')
+ targets = [options.target
+ ] if options.target else BENCHMARKS[benchmark].targets
+ for target in targets:
+ if options.skip_if_output_exists:
+ if options.outdir:
+ raise NotImplementedError
+ output = GetGSLocation(
+ GetArtifactLocation(benchmark, target, options.version,
+ 'result.json'))
+ if utils.cloud_storage_exists(output):
+ print(f'Skipping run, {output} already exists.')
+ continue
+
+ # Run benchmark.
+ benchmark_result_json_files = []
+ failed = False
+ for i in range(options.iterations):
+ utils.Print(
+ f'Benchmarking {benchmark} ({i+1}/{options.iterations})',
+ quiet=options.quiet)
+ benchhmark_result_file = os.path.join(
+ temp, f'result_file_{i}')
+ iteration_cmd = GetRunCmd(benchmark, target, options, [
+ '--iterations',
+ str(options.iterations_inner), '--output',
+ benchhmark_result_file, '--no-build'
+ ])
+ try:
+ subprocess.check_call(iteration_cmd)
+ benchmark_result_json_files.append(
+ benchhmark_result_file)
+ except subprocess.CalledProcessError as e:
+ failed = True
+ any_failed = True
+ break
+
+ if failed:
continue
- # Run benchmark.
- benchmark_result_json_files = []
- failed = False
- for i in range(options.iterations):
- utils.Print(f'Benchmarking {benchmark} ({i+1}/{options.iterations})',
- quiet=options.quiet)
- benchhmark_result_file = os.path.join(temp, f'result_file_{i}')
- iteration_cmd = GetRunCmd(benchmark, options, [
- '--iterations',
- str(options.iterations_inner), '--output',
- benchhmark_result_file, '--no-build'
- ])
- try:
- subprocess.check_call(iteration_cmd)
- benchmark_result_json_files.append(benchhmark_result_file)
- except subprocess.CalledProcessError as e:
- failed = True
- any_failed = True
- break
-
- if failed:
- continue
-
- # Merge results and write output.
- result_file = os.path.join(temp, 'result_file')
- with open(result_file, 'w') as f:
- json.dump(
- MergeBenchmarkResultJsonFiles(benchmark_result_json_files),
- f)
- ArchiveOutputFile(result_file,
- GetArtifactLocation(benchmark, options.target,
- options.version,
- 'result.json'),
- outdir=options.outdir)
-
- # Write metadata.
- if utils.is_bot():
- meta_file = os.path.join(temp, "meta")
- with open(meta_file, 'w') as f:
- f.write("Produced by: " + os.environ.get('SWARMING_BOT_ID'))
- ArchiveOutputFile(meta_file,
- GetArtifactLocation(benchmark, options.target,
- options.version, 'meta'),
+ # Merge results and write output.
+ result_file = os.path.join(temp, 'result_file')
+ with open(result_file, 'w') as f:
+ json.dump(
+ MergeBenchmarkResultJsonFiles(
+ benchmark_result_json_files), f)
+ ArchiveOutputFile(result_file,
+ GetArtifactLocation(benchmark, target,
+ options.version,
+ 'result.json'),
outdir=options.outdir)
+ # Write metadata.
+ if utils.is_bot():
+ meta_file = os.path.join(temp, "meta")
+ with open(meta_file, 'w') as f:
+ f.write("Produced by: " +
+ os.environ.get('SWARMING_BOT_ID'))
+ ArchiveOutputFile(meta_file,
+ GetArtifactLocation(
+ benchmark, target, options.version,
+ 'meta'),
+ outdir=options.outdir)
+
if utils.is_bot():
upload_benchmark_data_to_google_storage.run()
diff --git a/tools/perf/index.html b/tools/perf/r8.html
similarity index 99%
rename from tools/perf/index.html
rename to tools/perf/r8.html
index aa14c1b..857e607 100644
--- a/tools/perf/index.html
+++ b/tools/perf/r8.html
@@ -314,7 +314,7 @@
scales: scales.get()
};
- const commits = await state.importCommits("./benchmark_data.json");
+ const commits = await state.importCommits("./r8_benchmark_data.json");
state.initializeBenchmarks();
state.initializeLegends({
'Dex size': { default: true },
diff --git a/tools/perf/retrace.html b/tools/perf/retrace.html
new file mode 100644
index 0000000..00d5f48
--- /dev/null
+++ b/tools/perf/retrace.html
@@ -0,0 +1,202 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <meta charset="utf-8">
+ <title>Retrace perf</title>
+ <link rel="stylesheet" href="stylesheet.css">
+</head>
+<body>
+ <div id="benchmark-selectors"></div>
+ <div>
+ <canvas id="myChart"></canvas>
+ </div>
+ <div>
+ <div style="float: left; width: 50%">
+ <button type="button" id="show-more-left" disabled>⇐</button>
+ <button type="button" id="show-less-left">⇒</button>
+ </div>
+ <div style="float: left; text-align: right; width: 50%">
+ <button type="button" id="show-less-right">⇐</button>
+ <button type="button" id="show-more-right" disabled>⇒</button>
+ </div>
+ </div>
+ <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.3/dist/chart.umd.min.js"></script>
+ <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-datalabels@2.2.0"></script>
+ <script src="extensions.js"></script>
+ <script src="utils.js"></script>
+ <script type="module">
+ import chart from "./chart.js";
+ import dom from "./dom.js";
+ import scales from "./scales.js";
+ import state from "./state.js";
+
+ // Chart data provider.
+ function getData(filteredCommits) {
+ const labels = filteredCommits.map((c, i) => c.index);
+ const datasets = getDatasets(filteredCommits);
+ return {
+ labels: labels,
+ datasets: datasets
+ };
+ }
+
+ function getDatasets(filteredCommits) {
+ const datasets = [];
+ state.forEachSelectedBenchmark(
+ selectedBenchmark => {
+ const runtimeData =
+ filteredCommits.map(
+ (c, i) =>
+ selectedBenchmark in filteredCommits[i].benchmarks
+ ? getAllResults(selectedBenchmark, filteredCommits[i], "runtime")
+ .min()
+ .ns_to_s()
+ : NaN);
+ const runtimeScatterData = [];
+ for (const commit of filteredCommits.values()) {
+ if (!(selectedBenchmark in commit.benchmarks)) {
+ continue;
+ }
+ const runtimes = getAllResults(selectedBenchmark, commit, "runtime")
+ for (const runtime of runtimes.values()) {
+ runtimeScatterData.push({ x: commit.index, y: runtime.ns_to_s() });
+ }
+ }
+
+ const skipped = (ctx, value) => ctx.p0.skip || ctx.p1.skip ? value : undefined;
+ datasets.push(...[
+ {
+ benchmark: selectedBenchmark,
+ type: 'line',
+ label: 'Runtime',
+ data: runtimeData,
+ datalabels: {
+ labels: {
+ value: null
+ }
+ },
+ tension: 0.1,
+ yAxisID: 'y_runtime',
+ segment: {
+ borderColor: ctx =>
+ skipped(
+ ctx,
+ chart.get()
+ ? chart.get().data.datasets[ctx.datasetIndex].backgroundColor
+ : undefined),
+ borderDash: ctx => skipped(ctx, [6, 6]),
+ },
+ spanGaps: true
+ },
+ {
+ benchmark: selectedBenchmark,
+ type: 'scatter',
+ label: 'Runtime variance',
+ data: runtimeScatterData,
+ datalabels: {
+ labels: {
+ value: null
+ }
+ },
+ yAxisID: 'y_runtime'
+ }
+ ]);
+ });
+ return datasets;
+ }
+
+ // Chart options.
+ const options = {
+ onHover: (event, chartElement) =>
+ event.native.target.style.cursor =
+ chartElement[0] ? 'pointer' : 'default',
+ plugins: {
+ datalabels: {
+ backgroundColor: 'rgba(255, 255, 255, 0.7)',
+ borderColor: 'rgba(128, 128, 128, 0.7)',
+ borderRadius: 4,
+ borderWidth: 1,
+ color: context => chart.getDataPercentageChange(context) < 0 ? 'green' : 'red',
+ display: context => {
+ var percentageChange = chart.getDataPercentageChange(context);
+ return percentageChange !== null && Math.abs(percentageChange) >= 0.1;
+ },
+ font: {
+ size: 20,
+ weight: 'bold'
+ },
+ offset: 8,
+ formatter: chart.getDataLabelFormatter,
+ padding: 6
+ },
+ legend: {
+ labels: {
+ filter: (legendItem, data) => {
+ // Only retain the legends for the first selected benchmark. If
+ // multiple benchmarks are selected, then use the legends of the
+ // first selected benchmark to control all selected benchmarks.
+ const numUniqueLegends =
+ data.datasets.length / state.selectedBenchmarks.size;
+ return legendItem.datasetIndex < numUniqueLegends;
+ },
+ },
+ onClick: (e, legendItem, legend) => {
+ const clickedLegend = legendItem.text;
+ if (state.selectedLegends.has(clickedLegend)) {
+ state.selectedLegends.delete(clickedLegend);
+ } else {
+ state.selectedLegends.add(clickedLegend);
+ }
+ chart.update(false, true);
+ },
+ },
+ tooltip: {
+ callbacks: {
+ title: context => {
+ const elementInfo = context[0];
+ var commit;
+ if (elementInfo.dataset.type == 'line') {
+ commit = commits[state.zoom.left + elementInfo.dataIndex];
+ } else {
+ console.assert(elementInfo.dataset.type == 'scatter');
+ commit = commits[elementInfo.raw.x];
+ }
+ return commit.title;
+ },
+ footer: context => {
+ const elementInfo = context[0];
+ var commit;
+ if (elementInfo.dataset.type == 'line') {
+ commit = commits[state.zoom.left + elementInfo.dataIndex];
+ } else {
+ console.assert(elementInfo.dataset.type == 'scatter');
+ commit = commits[elementInfo.raw.x];
+ }
+ const dataset = chart.get().data.datasets[elementInfo.datasetIndex];
+ return `App: ${dataset.benchmark}\n`
+ + `Author: ${commit.author}\n`
+ + `Submitted: ${new Date(commit.submitted * 1000).toLocaleString()}\n`
+ + `Hash: ${commit.hash}\n`
+ + `Index: ${commit.index}`;
+ }
+ }
+ }
+ },
+ responsive: true,
+ scales: scales.get()
+ };
+
+ const commits = await state.importCommits("./retrace_benchmark_data.json");
+ state.initializeBenchmarks();
+ state.initializeLegends({
+ 'Runtime': { default: true },
+ 'Runtime variance': { default: true }
+ });
+ state.initializeZoom();
+ dom.initializeBenchmarkSelectors();
+ dom.initializeChartNavigation();
+ chart.setDataProvider(getData);
+ chart.initializeChart(options);
+ </script>
+</body>
+</html>
\ No newline at end of file
diff --git a/tools/run_benchmark.py b/tools/run_benchmark.py
index 783e695..436c046 100755
--- a/tools/run_benchmark.py
+++ b/tools/run_benchmark.py
@@ -47,7 +47,7 @@
help='The test target to run',
required=True,
# These should 1:1 with benchmarks/BenchmarkTarget.java
- choices=['d8', 'r8-full', 'r8-force', 'r8-compat'])
+ choices=['d8', 'r8-full', 'r8-force', 'r8-compat', 'retrace'])
result.add_argument(
'--debug-agent',
'--debug_agent',
@@ -165,8 +165,11 @@
def run(options, r8jar, testjars):
jdkhome = get_jdk_home(options, options.benchmark)
cmd = [
- jdk.GetJavaExecutable(jdkhome), '-Xms8g', '-Xmx8g',
- '-XX:+TieredCompilation', '-XX:TieredStopAtLevel=4',
+ jdk.GetJavaExecutable(jdkhome),
+ '-Xms8g',
+ '-Xmx8g',
+ '-XX:+TieredCompilation',
+ '-XX:TieredStopAtLevel=4',
'-DBENCHMARK_IGNORE_CODE_SIZE_DIFFERENCES',
f'-DBUILD_PROP_KEEPANNO_RUNTIME_PATH={utils.REPO_ROOT}/d8_r8/keepanno/build/classes/java/main',
# Since we change the working directory to a temp folder.
diff --git a/tools/upload_benchmark_data_to_google_storage.py b/tools/upload_benchmark_data_to_google_storage.py
index f201638..fdb1feb 100755
--- a/tools/upload_benchmark_data_to_google_storage.py
+++ b/tools/upload_benchmark_data_to_google_storage.py
@@ -20,7 +20,8 @@
'chart.js',
'dom.js',
'extensions.js',
- 'index.html',
+ 'r8.html',
+ 'retrace.html',
'scales.js',
'state.js',
'stylesheet.css',
@@ -50,6 +51,53 @@
return None
+def RecordBenchmarkResult(
+ commit, benchmark, benchmark_info, target, benchmarks):
+ if not target in benchmark_info.targets:
+ return
+ filename = perf.GetArtifactLocation(benchmark, target,
+ commit.hash(),
+ 'result.json')
+ benchmark_data = ParseJsonFromCloudStorage(
+ filename, local_bucket)
+ if benchmark_data:
+ benchmarks[benchmark] = benchmark_data
+
+
+def RecordBenchmarkResults(commit, benchmarks, benchmark_data):
+ if benchmarks or benchmark_data:
+ benchmark_data.append({
+ 'author': commit.author_name(),
+ 'hash': commit.hash(),
+ 'submitted': commit.committer_timestamp(),
+ 'title': commit.title(),
+ 'benchmarks': benchmarks
+ })
+
+
+def TrimBenchmarkResults(benchmark_data):
+ new_benchmark_data_len = len(benchmark_data)
+ while new_benchmark_data_len > 0:
+ candidate_len = new_benchmark_data_len - 1
+ if not benchmark_data[candidate_len]['benchmarks']:
+ new_benchmark_data_len = candidate_len
+ else:
+ break
+ return benchmark_data[0:new_benchmark_data_len]
+
+
+def ArchiveBenchmarkResults(benchmark_data, dest):
+ # Serialize JSON to temp file.
+ benchmark_data_file = os.path.join(temp, dest)
+ with open(benchmark_data_file, 'w') as f:
+ json.dump(benchmark_data, f)
+
+ # Write output files to public bucket.
+ perf.ArchiveOutputFile(benchmark_data_file,
+ dest,
+ header='Cache-Control:no-store')
+
+
def run():
# Get the N most recent commits sorted by newest first.
top = utils.get_sha1_from_revision('origin/main')
@@ -62,54 +110,38 @@
local_bucket = os.path.join(temp, perf.BUCKET)
DownloadCloudBucket(local_bucket)
- # Aggregate all the result.json files into a single benchmark_data.json file
- # that has the same format as tools/perf/benchmark_data.json.
- benchmark_data = []
+ # Aggregate all the result.json files into a single file that has the
+ # same format as tools/perf/benchmark_data.json.
+ r8_benchmark_data = []
+ retrace_benchmark_data = []
for commit in commits:
- benchmarks = {}
- for benchmark in BENCHMARKS:
- for target in TARGETS:
- filename = perf.GetArtifactLocation(benchmark, target,
- commit.hash(),
- 'result.json')
- single_benchmark_data = ParseJsonFromCloudStorage(
- filename, local_bucket)
- if single_benchmark_data:
- benchmarks[benchmark] = single_benchmark_data
- if benchmarks or benchmark_data:
- benchmark_data.append({
- 'author': commit.author_name(),
- 'hash': commit.hash(),
- 'submitted': commit.committer_timestamp(),
- 'title': commit.title(),
- 'benchmarks': benchmarks
- })
+ r8_benchmarks = {}
+ retrace_benchmarks = {}
+ for benchmark, benchmark_info in BENCHMARKS.items():
+ RecordBenchmarkResult(
+ commit, benchmark, benchmark_info, 'r8-full', r8_benchmarks)
+ RecordBenchmarkResult(
+ commit, benchmark, benchmark_info, 'retrace',
+ retrace_benchmarks)
+ RecordBenchmarkResults(commmit, r8_benchmarks, r8_benchmark_data)
+ RecordBenchmarkResults(
+ commmit, retrace_benchmarks, retrace_benchmark_data)
# Trim data.
- new_benchmark_data_len = len(benchmark_data)
- while new_benchmark_data_len > 0:
- candidate_len = new_benchmark_data_len - 1
- if not benchmark_data[candidate_len]['benchmarks']:
- new_benchmark_data_len = candidate_len
- else:
- break
- benchmark_data = benchmark_data[0:new_benchmark_data_len]
-
- # Serialize JSON to temp file.
- benchmark_data_file = os.path.join(temp, 'benchmark_data.json')
- with open(benchmark_data_file, 'w') as f:
- json.dump(benchmark_data, f)
+ r8_benchmark_data = TrimBenchmarkResults(r8_benchmark_data)
+ retrace_benchmark_data = TrimBenchmarkResults(retrace_benchmark_data)
# Write output files to public bucket.
- perf.ArchiveOutputFile(benchmark_data_file,
- 'benchmark_data.json',
- header='Cache-Control:no-store')
+ ArchiveBenchmarkResults(r8_benchmark_data, 'r8_benchmark_data.json')
+ ArchiveBenchmarkResults(
+ retrace_benchmark_data, 'retrace_benchmark_data.json'
+
+ # Write remaining files to public bucket.
for file in FILES:
dest = os.path.join(utils.TOOLS_DIR, 'perf', file)
perf.ArchiveOutputFile(dest, file)
-
def main():
run()
