Archive and aggregate perf try runs
When running jobs on the 'perf' bot, the option --patch-set should be set for try jobs. When this option is set, we now archive the results to gs://r8-perf-results/try/.
When aggregating the data in the bucket as part of creating a single json file for go/r8perf, we now look for hashes in gs://r8-perf-results/try/ and treat include these as try runs.
The aggregated try runs are archived to their own json file at gs://r8-perf-results/r8_benchmark_try_data.json.
Change-Id: Id7acc6ed7a021b22dbeb3616d3a9b84672c88713
diff --git a/tools/historic_run.py b/tools/historic_run.py
index 74bb69b..b153276 100755
--- a/tools/historic_run.py
+++ b/tools/historic_run.py
@@ -66,7 +66,9 @@
branches = subprocess.check_output(
['git', 'branch', '--contains',
self.hash(), '-r']).decode('utf-8').strip().splitlines()
- if len(branches) != 1:
+ if len(branches) == 0:
+ self._branch = None
+ elif len(branches) != 1:
self._branch = 'main'
else:
branch = branches[0].strip()
@@ -102,6 +104,9 @@
def committer_timestamp(self):
return self.timestamp
+ def parent_hash(self):
+ return utils.get_nth_sha1_from_revision(1, self.hash())
+
def version(self):
if self._version_is_computed:
return self._version
diff --git a/tools/perf.py b/tools/perf.py
index 4f69209..ab933c4 100755
--- a/tools/perf.py
+++ b/tools/perf.py
@@ -198,7 +198,7 @@
help='Output directory for running locally.')
result.add_argument('--patch-ref',
help='The patch ref for a try run. '
- 'Should only be used from rex.py.')
+ 'Should only be used from rex.py.')
result.add_argument('--skip-if-output-exists',
help='Skip if output exists.',
action='store_true',
@@ -223,6 +223,7 @@
options.benchmarks = INTERNAL_BENCHMARKS.keys()
else:
options.benchmarks = EXTERNAL_BENCHMARKS.keys()
+ options.is_try = options.patch_ref is not None
options.quiet = not options.verbose
del options.benchmark
return options, args
@@ -275,14 +276,25 @@
return json.loads(''.join(lines))
-def GetArtifactLocation(benchmark, target, version, filename, branch=None):
+def GetArtifactLocation(benchmark,
+ target,
+ version,
+ filename,
+ branch=None,
+ is_try=False):
if version:
+ if is_try:
+ assert branch is None
+ return f'try/{benchmark}/{target}/{version}/{filename}'
if branch and branch != 'main':
return f'branches/{branch}/{benchmark}/{target}/{version}/{filename}'
return f'{benchmark}/{target}/{version}/{filename}'
else:
commit = utils.get_HEAD_commit()
branch = commit.branch()
+ if is_try:
+ assert branch is None
+ return f'try/{benchmark}/{target}/{commit.hash()}/{filename}'
if branch == 'main':
return f'{benchmark}/{target}/{commit.hash()}/{filename}'
return f'branches/{branch}/{benchmark}/{target}/{commit.hash()}/{filename}'
@@ -298,8 +310,11 @@
with open(result_file, 'w') as f:
json.dump(MergeBenchmarkResultJsonFiles(benchmark_result_json_files), f)
ArchiveOutputFile(result_file,
- GetArtifactLocation(benchmark, target, options.version,
- 'result.json'),
+ GetArtifactLocation(benchmark,
+ target,
+ options.version,
+ 'result.json',
+ is_try=options.is_try),
outdir=options.outdir)
@@ -322,9 +337,6 @@
# --bottom 7486f01e0622cb5935b77a92b59ddf1ca8dbd2e2
def main():
options, args = ParseOptions()
- if options.patch_ref:
- print('Received patch ref', options.patch_ref)
- return
Build(options)
any_failed = False
with utils.TempDir() as temp:
@@ -348,8 +360,11 @@
if options.outdir:
raise NotImplementedError
output = GetGSLocation(
- GetArtifactLocation(benchmark, target, options.version,
- 'result.json'))
+ GetArtifactLocation(benchmark,
+ target,
+ options.version,
+ 'result.json',
+ is_try=options.is_try))
if utils.cloud_storage_exists(output):
print(f'Skipping run, {output} already exists.')
continue
@@ -424,8 +439,11 @@
os.environ.get('SWARMING_BOT_ID'))
ArchiveOutputFile(meta_file,
GetArtifactLocation(
- benchmark, target, options.version,
- 'meta'),
+ benchmark,
+ target,
+ options.version,
+ 'meta',
+ is_try=options.is_try),
outdir=options.outdir)
# Only upload benchmark data when running on the perf bot.
diff --git a/tools/upload_benchmark_data_to_google_storage.py b/tools/upload_benchmark_data_to_google_storage.py
index 7020a5e..4d6c534 100755
--- a/tools/upload_benchmark_data_to_google_storage.py
+++ b/tools/upload_benchmark_data_to_google_storage.py
@@ -70,8 +70,10 @@
def CmpVersions(x, y):
- semver_x = utils.check_basic_semver_version(x.version(), allowPrerelease=True)
- semver_y = utils.check_basic_semver_version(y.version(), allowPrerelease=True)
+ semver_x = utils.check_basic_semver_version(x.version(),
+ allowPrerelease=True)
+ semver_y = utils.check_basic_semver_version(y.version(),
+ allowPrerelease=True)
if semver_x.larger_than(semver_y):
return 1
if semver_y.larger_than(semver_x):
@@ -101,6 +103,17 @@
return release_commits
+def GetTryCommits(local_bucket_try_dict):
+ try_commits = []
+ for key, value in local_bucket_try_dict.items():
+ # The hash is the 4th component in the path:
+ # try/{benchmark}/{target}/{commit.hash()}/{filename}.
+ try_hash = key.split('/')[3]
+ try_commit = historic_run.git_commit_from_hash(try_hash)
+ try_commits.append(try_commit)
+ return try_commits
+
+
def ParseJsonFromCloudStorage(filename, local_bucket_dict):
if not filename in local_bucket_dict:
return None
@@ -108,7 +121,7 @@
def RecordBenchmarkResult(commit, benchmark, benchmark_info, local_bucket_dict,
- target, benchmarks):
+ target, benchmarks, is_try):
if not target in benchmark_info['targets']:
return
sub_benchmarks = benchmark_info.get('subBenchmarks', {})
@@ -116,25 +129,27 @@
if sub_benchmarks_for_target:
for sub_benchmark in sub_benchmarks_for_target:
RecordSingleBenchmarkResult(commit, benchmark + sub_benchmark,
- local_bucket_dict, target, benchmarks)
+ local_bucket_dict, target, benchmarks,
+ is_try)
else:
RecordSingleBenchmarkResult(commit, benchmark, local_bucket_dict,
- target, benchmarks)
+ target, benchmarks, is_try)
def RecordSingleBenchmarkResult(commit, benchmark, local_bucket_dict, target,
- benchmarks):
+ benchmarks, is_try):
filename = perf.GetArtifactLocation(benchmark,
target,
commit.hash(),
'result.json',
- branch=commit.branch())
+ branch=commit.branch(),
+ is_try=is_try)
benchmark_data = ParseJsonFromCloudStorage(filename, local_bucket_dict)
if benchmark_data:
benchmarks[benchmark] = benchmark_data
-def RecordBenchmarkResults(commit, benchmarks, benchmark_data):
+def RecordBenchmarkResults(commit, benchmarks, benchmark_data, is_try):
if benchmarks or benchmark_data:
data = {
'author': commit.author_name(),
@@ -143,6 +158,10 @@
'title': commit.title(),
'benchmarks': benchmarks
}
+ if is_try:
+ # TODO(christofferqa): We should find the first parent on main
+ # to support running try jobs for CL chains.
+ data['parent_hash'] = commit.parent_hash()
version = commit.version()
if version:
data['version'] = version
@@ -191,7 +210,7 @@
commit_hashes = set()
for benchmark in os.listdir(local_bucket):
benchmark_dir = os.path.join(local_bucket, benchmark)
- if not os.path.isdir(benchmark_dir):
+ if benchmark == 'try' or not os.path.isdir(benchmark_dir):
continue
for target in os.listdir(benchmark_dir):
target_dir = os.path.join(local_bucket, benchmark, target)
@@ -215,6 +234,7 @@
def run(commits, local_bucket, temp, outdir=None):
print('Loading bucket into memory')
local_bucket_dict = {}
+ local_bucket_try_dict = {}
for (root, dirs, files) in os.walk(local_bucket):
for file in files:
if file != 'result.json':
@@ -222,10 +242,28 @@
abs_path = os.path.join(root, file)
rel_path = os.path.relpath(abs_path, local_bucket)
with open(abs_path, 'r') as f:
- local_bucket_dict[rel_path] = f.read()
+ dict_or_try_dict = local_bucket_try_dict if rel_path.startswith(
+ 'try/') else local_bucket_dict
+ dict_or_try_dict[rel_path] = f.read()
# Aggregate all the result.json files into a single file that has the
# same format as tools/perf/benchmark_data.json.
+ process_commits(commits, local_bucket_dict, temp, outdir)
+ process_commits(GetTryCommits(local_bucket_try_dict),
+ local_bucket_try_dict,
+ temp,
+ outdir,
+ is_try=True)
+
+ # Write remaining files to public bucket.
+ print('Writing static files')
+ if outdir is None:
+ for file in FILES:
+ dest = os.path.join(utils.TOOLS_DIR, 'perf', file)
+ perf.ArchiveOutputFile(dest, file)
+
+
+def process_commits(commits, local_bucket_dict, temp, outdir, is_try=False):
print('Processing commits')
d8_benchmark_data = []
r8_benchmark_data = []
@@ -236,16 +274,18 @@
retrace_benchmarks = {}
for benchmark, benchmark_info in perf.ALL_BENCHMARKS.items():
RecordBenchmarkResult(commit, benchmark, benchmark_info,
- local_bucket_dict, 'd8', d8_benchmarks)
+ local_bucket_dict, 'd8', d8_benchmarks,
+ is_try)
RecordBenchmarkResult(commit, benchmark, benchmark_info,
- local_bucket_dict, 'r8-full', r8_benchmarks)
+ local_bucket_dict, 'r8-full', r8_benchmarks,
+ is_try)
RecordBenchmarkResult(commit, benchmark, benchmark_info,
local_bucket_dict, 'retrace',
- retrace_benchmarks)
- RecordBenchmarkResults(commit, d8_benchmarks, d8_benchmark_data)
- RecordBenchmarkResults(commit, r8_benchmarks, r8_benchmark_data)
+ retrace_benchmarks, is_try)
+ RecordBenchmarkResults(commit, d8_benchmarks, d8_benchmark_data, is_try)
+ RecordBenchmarkResults(commit, r8_benchmarks, r8_benchmark_data, is_try)
RecordBenchmarkResults(commit, retrace_benchmarks,
- retrace_benchmark_data)
+ retrace_benchmark_data, is_try)
# Trim data.
print('Trimming data')
@@ -256,19 +296,14 @@
# Write output JSON files to public bucket, or to tools/perf/ if running
# with --local-bucket.
print('Writing JSON')
- ArchiveBenchmarkResults(d8_benchmark_data, 'd8_benchmark_data.json', outdir,
- temp)
- ArchiveBenchmarkResults(r8_benchmark_data, 'r8_benchmark_data.json', outdir,
- temp)
+ data_file_suffix = '_try_data.json' if is_try else '_data.json'
+ ArchiveBenchmarkResults(d8_benchmark_data,
+ 'd8_benchmark' + data_file_suffix, outdir, temp)
+ ArchiveBenchmarkResults(r8_benchmark_data,
+ 'r8_benchmark' + data_file_suffix, outdir, temp)
ArchiveBenchmarkResults(retrace_benchmark_data,
- 'retrace_benchmark_data.json', outdir, temp)
-
- # Write remaining files to public bucket.
- print('Writing static files')
- if outdir is None:
- for file in FILES:
- dest = os.path.join(utils.TOOLS_DIR, 'perf', file)
- perf.ArchiveOutputFile(dest, file)
+ 'retrace_benchmark' + data_file_suffix, outdir,
+ temp)
def ParseOptions():
