Fix upload to GCS from perf
Change-Id: I2c4187155d069b7635c895fec7c671e2a9c2158a
diff --git a/tools/historic_run.py b/tools/historic_run.py
index 648e31d..996bc6d 100755
--- a/tools/historic_run.py
+++ b/tools/historic_run.py
@@ -44,10 +44,14 @@
class GitCommit(object):
def __init__(self, git_hash, destination_dir, destination, timestamp):
+ self._branch = None
+ self._branch_is_computed = False
self.git_hash = git_hash
self.destination_dir = destination_dir
self.destination = destination
self.timestamp = timestamp
+ self._version = None
+ self._version_is_computed = False
def __str__(self):
return '%s : %s (%s)' % (self.git_hash, self.destination,
@@ -57,15 +61,21 @@
return self.__str__()
def branch(self):
+ if self._branch_is_computed:
+ return self._branch
branches = subprocess.check_output(
- ['git', 'branch', '--contains', 'HEAD',
- '-r']).decode('utf-8').strip().splitlines()
+ ['git', 'branch', '--contains',
+ self.hash(), '-r']).decode('utf-8').strip().splitlines()
if len(branches) != 1:
- return 'main'
- branch = branches[0].strip()
- if 'main' in branch:
- return 'main'
- return branch[branch.find('/') + 1:]
+ self._branch = 'main'
+ else:
+ branch = branches[0].strip()
+ if 'main' in branch:
+ self._branch = 'main'
+ else:
+ self._branch = branch[branch.find('/') + 1:]
+ self._branch_is_computed = True
+ return self._branch
def hash(self):
return self.git_hash
@@ -92,12 +102,18 @@
return self.timestamp
def version(self):
+ if self._version_is_computed:
+ return self._version
title = self.title()
if title.startswith(
'Version '
- ) and 'src/main/java/com/android/tools/r8/Version.java' in changes_files:
- return title[len('Version '):]
- return None
+ ) and 'src/main/java/com/android/tools/r8/Version.java' in self.changed_files(
+ ):
+ self._version = title[len('Version '):]
+ else:
+ self._version = None
+ self._version_is_computed = True
+ return self._version
def git_commit_from_hash(hash):
diff --git a/tools/upload_benchmark_data_to_google_storage.py b/tools/upload_benchmark_data_to_google_storage.py
index 8570e8b..79e594e 100755
--- a/tools/upload_benchmark_data_to_google_storage.py
+++ b/tools/upload_benchmark_data_to_google_storage.py
@@ -89,20 +89,13 @@
return release_commits
-def ParseJsonFromCloudStorage(filename, local_bucket):
- abs_path = os.path.join(local_bucket, filename)
- if not os.path.exists(abs_path):
+def ParseJsonFromCloudStorage(filename, local_bucket_dict):
+ if not filename in local_bucket_dict:
return None
- with open(abs_path, 'r') as f:
- lines = f.readlines()
- content = ''.join(lines)
- try:
- return json.loads(content)
- except:
- return None
+ return json.loads(local_bucket_dict[filename])
-def RecordBenchmarkResult(commit, benchmark, benchmark_info, local_bucket,
+def RecordBenchmarkResult(commit, benchmark, benchmark_info, local_bucket_dict,
target, benchmarks):
if not target in benchmark_info['targets']:
return
@@ -111,20 +104,20 @@
if sub_benchmarks_for_target:
for sub_benchmark in sub_benchmarks_for_target:
RecordSingleBenchmarkResult(commit, benchmark + sub_benchmark,
- local_bucket, target, benchmarks)
+ local_bucket_dict, target, benchmarks)
else:
- RecordSingleBenchmarkResult(commit, benchmark, local_bucket, target,
- benchmarks)
+ RecordSingleBenchmarkResult(commit, benchmark, local_bucket_dict,
+ target, benchmarks)
-def RecordSingleBenchmarkResult(commit, benchmark, local_bucket, target,
+def RecordSingleBenchmarkResult(commit, benchmark, local_bucket_dict, target,
benchmarks):
filename = perf.GetArtifactLocation(benchmark,
target,
commit.hash(),
'result.json',
branch=commit.branch())
- benchmark_data = ParseJsonFromCloudStorage(filename, local_bucket)
+ benchmark_data = ParseJsonFromCloudStorage(filename, local_bucket_dict)
if benchmark_data:
benchmarks[benchmark] = benchmark_data
@@ -208,8 +201,20 @@
def run(commits, local_bucket, temp, outdir=None):
+ print('Loading bucket into memory')
+ local_bucket_dict = {}
+ for (root, dirs, files) in os.walk(local_bucket):
+ for file in files:
+ if file != 'result.json':
+ continue
+ abs_path = os.path.join(root, file)
+ rel_path = os.path.relpath(abs_path, local_bucket)
+ with open(abs_path, 'r') as f:
+ local_bucket_dict[rel_path] = f.read()
+
# Aggregate all the result.json files into a single file that has the
# same format as tools/perf/benchmark_data.json.
+ print('Processing commits')
d8_benchmark_data = []
r8_benchmark_data = []
retrace_benchmark_data = []
@@ -219,23 +224,26 @@
retrace_benchmarks = {}
for benchmark, benchmark_info in perf.ALL_BENCHMARKS.items():
RecordBenchmarkResult(commit, benchmark, benchmark_info,
- local_bucket, 'd8', d8_benchmarks)
+ local_bucket_dict, 'd8', d8_benchmarks)
RecordBenchmarkResult(commit, benchmark, benchmark_info,
- local_bucket, 'r8-full', r8_benchmarks)
+ local_bucket_dict, 'r8-full', r8_benchmarks)
RecordBenchmarkResult(commit, benchmark, benchmark_info,
- local_bucket, 'retrace', retrace_benchmarks)
+ local_bucket_dict, 'retrace',
+ retrace_benchmarks)
RecordBenchmarkResults(commit, d8_benchmarks, d8_benchmark_data)
RecordBenchmarkResults(commit, r8_benchmarks, r8_benchmark_data)
RecordBenchmarkResults(commit, retrace_benchmarks,
retrace_benchmark_data)
# Trim data.
+ print('Trimming data')
d8_benchmark_data = TrimBenchmarkResults(d8_benchmark_data)
r8_benchmark_data = TrimBenchmarkResults(r8_benchmark_data)
retrace_benchmark_data = TrimBenchmarkResults(retrace_benchmark_data)
# Write output JSON files to public bucket, or to tools/perf/ if running
# with --local-bucket.
+ print('Writing JSON')
ArchiveBenchmarkResults(d8_benchmark_data, 'd8_benchmark_data.json', outdir,
temp)
ArchiveBenchmarkResults(r8_benchmark_data, 'r8_benchmark_data.json', outdir,
@@ -244,6 +252,7 @@
'retrace_benchmark_data.json', outdir, temp)
# Write remaining files to public bucket.
+ print('Writing static files')
if outdir is None:
for file in FILES:
dest = os.path.join(utils.TOOLS_DIR, 'perf', file)
