Leverage historic_run.py for populating benchmark data
Change-Id: I1b9b67762aec19bcd4c58ebd167314ea72021b92
diff --git a/tools/historic_run.py b/tools/historic_run.py
index 37c3dae..35d12d9 100755
--- a/tools/historic_run.py
+++ b/tools/historic_run.py
@@ -34,6 +34,10 @@
result.add_option('--output',
default='build',
help='Directory where to output results')
+ result.add_option('--timeout',
+ default=1000,
+ help='Timeout in seconds (-1 for no timeout)',
+ type=int)
return result.parse_args(argv)
@@ -54,14 +58,9 @@
def git_commit_from_hash(hash):
- commit_timestamp = subprocess.check_output([
- 'git',
- 'show',
- '--no-patch',
- '--no-notes',
- '--pretty=\'%ct\'',
- hash
- ]).decode().strip().strip('\'')
+ commit_timestamp = subprocess.check_output(
+ ['git', 'show', '--no-patch', '--no-notes', '--pretty=\'%ct\'',
+ hash]).decode().strip().strip('\'')
destination_dir = '%s/%s/' % (MASTER_COMMITS, hash)
destination = '%s%s' % (destination_dir, 'r8.jar')
commit = GitCommit(hash, destination_dir, destination, commit_timestamp)
@@ -70,9 +69,12 @@
def enumerate_git_commits(top, bottom):
if bottom is None:
- output = subprocess.check_output(['git', 'rev-list', '--first-parent', '-n', 1000, top])
+ output = subprocess.check_output(
+ ['git', 'rev-list', '--first-parent', '-n', 1000, top])
else:
- output = subprocess.check_output(['git', 'rev-list', '--first-parent', '%s^..%s' % (bottom, top)])
+ output = subprocess.check_output(
+ ['git', 'rev-list', '--first-parent',
+ '%s^..%s' % (bottom, top)])
commits = []
for c in output.decode().splitlines():
commit_hash = c.strip()
@@ -81,8 +83,8 @@
def get_available_commits(commits):
- cloud_commits = subprocess.check_output(
- ['gsutil.py', 'ls', MASTER_COMMITS]).decode().splitlines()
+ cloud_commits = subprocess.check_output(['gsutil.py', 'ls', MASTER_COMMITS
+ ]).decode().splitlines()
available_commits = []
for commit in commits:
if commit.destination_dir in cloud_commits:
@@ -164,7 +166,8 @@
def run_cmd(options, commit):
- cmd = [options.cmd, commit.git_hash]
+ cmd = options.cmd.split(' ')
+ cmd.append(commit.git_hash)
output_path = options.output or 'build'
time_commit = '%s_%s' % (commit.timestamp, commit.git_hash)
time_commit_path = os.path.join(output_path, time_commit)
@@ -177,8 +180,8 @@
with open(stdout_path, 'w') as stdout:
with open(stderr_path, 'w') as stderr:
process = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
- timeout = 1000
- while process.poll() is None and timeout > 0:
+ timeout = options.timeout
+ while process.poll() is None and timeout != 0:
time.sleep(1)
timeout -= 1
if process.poll() is None:
diff --git a/tools/perf.py b/tools/perf.py
index 9827bd0..592172d 100755
--- a/tools/perf.py
+++ b/tools/perf.py
@@ -39,6 +39,10 @@
default=10)
result.add_argument('--outdir',
help='Output directory for running locally.')
+ result.add_argument('--skip-if-output-exists',
+ help='Skip if output exists.',
+ action='store_true',
+ default=False)
result.add_argument('--target',
help='Specific target to run on.',
default='r8-full',
@@ -47,6 +51,10 @@
help='To enable verbose logging.',
action='store_true',
default=False)
+ result.add_argument('--version',
+ '-v',
+ help='Use R8 hash for the run (default local build)',
+ default=None)
options, args = result.parse_known_args()
options.apps = options.app or ['NowInAndroidApp', 'TiviApp']
options.quiet = not options.verbose
@@ -82,8 +90,9 @@
return json.loads(''.join(lines))
-def GetArtifactLocation(app, target, filename):
- return f'{app}/{target}/{utils.get_HEAD_sha1()}/{filename}'
+def GetArtifactLocation(app, filename, options):
+ version = options.version or utils.get_HEAD_sha1()
+ return f'{app}/{options.target}/{version}/{filename}'
def GetGSLocation(filename):
@@ -99,14 +108,31 @@
utils.upload_file_to_cloud_storage(file, GetGSLocation(dest))
+# Usage with historic_run.py:
+# ./tools/historic_run.py
+# --cmd "perf.py --skip-if-output-exists --version"
+# --timeout -1
+# --top 3373fd18453835bf49bff9f02523a507a2ebf317
+# --bottom 7486f01e0622cb5935b77a92b59ddf1ca8dbd2e2
def main():
options, args = ParseOptions()
with utils.TempDir() as temp:
for app in options.apps:
+ if options.skip_if_output_exists:
+ if options.outdir:
+ raise NotImplementedError
+ output = GetGSLocation(
+ GetArtifactLocation(app, 'result.json', options))
+ if utils.cloud_storage_exists(output):
+ print(f'Skipping run, {output} already exists.')
+ continue
+
cmd = [
'tools/run_benchmark.py', '--benchmark', app, '--iterations',
'1', '--target', options.target
]
+ if options.version:
+ cmd.extend(['--version', options.version])
# Build and warmup
utils.Print(f'Preparing {app}', quiet=options.quiet)
@@ -117,7 +143,7 @@
for i in range(options.iterations):
utils.Print(f'Benchmarking {app} ({i+1}/{options.iterations})',
quiet=options.quiet)
- benchhmark_result_file = os.path.join(temp, f"result_file_{i}")
+ benchhmark_result_file = os.path.join(temp, f'result_file_{i}')
iteration_cmd = cmd + [
'--output', benchhmark_result_file, '--no-build'
]
@@ -130,18 +156,18 @@
json.dump(
MergeBenchmarkResultJsonFiles(benchmark_result_json_files),
f)
- ArchiveOutputFile(
- result_file, GetArtifactLocation(app, options.target,
- 'results'), options)
+ ArchiveOutputFile(result_file,
+ GetArtifactLocation(app, 'result.json', options),
+ options)
# Write metadata.
if os.environ.get('SWARMING_BOT_ID'):
meta_file = os.path.join(temp, "meta")
with open(meta_file, 'w') as f:
f.write("Produced by: " + os.environ.get('SWARMING_BOT_ID'))
- ArchiveOutputFile(
- meta_file, GetArtifactLocation(app, options.target, 'meta'),
- options)
+ ArchiveOutputFile(meta_file,
+ GetArtifactLocation(app, 'meta', options),
+ options)
if __name__ == '__main__':
diff --git a/tools/run_benchmark.py b/tools/run_benchmark.py
index 8948df7..65547a8 100755
--- a/tools/run_benchmark.py
+++ b/tools/run_benchmark.py
@@ -102,13 +102,14 @@
if options.nolib:
testBuildTargets = [
- utils.GRADLE_TASK_TEST_JAR,
- utils.GRADLE_TASK_TEST_DEPS_JAR,
+ utils.GRADLE_TASK_TEST_JAR, utils.GRADLE_TASK_TEST_DEPS_JAR,
utils.GRADLE_TASK_TEST_UNZIP_TESTBASE
]
buildTargets = [utils.GRADLE_TASK_R8] + testBuildTargets
r8jar = utils.R8_JAR
- testjars = [utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR, utils.R8_TESTBASE_JAR]
+ testjars = [
+ utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR, utils.R8_TESTBASE_JAR
+ ]
else:
testBuildTargets = GOLEM_BUILD_TARGETS_TESTS
buildTargets = GOLEM_BUILD_TARGETS
@@ -122,8 +123,8 @@
if options.version:
# r8 is downloaded so only test jar needs to be built.
buildTargets = testBuildTargets
- r8jar = compiledump.download_distribution(options.version,
- options.nolib, temp)
+ r8jar = compiledump.download_distribution(options.version, options,
+ temp)
if not options.no_build:
gradle.RunGradle(buildTargets + ['-Pno_internal'])
