Parallelize run_on_app.py
Bug: b/297302759
Change-Id: I454286c5d673c7cec41f14f0acad89f1842122cd
diff --git a/tools/run_on_app.py b/tools/run_on_app.py
index b4d0418..2340b46 100755
--- a/tools/run_on_app.py
+++ b/tools/run_on_app.py
@@ -18,6 +18,8 @@
import gmscore_data
import nest_data
from sanitize_libraries import SanitizeLibraries, SanitizeLibrariesInPgconf
+import thread_utils
+from thread_utils import print_thread
import toolhelper
import update_prebuilds_in_android
import utils
@@ -48,6 +50,11 @@
help='Compiler build to use',
choices=COMPILER_BUILDS,
default='lib')
+ result.add_option('--no-fail-fast',
+ help='Whether run_on_app.py should report all failures '
+ 'and not just the first one',
+ default=False,
+ action='store_true')
result.add_option('--hash',
help='The version of D8/R8 to use')
result.add_option('--app',
@@ -181,6 +188,10 @@
help='Disable compiler logging',
default=False,
action='store_true')
+ result.add_option('--workers',
+ help='Number of workers to use',
+ default=1,
+ type=int)
(options, args) = result.parse_args(argv)
assert not options.hash or options.no_build, (
'Argument --no-build is required when using --hash')
@@ -227,25 +238,53 @@
yield app, version, type, use_r8lib
def run_all(options, args):
+ # Build first so that each job won't.
+ if should_build(options):
+ gradle.RunGradle(['r8lib'])
+ options.no_build = True
+ assert not should_build(options)
+
# Args will be destroyed
assert len(args) == 0
+ jobs = []
for name, version, type, use_r8lib in get_permutations():
compiler = 'r8' if type == 'deploy' else 'd8'
compiler_build = 'lib' if use_r8lib else 'full'
- print('Executing %s/%s with %s %s %s' % (compiler, compiler_build, name,
- version, type))
-
fixed_options = copy.copy(options)
fixed_options.app = name
fixed_options.version = version
fixed_options.compiler = compiler
fixed_options.compiler_build = compiler_build
fixed_options.type = type
- exit_code = run_with_options(fixed_options, [])
- if exit_code != 0:
- print('Failed %s %s %s with %s/%s' % (name, version, type, compiler,
- compiler_build))
- exit(exit_code)
+ jobs.append(
+ create_job(
+ compiler, compiler_build, name, fixed_options, type, version))
+ exit_code = thread_utils.run_in_parallel(
+ jobs,
+ number_of_workers=options.workers,
+ stop_on_first_failure=not options.no_fail_fast)
+ exit(exit_code)
+
+def create_job(compiler, compiler_build, name, options, type, version):
+ return lambda worker_id: run_job(
+ compiler, compiler_build, name, options, type, version, worker_id)
+
+def run_job(
+ compiler, compiler_build, name, options, type, version, worker_id):
+ print_thread(
+ 'Executing %s/%s with %s %s %s'
+ % (compiler, compiler_build, name, version, type),
+ worker_id)
+ if worker_id is not None:
+ options.out = os.path.join(options.out, str(worker_id))
+ os.makedirs(options.out, exist_ok=True)
+ exit_code = run_with_options(options, [], worker_id=worker_id)
+ if exit_code:
+ print_thread(
+ 'Failed %s %s %s with %s/%s'
+ % (name, version, type, compiler, compiler_build),
+ worker_id)
+ return exit_code
def find_min_xmx(options, args):
# Args will be destroyed
@@ -492,7 +531,8 @@
os.path.join(android_java8_libs_output, 'classes.dex'),
os.path.join(outdir, dex_file_name))
-def run_with_options(options, args, extra_args=None, stdout=None, quiet=False):
+def run_with_options(
+ options, args, extra_args=None, stdout=None, quiet=False, worker_id=None):
if extra_args is None:
extra_args = []
app_provided_pg_conf = False;
@@ -550,9 +590,9 @@
if options.compiler == 'r8':
if 'pgconf' in values and not options.k:
+ sanitized_lib_path = os.path.join(
+ os.path.abspath(outdir), 'sanitized_lib.jar')
if has_injars_and_libraryjars(values['pgconf']):
- sanitized_lib_path = os.path.join(
- os.path.abspath(outdir), 'sanitized_lib.jar')
sanitized_pgconf_path = os.path.join(
os.path.abspath(outdir), 'sanitized.config')
SanitizeLibrariesInPgconf(
@@ -566,8 +606,6 @@
for pgconf in values['pgconf']:
args.extend(['--pg-conf', pgconf])
if 'sanitize_libraries' in values and values['sanitize_libraries']:
- sanitized_lib_path = os.path.join(
- os.path.abspath(outdir), 'sanitized_lib.jar')
SanitizeLibraries(
sanitized_lib_path, values['libraries'], values['inputs'])
libraries = [sanitized_lib_path]
@@ -693,7 +731,8 @@
cmd_prefix=[
'taskset', '-c', options.cpu_list] if options.cpu_list else [],
jar=jar,
- main=main)
+ main=main,
+ worker_id=worker_id)
if exit_code != 0:
with open(stderr_path) as stderr:
stderr_text = stderr.read()
