Minor updates to startup scripts
Change-Id: I397dfd95f3410d9285a50e221e560d3a0f7bcdac
diff --git a/tools/apk_utils.py b/tools/apk_utils.py
index f126f91..1da8d47 100755
--- a/tools/apk_utils.py
+++ b/tools/apk_utils.py
@@ -38,7 +38,8 @@
apk = args[0]
return (options, apk)
-def add_baseline_profile_to_apk(apk, baseline_profile, tmp_dir):
+def add_baseline_profile_to_apk(
+ apk, baseline_profile, baseline_profile_metadata, tmp_dir):
if baseline_profile is None:
return apk
ts = time.time_ns()
@@ -46,8 +47,13 @@
dest_apk_aligned = os.path.join(tmp_dir, 'app-aligned-%s.apk' % ts)
dest_apk_signed = os.path.join(tmp_dir, 'app-signed-%s.apk' % ts)
shutil.copy2(apk, dest_apk)
+ zip_utils.remove_files_from_zip(
+ ['assets/dexopt/baseline.prof', 'assets/dexopt/baseline.profm'], dest_apk)
zip_utils.add_file_to_zip(
baseline_profile, 'assets/dexopt/baseline.prof', dest_apk)
+ if baseline_profile_metadata is not None:
+ zip_utils.add_file_to_zip(
+ baseline_profile_metadata, 'assets/dexopt/baseline.profm', dest_apk)
align(dest_apk, dest_apk_aligned)
sign_with_apksigner(dest_apk_aligned, dest_apk_signed)
return dest_apk_signed
@@ -63,6 +69,15 @@
def default_keystore():
return os.path.join(os.getenv('HOME'), '.android', 'app.keystore')
+def get_min_api(apk):
+ aapt = os.path.join(utils.getAndroidBuildTools(), 'aapt')
+ cmd = [aapt, 'dump', 'badging', apk]
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ for line in stdout.splitlines():
+ if line.startswith('sdkVersion:\''):
+ return int(line[len('sdkVersion:\''): -1])
+ raise ValueError('Unexpected stdout: %s' % stdout)
+
def sign(unsigned_apk, signed_apk, keystore, quiet=False, logging=True):
utils.Print('Signing (ignore the warnings)', quiet=quiet)
cmd = ['zip', '-d', unsigned_apk, 'META-INF/*']
diff --git a/tools/startup/adb_utils.py b/tools/startup/adb_utils.py
index 3900ffc..d712ed5 100755
--- a/tools/startup/adb_utils.py
+++ b/tools/startup/adb_utils.py
@@ -281,7 +281,14 @@
build_apks_from_bundle(bundle, apks)
install_apks(apks, device_id)
-def install_profile(app_id, device_id=None):
+def install_profile_using_adb(app_id, host_profile_path, device_id=None):
+ device_profile_path = get_profile_path(app_id)
+ cmd = create_adb_cmd('push %s %s' % (host_profile_path, device_profile_path))
+ subprocess.check_call(cmd)
+ stop_app(app_id, device_id)
+ force_profile_compilation(app_id, device_id)
+
+def install_profile_using_profileinstaller(app_id, device_id=None):
# This assumes that the profileinstaller library has been added to the app,
# https://developer.android.com/jetpack/androidx/releases/profileinstaller.
action = 'androidx.profileinstaller.action.INSTALL_PROFILE'
diff --git a/tools/startup/generate_startup_descriptors.py b/tools/startup/generate_startup_descriptors.py
index d0af556..c94a59c 100755
--- a/tools/startup/generate_startup_descriptors.py
+++ b/tools/startup/generate_startup_descriptors.py
@@ -201,8 +201,12 @@
merged_flags = flags.copy()
other_flags = startup_descriptors_to_add[startup_descriptor]
assert not other_flags['conditional_startup']
- if other_flags['post_startup']:
- merged_flags['post_startup'] = True
+ merged_flags['hot'] = \
+ merged_flags['hot'] or other_flags['hot']
+ merged_flags['startup'] = \
+ merged_flags['startup'] or other_flags['startup']
+ merged_flags['post_startup'] = \
+ merged_flags['post_startup'] or other_flags['post_startup']
new_startup_descriptors[startup_descriptor] = merged_flags
else:
new_startup_descriptors[startup_descriptor] = flags.copy()
@@ -272,10 +276,12 @@
def startup_descriptor_to_string(startup_descriptor, flags):
result = ''
- if flags['conditional_startup']:
- pass # result += 'C'
+ if flags['hot']:
+ result += 'H'
+ if flags['startup']:
+ result += 'S'
if flags['post_startup']:
- pass # result += 'P'
+ result += 'P'
result += startup_descriptor
return result
diff --git a/tools/startup/instrument.py b/tools/startup/instrument.py
new file mode 100755
index 0000000..ecefe55
--- /dev/null
+++ b/tools/startup/instrument.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import apk_masseur
+import apk_utils
+import extractmarker
+import toolhelper
+import utils
+import zip_utils
+
+def parse_options(argv):
+ result = argparse.ArgumentParser(
+ description='Instrument the dex files of a given apk to print what is '
+ 'executed.')
+ result.add_argument('--apk',
+ help='Path to the .apk',
+ required=True)
+ result.add_argument('--dex-files',
+ action='append',
+ help='Name of dex files to instrument')
+ result.add_argument('--discard',
+ action='append',
+ help='Name of dex files to discard')
+ result.add_argument('--out',
+ help='Destination of resulting apk',
+ required=True)
+ options, args = result.parse_known_args(argv)
+ return options, args
+
+def add_instrumented_dex(dex_file, instrumented_dex_index, instrumented_dir):
+ dex_name = get_dex_name(instrumented_dex_index)
+ destination = os.path.join(instrumented_dir, dex_name)
+ shutil.move(dex_file, destination)
+
+def get_dex_name(dex_index):
+ assert dex_index > 0
+ return 'classes.dex' if dex_index == 1 else ('classes%s.dex' % dex_index)
+
+def instrument_dex_file(dex_file, include_instrumentation_server, options, tmp_dir):
+ d8_cmd = [
+ 'java',
+ '-cp', utils.R8_JAR,
+ '-Dcom.android.tools.r8.startup.instrumentation.instrument=1',
+ '-Dcom.android.tools.r8.startup.instrumentation.instrumentationtag=R8']
+ if not include_instrumentation_server:
+ # We avoid injecting the InstrumentationServer by specifying it should only
+ # be added if foo.bar.Baz is in the program.
+ d8_cmd.append(
+ '-Dcom.android.tools.r8.startup.instrumentation.instrumentationserversyntheticcontext=foo.bar.Baz')
+ d8_cmd.extend([
+ 'com.android.tools.r8.D8',
+ '--min-api', str(apk_utils.get_min_api(options.apk)),
+ '--output', tmp_dir,
+ '--release',
+ dex_file])
+ subprocess.check_call(d8_cmd)
+ instrumented_dex_files = []
+ instrumented_dex_index = 1
+ while True:
+ instrumented_dex_name = get_dex_name(instrumented_dex_index)
+ instrumented_dex_file = os.path.join(tmp_dir, instrumented_dex_name)
+ if not os.path.exists(instrumented_dex_file):
+ break
+ instrumented_dex_files.append(instrumented_dex_file)
+ instrumented_dex_index = instrumented_dex_index + 1
+ assert len(instrumented_dex_files) > 0
+ return instrumented_dex_files
+
+def should_discard_dex_file(dex_name, options):
+ return options.discard is not None and dex_name in options.discard
+
+def should_instrument_dex_file(dex_name, options):
+ return options.dex_files is not None and dex_name in options.dex_files
+
+def main(argv):
+ options, args = parse_options(argv)
+ with utils.TempDir() as tmp_dir:
+ # Extract the dex files of the apk.
+ uninstrumented_dir = os.path.join(tmp_dir, 'uninstrumented')
+ os.mkdir(uninstrumented_dir)
+
+ dex_predicate = \
+ lambda name : name.startswith('classes') and name.endswith('.dex')
+ zip_utils.extract_all_that_matches(
+ options.apk, uninstrumented_dir, dex_predicate)
+
+ # Instrument each dex one by one.
+ instrumented_dir = os.path.join(tmp_dir, 'instrumented')
+ os.mkdir(instrumented_dir)
+
+ include_instrumentation_server = True
+ instrumented_dex_index = 1
+ uninstrumented_dex_index = 1
+ while True:
+ dex_name = get_dex_name(uninstrumented_dex_index)
+ dex_file = os.path.join(uninstrumented_dir, dex_name)
+ if not os.path.exists(dex_file):
+ break
+ if not should_discard_dex_file(dex_name, options):
+ if should_instrument_dex_file(dex_name, options):
+ with utils.TempDir() as tmp_instrumentation_dir:
+ instrumented_dex_files = \
+ instrument_dex_file(
+ dex_file,
+ include_instrumentation_server,
+ options,
+ tmp_instrumentation_dir)
+ for instrumented_dex_file in instrumented_dex_files:
+ add_instrumented_dex(
+ instrumented_dex_file, instrumented_dex_index, instrumented_dir)
+ instrumented_dex_index = instrumented_dex_index + 1
+ include_instrumentation_server = False
+ else:
+ add_instrumented_dex(dex_file, instrumented_dex_index, instrumented_dir)
+ instrumented_dex_index = instrumented_dex_index + 1
+ uninstrumented_dex_index = uninstrumented_dex_index + 1
+
+ assert instrumented_dex_index > 1
+
+ # Masseur APK.
+ apk_masseur.masseur(options.apk, dex=instrumented_dir, out=options.out)
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/measure_startup.py b/tools/startup/measure_startup.py
index 1b94fad..99f988d 100755
--- a/tools/startup/measure_startup.py
+++ b/tools/startup/measure_startup.py
@@ -66,7 +66,9 @@
print('Average result:')
data_summary = compute_data_summary(data_total)
print(data_summary)
- write_data(options.out_dir, data_summary)
+ write_data_to_dir(options.out_dir, data_summary)
+ if options.out:
+ write_data_to_file(options.out, data_summary)
def compute_data_summary(data_total):
data_summary = {}
@@ -76,6 +78,8 @@
continue
data_summary['%s_avg' % key] = round(statistics.mean(value), 1)
data_summary['%s_med' % key] = statistics.median(value)
+ data_summary['%s_min' % key] = min(value)
+ data_summary['%s_max' % key] = max(value)
return data_summary
def setup_for_run(apk_or_apks, out_dir, options):
@@ -96,7 +100,13 @@
print('AOT compiling')
if options.baseline_profile:
adb_utils.clear_profile_data(options.app_id, options.device_id)
- adb_utils.install_profile(options.app_id, options.device_id)
+ if options.baseline_profile_install == 'adb':
+ adb_utils.install_profile_using_adb(
+ options.app_id, options.baseline_profile, options.device_id)
+ else:
+ assert options.baseline_profile_install == 'profileinstaller'
+ adb_utils.install_profile_using_profileinstaller(
+ options.app_id, options.device_id)
else:
adb_utils.force_compilation(options.app_id, options.device_id)
@@ -181,7 +191,7 @@
# Get minor and major page faults from app process.
data = compute_data(
launch_activity_result, logcat, perfetto_trace_path, options)
- write_data(out_dir, data)
+ write_data_to_dir(out_dir, data)
return data
def wait_until_fully_drawn(logcat_process, options):
@@ -305,9 +315,12 @@
# Return combined startup data.
return startup_data | perfetto_startup_data
-def write_data(out_dir, data):
+def write_data_to_dir(out_dir, data):
data_path = os.path.join(out_dir, 'data.txt')
- with open(data_path, 'w') as f:
+ write_data_to_file(data_path, data)
+
+def write_data_to_file(out_file, data):
+ with open(out_file, 'w') as f:
for key, value in data.items():
f.write('%s=%s\n' % (key, str(value)))
@@ -321,10 +334,6 @@
help='Enable force compilation',
default=False,
action='store_true')
- result.add_argument('--aot-profile',
- help='Enable force compilation using profiles',
- default=False,
- action='store_true')
result.add_argument('--apk',
help='Path to the .apk')
result.add_argument('--apks',
@@ -367,11 +376,21 @@
help='Disables perfetto trace generation',
action='store_true',
default=False)
+ result.add_argument('--out',
+ help='File to store result in')
result.add_argument('--out-dir',
help='Directory to store trace files in',
required=True)
result.add_argument('--baseline-profile',
- help='Baseline profile to install')
+ help='Baseline profile (.prof) in binary format')
+ result.add_argument('--baseline-profile-metadata',
+ help='Baseline profile metadata (.profm) in binary '
+ 'format')
+ result.add_argument('--baseline-profile-install',
+ help='Whether to install profile using adb or '
+ 'profileinstaller',
+ choices=['adb', 'profileinstaller'],
+ default='profileinstaller')
result.add_argument('--startup-duration',
help='Duration in seconds before shutting down app',
default=15,
@@ -424,10 +443,14 @@
(options, args) = parse_options(argv)
with utils.TempDir() as tmp_dir:
apk_or_apks = { 'apk': options.apk, 'apks': options.apks }
- if options.baseline_profile:
+ if options.baseline_profile \
+ and options.baseline_profile_install == 'profileinstaller':
assert not options.apks, 'Unimplemented'
apk_or_apks['apk'] = apk_utils.add_baseline_profile_to_apk(
- options.apk, options.baseline_profile, tmp_dir)
+ options.apk,
+ options.baseline_profile,
+ options.baseline_profile_metadata,
+ tmp_dir)
teardown_options = global_setup(options)
run_all(apk_or_apks, options, tmp_dir)
global_teardown(options, teardown_options)
diff --git a/tools/startup/profile_utils.py b/tools/startup/profile_utils.py
index 923326f..5c74a5c 100755
--- a/tools/startup/profile_utils.py
+++ b/tools/startup/profile_utils.py
@@ -36,7 +36,9 @@
flag_name = flags_to_name.get(flag_abbreviation)
flags[flag_name] = True
line = line[1:]
- assert line.startswith('L')
+ while line.startswith('['):
+ line = line[1:]
+ assert line.startswith('L'), line
descriptor = line
art_profile[descriptor] = flags
return art_profile
@@ -49,8 +51,9 @@
startup_descriptor) if generalize_synthetics else startup_descriptor
r8_startup_list[transformed_startup_descriptor] = {
'conditional_startup': False,
- 'post_startup': flags['post_startup'],
- 'startup': flags['startup']
+ 'hot': flags['hot'],
+ 'startup': flags['startup'],
+ 'post_startup': flags['post_startup']
}
return r8_startup_list
diff --git a/tools/startup/relayout.py b/tools/startup/relayout.py
index 0ace441..fee91aa 100755
--- a/tools/startup/relayout.py
+++ b/tools/startup/relayout.py
@@ -11,11 +11,14 @@
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import apk_masseur
+import apk_utils
import extractmarker
import toolhelper
import utils
import zip_utils
+LOWEST_SUPPORTED_MIN_API = 21 # Android L (native multi dex)
+
def parse_options(argv):
result = argparse.ArgumentParser(
description='Relayout a given APK using a startup profile.')
@@ -63,32 +66,23 @@
return '~~L8' in marker
return options.desugared_library == 'true'
-def get_min_api(apk):
- aapt = os.path.join(utils.getAndroidBuildTools(), 'aapt')
- cmd = [aapt, 'dump', 'badging', apk]
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- for line in stdout.splitlines():
- if line.startswith('sdkVersion:\''):
- return int(line[len('sdkVersion:\''): -1])
- raise ValueError('Unexpected stdout: %s' % stdout)
-
def main(argv):
(options, args) = parse_options(argv)
with utils.TempDir() as temp:
dex = os.path.join(temp, 'dex.zip')
d8_args = [
- '--min-api', str(get_min_api(options.apk)),
+ '--min-api',
+ str(max(apk_utils.get_min_api(options.apk), LOWEST_SUPPORTED_MIN_API)),
'--output', dex,
+ '--startup-profile', options.profile,
'--no-desugaring',
'--release']
dex_to_relayout, desugared_library_dex = get_dex_to_relayout(options, temp)
d8_args.extend(dex_to_relayout)
- extra_args = ['-Dcom.android.tools.r8.startup.profile=%s' % options.profile]
toolhelper.run(
'd8',
d8_args,
build=not options.no_build,
- extra_args=extra_args,
main='com.android.tools.r8.D8')
if desugared_library_dex is not None:
dex_files = [name for name in \
diff --git a/tools/zip_utils.py b/tools/zip_utils.py
index 795119b..33529de 100644
--- a/tools/zip_utils.py
+++ b/tools/zip_utils.py
@@ -3,6 +3,8 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
+import os
+import subprocess
import zipfile
def add_file_to_zip(file, destination, zip_file):
@@ -18,3 +20,8 @@
def get_names_that_matches(zip_file, predicate):
with zipfile.ZipFile(zip_file) as zip:
return [name for name in zip.namelist() if predicate(name)]
+
+def remove_files_from_zip(files, zip_file):
+ assert os.path.exists(zip_file)
+ cmd = ['zip', '-d', zip_file] + files
+ subprocess.run(cmd)
