Add support for recompiling generated APKs in run_on_as_app.py
Bug: 122584632
Change-Id: I391cf9f819e7dd9eb50d40de36af8b0a0df63f99
diff --git a/tools/apk-masseur.py b/tools/apk_masseur.py
similarity index 85%
rename from tools/apk-masseur.py
rename to tools/apk_masseur.py
index 4f24f0f..6013e6f 100755
--- a/tools/apk-masseur.py
+++ b/tools/apk_masseur.py
@@ -36,10 +36,6 @@
if len(args) != 1:
parser.error('Expected <apk> argument, got: ' + ' '.join(args))
apk = args[0]
- if not options.out:
- options.out = os.path.basename(apk)
- if not options.keystore:
- options.keystore = findKeystore()
return (options, apk)
def findKeystore():
@@ -81,28 +77,36 @@
subprocess.check_call(cmd)
return signed_apk
-def main():
- (options, apk) = parse_options()
+def masseur(
+ apk, dex=None, out=None, adb_options=None, keystore=None, install=False):
+ if not out:
+ out = os.path.basename(apk)
+ if not keystore:
+ keystore = findKeystore()
with utils.TempDir() as temp:
processed_apk = None
- if options.dex:
- processed_apk = repack(options.dex, apk, temp)
+ if dex:
+ processed_apk = repack(dex, apk, temp)
else:
print 'Signing original APK without modifying dex files'
processed_apk = os.path.join(temp, 'processed.apk')
shutil.copyfile(apk, processed_apk)
- signed_apk = sign(processed_apk, options.keystore, temp)
+ signed_apk = sign(processed_apk, keystore, temp)
aligned_apk = align(signed_apk, temp)
- print 'Writing result to', options.out
- shutil.copyfile(aligned_apk, options.out)
+ print 'Writing result to', out
+ shutil.copyfile(aligned_apk, out)
adb_cmd = ['adb']
- if options.adb_options:
+ if adb_options:
adb_cmd.extend(
- [option for option in options.adb_options.split(' ') if option])
- if options.install:
- adb_cmd.extend(['install', '-t', '-r', '-d', options.out]);
+ [option for option in adb_options.split(' ') if option])
+ if install:
+ adb_cmd.extend(['install', '-t', '-r', '-d', out]);
utils.PrintCmd(adb_cmd)
subprocess.check_call(adb_cmd)
+
+def main():
+ (options, apk) = parse_options()
+ masseur(apk, **vars(options))
return 0
if __name__ == '__main__':
diff --git a/tools/as_utils.py b/tools/as_utils.py
index d5db8a3..8b887bf 100644
--- a/tools/as_utils.py
+++ b/tools/as_utils.py
@@ -56,7 +56,8 @@
def remove_r8_dependency(checkout_dir):
build_file = os.path.join(checkout_dir, 'build.gradle')
- assert os.path.isfile(build_file), 'Expected a file to be present at {}'.format(build_file)
+ assert os.path.isfile(build_file), (
+ 'Expected a file to be present at {}'.format(build_file))
with open(build_file) as f:
lines = f.readlines()
with open(build_file, 'w') as f:
@@ -64,6 +65,28 @@
if (utils.R8_JAR not in line) and (utils.R8LIB_JAR not in line):
f.write(line)
+def SetPrintConfigurationDirective(app, config, checkout_dir, destination):
+ proguard_config_file = FindProguardConfigurationFile(
+ app, config, checkout_dir)
+ with open(proguard_config_file) as f:
+ lines = f.readlines()
+ with open(proguard_config_file, 'w') as f:
+ for line in lines:
+ if '-printconfiguration' not in line:
+ f.write(line)
+ f.write('-printconfiguration {}\n'.format(destination))
+
+def FindProguardConfigurationFile(app, config, checkout_dir):
+ app_module = config.get('app_module', 'app')
+ candidates = ['proguard-rules.pro', 'proguard-rules.txt', 'proguard.cfg']
+ for candidate in candidates:
+ proguard_config_file = os.path.join(checkout_dir, app_module, candidate)
+ if os.path.isfile(proguard_config_file):
+ return proguard_config_file
+ # Currently assuming that the Proguard configuration file can be found at
+ # one of the predefined locations.
+ assert False
+
def Move(src, dst):
print('Moving `{}` to `{}`'.format(src, dst))
dst_parent = os.path.dirname(dst)
diff --git a/tools/run_on_as_app.py b/tools/run_on_as_app.py
index 26907a0..deea382 100755
--- a/tools/run_on_as_app.py
+++ b/tools/run_on_as_app.py
@@ -3,12 +3,14 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
+import apk_masseur
import apk_utils
import gradle
import os
import optparse
import subprocess
import sys
+import tempfile
import time
import utils
import zipfile
@@ -128,6 +130,25 @@
subprocess.check_call(
['adb', '-s', emulator_id, 'install', '-r', '-d', apk_dest])
+def UninstallApkOnEmulator(app, config):
+ app_id = config.get('app_id')
+ process = subprocess.Popen(
+ ['adb', 'uninstall', app_id],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+
+ if stdout.strip() == 'Success':
+ # Successfully uninstalled
+ return
+
+ if 'Unknown package: {}'.format(app_id) in stderr:
+ # Application not installed
+ return
+
+ raise Exception(
+ 'Unexpected result from `adb uninstall {}\nStdout: {}\nStderr: {}'.format(
+ app_id, stdout, stderr))
+
def WaitForEmulator():
stdout = subprocess.check_output(['adb', 'devices'])
if '{}\tdevice'.format(emulator_id) in stdout:
@@ -190,8 +211,8 @@
apk_dest = None
result = {}
try:
- (apk_dest, profile_dest_dir) = BuildAppWithShrinker(
- app, config, shrinker, checkout_dir, options)
+ (apk_dest, profile_dest_dir, proguard_config_file) = \
+ BuildAppWithShrinker(app, config, shrinker, checkout_dir, options)
dex_size = ComputeSizeOfDexFilesInApk(apk_dest)
result['apk_dest'] = apk_dest,
result['build_status'] = 'success'
@@ -203,11 +224,36 @@
print('Error: ' + str(e))
result['build_status'] = 'failed'
- if options.monkey:
- if result.get('build_status') == 'success':
+ if result.get('build_status') == 'success':
+ if options.monkey:
result['monkey_status'] = 'success' if RunMonkey(
app, config, options, apk_dest) else 'failed'
+ if 'r8' in shrinker and options.r8_compilation_steps > 1:
+ recompilation_results = []
+ previous_apk = apk_dest
+ for i in range(1, options.r8_compilation_steps):
+ try:
+ recompiled_apk_dest = os.path.join(
+ checkout_dir, 'out', shrinker, 'app-release-{}.apk'.format(i))
+ RebuildAppWithShrinker(
+ previous_apk, recompiled_apk_dest, proguard_config_file, shrinker)
+ recompilation_result = {
+ 'apk_dest': recompiled_apk_dest,
+ 'build_status': 'success',
+ 'dex_size': ComputeSizeOfDexFilesInApk(recompiled_apk_dest)
+ }
+ if options.monkey:
+ recompilation_result['monkey_status'] = 'success' if RunMonkey(
+ app, config, options, recompiled_apk_dest) else 'failed'
+ recompilation_results.append(recompilation_result)
+ previous_apk = recompiled_apk_dest
+ except Exception as e:
+ warn('Failed to recompile {} with {}'.format(app, shrinker))
+ recompilation_results.append({ 'build_status': 'failed' })
+ break
+ result['recompilation_results'] = recompilation_results
+
result_per_shrinker[shrinker] = result
if IsTrackedByGit('gradle.properties'):
@@ -218,6 +264,7 @@
def BuildAppWithShrinker(app, config, shrinker, checkout_dir, options):
print('Building {} with {}'.format(app, shrinker))
+ # Add/remove 'r8.jar' from top-level build.gradle.
if options.disable_tot:
as_utils.remove_r8_dependency(checkout_dir)
else:
@@ -227,7 +274,7 @@
archives_base_name = config.get(' archives_base_name', app_module)
flavor = config.get('flavor')
- # Ensure that gradle.properties are not modified before modifying it to
+ # Ensure that gradle.properties is not modified before modifying it to
# select shrinker.
if IsTrackedByGit('gradle.properties'):
GitCheckout('gradle.properties')
@@ -244,6 +291,12 @@
if not os.path.exists(out):
os.makedirs(out)
+ # Set -printconfiguration in Proguard rules.
+ proguard_config_dest = os.path.abspath(
+ os.path.join(out, 'proguard-rules.pro'))
+ as_utils.SetPrintConfigurationDirective(
+ app, config, checkout_dir, proguard_config_dest)
+
env = os.environ.copy()
env['ANDROID_HOME'] = android_home
env['JAVA_OPTS'] = '-ea'
@@ -307,12 +360,33 @@
profile_dest_dir = os.path.join(out, 'profile')
as_utils.MoveProfileReportTo(profile_dest_dir, stdout)
- return (apk_dest, profile_dest_dir)
+ return (apk_dest, profile_dest_dir, proguard_config_dest)
+
+def RebuildAppWithShrinker(apk, apk_dest, proguard_config_file, shrinker):
+ assert 'r8' in shrinker
+ assert apk_dest.endswith('.apk')
+
+ # Compile given APK with shrinker to temporary zip file.
+ api = 28 # TODO(christofferqa): Should be the one from build.gradle
+ android_jar = os.path.join(utils.REPO_ROOT, utils.ANDROID_JAR.format(api=api))
+ r8_jar = utils.R8LIB_JAR if IsMinifiedR8(shrinker) else utils.R8_JAR
+ zip_dest = apk_dest[:-3] + '.zip'
+
+ cmd = ['java', '-ea', '-jar', r8_jar, '--release', '--pg-conf',
+ proguard_config_file, '--lib', android_jar, '--output', zip_dest, apk]
+ utils.PrintCmd(cmd)
+
+ subprocess.check_output(cmd)
+
+ # Make a copy of the given APK, move the newly generated dex files into the
+ # copied APK, and then sign the APK.
+ apk_masseur.masseur(apk, dex=zip_dest, out=apk_dest)
def RunMonkey(app, config, options, apk_dest):
if not WaitForEmulator():
return False
+ UninstallApkOnEmulator(app, config)
InstallApkOnEmulator(apk_dest)
app_id = config.get('app_id')
@@ -371,6 +445,21 @@
warn(' monkey: {}'.format(monkey_status))
else:
success(' monkey: {}'.format(monkey_status))
+ recompilation_results = result.get('recompilation_results', [])
+ i = 1
+ for recompilation_result in recompilation_results:
+ build_status = recompilation_result.get('build_status')
+ if build_status != 'success':
+ print(' recompilation #{}: {}'.format(i, build_status))
+ else:
+ dex_size = recompilation_result.get('dex_size')
+ print(' recompilation #{}'.format(i))
+ print(' dex size: {}'.format(dex_size))
+ if options.monkey:
+ monkey_status = recompilation_result.get('monkey_status')
+ msg = ' monkey: {}'.format(monkey_status)
+ success(msg) if monkey_status == 'success' else warn(msg)
+ i += 1
def ParseOptions(argv):
result = optparse.OptionParser()
@@ -396,6 +485,10 @@
result.add_option('--shrinker',
help='The shrinkers to use (by default, all are run)',
action='append')
+ result.add_option('--r8-compilation-steps', '--r8_compilation_steps',
+ help='Number of times R8 should be run on each app',
+ default=2,
+ type=int)
result.add_option('--disable-tot', '--disable_tot',
help='Whether to disable the use of the ToT version of R8',
default=False,
@@ -405,6 +498,9 @@
default=False,
action='store_true')
(options, args) = result.parse_args(argv)
+ if options.disable_tot:
+ # r8.jar is required for recompiling the generated APK
+ options.r8_compilation_steps = 1
if options.shrinker:
for shrinker in options.shrinker:
assert shrinker in SHRINKERS
