Add flag --quiet to run_on_as_app.py
Change-Id: I1fc6ce3a8e172c439e1297ee5ee104421c11197c
diff --git a/tools/apk_masseur.py b/tools/apk_masseur.py
index 6013e6f..fa46485 100755
--- a/tools/apk_masseur.py
+++ b/tools/apk_masseur.py
@@ -8,7 +8,6 @@
import optparse
import os
import shutil
-import subprocess
import sys
import utils
@@ -32,6 +31,9 @@
parser.add_option('--adb-options',
help='additional adb options when running adb',
default=None)
+ parser.add_option('--quiet',
+ help='disable verbose logging',
+ default=False)
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error('Expected <apk> argument, got: ' + ' '.join(args))
@@ -41,44 +43,44 @@
def findKeystore():
return os.path.join(os.getenv('HOME'), '.android', 'app.keystore')
-def repack(processed_out, original_apk, temp):
+def repack(processed_out, original_apk, temp, quiet):
processed_apk = os.path.join(temp, 'processed.apk')
shutil.copyfile(original_apk, processed_apk)
if not processed_out:
- print 'Using original APK as is'
+ utils.Print('Using original APK as is', quiet=quiet)
return processed_apk
- print 'Repacking APK with dex files from', processed_apk
- with utils.ChangedWorkingDirectory(temp):
+ utils.Print(
+ 'Repacking APK with dex files from {}'.format(processed_apk), quiet=quiet)
+ with utils.ChangedWorkingDirectory(temp, quiet=quiet):
cmd = ['zip', '-d', 'processed.apk', '*.dex']
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
+ utils.RunCmd(cmd, quiet=quiet)
if processed_out.endswith('.zip') or processed_out.endswith('.jar'):
cmd = ['unzip', processed_out, '-d', temp]
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
+ if quiet:
+ cmd.insert(1, '-q')
+ utils.RunCmd(cmd, quiet=quiet)
processed_out = temp
- with utils.ChangedWorkingDirectory(processed_out):
+ with utils.ChangedWorkingDirectory(processed_out, quiet=quiet):
dex = glob.glob('*.dex')
cmd = ['zip', '-u', '-9', processed_apk] + dex
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
+ utils.RunCmd(cmd, quiet=quiet)
return processed_apk
-def sign(unsigned_apk, keystore, temp):
+def sign(unsigned_apk, keystore, temp, quiet):
signed_apk = os.path.join(temp, 'unaligned.apk')
- apk_utils.sign(unsigned_apk, signed_apk, keystore)
+ apk_utils.sign(unsigned_apk, signed_apk, keystore, quiet=quiet)
return signed_apk
-def align(signed_apk, temp):
- print 'Aligning'
+def align(signed_apk, temp, quiet):
+ utils.Print('Aligning', quiet=quiet)
aligned_apk = os.path.join(temp, 'aligned.apk')
cmd = ['zipalign', '-f', '4', signed_apk, aligned_apk]
- print ' '.join(cmd)
- subprocess.check_call(cmd)
+ utils.RunCmd(cmd, quiet=quiet)
return signed_apk
def masseur(
- apk, dex=None, out=None, adb_options=None, keystore=None, install=False):
+ apk, dex=None, out=None, adb_options=None, keystore=None, install=False,
+ quiet=False):
if not out:
out = os.path.basename(apk)
if not keystore:
@@ -86,23 +88,23 @@
with utils.TempDir() as temp:
processed_apk = None
if dex:
- processed_apk = repack(dex, apk, temp)
+ processed_apk = repack(dex, apk, temp, quiet)
else:
- print 'Signing original APK without modifying dex files'
+ utils.Print(
+ 'Signing original APK without modifying dex files', quiet=quiet)
processed_apk = os.path.join(temp, 'processed.apk')
shutil.copyfile(apk, processed_apk)
- signed_apk = sign(processed_apk, keystore, temp)
- aligned_apk = align(signed_apk, temp)
- print 'Writing result to', out
+ signed_apk = sign(processed_apk, keystore, temp, quiet=quiet)
+ aligned_apk = align(signed_apk, temp, quiet=quiet)
+ utils.Print('Writing result to {}'.format(out), quiet=quiet)
shutil.copyfile(aligned_apk, out)
- adb_cmd = ['adb']
- if adb_options:
- adb_cmd.extend(
- [option for option in adb_options.split(' ') if option])
if install:
+ adb_cmd = ['adb']
+ if adb_options:
+ adb_cmd.extend(
+ [option for option in adb_options.split(' ') if option])
adb_cmd.extend(['install', '-t', '-r', '-d', out]);
- utils.PrintCmd(adb_cmd)
- subprocess.check_call(adb_cmd)
+ utils.RunCmd(adb_cmd, quiet=quiet)
def main():
(options, apk) = parse_options()
diff --git a/tools/apk_utils.py b/tools/apk_utils.py
index aaae1e4..5a6aa94 100644
--- a/tools/apk_utils.py
+++ b/tools/apk_utils.py
@@ -7,11 +7,10 @@
import subprocess
import utils
-def sign(unsigned_apk, signed_apk, keystore):
- print 'Signing (ignore the warnings)'
+def sign(unsigned_apk, signed_apk, keystore, quiet=False):
+ utils.Print('Signing (ignore the warnings)', quiet=quiet)
cmd = ['zip', '-d', unsigned_apk, 'META-INF/*']
- utils.PrintCmd(cmd)
- subprocess.call(cmd)
+ utils.RunCmd(cmd, quiet=quiet)
cmd = [
'jarsigner',
'-sigalg', 'SHA1withRSA',
@@ -22,8 +21,7 @@
unsigned_apk,
'androiddebugkey'
]
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
+ utils.RunCmd(cmd, quiet=quiet)
def sign_with_apksigner(build_tools_dir, unsigned_apk, signed_apk, keystore, password):
cmd = [
diff --git a/tools/as_utils.py b/tools/as_utils.py
index 9c2c8c8..10fb29f 100644
--- a/tools/as_utils.py
+++ b/tools/as_utils.py
@@ -30,23 +30,15 @@
'Unexpected line with \'dependencies {\'')
is_inside_dependencies = True
if is_inside_dependencies:
- if '/r8.jar' in stripped:
- if minified:
- # Skip line to avoid dependency on r8.jar
- continue
- added_r8_dependency = True
- elif '/r8lib.jar' in stripped:
- if not minified:
- # Skip line to avoid dependency on r8lib.jar
- continue
- added_r8_dependency = True
+ if '/r8.jar' in stripped or '/r8lib.jar' in stripped:
+ # Skip line to avoid dependency on r8.jar
+ continue
elif 'com.android.tools.build:gradle:' in stripped:
gradle_version = stripped[stripped.rindex(':')+1:-1]
- if not added_r8_dependency:
- indent = ''.ljust(line.index('classpath'))
- jar = os.path.join(temp_dir, 'r8lib.jar' if minified else 'r8.jar')
- f.write('{}classpath files(\'{}\')\n'.format(indent, jar))
- added_r8_dependency = True
+ indent = ''.ljust(line.index('classpath'))
+ jar = os.path.join(temp_dir, 'r8lib.jar' if minified else 'r8.jar')
+ f.write('{}classpath files(\'{}\')\n'.format(indent, jar))
+ added_r8_dependency = True
elif stripped == '}':
is_inside_dependencies = False
f.write(line)
@@ -147,8 +139,9 @@
# one of the predefined locations.
assert False
-def Move(src, dst):
- print('Moving `{}` to `{}`'.format(src, dst))
+def Move(src, dst, quiet=False):
+ if not quiet:
+ print('Moving `{}` to `{}`'.format(src, dst))
dst_parent = os.path.dirname(dst)
if not os.path.isdir(dst_parent):
os.makedirs(dst_parent)
@@ -158,15 +151,15 @@
os.remove(dst)
os.rename(src, dst)
-def MoveDir(src, dst):
+def MoveDir(src, dst, quiet=False):
assert os.path.isdir(src)
- Move(src, dst)
+ Move(src, dst, quiet=quiet)
-def MoveFile(src, dst):
+def MoveFile(src, dst, quiet=False):
assert os.path.isfile(src)
- Move(src, dst)
+ Move(src, dst, quiet=quiet)
-def MoveProfileReportTo(dest_dir, build_stdout):
+def MoveProfileReportTo(dest_dir, build_stdout, quiet=False):
html_file = None
profile_message = 'See the profiling report at: '
for line in build_stdout:
@@ -181,11 +174,12 @@
assert os.path.isfile(html_file), 'Expected to find HTML file at {}'.format(
html_file)
- MoveFile(html_file, os.path.join(dest_dir, 'index.html'))
+ MoveFile(html_file, os.path.join(dest_dir, 'index.html'), quiet=quiet)
html_dir = os.path.dirname(html_file)
for dir_name in ['css', 'js']:
- MoveDir(os.path.join(html_dir, dir_name), os.path.join(dest_dir, dir_name))
+ MoveDir(os.path.join(html_dir, dir_name), os.path.join(dest_dir, dir_name),
+ quiet=quiet)
def ParseProfileReport(profile_dir):
html_file = os.path.join(profile_dir, 'index.html')
diff --git a/tools/run_on_as_app.py b/tools/run_on_as_app.py
index 128b6e3..ae17feb 100755
--- a/tools/run_on_as_app.py
+++ b/tools/run_on_as_app.py
@@ -42,6 +42,8 @@
'app_id': 'de.danoeh.antennapod',
'git_repo': 'https://github.com/christofferqa/AntennaPod.git',
'flavor': 'play',
+ 'min_sdk': 14,
+ 'compile_sdk': 26
},
'apps-android-wikipedia': {
'app_id': 'org.wikipedia',
@@ -128,7 +130,7 @@
dex_size += z.getinfo(filename).file_size
return dex_size
-def IsBuiltWithR8(apk, temp_dir):
+def IsBuiltWithR8(apk, temp_dir, options):
r8_jar = os.path.join(temp_dir, 'r8.jar')
# Use the copy of r8.jar if it is there.
@@ -138,7 +140,7 @@
script = os.path.join(utils.TOOLS_DIR, 'extractmarker.py')
cmd = ['python', script, apk]
- utils.PrintCmd(cmd)
+ utils.PrintCmd(cmd, quiet=options.quiet)
return '~~R8' in subprocess.check_output(cmd).strip()
def IsMinifiedR8(shrinker):
@@ -157,9 +159,13 @@
def GitCheckout(file):
return subprocess.check_output(['git', 'checkout', file]).strip()
-def InstallApkOnEmulator(apk_dest):
- subprocess.check_call(
- ['adb', '-s', emulator_id, 'install', '-r', '-d', apk_dest])
+def InstallApkOnEmulator(apk_dest, options):
+ cmd = ['adb', '-s', emulator_id, 'install', '-r', '-d', apk_dest]
+ if options.quiet:
+ with open(os.devnull, 'w') as devnull:
+ subprocess.check_call(cmd, stdout=devnull)
+ else:
+ subprocess.check_call(cmd)
def PercentageDiffAsString(before, after):
if after < before:
@@ -167,7 +173,7 @@
else:
return '+' + str(round((after - before) / before * 100)) + '%'
-def UninstallApkOnEmulator(app, config):
+def UninstallApkOnEmulator(app, config, options):
app_id = config.get('app_id')
process = subprocess.Popen(
['adb', 'uninstall', app_id],
@@ -215,10 +221,10 @@
result = {}
if not os.path.exists(checkout_dir):
- with utils.ChangedWorkingDirectory(WORKING_DIR):
+ with utils.ChangedWorkingDirectory(WORKING_DIR, quiet=options.quiet):
GitClone(git_repo)
elif options.pull:
- with utils.ChangedWorkingDirectory(checkout_dir):
+ with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet):
# Checkout build.gradle to avoid merge conflicts.
if IsTrackedByGit('build.gradle'):
GitCheckout('build.gradle')
@@ -240,7 +246,7 @@
def BuildAppWithSelectedShrinkers(app, config, options, checkout_dir, temp_dir):
result_per_shrinker = {}
- with utils.ChangedWorkingDirectory(checkout_dir):
+ with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet):
for shrinker in SHRINKERS:
if options.shrinker and shrinker not in options.shrinker:
continue
@@ -280,15 +286,9 @@
# Build app with gradle using -D...keepRuleSynthesisForRecompilation=
# true.
out_dir = os.path.join(checkout_dir, 'out', shrinker + '-1')
- extra_env_vars = {
- 'JAVA_OPTS': ' '.join([
- '-ea:com.android.tools.r8...',
- '-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true'
- ])
- }
(apk_dest, profile_dest_dir, ext_proguard_config_file) = \
BuildAppWithShrinker(app, config, shrinker, checkout_dir, out_dir,
- temp_dir, options, extra_env_vars)
+ temp_dir, options, keepRuleSynthesisForRecompilation=True)
dex_size = ComputeSizeOfDexFilesInApk(apk_dest)
recompilation_result = {
'apk_dest': apk_dest,
@@ -314,8 +314,9 @@
recompiled_apk_dest = os.path.join(
checkout_dir, 'out', shrinker, 'app-release-{}.apk'.format(i))
RebuildAppWithShrinker(
- previous_apk, recompiled_apk_dest, ext_proguard_config_file,
- shrinker, min_sdk, compile_sdk, temp_dir)
+ app, previous_apk, recompiled_apk_dest,
+ ext_proguard_config_file, shrinker, min_sdk, compile_sdk,
+ options, temp_dir)
recompilation_result = {
'apk_dest': recompiled_apk_dest,
'build_status': 'success',
@@ -334,13 +335,20 @@
result_per_shrinker[shrinker] = result
+ if not options.app:
+ print('')
+ LogResultsForApp(app, result_per_shrinker, options)
+ print('')
+
return result_per_shrinker
def BuildAppWithShrinker(
app, config, shrinker, checkout_dir, out_dir, temp_dir, options,
- env_vars=None):
- print()
- print('Building {} with {}'.format(app, shrinker))
+ keepRuleSynthesisForRecompilation=False):
+ print('Building {} with {}{}'.format(
+ app,
+ shrinker,
+ ' for recompilation' if keepRuleSynthesisForRecompilation else ''))
# Add/remove 'r8.jar' from top-level build.gradle.
if options.disable_tot:
@@ -361,11 +369,9 @@
as_utils.SetPrintConfigurationDirective(
app, config, checkout_dir, proguard_config_dest)
- env = os.environ.copy()
+ env = {}
env['ANDROID_HOME'] = android_home
env['JAVA_OPTS'] = '-ea:com.android.tools.r8...'
- if env_vars:
- env.update(env_vars)
releaseTarget = config.get('releaseTarget')
if not releaseTarget:
@@ -381,20 +387,12 @@
'--profile', '--stacktrace',
'-Pandroid.enableR8=' + str(enableR8).lower(),
'-Pandroid.enableR8.fullMode=' + str(enableR8FullMode).lower()]
+ if keepRuleSynthesisForRecompilation:
+ cmd.append('-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true')
if options.gradle_flags:
cmd.extend(options.gradle_flags.split(' '))
- utils.PrintCmd(cmd)
- build_process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE)
- stdout = []
- while True:
- line = build_process.stdout.readline()
- if line != b'':
- stripped = line.rstrip()
- stdout.append(stripped)
- print(stripped)
- else:
- break
+ stdout = utils.RunCmd(cmd, env, quiet=options.quiet)
apk_base_name = (archives_base_name
+ (('-' + flavor) if flavor else '') + '-release')
@@ -425,25 +423,27 @@
if os.path.isfile(signed_apk):
apk_dest = os.path.join(out_dir, signed_apk_name)
- as_utils.MoveFile(signed_apk, apk_dest)
+ as_utils.MoveFile(signed_apk, apk_dest, quiet=options.quiet)
else:
apk_dest = os.path.join(out_dir, unsigned_apk_name)
- as_utils.MoveFile(unsigned_apk, apk_dest)
+ as_utils.MoveFile(unsigned_apk, apk_dest, quiet=options.quiet)
- assert IsBuiltWithR8(apk_dest, temp_dir) == ('r8' in shrinker), (
+ assert IsBuiltWithR8(apk_dest, temp_dir, options) == ('r8' in shrinker), (
'Unexpected marker in generated APK for {}'.format(shrinker))
profile_dest_dir = os.path.join(out_dir, 'profile')
- as_utils.MoveProfileReportTo(profile_dest_dir, stdout)
+ as_utils.MoveProfileReportTo(profile_dest_dir, stdout, quiet=options.quiet)
return (apk_dest, profile_dest_dir, proguard_config_dest)
def RebuildAppWithShrinker(
- apk, apk_dest, proguard_config_file, shrinker, min_sdk, compile_sdk,
- temp_dir):
+ app, apk, apk_dest, proguard_config_file, shrinker, min_sdk, compile_sdk,
+ options, temp_dir):
assert 'r8' in shrinker
assert apk_dest.endswith('.apk')
+ print('Rebuilding {} with {}'.format(app, shrinker))
+
# Compile given APK with shrinker to temporary zip file.
android_jar = utils.get_android_jar(compile_sdk)
r8_jar = os.path.join(
@@ -462,20 +462,19 @@
cmd.append('--lib')
cmd.append(android_optional_jar)
- utils.PrintCmd(cmd)
-
- subprocess.check_output(cmd)
+ utils.RunCmd(cmd, quiet=options.quiet)
# Make a copy of the given APK, move the newly generated dex files into the
# copied APK, and then sign the APK.
- apk_masseur.masseur(apk, dex=zip_dest, out=apk_dest)
+ apk_masseur.masseur(
+ apk, dex=zip_dest, out=apk_dest, quiet=options.quiet)
def RunMonkey(app, config, options, apk_dest):
if not WaitForEmulator():
return False
- UninstallApkOnEmulator(app, config)
- InstallApkOnEmulator(apk_dest)
+ UninstallApkOnEmulator(app, config, options)
+ InstallApkOnEmulator(apk_dest, options)
app_id = config.get('app_id')
number_of_events_to_generate = options.monkey_events
@@ -486,27 +485,28 @@
cmd = ['adb', 'shell', 'monkey', '-p', app_id, '-s', str(random_seed),
str(number_of_events_to_generate)]
- utils.PrintCmd(cmd)
try:
- stdout = subprocess.check_output(cmd)
+ stdout = utils.RunCmd(cmd, quiet=options.quiet)
succeeded = (
'Events injected: {}'.format(number_of_events_to_generate) in stdout)
except subprocess.CalledProcessError as e:
succeeded = False
- UninstallApkOnEmulator(app, config)
+ UninstallApkOnEmulator(app, config, options)
return succeeded
def LogResultsForApps(result_per_shrinker_per_app, options):
- for app, result_per_shrinker in result_per_shrinker_per_app.iteritems():
+ print('')
+ for app, result_per_shrinker in sorted(
+ result_per_shrinker_per_app.iteritems()):
LogResultsForApp(app, result_per_shrinker, options)
def LogResultsForApp(app, result_per_shrinker, options):
print(app + ':')
- if result_per_shrinker.get('status') != 'success':
+ if result_per_shrinker.get('status', 'success') != 'success':
error_message = result_per_shrinker.get('error_message')
print(' skipped ({})'.format(error_message))
return
@@ -613,6 +613,10 @@
action='store_true')
result.add_option('--gradle-flags', '--gradle_flags',
help='Flags to pass in to gradle')
+ result.add_option('--quiet',
+ help='Disable verbose logging',
+ default=False,
+ action='store_true')
(options, args) = result.parse_args(argv)
if options.disable_tot:
# r8.jar is required for recompiling the generated APK
@@ -653,7 +657,7 @@
result_per_shrinker_per_app[options.app] = GetResultsForApp(
options.app, APPS.get(options.app), options, temp_dir)
else:
- for app, config in APPS.iteritems():
+ for app, config in sorted(APPS.iteritems()):
if not config.get('skip', False):
result_per_shrinker_per_app[app] = GetResultsForApp(
app, config, options, temp_dir)
diff --git a/tools/utils.py b/tools/utils.py
index 72febdc..8c37a55 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -54,13 +54,86 @@
R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
RETRACE_JAR = os.path.join(THIRD_PARTY, 'proguard', 'proguard6.0.1', 'lib', 'retrace.jar')
-def PrintCmd(s):
- if type(s) is list:
- s = ' '.join(s)
- print 'Running: %s' % s
+def Print(s, quiet=False):
+ if quiet:
+ return
+ print(s)
+
+def Warn(message):
+ CRED = '\033[91m'
+ CEND = '\033[0m'
+ print(CRED + message + CEND)
+
+def PrintCmd(cmd, env=None, quiet=False):
+ if quiet:
+ return
+ if type(cmd) is list:
+ cmd = ' '.join(cmd)
+ if env:
+ env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
+ print('Running: {} {}'.format(env, cmd))
+ else:
+ print('Running: {}'.format(cmd))
# I know this will hit os on windows eventually if we don't do this.
sys.stdout.flush()
+class ProgressLogger(object):
+ CLEAR_LINE = '\033[K'
+ UP = '\033[F'
+
+ def __init__(self, quiet=False):
+ self._count = 0
+ self._has_printed = False
+ self._quiet = quiet
+
+ def log(self, text):
+ if self._quiet:
+ if self._has_printed:
+ sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
+ if len(text) > 140:
+ text = text[0:140] + '...'
+ print(text)
+ self._has_printed = True
+
+ def done(self):
+ if self._quiet and self._has_printed:
+ sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
+ print('')
+ sys.stdout.write(ProgressLogger.UP)
+
+def RunCmd(cmd, env_vars=None, quiet=False):
+ PrintCmd(cmd, env=env_vars, quiet=quiet)
+ env = os.environ.copy()
+ if env_vars:
+ env.update(env_vars)
+ process = subprocess.Popen(
+ cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout = []
+ logger = ProgressLogger(quiet=quiet)
+ failed = False
+ while True:
+ line = process.stdout.readline()
+ if line != b'':
+ stripped = line.rstrip()
+ stdout.append(stripped)
+ logger.log(stripped)
+
+ # TODO(christofferqa): r8 should fail with non-zero exit code.
+ if ('AssertionError' in stripped
+ or 'CompilationError' in stripped
+ or 'CompilationFailedException' in stripped
+ or 'Compilation failed' in stripped):
+ failed = True
+ else:
+ logger.done()
+ exit_code = process.poll()
+ if exit_code or failed:
+ for line in stdout:
+ Warn(line)
+ raise subprocess.CalledProcessError(
+ exit_code, cmd, output='\n'.join(stdout))
+ return stdout
+
def IsWindows():
return os.name == 'nt'
@@ -196,16 +269,19 @@
shutil.rmtree(self._temp_dir, ignore_errors=True)
class ChangedWorkingDirectory(object):
- def __init__(self, working_directory):
+ def __init__(self, working_directory, quiet=False):
+ self._quiet = quiet
self._working_directory = working_directory
def __enter__(self):
self._old_cwd = os.getcwd()
- print 'Enter directory = ', self._working_directory
+ if not self._quiet:
+ print 'Enter directory:', self._working_directory
os.chdir(self._working_directory)
def __exit__(self, *_):
- print 'Enter directory = ', self._old_cwd
+ if not self._quiet:
+ print 'Enter directory:', self._old_cwd
os.chdir(self._old_cwd)
# Reading Android CTS test_result.xml
