blob: 00f4380debf897f84c039a58ddac1a6d97503756 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import apk_masseur
import apk_utils
import as_utils
import golem
import jdk
import json
import optparse
import os
import shutil
import signal
import subprocess
import sys
import time
import update_prebuilds_in_android
import zipfile
from datetime import datetime
from xml.dom import minidom
import gradle
import utils
SHRINKERS = ['r8', 'r8-full', 'r8-nolib', 'r8-nolib-full', 'pg']
WORKING_DIR = os.path.join(utils.BUILD, 'opensource_apps')
GRADLE_USER_HOME = '.gradle_user_home'
if ('R8_BENCHMARK_DIR' in os.environ
and os.path.isdir(os.environ['R8_BENCHMARK_DIR'])):
WORKING_DIR = os.environ['R8_BENCHMARK_DIR']
class Repo(object):
def __init__(self, fields):
self.__dict__ = fields
# If there is only one app in this repository, then give the app the same
# name as the repository, if it does not already have one.
if len(self.apps) == 1:
app = self.apps[0]
if not app.name:
app.name = self.name
class App(object):
def __init__(self, fields):
module = fields.get('module', 'app')
defaults = {
'archives_base_name': module,
'build_dir': 'build',
'compile_sdk': None,
'dir': None,
'flavor': None,
'has_instrumentation_tests': False,
'main_dex_rules': None,
'module': module,
'min_sdk': None,
'name': None,
'releaseTarget': None,
'signed_apk_name': None,
'skip': False,
'has_lint_task': True,
'legacy_desugaring': False,
'compatibility': None
}
self.__dict__ = dict(defaults.items() + fields.items())
# For running on Golem all third-party repositories are bundled as an x20-
# dependency and then copied to WORKING_DIR. To update the app-bundle use
# 'run_on_as_app_x20_packager.py'.
# For showing benchmark data, also include the app in appSegmentBenchmarks in
# the file <golem_repo>/config/r8/benchmarks.dart.
APP_REPOSITORIES = [
# ...
# Repo({
# 'name': ...,
# 'url': ...,
# 'revision': ...,
# 'apps': [
# {
# 'id': ...,
# 'dir': ...,
# 'module': ... (default app)
# 'name': ...,
# 'archives_base_name': ... (default same as module)
# 'flavor': ... (default no flavor)
# 'releaseTarget': ... (default <module>:assemble<flavor>Release
# },
# ...
# ]
# }),
# ...
Repo({
'name': 'android-suite',
'url': 'https://github.com/christofferqa/android-suite',
'revision': '46c96f214711cf6cdcb72cc0c94520ef418e3739',
'apps': [
App({
'id': 'com.numix.calculator',
'dir': 'Calculator',
'name': 'numix-calculator',
'has_instrumentation_tests': True,
'has_lint_task': False
})
]
}),
Repo({
'name': 'AnExplorer',
'url': 'https://github.com/christofferqa/AnExplorer',
'revision': '365927477b8eab4052a1882d5e358057ae3dee4d',
'apps': [
App({
'id': 'dev.dworks.apps.anexplorer.pro',
'flavor': 'googleMobilePro',
'signed_apk_name': 'AnExplorer-googleMobileProRelease-4.0.3.apk',
'min_sdk': 17,
'legacy_desugaring': True,
'compatibility': '1.7',
})
]
}),
Repo({
'name': 'AntennaPod',
'url': 'https://github.com/christofferqa/AntennaPod.git',
'revision': '77e94f4783a16abe9cc5b78dc2d2b2b1867d8c06',
'apps': [
App({
'id': 'de.danoeh.antennapod',
'flavor': 'play',
'min_sdk': 14,
'compile_sdk': 26
})
]
}),
Repo({
'name': 'applymapping',
'url': 'https://github.com/mkj-gram/applymapping',
'revision': 'e3ae14b8c16fa4718e5dea8f7ad00937701b3c48',
'apps': [
App({
'id': 'com.example.applymapping',
'has_instrumentation_tests': True
})
]
}),
Repo({
'name': 'apps-android-wikipedia',
'url': 'https://github.com/christofferqa/apps-android-wikipedia',
'revision': '686e8aa5682af8e6a905054b935dd2daa57e63ee',
'apps': [
App({
'id': 'org.wikipedia',
'flavor': 'prod',
'signed_apk_name': 'app-prod-universal-release.apk'
})
]
}),
Repo({
'name': 'chanu',
'url': 'https://github.com/mkj-gram/chanu.git',
'revision': '6e53458f167b6d78398da60c20fd0da01a232617',
'apps': [
App({
'id': 'com.chanapps.four.activity',
'has_lint_task': False,
'min_sdk': 15,
'compatibility': '1.7',
})
]
}),
Repo({
'name': 'friendlyeats-android',
'url': 'https://github.com/christofferqa/friendlyeats-android.git',
'revision': '10091fa0ec37da12e66286559ad1b6098976b07b',
'apps': [
App({
'id': 'com.google.firebase.example.fireeats',
'min_sdk': 16,
'compatibility': '1.7',
})
]
}),
Repo({
'name': 'googlesamples',
'url': 'https://github.com/christofferqa/android-sunflower.git',
'revision': 'df0a082a0bcbeae253817e13daca3c7a7c54f67a',
'apps': [
App({
'id': 'com.google.samples.apps.sunflower',
'name': 'android-sunflower',
'min_sdk': 19,
'compile_sdk': 28
})
]
}),
Repo({
'name': 'Instabug-Android',
'url': 'https://github.com/christofferqa/Instabug-Android.git',
'revision': 'b8df78c96630a6537fbc07787b4990afc030cc0f',
'apps': [
App({
'id': 'com.example.instabug',
'min_sdk': 15,
'compatibility': '1.7',
})
]
}),
Repo({
'name': 'iosched',
'url': 'https://github.com/christofferqa/iosched.git',
'revision': '581cbbe2253711775dbccb753cdb53e7e506cb02',
'apps': [
App({
'id': 'com.google.samples.apps.iosched',
'module': 'mobile',
'min_sdk': 21,
'compile_sdk': 29,
'has_lint_task': False
})
]
}),
Repo({
'name': 'KISS',
'url': 'https://github.com/christofferqa/KISS',
'revision': '093da9ee0512e67192f62951c45a07a616fc3224',
'apps': [
App({
'id': 'fr.neamar.kiss',
'min_sdk': 15,
'compatibility': '1.7',
})
]
}),
Repo({
'name': 'materialistic',
'url': 'https://github.com/christofferqa/materialistic',
'revision': '2b2b2ee25ce9e672d5aab1dc90a354af1522b1d9',
'apps': [
App({
'id': 'io.github.hidroh.materialistic'
})
]
}),
Repo({
'name': 'Minimal-Todo',
'url': 'https://github.com/christofferqa/Minimal-Todo',
'revision': '9d8c73746762cd376b718858ec1e8783ca07ba7c',
'apps': [
App({
'id': 'com.avjindersinghsekhon.minimaltodo'
})
]
}),
Repo({
'name': 'muzei',
'url': 'https://github.com/sgjesse/muzei.git',
'revision': 'a1f1d9b119faa0db09b6bbffe2318c4ec3679418',
'apps': [
App({
'id': 'net.nurik.roman.muzei',
'module': 'main',
'archives_base_name': 'muzei',
'compile_sdk': 28,
})
]
}),
Repo({
'name': 'NewPipe',
'url': 'https://github.com/christofferqa/NewPipe',
'revision': 'ed543099c7823be00f15d9340f94bdb7cb37d1e6',
'apps': [
App({
'id': 'org.schabi.newpipe',
'has_lint_task': False
})
]
}),
Repo({
'name': 'rover-android',
'url': 'https://github.com/mkj-gram/rover-android.git',
'revision': 'a5e155a1ed7d19b1cecd9a7b075e2852623a06bf',
'apps': [
App({
'id': 'io.rover.app.debug',
'module': 'debug-app'
})
]
}),
Repo({
'name': 'santa-tracker-android',
'url': 'https://github.com/christofferqa/santa-tracker-android.git',
'revision': '8dee74be7d9ee33c69465a07088c53087d24a6dd',
'apps': [
App({
'id': 'com.google.android.apps.santatracker',
'module': 'santa-tracker',
'min_sdk': 21,
'compile_sdk': 28
})
]
}),
Repo({
'name': 'Signal-Android',
'url': 'https://github.com/mkj-gram/Signal-Android.git',
'revision': 'cd542cab9bf860e71504ecb1caaf0a8476ba3989',
'apps': [
App({
'id': 'org.thoughtcrime.securesms',
'module': '',
'flavor': 'play',
'main_dex_rules': 'multidex-config.pro',
'signed_apk_name': 'Signal-play-release-4.32.7.apk'
})
]
}),
Repo({
'name': 'Simple-Calendar',
'url': 'https://github.com/christofferqa/Simple-Calendar',
'revision': '82dad8c203eea5a0f0ddb513506d8f1de986ef2b',
'apps': [
App({
'id': 'com.simplemobiletools.calendar.pro',
'signed_apk_name': 'calendar-release.apk',
'has_lint_task': False
})
]
}),
Repo({
'name': 'Simple-Camera',
'url': 'https://github.com/jsjeon/Simple-Camera',
'revision': '451fe188ab123e6956413b42e89839b44c05ac14',
'apps': [
App({
'id': 'com.simplemobiletools.camera.pro',
'signed_apk_name': 'camera-release.apk',
'has_lint_task': False
})
]
}),
Repo({
'name': 'Simple-File-Manager',
'url': 'https://github.com/jsjeon/Simple-File-Manager',
'revision': '282b57d9e73f4d250cc844d8d73fd223509a141e',
'apps': [
App({
'id': 'com.simplemobiletools.filemanager.pro',
'signed_apk_name': 'file-manager-release.apk',
'has_lint_task': False
})
]
}),
Repo({
'name': 'Simple-Gallery',
'url': 'https://github.com/jsjeon/Simple-Gallery',
'revision': '679125601eee7e057dfdfecd7bea6c4a6ac73ef9',
'apps': [
App({
'id': 'com.simplemobiletools.gallery.pro',
'signed_apk_name': 'gallery-release.apk',
'has_lint_task': False
})
]
}),
Repo({
'name': 'sqldelight',
'url': 'https://github.com/christofferqa/sqldelight.git',
'revision': '2e67a1126b6df05e4119d1e3a432fde51d76cdc8',
'apps': [
App({
'id': 'com.example.sqldelight.hockey',
'module': 'sample/android',
'archives_base_name': 'android',
'min_sdk': 14,
'compile_sdk': 28
})
]
}),
Repo({
'name': 'tachiyomi',
'url': 'https://github.com/sgjesse/tachiyomi.git',
'revision': 'b15d2fe16864645055af6a745a62cc5566629798',
'apps': [
App({
'id': 'eu.kanade.tachiyomi',
'flavor': 'dev',
'min_sdk': 21,
'has_lint_task': False,
'compatibility': '1.7',
})
]
}),
Repo({
'name': 'tivi',
'url': 'https://github.com/sgjesse/tivi.git',
'revision': '25c52e3593e7c98da4e537b49b29f6f67f88754d',
'apps': [
App({
'id': 'app.tivi',
'min_sdk': 23,
'compile_sdk': 28,
'has_lint_task': False
})
]
}),
Repo({
'name': 'Tusky',
'url': 'https://github.com/mkj-gram/Tusky.git',
'revision': 'e7fbd190fb53bf9fde72253b816920cb6fe34518',
'apps': [
App({
'id': 'com.keylesspalace.tusky',
'flavor': 'blue'
})
]
}),
Repo({
'name': 'Vungle-Android-SDK',
'url': 'https://github.com/mkj-gram/Vungle-Android-SDK.git',
'revision': '138d3f18c027b61b195c98911f1c5ab7d87ad18b',
'apps': [
App({
'id': 'com.publisher.vungle.sample',
'skip': True, # TODO(b/144058031)
})
]
})
]
def signal_handler(signum, frame):
subprocess.call(['pkill', 'java'])
raise Exception('Got killed by %s' % signum)
class EnsureNoGradleAlive(object):
def __init__(self, active):
self.active = active
def __enter__(self):
if self.active:
# If we timeout and get a sigterm we should still kill all java
signal.signal(signal.SIGTERM, signal_handler)
print 'Running with wrapper that will kill java after'
def __exit__(self, *_):
if self.active:
subprocess.call(['pkill', 'java'])
def GetAllApps():
apps = []
for repo in APP_REPOSITORIES:
for app in repo.apps:
apps.append((app, repo))
return apps
def GetAllAppNames():
return [app.name for (app, repo) in GetAllApps()]
def GetAppWithName(query):
for (app, repo) in GetAllApps():
if app.name == query:
return (app, repo)
assert False
def ComputeSizeOfDexFilesInApk(apk):
dex_size = 0
z = zipfile.ZipFile(apk, 'r')
for filename in z.namelist():
if filename.endswith('.dex'):
dex_size += z.getinfo(filename).file_size
return dex_size
def ExtractMarker(apk, temp_dir, options):
r8_jar = os.path.join(temp_dir, 'r8.jar')
r8lib_jar = os.path.join(temp_dir, 'r8lib.jar')
# Use the copy of r8.jar or r8lib.jar if one is there.
if os.path.isfile(r8_jar):
cmd = [jdk.GetJavaExecutable(), '-ea', '-jar', r8_jar, 'extractmarker', apk]
elif os.path.isfile(r8lib_jar):
cmd = [jdk.GetJavaExecutable(), '-ea', '-cp', r8lib_jar,
'com.android.tools.r8.ExtractMarker', apk]
else:
script = os.path.join(utils.TOOLS_DIR, 'extractmarker.py')
cmd = ['python', script, apk]
utils.PrintCmd(cmd, quiet=options.quiet)
stdout = subprocess.check_output(cmd)
# Return the last line.
lines = stdout.strip().splitlines()
assert len(lines) >= 1
return lines[-1]
def CheckIsBuiltWithExpectedR8(apk, temp_dir, shrinker, options):
marker_raw = ExtractMarker(apk, temp_dir, options)
# Marker should be a string on the following format (no whitespace):
# ~~R8{"compilation-mode":"release",
# "min-api":16,
# "pg-map-id":"767707e",
# "sha-1":"7111a35bae6d5185dcfb338d61074aca8426c006",
# "version":"1.5.14-dev"}
if not marker_raw.startswith('~~R8'):
raise Exception(
'Expected marker to start with \'~~R8\' (was: {})'.format(marker_raw))
marker = json.loads(marker_raw[4:])
if options.hash:
actual_hash = marker.get('sha-1')
if actual_hash != options.hash:
raise Exception(
'Expected APK to be built with R8 version {} (was: {})'.format(
expected_hash, marker_raw))
return True
expected_version = (
options.version
if options.version
else utils.getR8Version(
os.path.join(
temp_dir,
'r8lib.jar' if IsMinifiedR8(shrinker) else 'r8.jar')))
actual_version = marker.get('version')
if actual_version != expected_version:
raise Exception(
'Expected APK to be built with R8 version {} (was: {})'.format(
expected_version, marker_raw))
return True
def IsR8(shrinker):
return 'r8' in shrinker
def IsR8FullMode(shrinker):
return shrinker == 'r8-full' or shrinker == 'r8-nolib-full'
def IsLoggingEnabledFor(app, options):
if options.no_logging:
return False
if options.app_logging_filter and app.name not in options.app_logging_filter:
return False
return True
def IsMinifiedR8(shrinker):
return 'nolib' not in shrinker
def IsTrackedByGit(file):
return subprocess.check_output(['git', 'ls-files', file]).strip() != ''
def GitClone(repo, checkout_dir, quiet):
result = subprocess.check_output(
['git', 'clone', repo.url, checkout_dir]).strip()
head_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir)
if repo.revision == head_rev:
return result
warn('Target revision is not head in {}.'.format(checkout_dir))
with utils.ChangedWorkingDirectory(checkout_dir, quiet=quiet):
subprocess.check_output(['git', 'reset', '--hard', repo.revision])
return result
def GitCheckout(file):
return subprocess.check_output(['git', 'checkout', file]).strip()
def InstallApkOnEmulator(apk_dest, options):
cmd = ['adb', '-s', options.emulator_id, 'install', '-r', '-d', apk_dest]
if options.quiet:
with open(os.devnull, 'w') as devnull:
subprocess.check_call(cmd, stdout=devnull)
else:
subprocess.check_call(cmd)
def PercentageDiffAsString(before, after):
if after < before:
return '-' + str(round((1.0 - after / before) * 100)) + '%'
else:
return '+' + str(round((after - before) / before * 100)) + '%'
def UninstallApkOnEmulator(app, options):
process = subprocess.Popen(
['adb', '-s', options.emulator_id, 'uninstall', app.id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if stdout.strip() == 'Success':
# Successfully uninstalled
return
if 'Unknown package: {}'.format(app.id) in stderr:
# Application not installed
return
raise Exception(
'Unexpected result from `adb uninstall {}\nStdout: {}\nStderr: {}'.format(
app.id, stdout, stderr))
def WaitForEmulator(options):
stdout = subprocess.check_output(['adb', 'devices'])
if '{}\tdevice'.format(options.emulator_id) in stdout:
return True
print('Emulator \'{}\' not connected; waiting for connection'.format(
options.emulator_id))
time_waited = 0
while True:
time.sleep(10)
time_waited += 10
stdout = subprocess.check_output(['adb', 'devices'])
if '{}\tdevice'.format(options.emulator_id) not in stdout:
print('... still waiting for connection')
if time_waited >= 5 * 60:
return False
else:
return True
def GetResultsForApp(app, repo, options, temp_dir):
# Checkout and build in the build directory.
repo_name = repo.name
repo_checkout_dir = os.path.join(WORKING_DIR, repo_name)
result = {}
if not os.path.exists(repo_checkout_dir) and not options.golem:
with utils.ChangedWorkingDirectory(WORKING_DIR, quiet=options.quiet):
GitClone(repo, repo_checkout_dir, options.quiet)
checkout_rev = utils.get_HEAD_sha1_for_checkout(repo_checkout_dir)
if repo.revision != checkout_rev:
msg = 'Checkout is not target revision for {} in {}.'.format(
app.name, repo_checkout_dir)
if options.ignore_versions:
warn(msg)
else:
raise Exception(msg)
result['status'] = 'success'
app_checkout_dir = (os.path.join(repo_checkout_dir, app.dir)
if app.dir else repo_checkout_dir)
result_per_shrinker = BuildAppWithSelectedShrinkers(
app, repo, options, app_checkout_dir, temp_dir)
for shrinker, shrinker_result in result_per_shrinker.iteritems():
result[shrinker] = shrinker_result
return result
def BuildAppWithSelectedShrinkers(
app, repo, options, checkout_dir, temp_dir):
result_per_shrinker = {}
with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet):
for shrinker in options.shrinker:
apk_dest = None
result = {}
proguard_config_file = None
if not options.r8_compilation_steps_only:
try:
out_dir = os.path.join(checkout_dir, 'out', shrinker)
(apk_dest, profile_dest_dir, res_proguard_config_file) = \
BuildAppWithShrinker(
app, repo, shrinker, checkout_dir, out_dir, temp_dir,
options)
proguard_config_file = res_proguard_config_file
dex_size = ComputeSizeOfDexFilesInApk(apk_dest)
result['apk_dest'] = apk_dest
result['build_status'] = 'success'
result['dex_size'] = dex_size
result['profile_dest_dir'] = profile_dest_dir
profile = as_utils.ParseProfileReport(profile_dest_dir)
result['profile'] = {
task_name:duration for task_name, duration in profile.iteritems()
if as_utils.IsGradleCompilerTask(task_name, shrinker)}
except Exception as e:
warn('Failed to build {} with {}'.format(app.name, shrinker))
if e:
print('Error: ' + str(e))
result['build_status'] = 'failed'
if result.get('build_status') == 'success':
if options.monkey:
result['monkey_status'] = 'success' if RunMonkey(
app, options, apk_dest) else 'failed'
if (result.get('build_status') == 'success'
or options.r8_compilation_steps_only):
if 'r8' in shrinker and options.r8_compilation_steps > 1:
result['recompilation_results'] = \
ComputeRecompilationResults(
app, repo, options, checkout_dir, temp_dir, shrinker,
proguard_config_file)
if result.get('build_status') == 'success':
if options.run_tests and app.has_instrumentation_tests:
result['instrumentation_test_results'] = \
ComputeInstrumentationTestResults(
app, options, checkout_dir, out_dir, shrinker)
result_per_shrinker[shrinker] = result
if len(options.apps) > 1:
print('')
LogResultsForApp(app, result_per_shrinker, options)
print('')
return result_per_shrinker
def BuildAppWithShrinker(
app, repo, shrinker, checkout_dir, out_dir, temp_dir, options,
keepRuleSynthesisForRecompilation=False):
print('[{}] Building {} with {}{}'.format(
datetime.now().strftime("%H:%M:%S"),
app.name,
shrinker,
' for recompilation' if keepRuleSynthesisForRecompilation else ''))
print('To compile locally: '
'tools/run_on_as_app.py --shrinker {} --r8-compilation-steps {} '
'--app {} {}'.format(
shrinker,
options.r8_compilation_steps,
app.name,
'--r8-compilation-steps-only'
if options.r8_compilation_steps_only else ''))
print('HINT: use --shrinker r8-nolib --no-build if you have a local R8.jar')
# Add settings.gradle file if it is not present to prevent gradle from finding
# the settings.gradle file in the r8 root when apps are placed under
# $R8/build.
as_utils.add_settings_gradle(checkout_dir, app.name)
# Add 'r8.jar' to top-level build.gradle.
as_utils.add_r8_dependency(checkout_dir, temp_dir, IsMinifiedR8(shrinker))
archives_base_name = app.archives_base_name
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# Set -printconfiguration in Proguard rules.
proguard_config_dest = os.path.abspath(
os.path.join(out_dir, 'proguard-rules.pro'))
as_utils.SetPrintConfigurationDirective(
app, checkout_dir, proguard_config_dest)
env_vars = {}
env_vars['JAVA_HOME'] = jdk.GetJdk8Home()
env_vars['ANDROID_HOME'] = utils.getAndroidHome()
if not options.disable_assertions:
env_vars['JAVA_OPTS'] = '-ea:com.android.tools.r8...'
release_target = app.releaseTarget
if not release_target:
app_module = app.module.replace('/', ':')
app_flavor = (app.flavor.capitalize() if app.flavor else '') + 'Release'
release_target = app_module + ':' + 'assemble' + app_flavor
# Build using gradle.
args = [release_target,
'-g=' + os.path.join(checkout_dir, GRADLE_USER_HOME),
'-Pandroid.enableR8=' + str(IsR8(shrinker)).lower(),
'-Pandroid.enableR8.fullMode=' + str(IsR8FullMode(shrinker)).lower()]
if app.has_lint_task:
args.extend(['-x', app_module + ':lintVital' + app_flavor])
if options.bot:
args.extend(['--console=plain', '--info'])
# Warm up gradle if pre_runs > 0. For posterity we generate the same sequence
# as the benchmarking at https://github.com/madsager/santa-tracker-android.
for i in range(0, options.gradle_pre_runs):
if i == 0:
utils.RunGradlew(
["--stop"],
env_vars=env_vars,
quiet=options.quiet,
logging=IsLoggingEnabledFor(app, options),
use_daemon=options.use_daemon)
utils.RunGradlew(
args,
env_vars=env_vars,
quiet=options.quiet,
clean=i > 0,
use_daemon=options.use_daemon,
logging=IsLoggingEnabledFor(app, options))
if keepRuleSynthesisForRecompilation:
args.append('-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true')
if options.gradle_flags:
args.extend(options.gradle_flags.split(' '))
args.append('--profile')
stdout = utils.RunGradlew(
args,
env_vars=env_vars,
quiet=options.quiet,
use_daemon=options.use_daemon,
logging=IsLoggingEnabledFor(app, options))
apk_base_name = (archives_base_name
+ (('-' + app.flavor) if app.flavor else '') + '-release')
signed_apk_name = (
app.signed_apk_name
if app.signed_apk_name
else apk_base_name + '.apk')
unsigned_apk_name = apk_base_name + '-unsigned.apk'
build_dir = os.path.join(app.module, app.build_dir)
build_output_apks = os.path.join(build_dir, 'outputs', 'apk')
if app.flavor:
build_output_apks = os.path.join(build_output_apks, app.flavor, 'release')
else:
build_output_apks = os.path.join(build_output_apks, 'release')
signed_apk = os.path.join(build_output_apks, signed_apk_name)
unsigned_apk = os.path.join(build_output_apks, unsigned_apk_name)
assert os.path.isfile(signed_apk) or os.path.isfile(unsigned_apk), (
"Expected a file to be present at {} or {}, found: {}\n"
"Standard out from compilation: {}".format(
signed_apk,
unsigned_apk,
', '.join(
as_utils.ListFiles(build_dir, lambda x : x.endswith('.apk'))),
stdout))
if options.sign_apks and not os.path.isfile(signed_apk):
assert os.path.isfile(unsigned_apk)
if options.sign_apks:
apk_utils.sign_with_apksigner(
unsigned_apk,
signed_apk,
options.keystore,
options.keystore_password,
quiet=options.quiet,
logging=IsLoggingEnabledFor(app, options))
if os.path.isfile(signed_apk):
apk_dest = os.path.join(out_dir, signed_apk_name)
as_utils.MoveFile(signed_apk, apk_dest, quiet=options.quiet)
else:
apk_dest = os.path.join(out_dir, unsigned_apk_name)
as_utils.MoveFile(unsigned_apk, apk_dest, quiet=options.quiet)
assert ('r8' not in shrinker
or CheckIsBuiltWithExpectedR8(apk_dest, temp_dir, shrinker, options))
profile_dest_dir = os.path.join(out_dir, 'profile')
as_utils.MoveProfileReportTo(profile_dest_dir, stdout, quiet=options.quiet)
# Ensure that the gradle daemon is stopped if we are running with it.
if options.use_daemon:
utils.RunGradlew(['--stop', '-g=' + os.path.join(checkout_dir, GRADLE_USER_HOME)])
return (apk_dest, profile_dest_dir, proguard_config_dest)
def ComputeInstrumentationTestResults(
app, options, checkout_dir, out_dir, shrinker):
args = ['connectedAndroidTest',
'-Pandroid.enableR8=' + str(IsR8(shrinker)).lower(),
'-Pandroid.enableR8.fullMode=' + str(IsR8FullMode(shrinker)).lower()]
env_vars = { 'ANDROID_SERIAL': options.emulator_id }
stdout = utils.RunGradlew(
args,
env_vars=env_vars,
quiet=options.quiet,
fail=False,
logging=IsLoggingEnabledFor(app, options),
use_daemon=options.use_daemon)
xml_test_result_dest = os.path.join(out_dir, 'test_result')
as_utils.MoveXMLTestResultFileTo(
xml_test_result_dest, stdout, quiet=options.quiet)
with open(xml_test_result_dest, 'r') as f:
xml_test_result_contents = f.read()
xml_document = minidom.parseString(xml_test_result_contents)
testsuite_element = xml_document.documentElement
return {
'xml_test_result_dest': xml_test_result_dest,
'tests': int(testsuite_element.getAttribute('tests')),
'failures': int(testsuite_element.getAttribute('failures')),
'errors': int(testsuite_element.getAttribute('errors')),
'skipped': int(testsuite_element.getAttribute('skipped'))
}
def ComputeRecompilationResults(
app, repo, options, checkout_dir, temp_dir, shrinker, proguard_config_file):
recompilation_results = []
# Build app with gradle using -D...keepRuleSynthesisForRecompilation=true.
out_dir = os.path.join(checkout_dir, 'out', shrinker + '-1')
(apk_dest, profile_dest_dir, ext_proguard_config_file) = \
BuildAppWithShrinker(
app, repo, shrinker, checkout_dir, out_dir,
temp_dir, options, keepRuleSynthesisForRecompilation=True)
recompilation_result = {
'apk_dest': apk_dest,
'build_status': 'success',
'dex_size': ComputeSizeOfDexFilesInApk(apk_dest),
'monkey_status': 'skipped'
}
recompilation_results.append(recompilation_result)
# Sanity check that keep rules have changed. If we are only doing
# recompilation, the passed in proguard_config_file is None.
if proguard_config_file:
with open(ext_proguard_config_file) as new:
with open(proguard_config_file) as old:
assert(
sum(1 for line in new
if line.strip() and '-printconfiguration' not in line)
>
sum(1 for line in old
if line.strip() and '-printconfiguration' not in line))
# Extract min-sdk and target-sdk
(min_sdk, compile_sdk) = \
as_utils.GetMinAndCompileSdk(app, checkout_dir, apk_dest)
# Now rebuild generated apk.
previous_apk = apk_dest
# We may need main dex rules when re-compiling with R8 as standalone.
main_dex_rules = None
if app.main_dex_rules:
main_dex_rules = os.path.join(checkout_dir, app.main_dex_rules)
for i in range(1, options.r8_compilation_steps):
try:
recompiled_apk_dest = os.path.join(
checkout_dir, 'out', shrinker, 'app-release-{}.apk'.format(i))
if not os.path.exists(os.path.dirname(recompiled_apk_dest)):
os.makedirs(os.path.dirname(recompiled_apk_dest))
RebuildAppWithShrinker(
app, previous_apk, recompiled_apk_dest,
ext_proguard_config_file, shrinker, min_sdk, compile_sdk,
options, temp_dir, main_dex_rules)
recompilation_result = {
'apk_dest': recompiled_apk_dest,
'build_status': 'success',
'dex_size': ComputeSizeOfDexFilesInApk(recompiled_apk_dest)
}
if options.monkey:
recompilation_result['monkey_status'] = 'success' if RunMonkey(
app, options, recompiled_apk_dest) else 'failed'
recompilation_results.append(recompilation_result)
previous_apk = recompiled_apk_dest
except Exception as e:
warn('Failed to recompile {} with {}'.format(
app.name, shrinker))
recompilation_results.append({ 'build_status': 'failed' })
break
return recompilation_results
def RebuildAppWithShrinker(
app, apk, apk_dest, proguard_config_file, shrinker, min_sdk, compile_sdk,
options, temp_dir, main_dex_rules):
assert 'r8' in shrinker
assert apk_dest.endswith('.apk')
print('Rebuilding {} with {}'.format(app.name, shrinker))
# Compile given APK with shrinker to temporary zip file.
android_jar = utils.get_android_jar(compile_sdk)
r8_jar = os.path.join(
temp_dir, 'r8lib.jar' if IsMinifiedR8(shrinker) else 'r8.jar')
zip_dest = apk_dest[:-4] + '.zip'
# TODO(christofferqa): Entry point should be CompatProguard if the shrinker
# is 'r8'.
entry_point = 'com.android.tools.r8.R8'
cmd = ([jdk.GetJavaExecutable()] +
(['-ea:com.android.tools.r8...']
if not options.disable_assertions
else []) +
['-cp', r8_jar, entry_point,
'--release', '--min-api', str(min_sdk),
'--pg-conf', proguard_config_file,
'--lib', android_jar,
'--output', zip_dest,
apk] +
(['--no-desugaring']
if app.legacy_desugaring or app.compatibility == '1.7'
else []))
for android_optional_jar in utils.get_android_optional_jars(compile_sdk):
cmd.append('--lib')
cmd.append(android_optional_jar)
if main_dex_rules:
cmd.append('--main-dex-rules')
cmd.append(main_dex_rules)
utils.RunCmd(
cmd, quiet=options.quiet, logging=IsLoggingEnabledFor(app, options))
# Make a copy of the given APK, move the newly generated dex files into the
# copied APK, and then sign the APK.
apk_masseur.masseur(
apk, dex=zip_dest, resources='META-INF/services/*', out=apk_dest,
quiet=options.quiet, logging=IsLoggingEnabledFor(app, options),
keystore=options.keystore)
def RunMonkey(app, options, apk_dest):
if not WaitForEmulator(options):
return False
UninstallApkOnEmulator(app, options)
InstallApkOnEmulator(apk_dest, options)
number_of_events_to_generate = options.monkey_events
# Intentionally using a constant seed such that the monkey generates the same
# event sequence for each shrinker.
random_seed = 42
cmd = ['adb', '-s', options.emulator_id, 'shell', 'monkey', '-p', app.id,
'-s', str(random_seed), str(number_of_events_to_generate)]
try:
stdout = utils.RunCmd(
cmd, quiet=options.quiet, logging=IsLoggingEnabledFor(app, options))
succeeded = (
'Events injected: {}'.format(number_of_events_to_generate) in stdout)
except subprocess.CalledProcessError as e:
succeeded = False
UninstallApkOnEmulator(app, options)
return succeeded
def LogResultsForApps(result_per_shrinker_per_app, options):
print('')
app_errors = 0
for (app, result_per_shrinker) in result_per_shrinker_per_app:
app_errors += (1 if LogResultsForApp(app, result_per_shrinker, options)
else 0)
return app_errors
def LogResultsForApp(app, result_per_shrinker, options):
if options.print_dexsegments:
LogSegmentsForApp(app, result_per_shrinker, options)
return False
else:
return LogComparisonResultsForApp(app, result_per_shrinker, options)
def LogSegmentsForApp(app, result_per_shrinker, options):
for shrinker in SHRINKERS:
if shrinker not in result_per_shrinker:
continue
result = result_per_shrinker[shrinker];
benchmark_name = '{}-{}'.format(options.print_dexsegments, app.name)
utils.print_dexsegments(benchmark_name, [result.get('apk_dest')])
duration = sum(result.get('profile').values())
print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration * 1000))
print('%s-Total(CodeSize): %s' % (benchmark_name, result.get('dex_size')))
def LogComparisonResultsForApp(app, result_per_shrinker, options):
print(app.name + ':')
app_error = False
if result_per_shrinker.get('status', 'success') != 'success':
error_message = result_per_shrinker.get('error_message')
print(' skipped ({})'.format(error_message))
return
proguard_result = result_per_shrinker.get('pg', {})
proguard_dex_size = float(proguard_result.get('dex_size', -1))
proguard_duration = sum(proguard_result.get('profile', {}).values())
for shrinker in SHRINKERS:
if shrinker not in result_per_shrinker:
continue
result = result_per_shrinker.get(shrinker)
build_status = result.get('build_status')
if build_status != 'success' and build_status is not None:
app_error = True
warn(' {}: {}'.format(shrinker, build_status))
continue
print(' {}:'.format(shrinker))
dex_size = result.get('dex_size')
msg = ' dex size: {}'.format(dex_size)
if dex_size != proguard_dex_size and proguard_dex_size >= 0:
msg = '{} ({}, {})'.format(
msg, dex_size - proguard_dex_size,
PercentageDiffAsString(proguard_dex_size, dex_size))
success(msg) if dex_size < proguard_dex_size else warn(msg)
else:
print(msg)
profile = result.get('profile')
if profile:
duration = sum(profile.values())
msg = ' performance: {}s'.format(duration)
if duration != proguard_duration and proguard_duration > 0:
msg = '{} ({}s, {})'.format(
msg, duration - proguard_duration,
PercentageDiffAsString(proguard_duration, duration))
success(msg) if duration < proguard_duration else warn(msg)
else:
print(msg)
if len(profile) >= 2:
for task_name, task_duration in profile.iteritems():
print(' {}: {}s'.format(task_name, task_duration))
if options.monkey:
monkey_status = result.get('monkey_status')
if monkey_status != 'success':
app_error = True
warn(' monkey: {}'.format(monkey_status))
else:
success(' monkey: {}'.format(monkey_status))
recompilation_results = result.get('recompilation_results', [])
i = 0
for recompilation_result in recompilation_results:
build_status = recompilation_result.get('build_status')
if build_status != 'success':
app_error = True
print(' recompilation #{}: {}'.format(i, build_status))
else:
dex_size = recompilation_result.get('dex_size')
print(' recompilation #{}'.format(i))
print(' dex size: {}'.format(dex_size))
if options.monkey:
monkey_status = recompilation_result.get('monkey_status')
msg = ' monkey: {}'.format(monkey_status)
if monkey_status == 'success':
success(msg)
elif monkey_status == 'skipped':
print(msg)
else:
warn(msg)
i += 1
if options.run_tests and 'instrumentation_test_results' in result:
instrumentation_test_results = \
result.get('instrumentation_test_results')
succeeded = (
instrumentation_test_results.get('failures')
+ instrumentation_test_results.get('errors')
+ instrumentation_test_results.get('skipped')) == 0
if succeeded:
success(' tests: succeeded')
else:
app_error = True
warn(
' tests: failed (failures: {}, errors: {}, skipped: {})'
.format(
instrumentation_test_results.get('failures'),
instrumentation_test_results.get('errors'),
instrumentation_test_results.get('skipped')))
return app_error
def ParseOptions(argv):
result = optparse.OptionParser()
result.add_option('--app',
help='What app to run on',
choices=GetAllAppNames(),
action='append')
result.add_option('--bot',
help='Running on bot, use third_party dependency.',
default=False,
action='store_true')
result.add_option('--disable-assertions', '--disable_assertions',
help='Disable assertions when compiling',
default=False,
action='store_true')
result.add_option('--download-only', '--download_only',
help='Whether to download apps without any compilation',
default=False,
action='store_true')
result.add_option('--emulator-id', '--emulator_id',
help='Id of the emulator to use',
default='emulator-5554')
result.add_option('--golem',
help='Running on golem, do not download',
default=False,
action='store_true')
result.add_option('--gradle-flags', '--gradle_flags',
help='Flags to pass in to gradle')
result.add_option('--gradle-pre-runs', '--gradle_pre_runs',
help='Do rounds of compilations to warm up gradle',
default=0,
type=int)
result.add_option('--hash',
help='The version of R8 to use')
result.add_option('--ignore-versions', '--ignore_versions',
help='Allow checked-out app to differ in revision from '
'pinned',
default=False,
action='store_true')
result.add_option('--keystore',
help='Path to app.keystore',
default=os.path.join(utils.TOOLS_DIR, 'debug.keystore'))
result.add_option('--keystore-password', '--keystore_password',
help='Password for app.keystore',
default='android')
result.add_option('--app-logging-filter', '--app_logging_filter',
help='The apps for which to turn on logging',
action='append')
result.add_option('--monkey',
help='Whether to install and run app(s) with monkey',
default=False,
action='store_true')
result.add_option('--monkey-events', '--monkey_events',
help='Number of events that the monkey should trigger',
default=250,
type=int)
result.add_option('--no-build', '--no_build',
help='Run without building ToT first (only when using ToT)',
default=False,
action='store_true')
result.add_option('--no-logging', '--no_logging',
help='Disable logging except for errors',
default=False,
action='store_true')
result.add_option('--print-dexsegments',
metavar='BENCHMARKNAME',
help='Print the sizes of individual dex segments as ' +
'\'<BENCHMARKNAME>-<APP>-<segment>(CodeSize): '
'<bytes>\'')
result.add_option('--quiet',
help='Disable verbose logging',
default=False,
action='store_true')
result.add_option('--r8-compilation-steps', '--r8_compilation_steps',
help='Number of times R8 should be run on each app',
default=2,
type=int)
result.add_option('--r8-compilation-steps-only', '--r8_compilation_steps_only',
help='Specify to only run compilation steps',
default=False,
action='store_true')
result.add_option('--run-tests', '--run_tests',
help='Whether to run instrumentation tests',
default=False,
action='store_true')
result.add_option('--sign-apks', '--sign_apks',
help='Whether the APKs should be signed',
default=False,
action='store_true')
result.add_option('--shrinker',
help='The shrinkers to use (by default, all are run)',
action='append')
result.add_option('--use-daemon', '--use_daemon',
help='Whether to use a gradle daemon',
default=False,
action='store_true')
result.add_option('--version',
help='The version of R8 to use (e.g., 1.4.51)')
(options, args) = result.parse_args(argv)
if options.app:
options.apps = [(app, repo) for (app, repo) in GetAllApps()
if app.name in options.app]
del options.app
else:
options.apps = GetAllApps()
if options.app_logging_filter:
for app_name in options.app_logging_filter:
assert any(app.name == app_name for (app, repo) in options.apps)
if options.shrinker:
for shrinker in options.shrinker:
assert shrinker in SHRINKERS
else:
options.shrinker = [shrinker for shrinker in SHRINKERS]
if options.hash or options.version:
# No need to build R8 if a specific version should be used.
options.no_build = True
if 'r8-nolib' in options.shrinker:
warn('Skipping shrinker r8-nolib because a specific version '
+ 'of r8 was specified')
options.shrinker.remove('r8-nolib')
if 'r8-nolib-full' in options.shrinker:
warn('Skipping shrinker r8-nolib-full because a specific version '
+ 'of r8 was specified')
options.shrinker.remove('r8-nolib-full')
assert not options.r8_compilation_steps_only \
or options.r8_compilation_steps > 1
return (options, args)
def clone_repositories(quiet):
# Clone repositories into WORKING_DIR.
with utils.ChangedWorkingDirectory(WORKING_DIR):
for repo in APP_REPOSITORIES:
repo_dir = os.path.join(WORKING_DIR, repo.name)
if not os.path.exists(repo_dir):
GitClone(repo, repo_dir, quiet)
def main(argv):
(options, args) = ParseOptions(argv)
if options.bot:
utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1',
bucket='r8-deps-internal',
auth=True)
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
# TODO(b/141081520): Set to True once fixed.
options.no_logging = False
# TODO(b/141081520): Remove logging filter once fixed.
options.app_logging_filter = ['sqldelight']
options.shrinker = ['r8', 'r8-full']
print(options.shrinker)
if options.golem:
golem.link_third_party()
if os.path.exists(WORKING_DIR):
shutil.rmtree(WORKING_DIR)
shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR)
os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join(
utils.ANDROID_SDK)
os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3'
options.disable_assertions = True
options.ignore_versions = True
options.no_build = True
options.r8_compilation_steps = 1
options.quiet = True
options.gradle_pre_runs = 2
options.use_daemon = True
options.no_logging = True
if not os.path.exists(WORKING_DIR):
os.makedirs(WORKING_DIR)
if options.download_only:
clone_repositories(options.quiet)
return
with utils.TempDir() as temp_dir:
if not (options.no_build or options.golem):
gradle.RunGradle(['r8', '-Pno_internal'])
build_r8lib = False
for shrinker in options.shrinker:
if IsMinifiedR8(shrinker):
build_r8lib = True
if build_r8lib:
gradle.RunGradle(['r8lib', '-Pno_internal'])
if options.hash:
# Download r8-<hash>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.hash)
update_prebuilds_in_android.download_hash(
temp_dir, 'com/android/tools/r8/' + options.hash, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
elif options.version:
# Download r8-<version>.jar from
# https://storage.googleapis.com/r8-releases/raw/.
target = 'r8-{}.jar'.format(options.version)
update_prebuilds_in_android.download_version(
temp_dir, 'com/android/tools/r8/' + options.version, target)
as_utils.MoveFile(
os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
quiet=options.quiet)
else:
# Make a copy of r8.jar and r8lib.jar such that they stay the same for
# the entire execution of this script.
if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar'
shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
result_per_shrinker_per_app = []
# If we are running on golem we kill all java processes after the run
# to ensure no hanging gradle daemons.
with EnsureNoGradleAlive(options.golem):
for (app, repo) in options.apps:
if app.skip:
continue
result_per_shrinker_per_app.append(
(app, GetResultsForApp(app, repo, options, temp_dir)))
return LogResultsForApps(result_per_shrinker_per_app, options)
def success(message):
CGREEN = '\033[32m'
CEND = '\033[0m'
print(CGREEN + message + CEND)
def warn(message):
CRED = '\033[91m'
CEND = '\033[0m'
print(CRED + message + CEND)
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))