| #!/usr/bin/env python | |
| # Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file | |
| # for details. All rights reserved. Use of this source code is governed by a | |
| # BSD-style license that can be found in the LICENSE file. | |
| import apk_masseur | |
| import apk_utils | |
| import golem | |
| import gradle | |
| import jdk | |
| import json | |
| import os | |
| import optparse | |
| import shutil | |
| import signal | |
| import subprocess | |
| import sys | |
| import time | |
| import utils | |
| import zipfile | |
| from xml.dom import minidom | |
| from datetime import datetime | |
| import as_utils | |
| import update_prebuilds_in_android | |
| import download_all_benchmark_dependencies | |
| SHRINKERS = ['r8', 'r8-full', 'r8-nolib', 'r8-nolib-full', 'pg'] | |
| WORKING_DIR = os.path.join(utils.BUILD, 'opensource_apps') | |
| GRADLE_USER_HOME = '.gradle_user_home' | |
| if ('R8_BENCHMARK_DIR' in os.environ | |
| and os.path.isdir(os.environ['R8_BENCHMARK_DIR'])): | |
| WORKING_DIR = os.environ['R8_BENCHMARK_DIR'] | |
| class Repo(object): | |
| def __init__(self, fields): | |
| self.__dict__ = fields | |
| # If there is only one app in this repository, then give the app the same | |
| # name as the repository, if it does not already have one. | |
| if len(self.apps) == 1: | |
| app = self.apps[0] | |
| if not app.name: | |
| app.name = self.name | |
| class App(object): | |
| def __init__(self, fields): | |
| module = fields.get('module', 'app') | |
| defaults = { | |
| 'archives_base_name': module, | |
| 'build_dir': 'build', | |
| 'compile_sdk': None, | |
| 'dir': None, | |
| 'flavor': None, | |
| 'has_instrumentation_tests': False, | |
| 'main_dex_rules': None, | |
| 'module': module, | |
| 'min_sdk': None, | |
| 'name': None, | |
| 'releaseTarget': None, | |
| 'signed_apk_name': None, | |
| 'skip': False, | |
| 'has_lint_task': True | |
| } | |
| self.__dict__ = dict(defaults.items() + fields.items()) | |
| # For running on Golem all third-party repositories are bundled as an x20- | |
| # dependency and then copied to WORKING_DIR. To update the app-bundle use | |
| # 'run_on_as_app_x20_packager.py'. | |
| # For showing benchmark data, also include the app in appSegmentBenchmarks in | |
| # the file <golem_repo>/config/r8/benchmarks.dart. | |
| APP_REPOSITORIES = [ | |
| # ... | |
| # Repo({ | |
| # 'name': ..., | |
| # 'url': ..., | |
| # 'revision': ..., | |
| # 'apps': [ | |
| # { | |
| # 'id': ..., | |
| # 'dir': ..., | |
| # 'module': ... (default app) | |
| # 'name': ..., | |
| # 'archives_base_name': ... (default same as module) | |
| # 'flavor': ... (default no flavor) | |
| # 'releaseTarget': ... (default <module>:assemble<flavor>Release | |
| # }, | |
| # ... | |
| # ] | |
| # }), | |
| # ... | |
| Repo({ | |
| 'name': 'android-suite', | |
| 'url': 'https://github.com/christofferqa/android-suite', | |
| 'revision': '46c96f214711cf6cdcb72cc0c94520ef418e3739', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.numix.calculator', | |
| 'dir': 'Calculator', | |
| 'name': 'numix-calculator', | |
| 'has_instrumentation_tests': True, | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'AnExplorer', | |
| 'url': 'https://github.com/christofferqa/AnExplorer', | |
| 'revision': '365927477b8eab4052a1882d5e358057ae3dee4d', | |
| 'apps': [ | |
| App({ | |
| 'id': 'dev.dworks.apps.anexplorer.pro', | |
| 'flavor': 'googleMobilePro', | |
| 'signed_apk_name': 'AnExplorer-googleMobileProRelease-4.0.3.apk', | |
| 'min_sdk': 17 | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'AntennaPod', | |
| 'url': 'https://github.com/christofferqa/AntennaPod.git', | |
| 'revision': '77e94f4783a16abe9cc5b78dc2d2b2b1867d8c06', | |
| 'apps': [ | |
| App({ | |
| 'id': 'de.danoeh.antennapod', | |
| 'flavor': 'play', | |
| 'min_sdk': 14, | |
| 'compile_sdk': 26 | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'applymapping', | |
| 'url': 'https://github.com/mkj-gram/applymapping', | |
| 'revision': 'e3ae14b8c16fa4718e5dea8f7ad00937701b3c48', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.example.applymapping', | |
| 'has_instrumentation_tests': True | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'apps-android-wikipedia', | |
| 'url': 'https://github.com/christofferqa/apps-android-wikipedia', | |
| 'revision': '686e8aa5682af8e6a905054b935dd2daa57e63ee', | |
| 'apps': [ | |
| App({ | |
| 'id': 'org.wikipedia', | |
| 'flavor': 'prod', | |
| 'signed_apk_name': 'app-prod-universal-release.apk' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'chanu', | |
| 'url': 'https://github.com/mkj-gram/chanu.git', | |
| 'revision': '6e53458f167b6d78398da60c20fd0da01a232617', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.chanapps.four.activity', | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'friendlyeats-android', | |
| 'url': 'https://github.com/christofferqa/friendlyeats-android.git', | |
| 'revision': '10091fa0ec37da12e66286559ad1b6098976b07b', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.google.firebase.example.fireeats' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'googlesamples', | |
| 'url': 'https://github.com/christofferqa/android-sunflower.git', | |
| 'revision': 'df0a082a0bcbeae253817e13daca3c7a7c54f67a', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.google.samples.apps.sunflower', | |
| 'name': 'android-sunflower', | |
| 'min_sdk': 19, | |
| 'compile_sdk': 28 | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Instabug-Android', | |
| 'url': 'https://github.com/christofferqa/Instabug-Android.git', | |
| 'revision': 'b8df78c96630a6537fbc07787b4990afc030cc0f', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.example.instabug' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'iosched', | |
| 'url': 'https://github.com/christofferqa/iosched.git', | |
| 'revision': '581cbbe2253711775dbccb753cdb53e7e506cb02', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.google.samples.apps.iosched', | |
| 'module': 'mobile', | |
| 'min_sdk': 21, | |
| 'compile_sdk': 29, | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'KISS', | |
| 'url': 'https://github.com/christofferqa/KISS', | |
| 'revision': '093da9ee0512e67192f62951c45a07a616fc3224', | |
| 'apps': [ | |
| App({ | |
| 'id': 'fr.neamar.kiss' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'materialistic', | |
| 'url': 'https://github.com/christofferqa/materialistic', | |
| 'revision': '2b2b2ee25ce9e672d5aab1dc90a354af1522b1d9', | |
| 'apps': [ | |
| App({ | |
| 'id': 'io.github.hidroh.materialistic' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Minimal-Todo', | |
| 'url': 'https://github.com/christofferqa/Minimal-Todo', | |
| 'revision': '9d8c73746762cd376b718858ec1e8783ca07ba7c', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.avjindersinghsekhon.minimaltodo' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'muzei', | |
| 'url': 'https://github.com/sgjesse/muzei.git', | |
| 'revision': 'a1f1d9b119faa0db09b6bbffe2318c4ec3679418', | |
| 'apps': [ | |
| App({ | |
| 'id': 'net.nurik.roman.muzei', | |
| 'module': 'main', | |
| 'archives_base_name': 'muzei', | |
| 'compile_sdk': 28, | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'NewPipe', | |
| 'url': 'https://github.com/christofferqa/NewPipe', | |
| 'revision': 'ed543099c7823be00f15d9340f94bdb7cb37d1e6', | |
| 'apps': [ | |
| App({ | |
| 'id': 'org.schabi.newpipe', | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'rover-android', | |
| 'url': 'https://github.com/mkj-gram/rover-android.git', | |
| 'revision': 'a5e155a1ed7d19b1cecd9a7b075e2852623a06bf', | |
| 'apps': [ | |
| App({ | |
| 'id': 'io.rover.app.debug', | |
| 'module': 'debug-app' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'santa-tracker-android', | |
| 'url': 'https://github.com/christofferqa/santa-tracker-android.git', | |
| 'revision': '8dee74be7d9ee33c69465a07088c53087d24a6dd', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.google.android.apps.santatracker', | |
| 'module': 'santa-tracker', | |
| 'min_sdk': 21, | |
| 'compile_sdk': 28 | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Signal-Android', | |
| 'url': 'https://github.com/mkj-gram/Signal-Android.git', | |
| 'revision': 'cd542cab9bf860e71504ecb1caaf0a8476ba3989', | |
| 'apps': [ | |
| App({ | |
| 'id': 'org.thoughtcrime.securesms', | |
| 'module': '', | |
| 'flavor': 'play', | |
| 'main_dex_rules': 'multidex-config.pro', | |
| 'signed_apk_name': 'Signal-play-release-4.32.7.apk' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Simple-Calendar', | |
| 'url': 'https://github.com/christofferqa/Simple-Calendar', | |
| 'revision': '82dad8c203eea5a0f0ddb513506d8f1de986ef2b', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.simplemobiletools.calendar.pro', | |
| 'signed_apk_name': 'calendar-release.apk', | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Simple-Camera', | |
| 'url': 'https://github.com/jsjeon/Simple-Camera', | |
| 'revision': '451fe188ab123e6956413b42e89839b44c05ac14', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.simplemobiletools.camera.pro', | |
| 'signed_apk_name': 'camera-release.apk', | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Simple-File-Manager', | |
| 'url': 'https://github.com/jsjeon/Simple-File-Manager', | |
| 'revision': '282b57d9e73f4d250cc844d8d73fd223509a141e', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.simplemobiletools.filemanager.pro', | |
| 'signed_apk_name': 'file-manager-release.apk', | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Simple-Gallery', | |
| 'url': 'https://github.com/jsjeon/Simple-Gallery', | |
| 'revision': '679125601eee7e057dfdfecd7bea6c4a6ac73ef9', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.simplemobiletools.gallery.pro', | |
| 'signed_apk_name': 'gallery-release.apk', | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'sqldelight', | |
| 'url': 'https://github.com/christofferqa/sqldelight.git', | |
| 'revision': '2e67a1126b6df05e4119d1e3a432fde51d76cdc8', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.example.sqldelight.hockey', | |
| 'module': 'sample/android', | |
| 'archives_base_name': 'android', | |
| 'min_sdk': 14, | |
| 'compile_sdk': 28 | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'tachiyomi', | |
| 'url': 'https://github.com/sgjesse/tachiyomi.git', | |
| 'revision': 'b15d2fe16864645055af6a745a62cc5566629798', | |
| 'apps': [ | |
| App({ | |
| 'id': 'eu.kanade.tachiyomi', | |
| 'flavor': 'dev', | |
| 'min_sdk': 16, | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'tivi', | |
| 'url': 'https://github.com/sgjesse/tivi.git', | |
| 'revision': '25c52e3593e7c98da4e537b49b29f6f67f88754d', | |
| 'apps': [ | |
| App({ | |
| 'id': 'app.tivi', | |
| 'min_sdk': 23, | |
| 'compile_sdk': 28, | |
| 'has_lint_task': False | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Tusky', | |
| 'url': 'https://github.com/mkj-gram/Tusky.git', | |
| 'revision': 'e7fbd190fb53bf9fde72253b816920cb6fe34518', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.keylesspalace.tusky', | |
| 'flavor': 'blue' | |
| }) | |
| ] | |
| }), | |
| Repo({ | |
| 'name': 'Vungle-Android-SDK', | |
| 'url': 'https://github.com/mkj-gram/Vungle-Android-SDK.git', | |
| 'revision': '138d3f18c027b61b195c98911f1c5ab7d87ad18b', | |
| 'apps': [ | |
| App({ | |
| 'id': 'com.publisher.vungle.sample', | |
| 'skip': True, # TODO(b/144058031) | |
| }) | |
| ] | |
| }) | |
| ] | |
| def signal_handler(signum, frame): | |
| subprocess.call(['pkill', 'java']) | |
| raise Exception('Got killed by %s' % signum) | |
| class EnsureNoGradleAlive(object): | |
| def __init__(self, active): | |
| self.active = active | |
| def __enter__(self): | |
| if self.active: | |
| # If we timeout and get a sigterm we should still kill all java | |
| signal.signal(signal.SIGTERM, signal_handler) | |
| print 'Running with wrapper that will kill java after' | |
| def __exit__(self, *_): | |
| if self.active: | |
| subprocess.call(['pkill', 'java']) | |
| def GetAllApps(): | |
| apps = [] | |
| for repo in APP_REPOSITORIES: | |
| for app in repo.apps: | |
| apps.append((app, repo)) | |
| return apps | |
| def GetAllAppNames(): | |
| return [app.name for (app, repo) in GetAllApps()] | |
| def GetAppWithName(query): | |
| for (app, repo) in GetAllApps(): | |
| if app.name == query: | |
| return (app, repo) | |
| assert False | |
| def ComputeSizeOfDexFilesInApk(apk): | |
| dex_size = 0 | |
| z = zipfile.ZipFile(apk, 'r') | |
| for filename in z.namelist(): | |
| if filename.endswith('.dex'): | |
| dex_size += z.getinfo(filename).file_size | |
| return dex_size | |
| def ExtractMarker(apk, temp_dir, options): | |
| r8_jar = os.path.join(temp_dir, 'r8.jar') | |
| r8lib_jar = os.path.join(temp_dir, 'r8lib.jar') | |
| # Use the copy of r8.jar or r8lib.jar if one is there. | |
| if os.path.isfile(r8_jar): | |
| cmd = [jdk.GetJavaExecutable(), '-ea', '-jar', r8_jar, 'extractmarker', apk] | |
| elif os.path.isfile(r8lib_jar): | |
| cmd = [jdk.GetJavaExecutable(), '-ea', '-cp', r8lib_jar, | |
| 'com.android.tools.r8.ExtractMarker', apk] | |
| else: | |
| script = os.path.join(utils.TOOLS_DIR, 'extractmarker.py') | |
| cmd = ['python', script, apk] | |
| utils.PrintCmd(cmd, quiet=options.quiet) | |
| stdout = subprocess.check_output(cmd) | |
| # Return the last line. | |
| lines = stdout.strip().splitlines() | |
| assert len(lines) >= 1 | |
| return lines[-1] | |
| def CheckIsBuiltWithExpectedR8(apk, temp_dir, shrinker, options): | |
| marker_raw = ExtractMarker(apk, temp_dir, options) | |
| # Marker should be a string on the following format (no whitespace): | |
| # ~~R8{"compilation-mode":"release", | |
| # "min-api":16, | |
| # "pg-map-id":"767707e", | |
| # "sha-1":"7111a35bae6d5185dcfb338d61074aca8426c006", | |
| # "version":"1.5.14-dev"} | |
| if not marker_raw.startswith('~~R8'): | |
| raise Exception( | |
| 'Expected marker to start with \'~~R8\' (was: {})'.format(marker_raw)) | |
| marker = json.loads(marker_raw[4:]) | |
| if options.hash: | |
| actual_hash = marker.get('sha-1') | |
| if actual_hash != options.hash: | |
| raise Exception( | |
| 'Expected APK to be built with R8 version {} (was: {})'.format( | |
| expected_hash, marker_raw)) | |
| return True | |
| expected_version = ( | |
| options.version | |
| if options.version | |
| else utils.getR8Version( | |
| os.path.join( | |
| temp_dir, | |
| 'r8lib.jar' if IsMinifiedR8(shrinker) else 'r8.jar'))) | |
| actual_version = marker.get('version') | |
| if actual_version != expected_version: | |
| raise Exception( | |
| 'Expected APK to be built with R8 version {} (was: {})'.format( | |
| expected_version, marker_raw)) | |
| return True | |
| def IsR8(shrinker): | |
| return 'r8' in shrinker | |
| def IsR8FullMode(shrinker): | |
| return shrinker == 'r8-full' or shrinker == 'r8-nolib-full' | |
| def IsLoggingEnabledFor(app, options): | |
| if options.no_logging: | |
| return False | |
| if options.app_logging_filter and app.name not in options.app_logging_filter: | |
| return False | |
| return True | |
| def IsMinifiedR8(shrinker): | |
| return 'nolib' not in shrinker | |
| def IsTrackedByGit(file): | |
| return subprocess.check_output(['git', 'ls-files', file]).strip() != '' | |
| def GitClone(repo, checkout_dir, quiet): | |
| result = subprocess.check_output( | |
| ['git', 'clone', repo.url, checkout_dir]).strip() | |
| head_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir) | |
| if repo.revision == head_rev: | |
| return result | |
| warn('Target revision is not head in {}.'.format(checkout_dir)) | |
| with utils.ChangedWorkingDirectory(checkout_dir, quiet=quiet): | |
| subprocess.check_output(['git', 'reset', '--hard', repo.revision]) | |
| return result | |
| def GitCheckout(file): | |
| return subprocess.check_output(['git', 'checkout', file]).strip() | |
| def InstallApkOnEmulator(apk_dest, options): | |
| cmd = ['adb', '-s', options.emulator_id, 'install', '-r', '-d', apk_dest] | |
| if options.quiet: | |
| with open(os.devnull, 'w') as devnull: | |
| subprocess.check_call(cmd, stdout=devnull) | |
| else: | |
| subprocess.check_call(cmd) | |
| def PercentageDiffAsString(before, after): | |
| if after < before: | |
| return '-' + str(round((1.0 - after / before) * 100)) + '%' | |
| else: | |
| return '+' + str(round((after - before) / before * 100)) + '%' | |
| def UninstallApkOnEmulator(app, options): | |
| process = subprocess.Popen( | |
| ['adb', '-s', options.emulator_id, 'uninstall', app.id], | |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| stdout, stderr = process.communicate() | |
| if stdout.strip() == 'Success': | |
| # Successfully uninstalled | |
| return | |
| if 'Unknown package: {}'.format(app.id) in stderr: | |
| # Application not installed | |
| return | |
| raise Exception( | |
| 'Unexpected result from `adb uninstall {}\nStdout: {}\nStderr: {}'.format( | |
| app.id, stdout, stderr)) | |
| def WaitForEmulator(options): | |
| stdout = subprocess.check_output(['adb', 'devices']) | |
| if '{}\tdevice'.format(options.emulator_id) in stdout: | |
| return True | |
| print('Emulator \'{}\' not connected; waiting for connection'.format( | |
| options.emulator_id)) | |
| time_waited = 0 | |
| while True: | |
| time.sleep(10) | |
| time_waited += 10 | |
| stdout = subprocess.check_output(['adb', 'devices']) | |
| if '{}\tdevice'.format(options.emulator_id) not in stdout: | |
| print('... still waiting for connection') | |
| if time_waited >= 5 * 60: | |
| return False | |
| else: | |
| return True | |
| def GetResultsForApp(app, repo, options, temp_dir): | |
| # Checkout and build in the build directory. | |
| repo_name = repo.name | |
| repo_checkout_dir = os.path.join(WORKING_DIR, repo_name) | |
| result = {} | |
| if not os.path.exists(repo_checkout_dir) and not options.golem: | |
| with utils.ChangedWorkingDirectory(WORKING_DIR, quiet=options.quiet): | |
| GitClone(repo, repo_checkout_dir, options.quiet) | |
| checkout_rev = utils.get_HEAD_sha1_for_checkout(repo_checkout_dir) | |
| if repo.revision != checkout_rev: | |
| msg = 'Checkout is not target revision for {} in {}.'.format( | |
| app.name, repo_checkout_dir) | |
| if options.ignore_versions: | |
| warn(msg) | |
| else: | |
| raise Exception(msg) | |
| result['status'] = 'success' | |
| app_checkout_dir = (os.path.join(repo_checkout_dir, app.dir) | |
| if app.dir else repo_checkout_dir) | |
| result_per_shrinker = BuildAppWithSelectedShrinkers( | |
| app, repo, options, app_checkout_dir, temp_dir) | |
| for shrinker, shrinker_result in result_per_shrinker.iteritems(): | |
| result[shrinker] = shrinker_result | |
| return result | |
| def BuildAppWithSelectedShrinkers( | |
| app, repo, options, checkout_dir, temp_dir): | |
| result_per_shrinker = {} | |
| with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet): | |
| for shrinker in options.shrinker: | |
| apk_dest = None | |
| result = {} | |
| proguard_config_file = None | |
| if not options.r8_compilation_steps_only: | |
| try: | |
| out_dir = os.path.join(checkout_dir, 'out', shrinker) | |
| (apk_dest, profile_dest_dir, res_proguard_config_file) = \ | |
| BuildAppWithShrinker( | |
| app, repo, shrinker, checkout_dir, out_dir, temp_dir, | |
| options) | |
| proguard_config_file = res_proguard_config_file | |
| dex_size = ComputeSizeOfDexFilesInApk(apk_dest) | |
| result['apk_dest'] = apk_dest | |
| result['build_status'] = 'success' | |
| result['dex_size'] = dex_size | |
| result['profile_dest_dir'] = profile_dest_dir | |
| profile = as_utils.ParseProfileReport(profile_dest_dir) | |
| result['profile'] = { | |
| task_name:duration for task_name, duration in profile.iteritems() | |
| if as_utils.IsGradleCompilerTask(task_name, shrinker)} | |
| except Exception as e: | |
| warn('Failed to build {} with {}'.format(app.name, shrinker)) | |
| if e: | |
| print('Error: ' + str(e)) | |
| result['build_status'] = 'failed' | |
| if result.get('build_status') == 'success': | |
| if options.monkey: | |
| result['monkey_status'] = 'success' if RunMonkey( | |
| app, options, apk_dest) else 'failed' | |
| if (result.get('build_status') == 'success' | |
| or options.r8_compilation_steps_only): | |
| if 'r8' in shrinker and options.r8_compilation_steps > 1: | |
| result['recompilation_results'] = \ | |
| ComputeRecompilationResults( | |
| app, repo, options, checkout_dir, temp_dir, shrinker, | |
| proguard_config_file) | |
| if result.get('build_status') == 'success': | |
| if options.run_tests and app.has_instrumentation_tests: | |
| result['instrumentation_test_results'] = \ | |
| ComputeInstrumentationTestResults( | |
| app, options, checkout_dir, out_dir, shrinker) | |
| result_per_shrinker[shrinker] = result | |
| if len(options.apps) > 1: | |
| print('') | |
| LogResultsForApp(app, result_per_shrinker, options) | |
| print('') | |
| return result_per_shrinker | |
| def BuildAppWithShrinker( | |
| app, repo, shrinker, checkout_dir, out_dir, temp_dir, options, | |
| keepRuleSynthesisForRecompilation=False): | |
| print('[{}] Building {} with {}{}'.format( | |
| datetime.now().strftime("%H:%M:%S"), | |
| app.name, | |
| shrinker, | |
| ' for recompilation' if keepRuleSynthesisForRecompilation else '')) | |
| # Add settings.gradle file if it is not present to prevent gradle from finding | |
| # the settings.gradle file in the r8 root when apps are placed under | |
| # $R8/build. | |
| as_utils.add_settings_gradle(checkout_dir, app.name) | |
| # Add 'r8.jar' to top-level build.gradle. | |
| as_utils.add_r8_dependency(checkout_dir, temp_dir, IsMinifiedR8(shrinker)) | |
| archives_base_name = app.archives_base_name | |
| if not os.path.exists(out_dir): | |
| os.makedirs(out_dir) | |
| # Set -printconfiguration in Proguard rules. | |
| proguard_config_dest = os.path.abspath( | |
| os.path.join(out_dir, 'proguard-rules.pro')) | |
| as_utils.SetPrintConfigurationDirective( | |
| app, checkout_dir, proguard_config_dest) | |
| env_vars = {} | |
| env_vars['JAVA_HOME'] = jdk.GetJdk8Home() | |
| env_vars['ANDROID_HOME'] = utils.getAndroidHome() | |
| if not options.disable_assertions: | |
| env_vars['JAVA_OPTS'] = '-ea:com.android.tools.r8...' | |
| release_target = app.releaseTarget | |
| if not release_target: | |
| app_module = app.module.replace('/', ':') | |
| app_flavor = (app.flavor.capitalize() if app.flavor else '') + 'Release' | |
| release_target = app_module + ':' + 'assemble' + app_flavor | |
| # Build using gradle. | |
| args = [release_target, | |
| '-g=' + os.path.join(checkout_dir, GRADLE_USER_HOME), | |
| '-Pandroid.enableR8=' + str(IsR8(shrinker)).lower(), | |
| '-Pandroid.enableR8.fullMode=' + str(IsR8FullMode(shrinker)).lower()] | |
| if app.has_lint_task: | |
| args.extend(['-x', app_module + ':lintVital' + app_flavor]) | |
| if options.bot: | |
| args.extend(['--console=plain', '--info']) | |
| # Warm up gradle if pre_runs > 0. For posterity we generate the same sequence | |
| # as the benchmarking at https://github.com/madsager/santa-tracker-android. | |
| for i in range(0, options.gradle_pre_runs): | |
| if i == 0: | |
| utils.RunGradlew( | |
| ["--stop"], | |
| env_vars=env_vars, | |
| quiet=options.quiet, | |
| logging=IsLoggingEnabledFor(app, options), | |
| use_daemon=options.use_daemon) | |
| utils.RunGradlew( | |
| args, | |
| env_vars=env_vars, | |
| quiet=options.quiet, | |
| clean=i > 0, | |
| use_daemon=options.use_daemon, | |
| logging=IsLoggingEnabledFor(app, options)) | |
| if keepRuleSynthesisForRecompilation: | |
| args.append('-Dcom.android.tools.r8.keepRuleSynthesisForRecompilation=true') | |
| if options.gradle_flags: | |
| args.extend(options.gradle_flags.split(' ')) | |
| args.append('--profile') | |
| stdout = utils.RunGradlew( | |
| args, | |
| env_vars=env_vars, | |
| quiet=options.quiet, | |
| use_daemon=options.use_daemon, | |
| logging=IsLoggingEnabledFor(app, options)) | |
| apk_base_name = (archives_base_name | |
| + (('-' + app.flavor) if app.flavor else '') + '-release') | |
| signed_apk_name = ( | |
| app.signed_apk_name | |
| if app.signed_apk_name | |
| else apk_base_name + '.apk') | |
| unsigned_apk_name = apk_base_name + '-unsigned.apk' | |
| build_dir = os.path.join(app.module, app.build_dir) | |
| build_output_apks = os.path.join(build_dir, 'outputs', 'apk') | |
| if app.flavor: | |
| build_output_apks = os.path.join(build_output_apks, app.flavor, 'release') | |
| else: | |
| build_output_apks = os.path.join(build_output_apks, 'release') | |
| signed_apk = os.path.join(build_output_apks, signed_apk_name) | |
| unsigned_apk = os.path.join(build_output_apks, unsigned_apk_name) | |
| assert os.path.isfile(signed_apk) or os.path.isfile(unsigned_apk), ( | |
| "Expected a file to be present at {} or {}, found: {}\n" | |
| "Standard out from compilation: {}".format( | |
| signed_apk, | |
| unsigned_apk, | |
| ', '.join( | |
| as_utils.ListFiles(build_dir, lambda x : x.endswith('.apk'))), | |
| stdout)) | |
| if options.sign_apks and not os.path.isfile(signed_apk): | |
| assert os.path.isfile(unsigned_apk) | |
| if options.sign_apks: | |
| apk_utils.sign_with_apksigner( | |
| unsigned_apk, | |
| signed_apk, | |
| options.keystore, | |
| options.keystore_password, | |
| quiet=options.quiet, | |
| logging=IsLoggingEnabledFor(app, options)) | |
| if os.path.isfile(signed_apk): | |
| apk_dest = os.path.join(out_dir, signed_apk_name) | |
| as_utils.MoveFile(signed_apk, apk_dest, quiet=options.quiet) | |
| else: | |
| apk_dest = os.path.join(out_dir, unsigned_apk_name) | |
| as_utils.MoveFile(unsigned_apk, apk_dest, quiet=options.quiet) | |
| assert ('r8' not in shrinker | |
| or CheckIsBuiltWithExpectedR8(apk_dest, temp_dir, shrinker, options)) | |
| profile_dest_dir = os.path.join(out_dir, 'profile') | |
| as_utils.MoveProfileReportTo(profile_dest_dir, stdout, quiet=options.quiet) | |
| # Ensure that the gradle daemon is stopped if we are running with it. | |
| if options.use_daemon: | |
| utils.RunGradlew(['--stop', '-g=' + os.path.join(checkout_dir, GRADLE_USER_HOME)]) | |
| return (apk_dest, profile_dest_dir, proguard_config_dest) | |
| def ComputeInstrumentationTestResults( | |
| app, options, checkout_dir, out_dir, shrinker): | |
| args = ['connectedAndroidTest', | |
| '-Pandroid.enableR8=' + str(IsR8(shrinker)).lower(), | |
| '-Pandroid.enableR8.fullMode=' + str(IsR8FullMode(shrinker)).lower()] | |
| env_vars = { 'ANDROID_SERIAL': options.emulator_id } | |
| stdout = utils.RunGradlew( | |
| args, | |
| env_vars=env_vars, | |
| quiet=options.quiet, | |
| fail=False, | |
| logging=IsLoggingEnabledFor(app, options), | |
| use_daemon=options.use_daemon) | |
| xml_test_result_dest = os.path.join(out_dir, 'test_result') | |
| as_utils.MoveXMLTestResultFileTo( | |
| xml_test_result_dest, stdout, quiet=options.quiet) | |
| with open(xml_test_result_dest, 'r') as f: | |
| xml_test_result_contents = f.read() | |
| xml_document = minidom.parseString(xml_test_result_contents) | |
| testsuite_element = xml_document.documentElement | |
| return { | |
| 'xml_test_result_dest': xml_test_result_dest, | |
| 'tests': int(testsuite_element.getAttribute('tests')), | |
| 'failures': int(testsuite_element.getAttribute('failures')), | |
| 'errors': int(testsuite_element.getAttribute('errors')), | |
| 'skipped': int(testsuite_element.getAttribute('skipped')) | |
| } | |
| def ComputeRecompilationResults( | |
| app, repo, options, checkout_dir, temp_dir, shrinker, proguard_config_file): | |
| recompilation_results = [] | |
| # Build app with gradle using -D...keepRuleSynthesisForRecompilation=true. | |
| out_dir = os.path.join(checkout_dir, 'out', shrinker + '-1') | |
| (apk_dest, profile_dest_dir, ext_proguard_config_file) = \ | |
| BuildAppWithShrinker( | |
| app, repo, shrinker, checkout_dir, out_dir, | |
| temp_dir, options, keepRuleSynthesisForRecompilation=True) | |
| recompilation_result = { | |
| 'apk_dest': apk_dest, | |
| 'build_status': 'success', | |
| 'dex_size': ComputeSizeOfDexFilesInApk(apk_dest), | |
| 'monkey_status': 'skipped' | |
| } | |
| recompilation_results.append(recompilation_result) | |
| # Sanity check that keep rules have changed. If we are only doing | |
| # recompilation, the passed in proguard_config_file is None. | |
| if proguard_config_file: | |
| with open(ext_proguard_config_file) as new: | |
| with open(proguard_config_file) as old: | |
| assert( | |
| sum(1 for line in new | |
| if line.strip() and '-printconfiguration' not in line) | |
| > | |
| sum(1 for line in old | |
| if line.strip() and '-printconfiguration' not in line)) | |
| # Extract min-sdk and target-sdk | |
| (min_sdk, compile_sdk) = \ | |
| as_utils.GetMinAndCompileSdk(app, checkout_dir, apk_dest) | |
| # Now rebuild generated apk. | |
| previous_apk = apk_dest | |
| # We may need main dex rules when re-compiling with R8 as standalone. | |
| main_dex_rules = None | |
| if app.main_dex_rules: | |
| main_dex_rules = os.path.join(checkout_dir, app.main_dex_rules) | |
| for i in range(1, options.r8_compilation_steps): | |
| try: | |
| recompiled_apk_dest = os.path.join( | |
| checkout_dir, 'out', shrinker, 'app-release-{}.apk'.format(i)) | |
| if not os.path.exists(os.path.dirname(recompiled_apk_dest)): | |
| os.makedirs(os.path.dirname(recompiled_apk_dest)) | |
| RebuildAppWithShrinker( | |
| app, previous_apk, recompiled_apk_dest, | |
| ext_proguard_config_file, shrinker, min_sdk, compile_sdk, | |
| options, temp_dir, main_dex_rules) | |
| recompilation_result = { | |
| 'apk_dest': recompiled_apk_dest, | |
| 'build_status': 'success', | |
| 'dex_size': ComputeSizeOfDexFilesInApk(recompiled_apk_dest) | |
| } | |
| if options.monkey: | |
| recompilation_result['monkey_status'] = 'success' if RunMonkey( | |
| app, options, recompiled_apk_dest) else 'failed' | |
| recompilation_results.append(recompilation_result) | |
| previous_apk = recompiled_apk_dest | |
| except Exception as e: | |
| warn('Failed to recompile {} with {}'.format( | |
| app.name, shrinker)) | |
| recompilation_results.append({ 'build_status': 'failed' }) | |
| break | |
| return recompilation_results | |
| def RebuildAppWithShrinker( | |
| app, apk, apk_dest, proguard_config_file, shrinker, min_sdk, compile_sdk, | |
| options, temp_dir, main_dex_rules): | |
| assert 'r8' in shrinker | |
| assert apk_dest.endswith('.apk') | |
| print('Rebuilding {} with {}'.format(app.name, shrinker)) | |
| # Compile given APK with shrinker to temporary zip file. | |
| android_jar = utils.get_android_jar(compile_sdk) | |
| r8_jar = os.path.join( | |
| temp_dir, 'r8lib.jar' if IsMinifiedR8(shrinker) else 'r8.jar') | |
| zip_dest = apk_dest[:-4] + '.zip' | |
| # TODO(christofferqa): Entry point should be CompatProguard if the shrinker | |
| # is 'r8'. | |
| entry_point = 'com.android.tools.r8.R8' | |
| cmd = ([jdk.GetJavaExecutable()] + | |
| (['-ea:com.android.tools.r8...'] | |
| if not options.disable_assertions | |
| else []) + | |
| ['-cp', r8_jar, entry_point, | |
| '--release', '--min-api', str(min_sdk), | |
| '--pg-conf', proguard_config_file, | |
| '--lib', android_jar, | |
| '--output', zip_dest, | |
| apk]) | |
| for android_optional_jar in utils.get_android_optional_jars(compile_sdk): | |
| cmd.append('--lib') | |
| cmd.append(android_optional_jar) | |
| if main_dex_rules: | |
| cmd.append('--main-dex-rules') | |
| cmd.append(main_dex_rules) | |
| utils.RunCmd( | |
| cmd, quiet=options.quiet, logging=IsLoggingEnabledFor(app, options)) | |
| # Make a copy of the given APK, move the newly generated dex files into the | |
| # copied APK, and then sign the APK. | |
| apk_masseur.masseur( | |
| apk, dex=zip_dest, resources='META-INF/services/*', out=apk_dest, | |
| quiet=options.quiet, logging=IsLoggingEnabledFor(app, options), | |
| keystore=options.keystore) | |
| def RunMonkey(app, options, apk_dest): | |
| if not WaitForEmulator(options): | |
| return False | |
| UninstallApkOnEmulator(app, options) | |
| InstallApkOnEmulator(apk_dest, options) | |
| number_of_events_to_generate = options.monkey_events | |
| # Intentionally using a constant seed such that the monkey generates the same | |
| # event sequence for each shrinker. | |
| random_seed = 42 | |
| cmd = ['adb', '-s', options.emulator_id, 'shell', 'monkey', '-p', app.id, | |
| '-s', str(random_seed), str(number_of_events_to_generate)] | |
| try: | |
| stdout = utils.RunCmd( | |
| cmd, quiet=options.quiet, logging=IsLoggingEnabledFor(app, options)) | |
| succeeded = ( | |
| 'Events injected: {}'.format(number_of_events_to_generate) in stdout) | |
| except subprocess.CalledProcessError as e: | |
| succeeded = False | |
| UninstallApkOnEmulator(app, options) | |
| return succeeded | |
| def LogResultsForApps(result_per_shrinker_per_app, options): | |
| print('') | |
| app_errors = 0 | |
| for (app, result_per_shrinker) in result_per_shrinker_per_app: | |
| app_errors += (1 if LogResultsForApp(app, result_per_shrinker, options) | |
| else 0) | |
| return app_errors | |
| def LogResultsForApp(app, result_per_shrinker, options): | |
| if options.print_dexsegments: | |
| LogSegmentsForApp(app, result_per_shrinker, options) | |
| return False | |
| else: | |
| return LogComparisonResultsForApp(app, result_per_shrinker, options) | |
| def LogSegmentsForApp(app, result_per_shrinker, options): | |
| for shrinker in SHRINKERS: | |
| if shrinker not in result_per_shrinker: | |
| continue | |
| result = result_per_shrinker[shrinker]; | |
| benchmark_name = '{}-{}'.format(options.print_dexsegments, app.name) | |
| utils.print_dexsegments(benchmark_name, [result.get('apk_dest')]) | |
| duration = sum(result.get('profile').values()) | |
| print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration * 1000)) | |
| print('%s-Total(CodeSize): %s' % (benchmark_name, result.get('dex_size'))) | |
| def LogComparisonResultsForApp(app, result_per_shrinker, options): | |
| print(app.name + ':') | |
| app_error = False | |
| if result_per_shrinker.get('status', 'success') != 'success': | |
| error_message = result_per_shrinker.get('error_message') | |
| print(' skipped ({})'.format(error_message)) | |
| return | |
| proguard_result = result_per_shrinker.get('pg', {}) | |
| proguard_dex_size = float(proguard_result.get('dex_size', -1)) | |
| proguard_duration = sum(proguard_result.get('profile', {}).values()) | |
| for shrinker in SHRINKERS: | |
| if shrinker not in result_per_shrinker: | |
| continue | |
| result = result_per_shrinker.get(shrinker) | |
| build_status = result.get('build_status') | |
| if build_status != 'success' and build_status is not None: | |
| app_error = True | |
| warn(' {}: {}'.format(shrinker, build_status)) | |
| continue | |
| print(' {}:'.format(shrinker)) | |
| dex_size = result.get('dex_size') | |
| msg = ' dex size: {}'.format(dex_size) | |
| if dex_size != proguard_dex_size and proguard_dex_size >= 0: | |
| msg = '{} ({}, {})'.format( | |
| msg, dex_size - proguard_dex_size, | |
| PercentageDiffAsString(proguard_dex_size, dex_size)) | |
| success(msg) if dex_size < proguard_dex_size else warn(msg) | |
| else: | |
| print(msg) | |
| profile = result.get('profile') | |
| if profile: | |
| duration = sum(profile.values()) | |
| msg = ' performance: {}s'.format(duration) | |
| if duration != proguard_duration and proguard_duration > 0: | |
| msg = '{} ({}s, {})'.format( | |
| msg, duration - proguard_duration, | |
| PercentageDiffAsString(proguard_duration, duration)) | |
| success(msg) if duration < proguard_duration else warn(msg) | |
| else: | |
| print(msg) | |
| if len(profile) >= 2: | |
| for task_name, task_duration in profile.iteritems(): | |
| print(' {}: {}s'.format(task_name, task_duration)) | |
| if options.monkey: | |
| monkey_status = result.get('monkey_status') | |
| if monkey_status != 'success': | |
| app_error = True | |
| warn(' monkey: {}'.format(monkey_status)) | |
| else: | |
| success(' monkey: {}'.format(monkey_status)) | |
| recompilation_results = result.get('recompilation_results', []) | |
| i = 0 | |
| for recompilation_result in recompilation_results: | |
| build_status = recompilation_result.get('build_status') | |
| if build_status != 'success': | |
| app_error = True | |
| print(' recompilation #{}: {}'.format(i, build_status)) | |
| else: | |
| dex_size = recompilation_result.get('dex_size') | |
| print(' recompilation #{}'.format(i)) | |
| print(' dex size: {}'.format(dex_size)) | |
| if options.monkey: | |
| monkey_status = recompilation_result.get('monkey_status') | |
| msg = ' monkey: {}'.format(monkey_status) | |
| if monkey_status == 'success': | |
| success(msg) | |
| elif monkey_status == 'skipped': | |
| print(msg) | |
| else: | |
| warn(msg) | |
| i += 1 | |
| if options.run_tests and 'instrumentation_test_results' in result: | |
| instrumentation_test_results = \ | |
| result.get('instrumentation_test_results') | |
| succeeded = ( | |
| instrumentation_test_results.get('failures') | |
| + instrumentation_test_results.get('errors') | |
| + instrumentation_test_results.get('skipped')) == 0 | |
| if succeeded: | |
| success(' tests: succeeded') | |
| else: | |
| app_error = True | |
| warn( | |
| ' tests: failed (failures: {}, errors: {}, skipped: {})' | |
| .format( | |
| instrumentation_test_results.get('failures'), | |
| instrumentation_test_results.get('errors'), | |
| instrumentation_test_results.get('skipped'))) | |
| return app_error | |
| def ParseOptions(argv): | |
| result = optparse.OptionParser() | |
| result.add_option('--app', | |
| help='What app to run on', | |
| choices=GetAllAppNames(), | |
| action='append') | |
| result.add_option('--bot', | |
| help='Running on bot, use third_party dependency.', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--disable-assertions', '--disable_assertions', | |
| help='Disable assertions when compiling', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--download-only', '--download_only', | |
| help='Whether to download apps without any compilation', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--emulator-id', '--emulator_id', | |
| help='Id of the emulator to use', | |
| default='emulator-5554') | |
| result.add_option('--golem', | |
| help='Running on golem, do not download', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--gradle-flags', '--gradle_flags', | |
| help='Flags to pass in to gradle') | |
| result.add_option('--gradle-pre-runs', '--gradle_pre_runs', | |
| help='Do rounds of compilations to warm up gradle', | |
| default=0, | |
| type=int) | |
| result.add_option('--hash', | |
| help='The version of R8 to use') | |
| result.add_option('--ignore-versions', '--ignore_versions', | |
| help='Allow checked-out app to differ in revision from ' | |
| 'pinned', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--keystore', | |
| help='Path to app.keystore', | |
| default=os.path.join(utils.TOOLS_DIR, 'debug.keystore')) | |
| result.add_option('--keystore-password', '--keystore_password', | |
| help='Password for app.keystore', | |
| default='android') | |
| result.add_option('--app-logging-filter', '--app_logging_filter', | |
| help='The apps for which to turn on logging', | |
| action='append') | |
| result.add_option('--monkey', | |
| help='Whether to install and run app(s) with monkey', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--monkey-events', '--monkey_events', | |
| help='Number of events that the monkey should trigger', | |
| default=250, | |
| type=int) | |
| result.add_option('--no-build', '--no_build', | |
| help='Run without building ToT first (only when using ToT)', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--no-logging', '--no_logging', | |
| help='Disable logging except for errors', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--print-dexsegments', | |
| metavar='BENCHMARKNAME', | |
| help='Print the sizes of individual dex segments as ' + | |
| '\'<BENCHMARKNAME>-<APP>-<segment>(CodeSize): ' | |
| '<bytes>\'') | |
| result.add_option('--quiet', | |
| help='Disable verbose logging', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--r8-compilation-steps', '--r8_compilation_steps', | |
| help='Number of times R8 should be run on each app', | |
| default=2, | |
| type=int) | |
| result.add_option('--r8-compilation-steps-only', '--r8_compilation_steps_only', | |
| help='Specify to only run compilation steps', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--run-tests', '--run_tests', | |
| help='Whether to run instrumentation tests', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--sign-apks', '--sign_apks', | |
| help='Whether the APKs should be signed', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--shrinker', | |
| help='The shrinkers to use (by default, all are run)', | |
| action='append') | |
| result.add_option('--use-daemon', '--use_daemon', | |
| help='Whether to use a gradle daemon', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--version', | |
| help='The version of R8 to use (e.g., 1.4.51)') | |
| (options, args) = result.parse_args(argv) | |
| if options.app: | |
| options.apps = [(app, repo) for (app, repo) in GetAllApps() | |
| if app.name in options.app] | |
| del options.app | |
| else: | |
| options.apps = GetAllApps() | |
| if options.app_logging_filter: | |
| for app_name in options.app_logging_filter: | |
| assert any(app.name == app_name for (app, repo) in options.apps) | |
| if options.shrinker: | |
| for shrinker in options.shrinker: | |
| assert shrinker in SHRINKERS | |
| else: | |
| options.shrinker = [shrinker for shrinker in SHRINKERS] | |
| if options.hash or options.version: | |
| # No need to build R8 if a specific version should be used. | |
| options.no_build = True | |
| if 'r8-nolib' in options.shrinker: | |
| warn('Skipping shrinker r8-nolib because a specific version ' | |
| + 'of r8 was specified') | |
| options.shrinker.remove('r8-nolib') | |
| if 'r8-nolib-full' in options.shrinker: | |
| warn('Skipping shrinker r8-nolib-full because a specific version ' | |
| + 'of r8 was specified') | |
| options.shrinker.remove('r8-nolib-full') | |
| assert not options.r8_compilation_steps_only \ | |
| or options.r8_compilation_steps > 1 | |
| return (options, args) | |
| def clone_repositories(quiet): | |
| # Clone repositories into WORKING_DIR. | |
| with utils.ChangedWorkingDirectory(WORKING_DIR): | |
| for repo in APP_REPOSITORIES: | |
| repo_dir = os.path.join(WORKING_DIR, repo.name) | |
| if not os.path.exists(repo_dir): | |
| GitClone(repo, repo_dir, quiet) | |
| def main(argv): | |
| (options, args) = ParseOptions(argv) | |
| if options.bot: | |
| utils.DownloadFromGoogleCloudStorage(utils.OPENSOURCE_APPS_SHA_FILE) | |
| utils.DownloadFromGoogleCloudStorage(utils.ANDROID_SDK + '.tar.gz.sha1', | |
| bucket='r8-deps-internal', | |
| auth=True) | |
| if os.path.exists(WORKING_DIR): | |
| shutil.rmtree(WORKING_DIR) | |
| shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR) | |
| os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join( | |
| utils.ANDROID_SDK) | |
| os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3' | |
| # TODO(b/141081520): Set to True once fixed. | |
| options.no_logging = False | |
| # TODO(b/141081520): Remove logging filter once fixed. | |
| options.app_logging_filter = ['sqldelight'] | |
| options.shrinker = ['r8', 'r8-full'] | |
| print(options.shrinker) | |
| if options.golem: | |
| golem.link_third_party() | |
| if os.path.exists(WORKING_DIR): | |
| shutil.rmtree(WORKING_DIR) | |
| shutil.copytree(utils.OPENSOURCE_APPS_FOLDER, WORKING_DIR) | |
| os.environ[utils.ANDROID_HOME_ENVIROMENT_NAME] = os.path.join( | |
| utils.ANDROID_SDK) | |
| os.environ[utils.ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME] = '28.0.3' | |
| options.disable_assertions = True | |
| options.ignore_versions = True | |
| options.no_build = True | |
| options.r8_compilation_steps = 1 | |
| options.quiet = True | |
| options.gradle_pre_runs = 2 | |
| options.use_daemon = True | |
| options.no_logging = True | |
| if not os.path.exists(WORKING_DIR): | |
| os.makedirs(WORKING_DIR) | |
| if options.download_only: | |
| clone_repositories(options.quiet) | |
| return | |
| with utils.TempDir() as temp_dir: | |
| if not (options.no_build or options.golem): | |
| gradle.RunGradle(['r8', '-Pno_internal']) | |
| build_r8lib = False | |
| for shrinker in options.shrinker: | |
| if IsMinifiedR8(shrinker): | |
| build_r8lib = True | |
| if build_r8lib: | |
| gradle.RunGradle(['r8lib', '-Pno_internal']) | |
| if options.hash: | |
| # Download r8-<hash>.jar from | |
| # https://storage.googleapis.com/r8-releases/raw/. | |
| target = 'r8-{}.jar'.format(options.hash) | |
| update_prebuilds_in_android.download_hash( | |
| temp_dir, 'com/android/tools/r8/' + options.hash, target) | |
| as_utils.MoveFile( | |
| os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'), | |
| quiet=options.quiet) | |
| elif options.version: | |
| # Download r8-<version>.jar from | |
| # https://storage.googleapis.com/r8-releases/raw/. | |
| target = 'r8-{}.jar'.format(options.version) | |
| update_prebuilds_in_android.download_version( | |
| temp_dir, 'com/android/tools/r8/' + options.version, target) | |
| as_utils.MoveFile( | |
| os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'), | |
| quiet=options.quiet) | |
| else: | |
| # Make a copy of r8.jar and r8lib.jar such that they stay the same for | |
| # the entire execution of this script. | |
| if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker: | |
| assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar' | |
| shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar')) | |
| if 'r8' in options.shrinker or 'r8-full' in options.shrinker: | |
| assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar' | |
| shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar')) | |
| result_per_shrinker_per_app = [] | |
| # If we are running on golem we kill all java processes after the run | |
| # to ensure no hanging gradle daemons. | |
| with EnsureNoGradleAlive(options.golem): | |
| for (app, repo) in options.apps: | |
| if app.skip: | |
| continue | |
| result_per_shrinker_per_app.append( | |
| (app, GetResultsForApp(app, repo, options, temp_dir))) | |
| return LogResultsForApps(result_per_shrinker_per_app, options) | |
| def success(message): | |
| CGREEN = '\033[32m' | |
| CEND = '\033[0m' | |
| print(CGREEN + message + CEND) | |
| def warn(message): | |
| CRED = '\033[91m' | |
| CEND = '\033[0m' | |
| print(CRED + message + CEND) | |
| if __name__ == '__main__': | |
| sys.exit(main(sys.argv[1:])) |