Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 1 | # Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file |
| 2 | # for details. All rights reserved. Use of this source code is governed by a |
| 3 | # BSD-style license that can be found in the LICENSE file. |
| 4 | |
| 5 | # Different utility functions used accross scripts |
| 6 | |
| 7 | import hashlib |
Søren Gjesse | 6e5e584 | 2019-09-03 08:48:30 +0200 | [diff] [blame] | 8 | import json |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 9 | import os |
Tamas Kenez | 82efeb5 | 2017-06-12 13:56:22 +0200 | [diff] [blame] | 10 | import re |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 11 | import shutil |
| 12 | import subprocess |
| 13 | import sys |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 14 | import tarfile |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 15 | import tempfile |
Morten Krogh-Jespersen | 16e925d | 2019-01-25 14:40:38 +0100 | [diff] [blame] | 16 | import zipfile |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 17 | |
Ian Zerny | 3709765 | 2019-04-11 13:13:27 +0200 | [diff] [blame] | 18 | import defines |
| 19 | import jdk |
| 20 | |
Christoffer Quist Adamsen | 17879c1 | 2019-01-22 16:13:54 +0100 | [diff] [blame] | 21 | ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}' |
| 22 | ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar') |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 23 | TOOLS_DIR = defines.TOOLS_DIR |
| 24 | REPO_ROOT = defines.REPO_ROOT |
| 25 | THIRD_PARTY = defines.THIRD_PARTY |
Morten Krogh-Jespersen | 220e570 | 2019-02-27 12:57:01 +0100 | [diff] [blame] | 26 | ANDROID_SDK = os.path.join(THIRD_PARTY, 'android_sdk') |
Tamas Kenez | fc34cd8 | 2017-07-13 12:43:57 +0200 | [diff] [blame] | 27 | MEMORY_USE_TMP_FILE = 'memory_use.tmp' |
Tamas Kenez | 02bff03 | 2017-07-18 12:13:58 +0200 | [diff] [blame] | 28 | DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)') |
Mads Ager | 12a56bc | 2017-11-27 11:51:25 +0100 | [diff] [blame] | 29 | BUILD = os.path.join(REPO_ROOT, 'build') |
Ian Zerny | 0f5fc73 | 2018-11-15 14:34:41 +0100 | [diff] [blame] | 30 | BUILD_DEPS_DIR = os.path.join(BUILD, 'deps') |
| 31 | BUILD_MAIN_DIR = os.path.join(BUILD, 'classes', 'main') |
| 32 | BUILD_TEST_DIR = os.path.join(BUILD, 'classes', 'test') |
Mads Ager | 12a56bc | 2017-11-27 11:51:25 +0100 | [diff] [blame] | 33 | LIBS = os.path.join(BUILD, 'libs') |
| 34 | GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense') |
Mads Ager | a4911eb | 2017-11-22 13:19:36 +0100 | [diff] [blame] | 35 | SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java') |
Søren Gjesse | 6e5e584 | 2019-09-03 08:48:30 +0200 | [diff] [blame] | 36 | TEST_ROOT = os.path.join(REPO_ROOT, 'src', 'test', 'java') |
Ian Zerny | 59dfa4c | 2019-10-25 10:34:36 +0200 | [diff] [blame] | 37 | REPO_SOURCE = 'https://r8.googlesource.com/r8' |
Søren Gjesse | dc9d8a2 | 2017-10-12 12:40:59 +0200 | [diff] [blame] | 38 | |
| 39 | D8 = 'd8' |
| 40 | R8 = 'r8' |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 41 | R8LIB = 'r8lib' |
Morten Krogh-Jespersen | e28db46 | 2019-01-09 13:32:15 +0100 | [diff] [blame] | 42 | R8LIB_NO_DEPS = 'r8LibNoDeps' |
Mads Ager | b10c07f | 2017-11-27 13:25:52 +0100 | [diff] [blame] | 43 | R8_SRC = 'sourceJar' |
Søren Gjesse | dc9d8a2 | 2017-10-12 12:40:59 +0200 | [diff] [blame] | 44 | COMPATDX = 'compatdx' |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 45 | COMPATDXLIB = 'compatdxlib' |
Søren Gjesse | dc9d8a2 | 2017-10-12 12:40:59 +0200 | [diff] [blame] | 46 | COMPATPROGUARD = 'compatproguard' |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 47 | COMPATPROGUARDLIB = 'compatproguardlib' |
Søren Gjesse | dc9d8a2 | 2017-10-12 12:40:59 +0200 | [diff] [blame] | 48 | |
Rico Wind | 74fab30 | 2017-10-02 07:25:33 +0200 | [diff] [blame] | 49 | D8_JAR = os.path.join(LIBS, 'd8.jar') |
| 50 | R8_JAR = os.path.join(LIBS, 'r8.jar') |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 51 | R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar') |
Mads Ager | b10c07f | 2017-11-27 13:25:52 +0100 | [diff] [blame] | 52 | R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar') |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 53 | R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar') |
Tamas Kenez | 180be09 | 2018-12-05 15:23:06 +0100 | [diff] [blame] | 54 | R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar') |
Rico Wind | 74fab30 | 2017-10-02 07:25:33 +0200 | [diff] [blame] | 55 | COMPATDX_JAR = os.path.join(LIBS, 'compatdx.jar') |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 56 | COMPATDXLIB_JAR = os.path.join(LIBS, 'compatdxlib.jar') |
Søren Gjesse | dc9d8a2 | 2017-10-12 12:40:59 +0200 | [diff] [blame] | 57 | COMPATPROGUARD_JAR = os.path.join(LIBS, 'compatproguard.jar') |
Tamas Kenez | 03ab76f | 2018-12-07 14:33:25 +0100 | [diff] [blame] | 58 | COMPATPROGUARDLIB_JAR = os.path.join(LIBS, 'compatproguardlib.jar') |
Mads Ager | 12a56bc | 2017-11-27 11:51:25 +0100 | [diff] [blame] | 59 | MAVEN_ZIP = os.path.join(LIBS, 'r8.zip') |
Rico Wind | 8fc8bfa | 2019-03-22 09:57:36 +0100 | [diff] [blame] | 60 | MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip') |
Søren Gjesse | 6e5e584 | 2019-09-03 08:48:30 +0200 | [diff] [blame] | 61 | # TODO(b/134732760): The JSON configuration should be moved. |
| 62 | DESUGAR_CONFIGURATION = os.path.join( |
| 63 | TEST_ROOT, |
| 64 | 'com', |
| 65 | 'android', |
| 66 | 'tools', |
| 67 | 'r8', |
| 68 | 'desugar', |
| 69 | 'corelib', |
| 70 | 'desugar_jdk_libs.json') |
| 71 | DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join( |
| 72 | LIBS, 'desugar_jdk_libs_configuration.zip') |
Mads Ager | 12a56bc | 2017-11-27 11:51:25 +0100 | [diff] [blame] | 73 | GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE') |
Mathias Rav | 3fb4a3a | 2018-05-29 15:41:36 +0200 | [diff] [blame] | 74 | RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar') |
Mathias Rav | b46dc00 | 2018-06-06 09:37:11 +0200 | [diff] [blame] | 75 | R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt') |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 76 | RETRACE_JAR = os.path.join( |
| 77 | THIRD_PARTY, |
| 78 | 'proguard', |
| 79 | 'proguard6.0.1', |
| 80 | 'lib', |
| 81 | 'retrace.jar') |
Ian Zerny | 3709765 | 2019-04-11 13:13:27 +0200 | [diff] [blame] | 82 | PROGUARD_JAR = os.path.join( |
| 83 | THIRD_PARTY, |
| 84 | 'proguard', |
| 85 | 'proguard6.0.1', |
| 86 | 'lib', |
| 87 | 'proguard.jar') |
Morten Krogh-Jespersen | 480784d | 2019-02-05 08:10:46 +0100 | [diff] [blame] | 88 | CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments') |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 89 | PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar') |
| 90 | PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar') |
Ian Zerny | fbb1f7a | 2019-05-02 14:34:13 +0200 | [diff] [blame] | 91 | SAMPLE_LIBRARIES_SHA_FILE = os.path.join( |
| 92 | THIRD_PARTY, 'sample_libraries.tar.gz.sha1') |
Morten Krogh-Jespersen | 6474020 | 2019-02-07 13:06:04 +0100 | [diff] [blame] | 93 | OPENSOURCE_APPS_SHA_FILE = os.path.join( |
Rico Wind | ec283b5 | 2019-04-03 15:16:55 +0200 | [diff] [blame] | 94 | THIRD_PARTY, 'opensource_apps.tar.gz.sha1') |
| 95 | OPENSOURCE_APPS_FOLDER = os.path.join(THIRD_PARTY, 'opensource_apps') |
Søren Gjesse | 1c115b5 | 2019-08-14 12:43:57 +0200 | [diff] [blame] | 96 | BAZEL_SHA_FILE = os.path.join(THIRD_PARTY, 'bazel.tar.gz.sha1') |
| 97 | BAZEL_TOOL = os.path.join(THIRD_PARTY, 'bazel') |
Søren Gjesse | 699f636 | 2019-10-09 14:56:33 +0200 | [diff] [blame] | 98 | JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86.tar.gz.sha1') |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 99 | |
Morten Krogh-Jespersen | 220e570 | 2019-02-27 12:57:01 +0100 | [diff] [blame] | 100 | ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME" |
| 101 | ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION" |
Morten Krogh-Jespersen | c8efedd | 2019-01-28 11:36:17 +0100 | [diff] [blame] | 102 | USER_HOME = os.path.expanduser('~') |
Morten Krogh-Jespersen | 220e570 | 2019-02-27 12:57:01 +0100 | [diff] [blame] | 103 | |
Morten Krogh-Jespersen | 0981b72 | 2019-10-09 10:00:33 +0200 | [diff] [blame] | 104 | R8_TEST_RESULTS_BUCKET = 'r8-test-results' |
| 105 | |
| 106 | def archive_file(name, gs_dir, src_file): |
| 107 | gs_file = '%s/%s' % (gs_dir, name) |
| 108 | upload_file_to_cloud_storage(src_file, gs_file, public_read=False) |
| 109 | |
| 110 | def archive_value(name, gs_dir, value): |
| 111 | with TempDir() as temp: |
| 112 | tempfile = os.path.join(temp, name); |
| 113 | with open(tempfile, 'w') as f: |
| 114 | f.write(str(value)) |
| 115 | archive_file(name, gs_dir, tempfile) |
| 116 | |
Morten Krogh-Jespersen | 220e570 | 2019-02-27 12:57:01 +0100 | [diff] [blame] | 117 | def getAndroidHome(): |
| 118 | return os.environ.get( |
| 119 | ANDROID_HOME_ENVIROMENT_NAME, os.path.join(USER_HOME, 'Android', 'Sdk')) |
| 120 | |
| 121 | def getAndroidBuildTools(): |
| 122 | version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME, '28.0.3') |
| 123 | return os.path.join(getAndroidHome(), 'build-tools', version) |
Morten Krogh-Jespersen | c8efedd | 2019-01-28 11:36:17 +0100 | [diff] [blame] | 124 | |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 125 | def Print(s, quiet=False): |
| 126 | if quiet: |
| 127 | return |
| 128 | print(s) |
| 129 | |
| 130 | def Warn(message): |
| 131 | CRED = '\033[91m' |
| 132 | CEND = '\033[0m' |
| 133 | print(CRED + message + CEND) |
| 134 | |
| 135 | def PrintCmd(cmd, env=None, quiet=False): |
| 136 | if quiet: |
| 137 | return |
| 138 | if type(cmd) is list: |
| 139 | cmd = ' '.join(cmd) |
| 140 | if env: |
| 141 | env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()]) |
| 142 | print('Running: {} {}'.format(env, cmd)) |
| 143 | else: |
| 144 | print('Running: {}'.format(cmd)) |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 145 | # I know this will hit os on windows eventually if we don't do this. |
| 146 | sys.stdout.flush() |
| 147 | |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 148 | class ProgressLogger(object): |
| 149 | CLEAR_LINE = '\033[K' |
| 150 | UP = '\033[F' |
| 151 | |
| 152 | def __init__(self, quiet=False): |
| 153 | self._count = 0 |
| 154 | self._has_printed = False |
| 155 | self._quiet = quiet |
| 156 | |
| 157 | def log(self, text): |
| 158 | if self._quiet: |
| 159 | if self._has_printed: |
| 160 | sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE) |
| 161 | if len(text) > 140: |
| 162 | text = text[0:140] + '...' |
| 163 | print(text) |
| 164 | self._has_printed = True |
| 165 | |
| 166 | def done(self): |
| 167 | if self._quiet and self._has_printed: |
| 168 | sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE) |
| 169 | print('') |
| 170 | sys.stdout.write(ProgressLogger.UP) |
| 171 | |
Morten Krogh-Jespersen | 7cdd3a7 | 2019-03-13 14:58:25 +0100 | [diff] [blame] | 172 | def RunCmd(cmd, env_vars=None, quiet=False, fail=True, logging=True): |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 173 | PrintCmd(cmd, env=env_vars, quiet=quiet) |
| 174 | env = os.environ.copy() |
| 175 | if env_vars: |
| 176 | env.update(env_vars) |
| 177 | process = subprocess.Popen( |
| 178 | cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| 179 | stdout = [] |
Morten Krogh-Jespersen | 7cdd3a7 | 2019-03-13 14:58:25 +0100 | [diff] [blame] | 180 | logger = ProgressLogger(quiet=quiet) if logging else None |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 181 | failed = False |
| 182 | while True: |
| 183 | line = process.stdout.readline() |
| 184 | if line != b'': |
| 185 | stripped = line.rstrip() |
| 186 | stdout.append(stripped) |
Morten Krogh-Jespersen | 7cdd3a7 | 2019-03-13 14:58:25 +0100 | [diff] [blame] | 187 | if logger: |
| 188 | logger.log(stripped) |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 189 | |
| 190 | # TODO(christofferqa): r8 should fail with non-zero exit code. |
Morten Krogh-Jespersen | 121c47b | 2019-01-25 09:57:21 +0100 | [diff] [blame] | 191 | if ('AssertionError:' in stripped |
| 192 | or 'CompilationError:' in stripped |
| 193 | or 'CompilationFailedException:' in stripped |
Morten Krogh-Jespersen | 5d02a6b | 2019-10-29 14:48:56 +0100 | [diff] [blame] | 194 | or 'Compilation failed' in stripped |
| 195 | or 'FAILURE:' in stripped |
| 196 | or 'org.gradle.api.ProjectConfigurationException' in stripped |
| 197 | or 'BUILD FAILED' in stripped): |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 198 | failed = True |
| 199 | else: |
Morten Krogh-Jespersen | 7cdd3a7 | 2019-03-13 14:58:25 +0100 | [diff] [blame] | 200 | if logger: |
| 201 | logger.done() |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 202 | exit_code = process.poll() |
| 203 | if exit_code or failed: |
| 204 | for line in stdout: |
| 205 | Warn(line) |
Christoffer Quist Adamsen | 7cf4c56 | 2019-03-07 10:57:33 +0100 | [diff] [blame] | 206 | if fail: |
| 207 | raise subprocess.CalledProcessError( |
| 208 | exit_code or -1, cmd, output='\n'.join(stdout)) |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 209 | return stdout |
| 210 | |
Christoffer Quist Adamsen | 7cf4c56 | 2019-03-07 10:57:33 +0100 | [diff] [blame] | 211 | def RunGradlew( |
| 212 | args, clean=True, stacktrace=True, use_daemon=False, env_vars=None, |
Morten Krogh-Jespersen | 7cdd3a7 | 2019-03-13 14:58:25 +0100 | [diff] [blame] | 213 | quiet=False, fail=True, logging=True): |
Christoffer Quist Adamsen | 7cf4c56 | 2019-03-07 10:57:33 +0100 | [diff] [blame] | 214 | cmd = ['./gradlew'] |
| 215 | if clean: |
| 216 | assert 'clean' not in args |
| 217 | cmd.append('clean') |
| 218 | if stacktrace: |
| 219 | assert '--stacktrace' not in args |
| 220 | cmd.append('--stacktrace') |
| 221 | if not use_daemon: |
| 222 | assert '--no-daemon' not in args |
| 223 | cmd.append('--no-daemon') |
| 224 | cmd.extend(args) |
Morten Krogh-Jespersen | 7cdd3a7 | 2019-03-13 14:58:25 +0100 | [diff] [blame] | 225 | return RunCmd(cmd, env_vars=env_vars, quiet=quiet, fail=fail, logging=logging) |
Christoffer Quist Adamsen | 7cf4c56 | 2019-03-07 10:57:33 +0100 | [diff] [blame] | 226 | |
Rico Wind | f80f5a2 | 2017-06-16 09:15:57 +0200 | [diff] [blame] | 227 | def IsWindows(): |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 228 | return defines.IsWindows() |
Ian Zerny | 5fffb0a | 2019-02-11 13:54:22 +0100 | [diff] [blame] | 229 | |
| 230 | def IsLinux(): |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 231 | return defines.IsLinux() |
Ian Zerny | 5fffb0a | 2019-02-11 13:54:22 +0100 | [diff] [blame] | 232 | |
| 233 | def IsOsX(): |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 234 | return defines.IsOsX() |
Ian Zerny | 5fffb0a | 2019-02-11 13:54:22 +0100 | [diff] [blame] | 235 | |
| 236 | def EnsureDepFromGoogleCloudStorage(dep, tgz, sha1, msg): |
| 237 | if not os.path.exists(dep) or os.path.getmtime(tgz) < os.path.getmtime(sha1): |
| 238 | DownloadFromGoogleCloudStorage(sha1) |
| 239 | # Update the mtime of the tar file to make sure we do not run again unless |
| 240 | # there is an update. |
| 241 | os.utime(tgz, None) |
| 242 | else: |
| 243 | print 'Ensure cloud dependency:', msg, 'present' |
Rico Wind | f80f5a2 | 2017-06-16 09:15:57 +0200 | [diff] [blame] | 244 | |
Jean-Marie Henaff | e4e36d1 | 2018-04-05 10:33:50 +0200 | [diff] [blame] | 245 | def DownloadFromX20(sha1_file): |
| 246 | download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py') |
| 247 | cmd = [download_script, sha1_file] |
| 248 | PrintCmd(cmd) |
| 249 | subprocess.check_call(cmd) |
| 250 | |
Rico Wind | 533e3ce | 2019-04-04 10:26:12 +0200 | [diff] [blame] | 251 | def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps', auth=False): |
Rico Wind | f80f5a2 | 2017-06-16 09:15:57 +0200 | [diff] [blame] | 252 | suffix = '.bat' if IsWindows() else '' |
| 253 | download_script = 'download_from_google_storage%s' % suffix |
Rico Wind | 533e3ce | 2019-04-04 10:26:12 +0200 | [diff] [blame] | 254 | cmd = [download_script] |
| 255 | if not auth: |
| 256 | cmd.append('-n') |
| 257 | cmd.extend(['-b', bucket, '-u', '-s', sha1_file]) |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 258 | PrintCmd(cmd) |
| 259 | subprocess.check_call(cmd) |
| 260 | |
| 261 | def get_sha1(filename): |
| 262 | sha1 = hashlib.sha1() |
| 263 | with open(filename, 'rb') as f: |
| 264 | while True: |
| 265 | chunk = f.read(1024*1024) |
| 266 | if not chunk: |
| 267 | break |
| 268 | sha1.update(chunk) |
| 269 | return sha1.hexdigest() |
| 270 | |
Rico Wind | 1b09c56 | 2019-01-17 08:53:09 +0100 | [diff] [blame] | 271 | def is_master(): |
| 272 | remotes = subprocess.check_output(['git', 'branch', '-r', '--contains', |
| 273 | 'HEAD']) |
| 274 | return 'origin/master' in remotes |
| 275 | |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 276 | def get_HEAD_sha1(): |
Morten Krogh-Jespersen | e0ce6a3 | 2019-02-07 11:44:45 +0100 | [diff] [blame] | 277 | return get_HEAD_sha1_for_checkout(REPO_ROOT) |
| 278 | |
| 279 | def get_HEAD_sha1_for_checkout(checkout): |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 280 | cmd = ['git', 'rev-parse', 'HEAD'] |
| 281 | PrintCmd(cmd) |
Morten Krogh-Jespersen | e0ce6a3 | 2019-02-07 11:44:45 +0100 | [diff] [blame] | 282 | with ChangedWorkingDirectory(checkout): |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 283 | return subprocess.check_output(cmd).strip() |
| 284 | |
Tamas Kenez | 971eec6 | 2017-05-24 11:08:40 +0200 | [diff] [blame] | 285 | def makedirs_if_needed(path): |
| 286 | try: |
| 287 | os.makedirs(path) |
| 288 | except OSError: |
| 289 | if not os.path.isdir(path): |
| 290 | raise |
| 291 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 292 | def upload_dir_to_cloud_storage(directory, destination, is_html=False, public_read=True): |
Rico Wind | a94f01c | 2017-06-27 10:32:34 +0200 | [diff] [blame] | 293 | # Upload and make the content encoding right for viewing directly |
Rico Wind | d55ce27 | 2019-03-26 08:02:09 +0100 | [diff] [blame] | 294 | cmd = ['gsutil.py', '-m', 'cp'] |
Rico Wind | d0d88cf | 2018-02-09 09:46:11 +0100 | [diff] [blame] | 295 | if is_html: |
| 296 | cmd += ['-z', 'html'] |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 297 | if public_read: |
| 298 | cmd += ['-a', 'public-read'] |
| 299 | cmd += ['-R', directory, destination] |
Rico Wind | a94f01c | 2017-06-27 10:32:34 +0200 | [diff] [blame] | 300 | PrintCmd(cmd) |
| 301 | subprocess.check_call(cmd) |
| 302 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 303 | def upload_file_to_cloud_storage(source, destination, public_read=True): |
| 304 | cmd = ['gsutil.py', 'cp'] |
| 305 | if public_read: |
| 306 | cmd += ['-a', 'public-read'] |
| 307 | cmd += [source, destination] |
Rico Wind | b4621c1 | 2017-08-28 12:48:53 +0200 | [diff] [blame] | 308 | PrintCmd(cmd) |
| 309 | subprocess.check_call(cmd) |
| 310 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 311 | def delete_file_from_cloud_storage(destination): |
| 312 | cmd = ['gsutil.py', 'rm', destination] |
| 313 | PrintCmd(cmd) |
| 314 | subprocess.check_call(cmd) |
| 315 | |
Rico Wind | 4fd2dda | 2018-09-26 17:41:45 +0200 | [diff] [blame] | 316 | def ls_files_on_cloud_storage(destination): |
| 317 | cmd = ['gsutil.py', 'ls', destination] |
| 318 | PrintCmd(cmd) |
| 319 | return subprocess.check_output(cmd) |
| 320 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 321 | def cat_file_on_cloud_storage(destination, ignore_errors=False): |
| 322 | cmd = ['gsutil.py', 'cat', destination] |
| 323 | PrintCmd(cmd) |
| 324 | try: |
| 325 | return subprocess.check_output(cmd) |
| 326 | except subprocess.CalledProcessError as e: |
| 327 | if ignore_errors: |
| 328 | return '' |
| 329 | else: |
| 330 | raise e |
| 331 | |
| 332 | def file_exists_on_cloud_storage(destination): |
| 333 | cmd = ['gsutil.py', 'ls', destination] |
| 334 | PrintCmd(cmd) |
| 335 | return subprocess.call(cmd) == 0 |
| 336 | |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 337 | def download_file_from_cloud_storage(source, destination): |
| 338 | cmd = ['gsutil.py', 'cp', source, destination] |
| 339 | PrintCmd(cmd) |
| 340 | subprocess.check_call(cmd) |
| 341 | |
Morten Krogh-Jespersen | 220e570 | 2019-02-27 12:57:01 +0100 | [diff] [blame] | 342 | def create_archive(name, sources=None): |
| 343 | if not sources: |
| 344 | sources = [name] |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 345 | tarname = '%s.tar.gz' % name |
| 346 | with tarfile.open(tarname, 'w:gz') as tar: |
Morten Krogh-Jespersen | 5409086 | 2019-02-19 11:31:10 +0100 | [diff] [blame] | 347 | for source in sources: |
| 348 | tar.add(source) |
Jean-Marie Henaff | 7a64eec | 2018-05-31 15:30:35 +0200 | [diff] [blame] | 349 | return tarname |
| 350 | |
| 351 | def extract_dir(filename): |
| 352 | return filename[0:len(filename) - len('.tar.gz')] |
| 353 | |
| 354 | def unpack_archive(filename): |
| 355 | dest_dir = extract_dir(filename) |
| 356 | if os.path.exists(dest_dir): |
| 357 | print 'Deleting existing dir %s' % dest_dir |
| 358 | shutil.rmtree(dest_dir) |
| 359 | dirname = os.path.dirname(os.path.abspath(filename)) |
| 360 | with tarfile.open(filename, 'r:gz') as tar: |
| 361 | tar.extractall(path=dirname) |
| 362 | |
Morten Krogh-Jespersen | 5409086 | 2019-02-19 11:31:10 +0100 | [diff] [blame] | 363 | def check_prodacces(): |
| 364 | subprocess.check_call(['prodaccess']) |
| 365 | |
Rico Wind | 1a29c4f | 2018-01-25 08:43:08 +0100 | [diff] [blame] | 366 | # Note that gcs is eventually consistent with regards to list operations. |
| 367 | # This is not a problem in our case, but don't ever use this method |
| 368 | # for synchronization. |
| 369 | def cloud_storage_exists(destination): |
| 370 | cmd = ['gsutil.py', 'ls', destination] |
| 371 | PrintCmd(cmd) |
| 372 | exit_code = subprocess.call(cmd) |
| 373 | return exit_code == 0 |
| 374 | |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 375 | class TempDir(object): |
Søren Gjesse | cbeae78 | 2019-05-21 14:14:25 +0200 | [diff] [blame] | 376 | def __init__(self, prefix='', delete=True): |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 377 | self._temp_dir = None |
| 378 | self._prefix = prefix |
Søren Gjesse | cbeae78 | 2019-05-21 14:14:25 +0200 | [diff] [blame] | 379 | self._delete = delete |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 380 | |
| 381 | def __enter__(self): |
| 382 | self._temp_dir = tempfile.mkdtemp(self._prefix) |
| 383 | return self._temp_dir |
| 384 | |
| 385 | def __exit__(self, *_): |
Søren Gjesse | cbeae78 | 2019-05-21 14:14:25 +0200 | [diff] [blame] | 386 | if self._delete: |
| 387 | shutil.rmtree(self._temp_dir, ignore_errors=True) |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 388 | |
| 389 | class ChangedWorkingDirectory(object): |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 390 | def __init__(self, working_directory, quiet=False): |
| 391 | self._quiet = quiet |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 392 | self._working_directory = working_directory |
| 393 | |
| 394 | def __enter__(self): |
| 395 | self._old_cwd = os.getcwd() |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 396 | if not self._quiet: |
| 397 | print 'Enter directory:', self._working_directory |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 398 | os.chdir(self._working_directory) |
| 399 | |
| 400 | def __exit__(self, *_): |
Christoffer Quist Adamsen | 1de3dde | 2019-01-24 13:17:46 +0100 | [diff] [blame] | 401 | if not self._quiet: |
| 402 | print 'Enter directory:', self._old_cwd |
Mads Ager | 418d1ca | 2017-05-22 09:35:49 +0200 | [diff] [blame] | 403 | os.chdir(self._old_cwd) |
Tamas Kenez | 82efeb5 | 2017-06-12 13:56:22 +0200 | [diff] [blame] | 404 | |
| 405 | # Reading Android CTS test_result.xml |
| 406 | |
| 407 | class CtsModule(object): |
| 408 | def __init__(self, module_name): |
| 409 | self.name = module_name |
| 410 | |
| 411 | class CtsTestCase(object): |
| 412 | def __init__(self, test_case_name): |
| 413 | self.name = test_case_name |
| 414 | |
| 415 | class CtsTest(object): |
| 416 | def __init__(self, test_name, outcome): |
| 417 | self.name = test_name |
| 418 | self.outcome = outcome |
| 419 | |
| 420 | # Generator yielding CtsModule, CtsTestCase or CtsTest from |
| 421 | # reading through a CTS test_result.xml file. |
| 422 | def read_cts_test_result(file_xml): |
| 423 | re_module = re.compile('<Module name="([^"]*)"') |
| 424 | re_test_case = re.compile('<TestCase name="([^"]*)"') |
| 425 | re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"') |
| 426 | with open(file_xml) as f: |
| 427 | for line in f: |
| 428 | m = re_module.search(line) |
| 429 | if m: |
| 430 | yield CtsModule(m.groups()[0]) |
| 431 | continue |
| 432 | m = re_test_case.search(line) |
| 433 | if m: |
| 434 | yield CtsTestCase(m.groups()[0]) |
| 435 | continue |
| 436 | m = re_test.search(line) |
| 437 | if m: |
| 438 | outcome = m.groups()[0] |
Rico Wind | f80f5a2 | 2017-06-16 09:15:57 +0200 | [diff] [blame] | 439 | assert outcome in ['fail', 'pass'] |
Tamas Kenez | 82efeb5 | 2017-06-12 13:56:22 +0200 | [diff] [blame] | 440 | yield CtsTest(m.groups()[1], outcome == 'pass') |
Tamas Kenez | fc34cd8 | 2017-07-13 12:43:57 +0200 | [diff] [blame] | 441 | |
| 442 | def grep_memoryuse(logfile): |
| 443 | re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)') |
| 444 | result = None |
| 445 | with open(logfile) as f: |
| 446 | for line in f: |
| 447 | m = re_vmhwm.search(line) |
| 448 | if m: |
| 449 | groups = m.groups() |
| 450 | s = len(groups) |
| 451 | if s >= 1: |
| 452 | result = int(groups[0]) |
| 453 | if s >= 2: |
| 454 | unit = groups[1] |
| 455 | if unit == 'kB': |
| 456 | result *= 1024 |
| 457 | elif unit != '': |
| 458 | raise Exception('Unrecognized unit in memory usage log: {}' |
| 459 | .format(unit)) |
| 460 | if result is None: |
| 461 | raise Exception('No memory usage found in log: {}'.format(logfile)) |
Tamas Kenez | 02bff03 | 2017-07-18 12:13:58 +0200 | [diff] [blame] | 462 | return result |
| 463 | |
| 464 | # Return a dictionary: {segment_name -> segments_size} |
| 465 | def getDexSegmentSizes(dex_files): |
| 466 | assert len(dex_files) > 0 |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 467 | cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments'] |
Tamas Kenez | 02bff03 | 2017-07-18 12:13:58 +0200 | [diff] [blame] | 468 | cmd.extend(dex_files) |
| 469 | PrintCmd(cmd) |
| 470 | output = subprocess.check_output(cmd) |
| 471 | |
| 472 | matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output) |
| 473 | |
| 474 | if matches is None or len(matches) == 0: |
| 475 | raise Exception('DexSegments failed to return any output for' \ |
| 476 | ' these files: {}'.format(dex_files)) |
| 477 | |
| 478 | result = {} |
| 479 | |
| 480 | for match in matches: |
| 481 | result[match[0]] = int(match[1]) |
| 482 | |
| 483 | return result |
| 484 | |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 485 | # Return a dictionary: {segment_name -> segments_size} |
| 486 | def getCfSegmentSizes(cfFile): |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 487 | cmd = [jdk.GetJavaExecutable(), |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 488 | '-cp', |
Morten Krogh-Jespersen | 480784d | 2019-02-05 08:10:46 +0100 | [diff] [blame] | 489 | CF_SEGMENTS_TOOL, |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 490 | 'com.android.tools.r8.cf_segments.MeasureLib', |
| 491 | cfFile] |
| 492 | PrintCmd(cmd) |
| 493 | output = subprocess.check_output(cmd) |
| 494 | |
| 495 | matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output) |
| 496 | |
| 497 | if matches is None or len(matches) == 0: |
| 498 | raise Exception('CfSegments failed to return any output for' \ |
| 499 | ' the file: ' + cfFile) |
| 500 | |
| 501 | result = {} |
| 502 | |
| 503 | for match in matches: |
| 504 | result[match[0]] = int(match[1]) |
| 505 | |
| 506 | return result |
| 507 | |
Søren Gjesse | 1c115b5 | 2019-08-14 12:43:57 +0200 | [diff] [blame] | 508 | def get_maven_path(artifact, version): |
| 509 | return os.path.join('com', 'android', 'tools', artifact, version) |
Rico Wind | c0b1638 | 2018-05-17 13:23:43 +0200 | [diff] [blame] | 510 | |
Morten Krogh-Jespersen | 38c7ca0 | 2019-02-04 10:39:57 +0100 | [diff] [blame] | 511 | def print_cfsegments(prefix, cf_files): |
| 512 | for cf_file in cf_files: |
| 513 | for segment_name, size in getCfSegmentSizes(cf_file).items(): |
| 514 | print('{}-{}(CodeSize): {}' |
| 515 | .format(prefix, segment_name, size)) |
| 516 | |
Tamas Kenez | 02bff03 | 2017-07-18 12:13:58 +0200 | [diff] [blame] | 517 | def print_dexsegments(prefix, dex_files): |
| 518 | for segment_name, size in getDexSegmentSizes(dex_files).items(): |
| 519 | print('{}-{}(CodeSize): {}' |
| 520 | .format(prefix, segment_name, size)) |
Tamas Kenez | 2cf47cf | 2017-07-25 10:22:52 +0200 | [diff] [blame] | 521 | |
Mads Ager | bc7b2ce | 2018-02-05 11:28:47 +0100 | [diff] [blame] | 522 | # Ensure that we are not benchmarking with a google jvm. |
Tamas Kenez | 2cf47cf | 2017-07-25 10:22:52 +0200 | [diff] [blame] | 523 | def check_java_version(): |
Ian Zerny | 3f54e22 | 2019-02-12 10:51:17 +0100 | [diff] [blame] | 524 | cmd= [jdk.GetJavaExecutable(), '-version'] |
Tamas Kenez | 2cf47cf | 2017-07-25 10:22:52 +0200 | [diff] [blame] | 525 | output = subprocess.check_output(cmd, stderr = subprocess.STDOUT) |
| 526 | m = re.search('openjdk version "([^"]*)"', output) |
| 527 | if m is None: |
| 528 | raise Exception("Can't check java version: no version string in output" |
| 529 | " of 'java -version': '{}'".format(output)) |
| 530 | version = m.groups(0)[0] |
Mads Ager | bc7b2ce | 2018-02-05 11:28:47 +0100 | [diff] [blame] | 531 | m = re.search('google', version) |
| 532 | if m is not None: |
| 533 | raise Exception("Do not use google JVM for benchmarking: " + version) |
Tamas Kenez | 0cad51c | 2017-08-21 14:42:01 +0200 | [diff] [blame] | 534 | |
Christoffer Quist Adamsen | 17879c1 | 2019-01-22 16:13:54 +0100 | [diff] [blame] | 535 | def get_android_jar_dir(api): |
| 536 | return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api)) |
| 537 | |
Rico Wind | 9d70f61 | 2018-08-31 09:17:43 +0200 | [diff] [blame] | 538 | def get_android_jar(api): |
| 539 | return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api)) |
Rico Wind | da6836e | 2018-12-07 12:32:03 +0100 | [diff] [blame] | 540 | |
Christoffer Quist Adamsen | 17879c1 | 2019-01-22 16:13:54 +0100 | [diff] [blame] | 541 | def get_android_optional_jars(api): |
| 542 | android_optional_jars_dir = os.path.join(get_android_jar_dir(api), 'optional') |
| 543 | android_optional_jars = [ |
| 544 | os.path.join(android_optional_jars_dir, 'android.test.base.jar'), |
| 545 | os.path.join(android_optional_jars_dir, 'android.test.mock.jar'), |
| 546 | os.path.join(android_optional_jars_dir, 'android.test.runner.jar'), |
| 547 | os.path.join(android_optional_jars_dir, 'org.apache.http.legacy.jar') |
| 548 | ] |
| 549 | return [ |
| 550 | android_optional_jar for android_optional_jar in android_optional_jars |
| 551 | if os.path.isfile(android_optional_jar)] |
| 552 | |
Rico Wind | faaac01 | 2019-02-25 11:24:05 +0100 | [diff] [blame] | 553 | def is_bot(): |
Rico Wind | e11f439 | 2019-03-18 07:59:37 +0100 | [diff] [blame] | 554 | return 'SWARMING_BOT_ID' in os.environ |
Morten Krogh-Jespersen | 16e925d | 2019-01-25 14:40:38 +0100 | [diff] [blame] | 555 | |
| 556 | def uncompressed_size(path): |
| 557 | return sum(z.file_size for z in zipfile.ZipFile(path).infolist()) |
Morten Krogh-Jespersen | 0de1373 | 2019-03-01 08:56:39 +0100 | [diff] [blame] | 558 | |
| 559 | def getR8Version(path): |
| 560 | cmd = [jdk.GetJavaExecutable(), '-cp', path, 'com.android.tools.r8.R8', |
| 561 | '--version'] |
| 562 | output = subprocess.check_output(cmd, stderr = subprocess.STDOUT) |
| 563 | # output is on form 'R8 <version>' so we just strip of 'R8 '. |
| 564 | return output.splitlines()[0][3:] |
Søren Gjesse | 6e5e584 | 2019-09-03 08:48:30 +0200 | [diff] [blame] | 565 | |
| 566 | def desugar_configuration_version(): |
| 567 | with open(DESUGAR_CONFIGURATION, 'r') as f: |
| 568 | configuration_json = json.loads(f.read()) |
| 569 | configuration_format_version = \ |
| 570 | configuration_json.get('configuration_format_version') |
| 571 | if (configuration_format_version != 1): |
| 572 | raise Exception( |
| 573 | 'Unsupported "configuration_format_version" ' |
| 574 | + configuration_format_version) |
| 575 | version = configuration_json.get('version') |
| 576 | if not version: |
| 577 | raise Exception( |
| 578 | 'No "version" found in ' + utils.DESUGAR_CONFIGURATION) |
Søren Gjesse | 1e17153 | 2019-09-03 09:44:22 +0200 | [diff] [blame] | 579 | check_basic_semver_version(version, 'in ' + DESUGAR_CONFIGURATION) |
Søren Gjesse | 6e5e584 | 2019-09-03 08:48:30 +0200 | [diff] [blame] | 580 | return version |
Søren Gjesse | 1e17153 | 2019-09-03 09:44:22 +0200 | [diff] [blame] | 581 | |
Søren Gjesse | 4e5c6fe | 2019-11-05 17:17:43 +0100 | [diff] [blame] | 582 | class SemanticVersion: |
| 583 | def __init__(self, major, minor, patch): |
| 584 | self.major = major |
| 585 | self.minor = minor |
| 586 | self.patch = patch |
| 587 | # Build metadata currently not suppported |
| 588 | |
| 589 | def larger_than(self, other): |
| 590 | if self.major > other.major: |
| 591 | return True |
| 592 | if self.major == other.major and self.minor > other.minor: |
| 593 | return True |
| 594 | if self.patch: |
| 595 | return (self.major == other.major |
| 596 | and self.minor == other.minor |
| 597 | and self.patch > other.patch) |
| 598 | else: |
| 599 | return False |
| 600 | |
| 601 | |
Søren Gjesse | 1e17153 | 2019-09-03 09:44:22 +0200 | [diff] [blame] | 602 | # Check that the passed string is formatted as a basic semver version (x.y.z) |
| 603 | # See https://semver.org/. |
Søren Gjesse | 4e5c6fe | 2019-11-05 17:17:43 +0100 | [diff] [blame] | 604 | def check_basic_semver_version(version, error_context = '', components = 3): |
| 605 | regexp = '^' |
| 606 | for x in range(components): |
| 607 | regexp += '([0-9]+)' |
| 608 | if x < components - 1: |
| 609 | regexp += '\\.' |
| 610 | regexp += '$' |
| 611 | reg = re.compile(regexp) |
| 612 | match = reg.match(version) |
| 613 | if not match: |
Søren Gjesse | 1e17153 | 2019-09-03 09:44:22 +0200 | [diff] [blame] | 614 | raise Exception("Invalid version '" |
| 615 | + version |
| 616 | + "'" |
| 617 | + (' ' + error_context) if len(error_context) > 0 else '') |
Søren Gjesse | 4e5c6fe | 2019-11-05 17:17:43 +0100 | [diff] [blame] | 618 | if components == 2: |
| 619 | return SemanticVersion(int(match.group(1)), int(match.group(2)), None) |
| 620 | elif components == 3: |
| 621 | return SemanticVersion( |
| 622 | int(match.group(1)), int(match.group(2)), int(match.group(3))) |
| 623 | else: |
| 624 | raise Exception('Argument "components" must be 2 or 3') |