blob: 64c9a49e5615c6124c7d84e9fe61a01b52e91c6f [file] [log] [blame]
Mads Ager418d1ca2017-05-22 09:35:49 +02001# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
2# for details. All rights reserved. Use of this source code is governed by a
3# BSD-style license that can be found in the LICENSE file.
4
5# Different utility functions used accross scripts
6
7import hashlib
Søren Gjesse6e5e5842019-09-03 08:48:30 +02008import json
Mads Ager418d1ca2017-05-22 09:35:49 +02009import os
Tamas Kenez82efeb52017-06-12 13:56:22 +020010import re
Mads Ager418d1ca2017-05-22 09:35:49 +020011import shutil
12import subprocess
13import sys
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +020014import tarfile
Mads Ager418d1ca2017-05-22 09:35:49 +020015import tempfile
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +010016import zipfile
Mads Ager418d1ca2017-05-22 09:35:49 +020017
Ian Zerny37097652019-04-11 13:13:27 +020018import defines
19import jdk
20
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +010021ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}'
22ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar')
Ian Zerny3f54e222019-02-12 10:51:17 +010023TOOLS_DIR = defines.TOOLS_DIR
24REPO_ROOT = defines.REPO_ROOT
25THIRD_PARTY = defines.THIRD_PARTY
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +010026ANDROID_SDK = os.path.join(THIRD_PARTY, 'android_sdk')
Tamas Kenezfc34cd82017-07-13 12:43:57 +020027MEMORY_USE_TMP_FILE = 'memory_use.tmp'
Tamas Kenez02bff032017-07-18 12:13:58 +020028DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)')
Mads Ager12a56bc2017-11-27 11:51:25 +010029BUILD = os.path.join(REPO_ROOT, 'build')
Ian Zerny0f5fc732018-11-15 14:34:41 +010030BUILD_DEPS_DIR = os.path.join(BUILD, 'deps')
31BUILD_MAIN_DIR = os.path.join(BUILD, 'classes', 'main')
Morten Krogh-Jespersen51db2b02020-11-11 12:49:26 +010032BUILD_JAVA_MAIN_DIR = os.path.join(BUILD, 'classes', 'java', 'main')
Ian Zerny0f5fc732018-11-15 14:34:41 +010033BUILD_TEST_DIR = os.path.join(BUILD, 'classes', 'test')
Mads Ager12a56bc2017-11-27 11:51:25 +010034LIBS = os.path.join(BUILD, 'libs')
35GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense')
Mads Agera4911eb2017-11-22 13:19:36 +010036SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java')
Søren Gjesse6e5e5842019-09-03 08:48:30 +020037TEST_ROOT = os.path.join(REPO_ROOT, 'src', 'test', 'java')
Ian Zerny59dfa4c2019-10-25 10:34:36 +020038REPO_SOURCE = 'https://r8.googlesource.com/r8'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020039
40D8 = 'd8'
41R8 = 'r8'
Tamas Kenez03ab76f2018-12-07 14:33:25 +010042R8LIB = 'r8lib'
Morten Krogh-Jespersene28db462019-01-09 13:32:15 +010043R8LIB_NO_DEPS = 'r8LibNoDeps'
Morten Krogh-Jespersen98ee89a2021-10-25 20:59:02 +020044R8RETRACE = 'R8Retrace'
45R8RETRACE_NO_DEPS = 'R8RetraceNoDeps'
Mads Agerb10c07f2017-11-27 13:25:52 +010046R8_SRC = 'sourceJar'
Søren Gjesse17fc67d2019-12-04 14:50:17 +010047LIBRARY_DESUGAR_CONVERSIONS = 'buildLibraryDesugarConversions'
Ian Zerny161ff742022-01-20 12:39:40 +010048R8_TESTS_TARGET = 'TestJar'
49R8_TESTS_DEPS_TARGET = 'RepackageTestDeps'
50R8LIB_TESTS_TARGET = 'configureTestForR8Lib'
51R8LIB_TESTS_DEPS_TARGET = R8_TESTS_DEPS_TARGET
Søren Gjessedc9d8a22017-10-12 12:40:59 +020052
Morten Krogh-Jespersen51db2b02020-11-11 12:49:26 +010053ALL_DEPS_JAR = os.path.join(LIBS, 'deps_all.jar')
Rico Wind74fab302017-10-02 07:25:33 +020054D8_JAR = os.path.join(LIBS, 'd8.jar')
55R8_JAR = os.path.join(LIBS, 'r8.jar')
Ian Zerny161ff742022-01-20 12:39:40 +010056R8_WITH_RELOCATED_DEPS_JAR = os.path.join(LIBS, 'r8_with_relocated_deps.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010057R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar')
Ian Zerny5ffa58f2020-02-26 08:37:14 +010058R8LIB_MAP = os.path.join(LIBS, 'r8lib.jar.map')
Mads Agerb10c07f2017-11-27 13:25:52 +010059R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010060R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar')
Tamas Kenez180be092018-12-05 15:23:06 +010061R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar')
Morten Krogh-Jespersen98ee89a2021-10-25 20:59:02 +020062R8RETRACE_JAR = os.path.join(LIBS, 'r8retrace.jar')
63R8RETRACE_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8retrace-exclude-deps.jar')
Ian Zerny161ff742022-01-20 12:39:40 +010064R8_TESTS_JAR = os.path.join(LIBS, 'r8tests.jar')
65R8LIB_TESTS_JAR = os.path.join(LIBS, 'r8libtestdeps-cf.jar')
66R8_TESTS_DEPS_JAR = os.path.join(LIBS, 'test_deps_all.jar')
67R8LIB_TESTS_DEPS_JAR = R8_TESTS_DEPS_JAR
Mads Ager12a56bc2017-11-27 11:51:25 +010068MAVEN_ZIP = os.path.join(LIBS, 'r8.zip')
Rico Wind8fc8bfa2019-03-22 09:57:36 +010069MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip')
Søren Gjesse17fc67d2019-12-04 14:50:17 +010070LIBRARY_DESUGAR_CONVERSIONS_ZIP = os.path.join(LIBS, 'library_desugar_conversions.zip')
71
Søren Gjesse6e5e5842019-09-03 08:48:30 +020072DESUGAR_CONFIGURATION = os.path.join(
Søren Gjesse927a92e2019-12-04 15:18:06 +010073 'src', 'library_desugar', 'desugar_jdk_libs.json')
Søren Gjesseee086b22020-10-30 11:46:39 +010074DESUGAR_IMPLEMENTATION = os.path.join(
Søren Gjesse3dc207b2021-02-15 09:45:30 +010075 'third_party', 'openjdk', 'desugar_jdk_libs', 'desugar_jdk_libs.jar')
Søren Gjesse705a3b12022-03-17 11:37:30 +010076DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join(
77 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
78DESUGAR_IMPLEMENTATION_JDK11 = os.path.join(
79 'third_party', 'openjdk', 'desugar_jdk_libs_11', 'desugar_jdk_libs.jar')
Søren Gjesse6e5e5842019-09-03 08:48:30 +020080DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
81 LIBS, 'desugar_jdk_libs_configuration.zip')
Søren Gjesse705a3b12022-03-17 11:37:30 +010082DESUGAR_CONFIGURATION_LEGACY_JDK11_MAVEN_ZIP = os.path.join(
83 LIBS, 'desugar_jdk_libs_configuration_legacy_jdk11.zip')
Mads Ager12a56bc2017-11-27 11:51:25 +010084GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
Mathias Rav3fb4a3a2018-05-29 15:41:36 +020085RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
Mathias Ravb46dc002018-06-06 09:37:11 +020086R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +010087CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +010088PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar')
89PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar')
Ian Zernyfbb1f7a2019-05-02 14:34:13 +020090SAMPLE_LIBRARIES_SHA_FILE = os.path.join(
91 THIRD_PARTY, 'sample_libraries.tar.gz.sha1')
Morten Krogh-Jespersen45d7a7b2020-11-02 08:31:09 +010092OPENSOURCE_DUMPS_DIR = os.path.join(THIRD_PARTY, 'opensource-apps')
Morten Krogh-Jespersen86222742021-03-02 11:13:33 +010093INTERNAL_DUMPS_DIR = os.path.join(THIRD_PARTY, 'internal-apps')
Søren Gjesse1c115b52019-08-14 12:43:57 +020094BAZEL_SHA_FILE = os.path.join(THIRD_PARTY, 'bazel.tar.gz.sha1')
95BAZEL_TOOL = os.path.join(THIRD_PARTY, 'bazel')
Søren Gjesse699f6362019-10-09 14:56:33 +020096JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86.tar.gz.sha1')
Søren Gjesseef195772021-03-11 16:04:42 +010097JAVA11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11', 'linux.tar.gz.sha1')
Christoffer Quist Adamsen1ca046c2021-02-21 11:25:16 +010098IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test', 'ignorewarnings.rules')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +010099
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100100ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME"
101ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION"
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100102USER_HOME = os.path.expanduser('~')
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100103
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200104R8_TEST_RESULTS_BUCKET = 'r8-test-results'
Rico Wind0aea5152022-04-25 12:29:45 +0200105R8_INTERNAL_TEST_RESULTS_BUCKET = 'r8-internal-test-results'
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200106
107def archive_file(name, gs_dir, src_file):
108 gs_file = '%s/%s' % (gs_dir, name)
109 upload_file_to_cloud_storage(src_file, gs_file, public_read=False)
110
111def archive_value(name, gs_dir, value):
112 with TempDir() as temp:
113 tempfile = os.path.join(temp, name);
114 with open(tempfile, 'w') as f:
115 f.write(str(value))
116 archive_file(name, gs_dir, tempfile)
117
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200118def find_cloud_storage_file_from_options(name, options, orElse=None):
119 # Import archive on-demand since archive depends on utils.
120 from archive import GetUploadDestination
121 hash_or_version = find_hash_or_version_from_options(options)
122 if not hash_or_version:
123 return orElse
124 is_hash = options.commit_hash is not None
125 download_path = GetUploadDestination(hash_or_version, name, is_hash)
126 if file_exists_on_cloud_storage(download_path):
127 out = tempfile.NamedTemporaryFile().name
128 download_file_from_cloud_storage(download_path, out)
129 return out
130 else:
131 raise Exception('Could not find file {} from hash/version: {}.'
132 .format(name, hash_or_version))
133
134def find_r8_jar_from_options(options):
135 return find_cloud_storage_file_from_options('r8.jar', options)
136
137def find_r8_lib_jar_from_options(options):
138 return find_cloud_storage_file_from_options('r8lib.jar', options)
139
140def find_hash_or_version_from_options(options):
141 if options.tag:
142 return find_hash_or_version_from_tag(options.tag)
143 else:
144 return options.commit_hash or options.version
145
146def find_hash_or_version_from_tag(tag_or_hash):
Rico Windfd186372022-02-28 08:55:48 +0100147 info = subprocess.check_output([
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200148 'git',
149 'show',
150 tag_or_hash,
151 '-s',
Rico Windfd186372022-02-28 08:55:48 +0100152 '--format=oneline']).decode('utf-8').splitlines()[-1].split()
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200153 # The info should be on the following form [hash,"Version",version]
154 if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version":
155 return info[2]
156 return None
157
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100158def getAndroidHome():
159 return os.environ.get(
160 ANDROID_HOME_ENVIROMENT_NAME, os.path.join(USER_HOME, 'Android', 'Sdk'))
161
162def getAndroidBuildTools():
163 version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME, '28.0.3')
164 return os.path.join(getAndroidHome(), 'build-tools', version)
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100165
Christoffer Quist Adamsen5c9ded12021-01-14 14:29:37 +0100166def is_python3():
167 return sys.version_info.major == 3
168
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100169def Print(s, quiet=False):
170 if quiet:
171 return
172 print(s)
173
174def Warn(message):
175 CRED = '\033[91m'
176 CEND = '\033[0m'
177 print(CRED + message + CEND)
178
179def PrintCmd(cmd, env=None, quiet=False):
180 if quiet:
181 return
182 if type(cmd) is list:
183 cmd = ' '.join(cmd)
184 if env:
185 env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
186 print('Running: {} {}'.format(env, cmd))
187 else:
188 print('Running: {}'.format(cmd))
Mads Ager418d1ca2017-05-22 09:35:49 +0200189 # I know this will hit os on windows eventually if we don't do this.
190 sys.stdout.flush()
191
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100192class ProgressLogger(object):
193 CLEAR_LINE = '\033[K'
194 UP = '\033[F'
195
196 def __init__(self, quiet=False):
197 self._count = 0
198 self._has_printed = False
199 self._quiet = quiet
200
201 def log(self, text):
202 if self._quiet:
203 if self._has_printed:
204 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
205 if len(text) > 140:
206 text = text[0:140] + '...'
207 print(text)
208 self._has_printed = True
209
210 def done(self):
211 if self._quiet and self._has_printed:
212 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
213 print('')
214 sys.stdout.write(ProgressLogger.UP)
215
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100216def RunCmd(cmd, env_vars=None, quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100217 PrintCmd(cmd, env=env_vars, quiet=quiet)
218 env = os.environ.copy()
219 if env_vars:
220 env.update(env_vars)
221 process = subprocess.Popen(
222 cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
223 stdout = []
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100224 logger = ProgressLogger(quiet=quiet) if logging else None
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100225 failed = False
226 while True:
Rico Wind744ba752021-01-22 06:24:49 +0100227 line = process.stdout.readline().decode('utf-8')
228 if line != '':
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100229 stripped = line.rstrip()
230 stdout.append(stripped)
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100231 if logger:
232 logger.log(stripped)
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100233 # TODO(christofferqa): r8 should fail with non-zero exit code.
Morten Krogh-Jespersen121c47b2019-01-25 09:57:21 +0100234 if ('AssertionError:' in stripped
235 or 'CompilationError:' in stripped
236 or 'CompilationFailedException:' in stripped
Morten Krogh-Jespersen5d02a6b2019-10-29 14:48:56 +0100237 or 'Compilation failed' in stripped
238 or 'FAILURE:' in stripped
239 or 'org.gradle.api.ProjectConfigurationException' in stripped
240 or 'BUILD FAILED' in stripped):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100241 failed = True
242 else:
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100243 if logger:
244 logger.done()
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100245 exit_code = process.poll()
246 if exit_code or failed:
247 for line in stdout:
248 Warn(line)
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100249 if fail:
250 raise subprocess.CalledProcessError(
251 exit_code or -1, cmd, output='\n'.join(stdout))
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100252 return stdout
253
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100254def RunGradlew(
255 args, clean=True, stacktrace=True, use_daemon=False, env_vars=None,
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100256 quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100257 cmd = ['./gradlew']
258 if clean:
259 assert 'clean' not in args
260 cmd.append('clean')
261 if stacktrace:
262 assert '--stacktrace' not in args
263 cmd.append('--stacktrace')
264 if not use_daemon:
265 assert '--no-daemon' not in args
266 cmd.append('--no-daemon')
267 cmd.extend(args)
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100268 return RunCmd(cmd, env_vars=env_vars, quiet=quiet, fail=fail, logging=logging)
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100269
Rico Windf80f5a22017-06-16 09:15:57 +0200270def IsWindows():
Ian Zerny3f54e222019-02-12 10:51:17 +0100271 return defines.IsWindows()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100272
273def IsLinux():
Ian Zerny3f54e222019-02-12 10:51:17 +0100274 return defines.IsLinux()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100275
276def IsOsX():
Ian Zerny3f54e222019-02-12 10:51:17 +0100277 return defines.IsOsX()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100278
279def EnsureDepFromGoogleCloudStorage(dep, tgz, sha1, msg):
280 if not os.path.exists(dep) or os.path.getmtime(tgz) < os.path.getmtime(sha1):
281 DownloadFromGoogleCloudStorage(sha1)
282 # Update the mtime of the tar file to make sure we do not run again unless
283 # there is an update.
284 os.utime(tgz, None)
285 else:
Rico Wind3d369b42021-01-12 10:26:24 +0100286 print('Ensure cloud dependency:', msg, 'present')
Rico Windf80f5a22017-06-16 09:15:57 +0200287
Jean-Marie Henaffe4e36d12018-04-05 10:33:50 +0200288def DownloadFromX20(sha1_file):
289 download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
290 cmd = [download_script, sha1_file]
291 PrintCmd(cmd)
292 subprocess.check_call(cmd)
293
Rico Wind59593922021-03-03 09:12:36 +0100294def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps', auth=False,
295 quiet=False):
Rico Windf80f5a22017-06-16 09:15:57 +0200296 suffix = '.bat' if IsWindows() else ''
297 download_script = 'download_from_google_storage%s' % suffix
Rico Wind533e3ce2019-04-04 10:26:12 +0200298 cmd = [download_script]
299 if not auth:
300 cmd.append('-n')
301 cmd.extend(['-b', bucket, '-u', '-s', sha1_file])
Rico Wind59593922021-03-03 09:12:36 +0100302 if not quiet:
303 PrintCmd(cmd)
304 subprocess.check_call(cmd)
305 else:
306 subprocess.check_output(cmd)
Mads Ager418d1ca2017-05-22 09:35:49 +0200307
308def get_sha1(filename):
309 sha1 = hashlib.sha1()
310 with open(filename, 'rb') as f:
311 while True:
312 chunk = f.read(1024*1024)
313 if not chunk:
314 break
315 sha1.update(chunk)
316 return sha1.hexdigest()
317
Rico Wind1b52acf2021-03-21 12:36:55 +0100318def is_main():
Rico Windfd186372022-02-28 08:55:48 +0100319 remotes = subprocess.check_output(['git', 'branch', '-r', '--contains',
320 'HEAD']).decode('utf-8')
Rico Wind1b52acf2021-03-21 12:36:55 +0100321 return 'origin/main' in remotes
Rico Wind1b09c562019-01-17 08:53:09 +0100322
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200323def get_HEAD_sha1():
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100324 return get_HEAD_sha1_for_checkout(REPO_ROOT)
325
326def get_HEAD_sha1_for_checkout(checkout):
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200327 cmd = ['git', 'rev-parse', 'HEAD']
328 PrintCmd(cmd)
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100329 with ChangedWorkingDirectory(checkout):
Christoffer Quist Adamsen50930e42021-01-20 11:55:12 +0100330 return subprocess.check_output(cmd).decode('utf-8').strip()
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200331
Tamas Kenez971eec62017-05-24 11:08:40 +0200332def makedirs_if_needed(path):
333 try:
334 os.makedirs(path)
335 except OSError:
336 if not os.path.isdir(path):
337 raise
338
Rico Windef7420c2021-10-14 13:16:01 +0200339def get_gsutil():
340 return 'gsutil.py' if os.name != 'nt' else 'gsutil.py.bat'
341
Rico Wind800fd712018-09-24 11:29:33 +0200342def upload_dir_to_cloud_storage(directory, destination, is_html=False, public_read=True):
Rico Winda94f01c2017-06-27 10:32:34 +0200343 # Upload and make the content encoding right for viewing directly
Rico Windef7420c2021-10-14 13:16:01 +0200344 cmd = [get_gsutil(), '-m', 'cp']
Rico Windd0d88cf2018-02-09 09:46:11 +0100345 if is_html:
346 cmd += ['-z', 'html']
Rico Wind800fd712018-09-24 11:29:33 +0200347 cmd += ['-R', directory, destination]
Rico Winda94f01c2017-06-27 10:32:34 +0200348 PrintCmd(cmd)
349 subprocess.check_call(cmd)
350
Rico Wind800fd712018-09-24 11:29:33 +0200351def upload_file_to_cloud_storage(source, destination, public_read=True):
Rico Windef7420c2021-10-14 13:16:01 +0200352 cmd = [get_gsutil(), 'cp']
Rico Wind800fd712018-09-24 11:29:33 +0200353 cmd += [source, destination]
Rico Windb4621c12017-08-28 12:48:53 +0200354 PrintCmd(cmd)
355 subprocess.check_call(cmd)
356
Rico Wind139eece2018-09-25 09:42:09 +0200357def delete_file_from_cloud_storage(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200358 cmd = [get_gsutil(), 'rm', destination]
Rico Wind139eece2018-09-25 09:42:09 +0200359 PrintCmd(cmd)
360 subprocess.check_call(cmd)
361
Rico Wind4fd2dda2018-09-26 17:41:45 +0200362def ls_files_on_cloud_storage(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200363 cmd = [get_gsutil(), 'ls', destination]
Rico Wind4fd2dda2018-09-26 17:41:45 +0200364 PrintCmd(cmd)
Rico Windfd186372022-02-28 08:55:48 +0100365 return subprocess.check_output(cmd).decode('utf-8')
Rico Wind4fd2dda2018-09-26 17:41:45 +0200366
Rico Wind139eece2018-09-25 09:42:09 +0200367def cat_file_on_cloud_storage(destination, ignore_errors=False):
Rico Windef7420c2021-10-14 13:16:01 +0200368 cmd = [get_gsutil(), 'cat', destination]
Rico Wind139eece2018-09-25 09:42:09 +0200369 PrintCmd(cmd)
370 try:
Rico Wind8dadb312022-02-28 08:40:20 +0100371 return subprocess.check_output(cmd).decode('utf-8').strip()
Rico Wind139eece2018-09-25 09:42:09 +0200372 except subprocess.CalledProcessError as e:
373 if ignore_errors:
374 return ''
375 else:
376 raise e
377
378def file_exists_on_cloud_storage(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200379 cmd = [get_gsutil(), 'ls', destination]
Rico Wind139eece2018-09-25 09:42:09 +0200380 PrintCmd(cmd)
381 return subprocess.call(cmd) == 0
382
Christoffer Quist Adamsen870fa462020-12-15 10:50:54 +0100383def download_file_from_cloud_storage(source, destination, quiet=False):
Rico Windef7420c2021-10-14 13:16:01 +0200384 cmd = [get_gsutil(), 'cp', source, destination]
Christoffer Quist Adamsen870fa462020-12-15 10:50:54 +0100385 PrintCmd(cmd, quiet=quiet)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200386 subprocess.check_call(cmd)
387
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100388def create_archive(name, sources=None):
389 if not sources:
390 sources = [name]
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200391 tarname = '%s.tar.gz' % name
392 with tarfile.open(tarname, 'w:gz') as tar:
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100393 for source in sources:
394 tar.add(source)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200395 return tarname
396
397def extract_dir(filename):
398 return filename[0:len(filename) - len('.tar.gz')]
399
400def unpack_archive(filename):
401 dest_dir = extract_dir(filename)
402 if os.path.exists(dest_dir):
Rico Wind3d369b42021-01-12 10:26:24 +0100403 print('Deleting existing dir %s' % dest_dir)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200404 shutil.rmtree(dest_dir)
405 dirname = os.path.dirname(os.path.abspath(filename))
406 with tarfile.open(filename, 'r:gz') as tar:
407 tar.extractall(path=dirname)
408
Morten Krogh-Jespersenec3047b2020-08-18 13:09:06 +0200409def check_gcert():
Ian Zerny86c4cfd2022-01-17 13:43:37 +0100410 status = subprocess.call(['gcertstatus'])
411 if status != 0:
412 subprocess.check_call(['gcert'])
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100413
Rico Wind1a29c4f2018-01-25 08:43:08 +0100414# Note that gcs is eventually consistent with regards to list operations.
415# This is not a problem in our case, but don't ever use this method
416# for synchronization.
417def cloud_storage_exists(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200418 cmd = [get_gsutil(), 'ls', destination]
Rico Wind1a29c4f2018-01-25 08:43:08 +0100419 PrintCmd(cmd)
420 exit_code = subprocess.call(cmd)
421 return exit_code == 0
422
Mads Ager418d1ca2017-05-22 09:35:49 +0200423class TempDir(object):
Søren Gjessecbeae782019-05-21 14:14:25 +0200424 def __init__(self, prefix='', delete=True):
Mads Ager418d1ca2017-05-22 09:35:49 +0200425 self._temp_dir = None
426 self._prefix = prefix
Søren Gjessecbeae782019-05-21 14:14:25 +0200427 self._delete = delete
Mads Ager418d1ca2017-05-22 09:35:49 +0200428
429 def __enter__(self):
430 self._temp_dir = tempfile.mkdtemp(self._prefix)
431 return self._temp_dir
432
433 def __exit__(self, *_):
Søren Gjessecbeae782019-05-21 14:14:25 +0200434 if self._delete:
435 shutil.rmtree(self._temp_dir, ignore_errors=True)
Mads Ager418d1ca2017-05-22 09:35:49 +0200436
437class ChangedWorkingDirectory(object):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100438 def __init__(self, working_directory, quiet=False):
439 self._quiet = quiet
Mads Ager418d1ca2017-05-22 09:35:49 +0200440 self._working_directory = working_directory
441
442 def __enter__(self):
443 self._old_cwd = os.getcwd()
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100444 if not self._quiet:
Rico Wind3d369b42021-01-12 10:26:24 +0100445 print('Enter directory:', self._working_directory)
Mads Ager418d1ca2017-05-22 09:35:49 +0200446 os.chdir(self._working_directory)
447
448 def __exit__(self, *_):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100449 if not self._quiet:
Rico Wind3d369b42021-01-12 10:26:24 +0100450 print('Enter directory:', self._old_cwd)
Mads Ager418d1ca2017-05-22 09:35:49 +0200451 os.chdir(self._old_cwd)
Tamas Kenez82efeb52017-06-12 13:56:22 +0200452
453# Reading Android CTS test_result.xml
454
455class CtsModule(object):
456 def __init__(self, module_name):
457 self.name = module_name
458
459class CtsTestCase(object):
460 def __init__(self, test_case_name):
461 self.name = test_case_name
462
463class CtsTest(object):
464 def __init__(self, test_name, outcome):
465 self.name = test_name
466 self.outcome = outcome
467
468# Generator yielding CtsModule, CtsTestCase or CtsTest from
469# reading through a CTS test_result.xml file.
470def read_cts_test_result(file_xml):
471 re_module = re.compile('<Module name="([^"]*)"')
472 re_test_case = re.compile('<TestCase name="([^"]*)"')
473 re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
474 with open(file_xml) as f:
475 for line in f:
476 m = re_module.search(line)
477 if m:
478 yield CtsModule(m.groups()[0])
479 continue
480 m = re_test_case.search(line)
481 if m:
482 yield CtsTestCase(m.groups()[0])
483 continue
484 m = re_test.search(line)
485 if m:
486 outcome = m.groups()[0]
Rico Windf80f5a22017-06-16 09:15:57 +0200487 assert outcome in ['fail', 'pass']
Tamas Kenez82efeb52017-06-12 13:56:22 +0200488 yield CtsTest(m.groups()[1], outcome == 'pass')
Tamas Kenezfc34cd82017-07-13 12:43:57 +0200489
490def grep_memoryuse(logfile):
491 re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
492 result = None
493 with open(logfile) as f:
494 for line in f:
495 m = re_vmhwm.search(line)
496 if m:
497 groups = m.groups()
498 s = len(groups)
499 if s >= 1:
500 result = int(groups[0])
501 if s >= 2:
502 unit = groups[1]
503 if unit == 'kB':
504 result *= 1024
505 elif unit != '':
506 raise Exception('Unrecognized unit in memory usage log: {}'
507 .format(unit))
508 if result is None:
509 raise Exception('No memory usage found in log: {}'.format(logfile))
Tamas Kenez02bff032017-07-18 12:13:58 +0200510 return result
511
512# Return a dictionary: {segment_name -> segments_size}
513def getDexSegmentSizes(dex_files):
514 assert len(dex_files) > 0
Ian Zerny3f54e222019-02-12 10:51:17 +0100515 cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments']
Tamas Kenez02bff032017-07-18 12:13:58 +0200516 cmd.extend(dex_files)
517 PrintCmd(cmd)
Morten Krogh-Jespersenf54dd782021-02-09 09:06:08 +0100518 output = subprocess.check_output(cmd).decode('utf-8')
Tamas Kenez02bff032017-07-18 12:13:58 +0200519
520 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
521
522 if matches is None or len(matches) == 0:
523 raise Exception('DexSegments failed to return any output for' \
524 ' these files: {}'.format(dex_files))
525
526 result = {}
527
528 for match in matches:
529 result[match[0]] = int(match[1])
530
531 return result
532
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100533# Return a dictionary: {segment_name -> segments_size}
534def getCfSegmentSizes(cfFile):
Ian Zerny3f54e222019-02-12 10:51:17 +0100535 cmd = [jdk.GetJavaExecutable(),
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100536 '-cp',
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +0100537 CF_SEGMENTS_TOOL,
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100538 'com.android.tools.r8.cf_segments.MeasureLib',
539 cfFile]
540 PrintCmd(cmd)
Rico Windfd186372022-02-28 08:55:48 +0100541 output = subprocess.check_output(cmd).decode('utf-8')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100542
543 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
544
545 if matches is None or len(matches) == 0:
546 raise Exception('CfSegments failed to return any output for' \
547 ' the file: ' + cfFile)
548
549 result = {}
550
551 for match in matches:
552 result[match[0]] = int(match[1])
553
554 return result
555
Søren Gjesse1c115b52019-08-14 12:43:57 +0200556def get_maven_path(artifact, version):
557 return os.path.join('com', 'android', 'tools', artifact, version)
Rico Windc0b16382018-05-17 13:23:43 +0200558
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100559def print_cfsegments(prefix, cf_files):
560 for cf_file in cf_files:
561 for segment_name, size in getCfSegmentSizes(cf_file).items():
562 print('{}-{}(CodeSize): {}'
563 .format(prefix, segment_name, size))
564
Tamas Kenez02bff032017-07-18 12:13:58 +0200565def print_dexsegments(prefix, dex_files):
566 for segment_name, size in getDexSegmentSizes(dex_files).items():
567 print('{}-{}(CodeSize): {}'
568 .format(prefix, segment_name, size))
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200569
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100570# Ensure that we are not benchmarking with a google jvm.
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200571def check_java_version():
Ian Zerny3f54e222019-02-12 10:51:17 +0100572 cmd= [jdk.GetJavaExecutable(), '-version']
Rico Windfd186372022-02-28 08:55:48 +0100573 output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
574 m = re.search('openjdk version "([^"]*)"', output)
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200575 if m is None:
576 raise Exception("Can't check java version: no version string in output"
577 " of 'java -version': '{}'".format(output))
578 version = m.groups(0)[0]
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100579 m = re.search('google', version)
580 if m is not None:
581 raise Exception("Do not use google JVM for benchmarking: " + version)
Tamas Kenez0cad51c2017-08-21 14:42:01 +0200582
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100583def get_android_jar_dir(api):
584 return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api))
585
Rico Wind9d70f612018-08-31 09:17:43 +0200586def get_android_jar(api):
587 return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
Rico Windda6836e2018-12-07 12:32:03 +0100588
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100589def get_android_optional_jars(api):
590 android_optional_jars_dir = os.path.join(get_android_jar_dir(api), 'optional')
591 android_optional_jars = [
592 os.path.join(android_optional_jars_dir, 'android.test.base.jar'),
593 os.path.join(android_optional_jars_dir, 'android.test.mock.jar'),
594 os.path.join(android_optional_jars_dir, 'android.test.runner.jar'),
595 os.path.join(android_optional_jars_dir, 'org.apache.http.legacy.jar')
596 ]
597 return [
598 android_optional_jar for android_optional_jar in android_optional_jars
599 if os.path.isfile(android_optional_jar)]
600
Rico Windfaaac012019-02-25 11:24:05 +0100601def is_bot():
Rico Winde11f4392019-03-18 07:59:37 +0100602 return 'SWARMING_BOT_ID' in os.environ
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +0100603
604def uncompressed_size(path):
605 return sum(z.file_size for z in zipfile.ZipFile(path).infolist())
Morten Krogh-Jespersen0de13732019-03-01 08:56:39 +0100606
607def getR8Version(path):
608 cmd = [jdk.GetJavaExecutable(), '-cp', path, 'com.android.tools.r8.R8',
609 '--version']
Rico Windfd186372022-02-28 08:55:48 +0100610 output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
Jinseong Jeona2394232019-11-26 22:17:55 -0800611 # output is of the form 'R8 <version> (with additional info)'
612 # so we split on '('; clean up tailing spaces; and strip off 'R8 '.
613 return output.split('(')[0].strip()[3:]
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200614
Søren Gjesse705a3b12022-03-17 11:37:30 +0100615def desugar_configuration_version(configuration):
616 with open(configuration, 'r') as f:
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200617 configuration_json = json.loads(f.read())
618 configuration_format_version = \
619 configuration_json.get('configuration_format_version')
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200620 version = configuration_json.get('version')
621 if not version:
622 raise Exception(
Søren Gjesse705a3b12022-03-17 11:37:30 +0100623 'No "version" found in ' + configuration)
624 check_basic_semver_version(version, 'in ' + configuration, allowPrerelease = True)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200625 return version
Søren Gjesse1e171532019-09-03 09:44:22 +0200626
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100627class SemanticVersion:
Søren Gjesse705a3b12022-03-17 11:37:30 +0100628 def __init__(self, major, minor, patch, prerelease):
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100629 self.major = major
630 self.minor = minor
631 self.patch = patch
Søren Gjesse705a3b12022-03-17 11:37:30 +0100632 self.prerelease = prerelease
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100633 # Build metadata currently not suppported
634
635 def larger_than(self, other):
Søren Gjesse705a3b12022-03-17 11:37:30 +0100636 if self.prepelease or other.prepelease:
637 raise Exception("Comparison with prerelease not implemented")
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100638 if self.major > other.major:
639 return True
640 if self.major == other.major and self.minor > other.minor:
641 return True
642 if self.patch:
643 return (self.major == other.major
644 and self.minor == other.minor
645 and self.patch > other.patch)
646 else:
647 return False
648
649
Søren Gjesse705a3b12022-03-17 11:37:30 +0100650# Check that the passed string is formatted as a basic semver version (x.y.z or x.y.z-prerelease
651# depending on the value of allowPrerelease).
652# See https://semver.org/. The regexp parts used are not all complient with what is suggested
653# on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string.
654def check_basic_semver_version(version, error_context = '', components = 3, allowPrerelease = False):
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100655 regexp = '^'
656 for x in range(components):
657 regexp += '([0-9]+)'
658 if x < components - 1:
659 regexp += '\\.'
Søren Gjesse705a3b12022-03-17 11:37:30 +0100660 if allowPrerelease:
661 # This part is from
662 # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
663 regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?'
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100664 regexp += '$'
665 reg = re.compile(regexp)
666 match = reg.match(version)
667 if not match:
Søren Gjesse1e171532019-09-03 09:44:22 +0200668 raise Exception("Invalid version '"
669 + version
670 + "'"
671 + (' ' + error_context) if len(error_context) > 0 else '')
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100672 if components == 2:
Søren Gjesse705a3b12022-03-17 11:37:30 +0100673 return SemanticVersion(int(match.group(1)), int(match.group(2)), None, None)
674 elif components == 3 and not allowPrerelease:
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100675 return SemanticVersion(
Søren Gjesse705a3b12022-03-17 11:37:30 +0100676 int(match.group(1)), int(match.group(2)), int(match.group(3)), None)
677 elif components == 3 and allowPrerelease:
678 return SemanticVersion(
679 int(match.group(1)), int(match.group(2)), int(match.group(3)), match.group('prerelease'))
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100680 else:
681 raise Exception('Argument "components" must be 2 or 3')