blob: 86d7e8ded06a800329b9a14cb0f1695873655644 [file] [log] [blame]
Mads Ager418d1ca2017-05-22 09:35:49 +02001# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
2# for details. All rights reserved. Use of this source code is governed by a
3# BSD-style license that can be found in the LICENSE file.
4
5# Different utility functions used accross scripts
6
7import hashlib
Clément Béra3718ad02023-09-05 14:12:48 +02008import jdk
Søren Gjesse6e5e5842019-09-03 08:48:30 +02009import json
Mads Ager418d1ca2017-05-22 09:35:49 +020010import os
Tamas Kenez82efeb52017-06-12 13:56:22 +020011import re
Mads Ager418d1ca2017-05-22 09:35:49 +020012import shutil
13import subprocess
14import sys
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +020015import tarfile
Mads Ager418d1ca2017-05-22 09:35:49 +020016import tempfile
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +010017import zipfile
Mads Ager418d1ca2017-05-22 09:35:49 +020018
Ian Zerny37097652019-04-11 13:13:27 +020019import defines
Christoffer Quist Adamsen65ef2982023-08-24 08:45:39 +020020from thread_utils import print_thread
Ian Zerny37097652019-04-11 13:13:27 +020021
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +010022ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}'
23ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar')
Ian Zerny3f54e222019-02-12 10:51:17 +010024TOOLS_DIR = defines.TOOLS_DIR
25REPO_ROOT = defines.REPO_ROOT
26THIRD_PARTY = defines.THIRD_PARTY
Christoffer Quist Adamsen4d9fc512022-08-11 19:59:44 +020027BUNDLETOOL_JAR_DIR = os.path.join(THIRD_PARTY, 'bundletool/bundletool-1.11.0')
28BUNDLETOOL_JAR = os.path.join(BUNDLETOOL_JAR_DIR, 'bundletool-all-1.11.0.jar')
Tamas Kenezfc34cd82017-07-13 12:43:57 +020029MEMORY_USE_TMP_FILE = 'memory_use.tmp'
Tamas Kenez02bff032017-07-18 12:13:58 +020030DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)')
Rico Windaf9e8fb2023-10-10 09:25:26 +020031
Rico Windb65d75c2023-10-10 11:14:52 +020032DEPENDENCIES_DIR = os.path.join(THIRD_PARTY, 'dependencies')
33
Mads Ager12a56bc2017-11-27 11:51:25 +010034BUILD = os.path.join(REPO_ROOT, 'build')
Morten Krogh-Jespersen51db2b02020-11-11 12:49:26 +010035BUILD_JAVA_MAIN_DIR = os.path.join(BUILD, 'classes', 'java', 'main')
Mads Ager12a56bc2017-11-27 11:51:25 +010036LIBS = os.path.join(BUILD, 'libs')
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020037CUSTOM_CONVERSION_DIR = os.path.join(THIRD_PARTY, 'openjdk',
38 'custom_conversion')
Mads Ager12a56bc2017-11-27 11:51:25 +010039GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense')
Mads Agera4911eb2017-11-22 13:19:36 +010040SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java')
Ian Zerny59dfa4c2019-10-25 10:34:36 +020041REPO_SOURCE = 'https://r8.googlesource.com/r8'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020042
Rico Wind93755fb2023-10-09 13:29:29 +020043GRADLE_TASK_CLEAN_TEST = ':test:cleanTest'
44GRADLE_TASK_CONSOLIDATED_LICENSE = ':main:consolidatedLicense'
45GRADLE_TASK_KEEP_ANNO_JAR = ':keepanno:keepAnnoAnnotationsJar'
Ian Zernya6af7772023-11-21 14:05:57 +010046GRADLE_TASK_KEEP_ANNO_DOC = ':keepanno:keepAnnoAnnotationsDoc'
Rico Wind93755fb2023-10-09 13:29:29 +020047GRADLE_TASK_R8 = ':main:r8WithRelocatedDeps'
Christoffer Quist Adamsen31237c32023-10-13 10:59:53 +020048GRADLE_TASK_R8LIB = ':test:assembleR8LibWithRelocatedDeps'
49GRADLE_TASK_R8LIB_NO_DEPS = ':test:assembleR8LibNoDeps'
Ian Zerny077a73f2023-10-31 09:05:47 +010050GRADLE_TASK_THREADING_MODULE_BLOCKING = ':main:threadingModuleBlockingJar'
51GRADLE_TASK_THREADING_MODULE_SINGLE_THREADED = ':main:threadingModuleSingleThreadedJar'
Christoffer Quist Adamsen31237c32023-10-13 10:59:53 +020052GRADLE_TASK_SOURCE_JAR = ':test:packageSources'
Rico Wind93755fb2023-10-09 13:29:29 +020053GRADLE_TASK_SWISS_ARMY_KNIFE = ':main:swissArmyKnife'
54GRADLE_TASK_TEST = ':test:test'
Søren Gjesse397f1a12023-10-13 13:22:31 +020055GRADLE_TASK_ALL_TESTS_WITH_APPLY_MAPPING_JAR = ':test:rewriteTestsForR8LibWithRelocatedDeps'
Rico Wind9365e942024-03-27 12:58:05 +010056GRADLE_TASK_TESTBASE_WITH_APPLY_MAPPING_JAR = ':test:rewriteTestBaseForR8LibWithRelocatedDeps'
Søren Gjesse397f1a12023-10-13 13:22:31 +020057GRADLE_TASK_TEST_DEPS_JAR = ':test:packageTestDeps'
Søren Gjesse1da57172024-07-31 11:10:37 +020058GRADLE_TASK_TEST_BASE_JAR = ':test:relocateTestBaseForR8LibWithRelocatedDeps'
Søren Gjessea28c4612023-11-01 14:55:27 +010059GRADLE_TASK_TEST_JAR = ':test:relocateTestsForR8LibWithRelocatedDeps'
Christoffer Adamsend58f0472024-06-04 14:28:12 +020060GRADLE_TASK_TEST_UNZIP_TESTBASE = ':test:unzipTestBase'
Rico Wind93755fb2023-10-09 13:29:29 +020061
Søren Gjessedc9d8a22017-10-12 12:40:59 +020062R8 = 'r8'
Tamas Kenez03ab76f2018-12-07 14:33:25 +010063R8LIB = 'r8lib'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020064
Morten Krogh-Jespersen51db2b02020-11-11 12:49:26 +010065ALL_DEPS_JAR = os.path.join(LIBS, 'deps_all.jar')
Rico Wind74fab302017-10-02 07:25:33 +020066R8_JAR = os.path.join(LIBS, 'r8.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010067R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar')
Rico Wind158ef9f2022-05-19 11:08:30 +020068R8LIB_MAP = '%s.map' % R8LIB_JAR
Mads Agerb10c07f2017-11-27 13:25:52 +010069R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010070R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar')
Tamas Kenez180be092018-12-05 15:23:06 +010071R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar')
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +020072THREADING_MODULE_BLOCKING_JAR = os.path.join(LIBS,
73 'threading-module-blocking.jar')
74THREADING_MODULE_SINGLE_THREADED_JAR = os.path.join(
75 LIBS, 'threading-module-single-threaded.jar')
Ian Zerny161ff742022-01-20 12:39:40 +010076R8_TESTS_JAR = os.path.join(LIBS, 'r8tests.jar')
Rico Wind9365e942024-03-27 12:58:05 +010077R8_TESTBASE_JAR = os.path.join(LIBS, 'r8test_base.jar')
78R8LIB_TESTBASE_JAR = os.path.join(LIBS, 'r8libtestbase-cf.jar')
Ian Zerny161ff742022-01-20 12:39:40 +010079R8LIB_TESTS_JAR = os.path.join(LIBS, 'r8libtestdeps-cf.jar')
80R8_TESTS_DEPS_JAR = os.path.join(LIBS, 'test_deps_all.jar')
81R8LIB_TESTS_DEPS_JAR = R8_TESTS_DEPS_JAR
Rico Wind8fc8bfa2019-03-22 09:57:36 +010082MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip')
Clément Béra3718ad02023-09-05 14:12:48 +020083LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP = os.path.join(
84 CUSTOM_CONVERSION_DIR, 'library_desugar_conversions_legacy.jar')
85LIBRARY_DESUGAR_CONVERSIONS_ZIP = os.path.join(
86 CUSTOM_CONVERSION_DIR, 'library_desugar_conversions.jar')
Ian Zernyf13d18f2023-05-24 12:50:37 +020087KEEPANNO_ANNOTATIONS_JAR = os.path.join(LIBS, 'keepanno-annotations.jar')
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +020088KEEPANNO_ANNOTATIONS_DOC = os.path.join('d8_r8', 'keepanno', 'build', 'docs',
89 'javadoc')
Søren Gjesse17fc67d2019-12-04 14:50:17 +010090
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020091DESUGAR_CONFIGURATION = os.path.join('src', 'library_desugar',
92 'desugar_jdk_libs.json')
93DESUGAR_IMPLEMENTATION = os.path.join('third_party', 'openjdk',
94 'desugar_jdk_libs',
95 'desugar_jdk_libs.jar')
Søren Gjesse705a3b12022-03-17 11:37:30 +010096DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020097 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
Søren Gjesse2b047692022-08-19 16:34:38 +020098DESUGAR_CONFIGURATION_JDK11_MINIMAL = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020099 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json')
100DESUGAR_CONFIGURATION_JDK11 = os.path.join('src', 'library_desugar', 'jdk11',
101 'desugar_jdk_libs.json')
102DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join('src', 'library_desugar',
103 'jdk11',
104 'desugar_jdk_libs_nio.json')
105DESUGAR_IMPLEMENTATION_JDK11 = os.path.join('third_party', 'openjdk',
106 'desugar_jdk_libs_11',
107 'desugar_jdk_libs.jar')
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200108DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200109 LIBS, 'desugar_jdk_libs_configuration.zip')
Søren Gjessee18fa6e2022-06-24 15:14:53 +0200110DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200111 LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip')
Søren Gjesse2b047692022-08-19 16:34:38 +0200112DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200113 LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip')
Søren Gjesse2b047692022-08-19 16:34:38 +0200114DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200115 LIBS, 'desugar_jdk_libs_configuration_jdk11.zip')
Søren Gjesse2b047692022-08-19 16:34:38 +0200116DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP = os.path.join(
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200117 LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip')
Mads Ager12a56bc2017-11-27 11:51:25 +0100118GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
Mathias Rav3fb4a3a2018-05-29 15:41:36 +0200119RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
Mathias Ravb46dc002018-06-06 09:37:11 +0200120R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +0100121CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100122PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar')
123PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar')
Rico Wind8bd693f2023-10-13 11:29:32 +0200124
Morten Krogh-Jespersen45d7a7b2020-11-02 08:31:09 +0100125OPENSOURCE_DUMPS_DIR = os.path.join(THIRD_PARTY, 'opensource-apps')
Morten Krogh-Jespersen86222742021-03-02 11:13:33 +0100126INTERNAL_DUMPS_DIR = os.path.join(THIRD_PARTY, 'internal-apps')
Søren Gjesse1c115b52019-08-14 12:43:57 +0200127BAZEL_SHA_FILE = os.path.join(THIRD_PARTY, 'bazel.tar.gz.sha1')
128BAZEL_TOOL = os.path.join(THIRD_PARTY, 'bazel')
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200129JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8',
130 'linux-x86.tar.gz.sha1')
131JAVA11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11',
132 'linux.tar.gz.sha1')
133DESUGAR_JDK_LIBS_11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk',
134 'desugar_jdk_libs_11.tar.gz.sha1')
135IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test',
136 'ignorewarnings.rules')
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100137ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME"
138ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION"
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100139USER_HOME = os.path.expanduser('~')
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100140
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200141R8_TEST_RESULTS_BUCKET = 'r8-test-results'
Rico Wind635b2de2022-04-25 10:35:14 +0200142R8_INTERNAL_TEST_RESULTS_BUCKET = 'r8-internal-test-results'
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200143
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200144
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200145def archive_file(name, gs_dir, src_file):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200146 gs_file = '%s/%s' % (gs_dir, name)
147 upload_file_to_cloud_storage(src_file, gs_file)
148
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200149
150def archive_value(name, gs_dir, value):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200151 with TempDir() as temp:
152 temparchive = os.path.join(temp, name)
153 with open(temparchive, 'w') as f:
154 f.write(str(value))
155 archive_file(name, gs_dir, temparchive)
156
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200157
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200158def find_cloud_storage_file_from_options(name, options, orElse=None):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200159 # Import archive on-demand since archive depends on utils.
160 from archive import GetUploadDestination
161 hash_or_version = find_hash_or_version_from_options(options)
162 if not hash_or_version:
163 return orElse
164 is_hash = options.commit_hash is not None
165 download_path = GetUploadDestination(hash_or_version, name, is_hash)
166 if file_exists_on_cloud_storage(download_path):
167 out = tempfile.NamedTemporaryFile().name
168 download_file_from_cloud_storage(download_path, out)
169 return out
170 else:
171 raise Exception('Could not find file {} from hash/version: {}.'.format(
172 name, hash_or_version))
173
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200174
175def find_r8_jar_from_options(options):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200176 return find_cloud_storage_file_from_options('r8.jar', options)
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200177
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200178
179def find_hash_or_version_from_options(options):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200180 if options.tag:
181 return find_hash_or_version_from_tag(options.tag)
182 else:
183 return options.commit_hash or options.version
184
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200185
186def find_hash_or_version_from_tag(tag_or_hash):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200187 info = subprocess.check_output(
188 ['git', 'show', tag_or_hash, '-s',
189 '--format=oneline']).decode('utf-8').splitlines()[-1].split()
190 # The info should be on the following form [hash,"Version",version]
191 if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version":
192 return info[2]
193 return None
194
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200195
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100196def getAndroidHome():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200197 return os.environ.get(ANDROID_HOME_ENVIROMENT_NAME,
198 os.path.join(USER_HOME, 'Android', 'Sdk'))
199
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100200
201def getAndroidBuildTools():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200202 if ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME in os.environ:
203 version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME)
204 build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
205 assert os.path.exists(build_tools_dir)
Christoffer Quist Adamsen8c803b42022-05-31 10:36:17 +0200206 return build_tools_dir
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200207 else:
Christoffer Adamsen500a6522024-02-27 12:27:29 +0000208 versions = ['34.0.0', '33.0.1', '32.0.0']
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200209 for version in versions:
210 build_tools_dir = os.path.join(getAndroidHome(), 'build-tools',
211 version)
212 if os.path.exists(build_tools_dir):
213 return build_tools_dir
214 raise Exception('Unable to find Android build-tools')
215
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100216
Christoffer Quist Adamsen5c9ded12021-01-14 14:29:37 +0100217def is_python3():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200218 return sys.version_info.major == 3
219
Christoffer Quist Adamsen5c9ded12021-01-14 14:29:37 +0100220
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100221def Print(s, quiet=False):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200222 if quiet:
223 return
224 print(s)
225
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100226
227def Warn(message):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200228 CRED = '\033[91m'
229 CEND = '\033[0m'
230 print(CRED + message + CEND)
231
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100232
Christoffer Quist Adamsen65ef2982023-08-24 08:45:39 +0200233def PrintCmd(cmd, env=None, quiet=False, worker_id=None):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200234 if quiet:
235 return
236 if type(cmd) is list:
237 cmd = ' '.join(cmd)
238 if env:
239 env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
240 print_thread('Running: {} {}'.format(env, cmd), worker_id)
241 else:
242 print_thread('Running: {}'.format(cmd), worker_id)
243 # I know this will hit os on windows eventually if we don't do this.
244 sys.stdout.flush()
245
Mads Ager418d1ca2017-05-22 09:35:49 +0200246
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100247class ProgressLogger(object):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200248 CLEAR_LINE = '\033[K'
249 UP = '\033[F'
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100250
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200251 def __init__(self, quiet=False):
252 self._count = 0
253 self._has_printed = False
254 self._quiet = quiet
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100255
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200256 def log(self, text):
257 if len(text.strip()) == 0:
258 return
259 if self._quiet:
260 if self._has_printed:
261 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
262 if len(text) > 140:
263 text = text[0:140] + '...'
264 print(text)
265 self._has_printed = True
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100266
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200267 def done(self):
268 if self._quiet and self._has_printed:
269 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
270 print('')
271 sys.stdout.write(ProgressLogger.UP)
272
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100273
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100274def RunCmd(cmd, env_vars=None, quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200275 PrintCmd(cmd, env=env_vars, quiet=quiet)
276 env = os.environ.copy()
277 if env_vars:
278 env.update(env_vars)
279 process = subprocess.Popen(cmd,
280 env=env,
281 stdout=subprocess.PIPE,
282 stderr=subprocess.STDOUT)
283 stdout = []
284 logger = ProgressLogger(quiet=quiet) if logging else None
285 failed = False
286 while True:
287 line = process.stdout.readline().decode('utf-8')
288 if line != '':
289 stripped = line.rstrip()
290 stdout.append(stripped)
291 if logger:
292 logger.log(stripped)
293 # TODO(christofferqa): r8 should fail with non-zero exit code.
294 if ('AssertionError:' in stripped or
295 'CompilationError:' in stripped or
296 'CompilationFailedException:' in stripped or
297 'Compilation failed' in stripped or
298 'FAILURE:' in stripped or
299 'org.gradle.api.ProjectConfigurationException' in stripped
300 or 'BUILD FAILED' in stripped):
301 failed = True
302 else:
303 if logger:
304 logger.done()
305 exit_code = process.poll()
306 if exit_code or failed:
307 for line in stdout:
308 Warn(line)
309 if fail:
310 raise subprocess.CalledProcessError(
311 exit_code or -1, cmd, output='\n'.join(stdout))
312 return stdout
313
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100314
Rico Windf80f5a22017-06-16 09:15:57 +0200315def IsWindows():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200316 return defines.IsWindows()
317
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100318
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100319def EnsureDepFromGoogleCloudStorage(dep, tgz, sha1, msg):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200320 if (not os.path.exists(dep) or not os.path.exists(tgz) or
321 os.path.getmtime(tgz) < os.path.getmtime(sha1)):
322 DownloadFromGoogleCloudStorage(sha1)
323 # Update the mtime of the tar file to make sure we do not run again unless
324 # there is an update.
325 os.utime(tgz, None)
326 else:
327 print('Ensure cloud dependency:', msg, 'present')
328
Rico Windf80f5a22017-06-16 09:15:57 +0200329
Jean-Marie Henaffe4e36d12018-04-05 10:33:50 +0200330def DownloadFromX20(sha1_file):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200331 download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
332 cmd = [download_script, sha1_file]
Rico Wind59593922021-03-03 09:12:36 +0100333 PrintCmd(cmd)
334 subprocess.check_call(cmd)
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200335
336
337def DownloadFromGoogleCloudStorage(sha1_file,
Rico Wind5e1580a2024-07-01 08:32:39 +0000338 bucket='r8-deps',
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200339 auth=False,
340 quiet=False):
341 suffix = '.bat' if IsWindows() else ''
342 download_script = 'download_from_google_storage%s' % suffix
343 cmd = [download_script]
344 if not auth:
345 cmd.append('-n')
346 cmd.extend(['-b', bucket, '-u', '-s', sha1_file])
347 if not quiet:
348 PrintCmd(cmd)
349 subprocess.check_call(cmd)
350 else:
351 subprocess.check_output(cmd)
352
Mads Ager418d1ca2017-05-22 09:35:49 +0200353
Christoffer Adamsenc09ba6f2024-06-12 18:48:36 +0200354def get_nth_sha1_from_revision(n, revision):
355 result = subprocess.check_output([
356 'git', 'log', revision, f'--skip={n}', '--max-count=1',
357 '--pretty=format:%H'
358 ]).decode('utf-8')
Christoffer Adamsen30283712024-06-12 13:08:14 +0200359 return result.strip()
360
361
Mads Ager418d1ca2017-05-22 09:35:49 +0200362def get_sha1(filename):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200363 sha1 = hashlib.sha1()
364 with open(filename, 'rb') as f:
365 while True:
366 chunk = f.read(1024 * 1024)
367 if not chunk:
368 break
369 sha1.update(chunk)
370 return sha1.hexdigest()
371
Mads Ager418d1ca2017-05-22 09:35:49 +0200372
Christoffer Adamsenc09ba6f2024-06-12 18:48:36 +0200373def get_sha1_from_revision(revision):
374 cmd = ['git', 'rev-parse', revision]
375 PrintCmd(cmd)
376 with ChangedWorkingDirectory(REPO_ROOT):
377 return subprocess.check_output(cmd).decode('utf-8').strip()
378
379
Ian Zernyc2de7b72023-09-06 20:52:16 +0200380def get_HEAD_branch():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200381 result = subprocess.check_output(
382 ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8')
383 return result.strip()
384
Ian Zernyc2de7b72023-09-06 20:52:16 +0200385
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200386def get_HEAD_sha1():
Christoffer Adamsenc09ba6f2024-06-12 18:48:36 +0200387 return get_sha1_from_revision('HEAD')
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200388
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100389
Ian Zerny9f0345d2023-09-07 11:15:00 +0200390def get_HEAD_diff_stat():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200391 return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8')
392
Ian Zerny9f0345d2023-09-07 11:15:00 +0200393
Tamas Kenez971eec62017-05-24 11:08:40 +0200394def makedirs_if_needed(path):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200395 try:
396 os.makedirs(path)
397 except OSError:
398 if not os.path.isdir(path):
399 raise
400
Tamas Kenez971eec62017-05-24 11:08:40 +0200401
Rico Windef7420c2021-10-14 13:16:01 +0200402def get_gsutil():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200403 return 'gsutil.py' if os.name != 'nt' else 'gsutil.py.bat'
404
Rico Windef7420c2021-10-14 13:16:01 +0200405
Christoffer Adamsenb43ff592024-06-12 18:49:01 +0200406def upload_file_to_cloud_storage(source, destination, header=None):
407 cmd = [get_gsutil()]
408 if header:
409 cmd.extend(['-h', header])
410 cmd.extend(['cp', source, destination])
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200411 PrintCmd(cmd)
412 subprocess.check_call(cmd)
413
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +0200414
Ian Zerny566046e2024-01-04 14:24:56 +0100415def check_dir_args(source, destination):
416 # We require that the dirname of the paths coincide, e.g., src/dirname and dst/dirname
417 # The target is then stripped so the upload command will be: cp -R src/dirname dst/
418 (destination_parent, destination_file) = os.path.split(destination)
419 if os.path.basename(source) != destination_file:
420 raise Exception(
421 'Attempt to upload directory with non-matching directory name: ' +
422 f'{source} and {destination}')
423 if len(destination_parent.strip()) == 0:
424 raise Exception(
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +0200425 'Attempt to upload directory to empty destination directory: ' +
426 destination)
Ian Zerny566046e2024-01-04 14:24:56 +0100427 return destination_parent
428
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +0200429
Ian Zernya6af7772023-11-21 14:05:57 +0100430def upload_directory_to_cloud_storage(source, destination, parallel=True):
Ian Zerny566046e2024-01-04 14:24:56 +0100431 destination_parent = check_dir_args(source, destination)
Ian Zernya6af7772023-11-21 14:05:57 +0100432 cmd = [get_gsutil()]
433 if parallel:
434 cmd += ['-m']
435 cmd += ['cp', '-R']
Ian Zerny566046e2024-01-04 14:24:56 +0100436 cmd += [source, destination_parent + '/']
437 PrintCmd(cmd)
438 subprocess.check_call(cmd)
439
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +0200440
Ian Zerny566046e2024-01-04 14:24:56 +0100441def rsync_directory_to_cloud_storage(source, destination, parallel=True):
442 check_dir_args(source, destination)
443 cmd = [get_gsutil()]
444 if parallel:
445 cmd += ['-m']
446 cmd += ['rsync', '-R']
Ian Zernya6af7772023-11-21 14:05:57 +0100447 cmd += [source, destination]
448 PrintCmd(cmd)
449 subprocess.check_call(cmd)
Rico Windb4621c12017-08-28 12:48:53 +0200450
Christoffer Adamsene3b8b6d2024-06-12 13:07:49 +0200451
Rico Wind139eece2018-09-25 09:42:09 +0200452def delete_file_from_cloud_storage(destination):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200453 cmd = [get_gsutil(), 'rm', destination]
454 PrintCmd(cmd)
455 subprocess.check_call(cmd)
456
Rico Wind139eece2018-09-25 09:42:09 +0200457
Rico Wind4fd2dda2018-09-26 17:41:45 +0200458def ls_files_on_cloud_storage(destination):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200459 cmd = [get_gsutil(), 'ls', destination]
460 PrintCmd(cmd)
461 return subprocess.check_output(cmd).decode('utf-8')
462
Rico Wind4fd2dda2018-09-26 17:41:45 +0200463
Rico Wind139eece2018-09-25 09:42:09 +0200464def cat_file_on_cloud_storage(destination, ignore_errors=False):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200465 cmd = [get_gsutil(), 'cat', destination]
466 PrintCmd(cmd)
467 try:
468 return subprocess.check_output(cmd).decode('utf-8').strip()
469 except subprocess.CalledProcessError as e:
470 if ignore_errors:
471 return ''
472 else:
473 raise e
474
Rico Wind139eece2018-09-25 09:42:09 +0200475
476def file_exists_on_cloud_storage(destination):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200477 cmd = [get_gsutil(), 'ls', destination]
478 PrintCmd(cmd)
479 return subprocess.call(cmd) == 0
480
Rico Wind139eece2018-09-25 09:42:09 +0200481
Christoffer Adamsenc09ba6f2024-06-12 18:48:36 +0200482def download_file_from_cloud_storage(source,
483 destination,
484 concurrent=False,
485 flags=None,
486 quiet=False):
487 cmd = [get_gsutil()]
488 if concurrent:
489 cmd.append('-m')
490 cmd.append('cp')
491 if flags:
492 cmd.extend(flags)
493 cmd.extend([source, destination])
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200494 PrintCmd(cmd, quiet=quiet)
495 subprocess.check_call(cmd)
496
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200497
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100498def create_archive(name, sources=None):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200499 if not sources:
500 sources = [name]
501 tarname = '%s.tar.gz' % name
502 with tarfile.open(tarname, 'w:gz') as tar:
503 for source in sources:
504 tar.add(source)
505 return tarname
506
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200507
508def extract_dir(filename):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200509 return filename[0:len(filename) - len('.tar.gz')]
510
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200511
512def unpack_archive(filename):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200513 dest_dir = extract_dir(filename)
514 if os.path.exists(dest_dir):
515 print('Deleting existing dir %s' % dest_dir)
516 shutil.rmtree(dest_dir)
517 dirname = os.path.dirname(os.path.abspath(filename))
518 with tarfile.open(filename, 'r:gz') as tar:
519 tar.extractall(path=dirname)
520
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200521
Morten Krogh-Jespersenec3047b2020-08-18 13:09:06 +0200522def check_gcert():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200523 status = subprocess.call(['gcertstatus'])
524 if status != 0:
525 subprocess.check_call(['gcert'])
526
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100527
Rico Wind1a29c4f2018-01-25 08:43:08 +0100528# Note that gcs is eventually consistent with regards to list operations.
529# This is not a problem in our case, but don't ever use this method
530# for synchronization.
531def cloud_storage_exists(destination):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200532 cmd = [get_gsutil(), 'ls', destination]
533 PrintCmd(cmd)
534 exit_code = subprocess.call(cmd)
535 return exit_code == 0
536
Rico Wind1a29c4f2018-01-25 08:43:08 +0100537
Mads Ager418d1ca2017-05-22 09:35:49 +0200538class TempDir(object):
Mads Ager418d1ca2017-05-22 09:35:49 +0200539
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200540 def __init__(self, prefix='', delete=True):
541 self._temp_dir = None
542 self._prefix = prefix
543 self._delete = delete
Mads Ager418d1ca2017-05-22 09:35:49 +0200544
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200545 def __enter__(self):
546 self._temp_dir = tempfile.mkdtemp(self._prefix)
547 return self._temp_dir
548
549 def __exit__(self, *_):
550 if self._delete:
551 shutil.rmtree(self._temp_dir, ignore_errors=True)
552
Mads Ager418d1ca2017-05-22 09:35:49 +0200553
554class ChangedWorkingDirectory(object):
Mads Ager418d1ca2017-05-22 09:35:49 +0200555
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200556 def __init__(self, working_directory, quiet=False):
557 self._quiet = quiet
558 self._working_directory = working_directory
Mads Ager418d1ca2017-05-22 09:35:49 +0200559
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200560 def __enter__(self):
561 self._old_cwd = os.getcwd()
562 if not self._quiet:
563 print('Enter directory:', self._working_directory)
564 os.chdir(self._working_directory)
565
566 def __exit__(self, *_):
567 if not self._quiet:
568 print('Enter directory:', self._old_cwd)
569 os.chdir(self._old_cwd)
570
Tamas Kenez82efeb52017-06-12 13:56:22 +0200571
572# Reading Android CTS test_result.xml
573
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200574
Tamas Kenez82efeb52017-06-12 13:56:22 +0200575class CtsModule(object):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200576
577 def __init__(self, module_name):
578 self.name = module_name
579
Tamas Kenez82efeb52017-06-12 13:56:22 +0200580
581class CtsTestCase(object):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200582
583 def __init__(self, test_case_name):
584 self.name = test_case_name
585
Tamas Kenez82efeb52017-06-12 13:56:22 +0200586
587class CtsTest(object):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200588
589 def __init__(self, test_name, outcome):
590 self.name = test_name
591 self.outcome = outcome
592
Tamas Kenez82efeb52017-06-12 13:56:22 +0200593
594# Generator yielding CtsModule, CtsTestCase or CtsTest from
595# reading through a CTS test_result.xml file.
596def read_cts_test_result(file_xml):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200597 re_module = re.compile('<Module name="([^"]*)"')
598 re_test_case = re.compile('<TestCase name="([^"]*)"')
599 re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
600 with open(file_xml) as f:
601 for line in f:
602 m = re_module.search(line)
603 if m:
604 yield CtsModule(m.groups()[0])
605 continue
606 m = re_test_case.search(line)
607 if m:
608 yield CtsTestCase(m.groups()[0])
609 continue
610 m = re_test.search(line)
611 if m:
612 outcome = m.groups()[0]
613 assert outcome in ['fail', 'pass']
614 yield CtsTest(m.groups()[1], outcome == 'pass')
615
Tamas Kenezfc34cd82017-07-13 12:43:57 +0200616
617def grep_memoryuse(logfile):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200618 re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
619 result = None
620 with open(logfile) as f:
621 for line in f:
622 m = re_vmhwm.search(line)
623 if m:
624 groups = m.groups()
625 s = len(groups)
626 if s >= 1:
627 result = int(groups[0])
628 if s >= 2:
629 unit = groups[1]
630 if unit == 'kB':
631 result *= 1024
632 elif unit != '':
633 raise Exception(
634 'Unrecognized unit in memory usage log: {}'.
635 format(unit))
636 if result is None:
637 raise Exception('No memory usage found in log: {}'.format(logfile))
638 return result
639
Tamas Kenez02bff032017-07-18 12:13:58 +0200640
641# Return a dictionary: {segment_name -> segments_size}
642def getDexSegmentSizes(dex_files):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200643 assert len(dex_files) > 0
644 cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments']
645 cmd.extend(dex_files)
646 PrintCmd(cmd)
647 output = subprocess.check_output(cmd).decode('utf-8')
Tamas Kenez02bff032017-07-18 12:13:58 +0200648
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200649 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
Tamas Kenez02bff032017-07-18 12:13:58 +0200650
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200651 if matches is None or len(matches) == 0:
652 raise Exception('DexSegments failed to return any output for' \
653 ' these files: {}'.format(dex_files))
Tamas Kenez02bff032017-07-18 12:13:58 +0200654
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200655 result = {}
Tamas Kenez02bff032017-07-18 12:13:58 +0200656
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200657 for match in matches:
658 result[match[0]] = int(match[1])
Tamas Kenez02bff032017-07-18 12:13:58 +0200659
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200660 return result
661
Tamas Kenez02bff032017-07-18 12:13:58 +0200662
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100663# Return a dictionary: {segment_name -> segments_size}
664def getCfSegmentSizes(cfFile):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200665 cmd = [
666 jdk.GetJavaExecutable(), '-cp', CF_SEGMENTS_TOOL,
667 'com.android.tools.r8.cf_segments.MeasureLib', cfFile
668 ]
669 PrintCmd(cmd)
670 output = subprocess.check_output(cmd).decode('utf-8')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100671
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200672 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100673
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200674 if matches is None or len(matches) == 0:
675 raise Exception('CfSegments failed to return any output for' \
676 ' the file: ' + cfFile)
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100677
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200678 result = {}
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100679
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200680 for match in matches:
681 result[match[0]] = int(match[1])
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100682
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200683 return result
684
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100685
Søren Gjesse1c115b52019-08-14 12:43:57 +0200686def get_maven_path(artifact, version):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200687 return os.path.join('com', 'android', 'tools', artifact, version)
688
Rico Windc0b16382018-05-17 13:23:43 +0200689
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100690def print_cfsegments(prefix, cf_files):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200691 for cf_file in cf_files:
692 for segment_name, size in getCfSegmentSizes(cf_file).items():
693 print('{}-{}(CodeSize): {}'.format(prefix, segment_name, size))
694
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100695
Christoffer Quist Adamsenbe6661d2023-08-24 11:01:12 +0200696def print_dexsegments(prefix, dex_files, worker_id=None):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200697 for segment_name, size in getDexSegmentSizes(dex_files).items():
698 print_thread('{}-{}(CodeSize): {}'.format(prefix, segment_name, size),
699 worker_id)
700
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200701
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100702# Ensure that we are not benchmarking with a google jvm.
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200703def check_java_version():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200704 cmd = [jdk.GetJavaExecutable(), '-version']
705 output = subprocess.check_output(cmd,
706 stderr=subprocess.STDOUT).decode('utf-8')
707 m = re.search('openjdk version "([^"]*)"', output)
708 if m is None:
709 raise Exception("Can't check java version: no version string in output"
710 " of 'java -version': '{}'".format(output))
711 version = m.groups(0)[0]
712 m = re.search('google', version)
713 if m is not None:
714 raise Exception("Do not use google JVM for benchmarking: " + version)
715
Tamas Kenez0cad51c2017-08-21 14:42:01 +0200716
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100717def get_android_jar_dir(api):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200718 return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api))
719
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100720
Rico Wind9d70f612018-08-31 09:17:43 +0200721def get_android_jar(api):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200722 return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
723
Rico Windda6836e2018-12-07 12:32:03 +0100724
Rico Windfaaac012019-02-25 11:24:05 +0100725def is_bot():
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200726 return 'SWARMING_BOT_ID' in os.environ
727
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +0100728
729def uncompressed_size(path):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200730 return sum(z.file_size for z in zipfile.ZipFile(path).infolist())
731
Morten Krogh-Jespersen0de13732019-03-01 08:56:39 +0100732
Søren Gjesse2b047692022-08-19 16:34:38 +0200733def desugar_configuration_name_and_version(configuration, is_for_maven):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200734 name = 'desugar_jdk_libs_configuration'
735 with open(configuration, 'r') as f:
736 configuration_json = json.loads(f.read())
737 configuration_format_version = \
738 configuration_json.get('configuration_format_version')
739 if (not configuration_format_version):
740 raise Exception('No "configuration_format_version" found in ' +
741 configuration)
742 if (configuration_format_version != 3 and
743 configuration_format_version != 5 and
Clément Béra43b5cf02023-11-06 14:14:52 +0100744 configuration_format_version != (200 if is_for_maven else 101)):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200745 raise Exception(
746 'Unsupported "configuration_format_version" "%s" found in %s' %
747 (configuration_format_version, configuration))
748 version = configuration_json.get('version')
749 if not version:
Clément Béra43b5cf02023-11-06 14:14:52 +0100750 if configuration_format_version == (200 if is_for_maven else 101):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200751 identifier = configuration_json.get('identifier')
752 if not identifier:
753 raise Exception('No "identifier" found in ' + configuration)
754 identifier_split = identifier.split(':')
755 if (len(identifier_split) != 3):
756 raise Exception('Invalid "identifier" found in ' +
757 configuration)
758 if (identifier_split[0] != 'com.tools.android'):
759 raise Exception('Invalid "identifier" found in ' +
760 configuration)
761 if not identifier_split[1].startswith(
762 'desugar_jdk_libs_configuration'):
763 raise Exception('Invalid "identifier" found in ' +
764 configuration)
765 name = identifier_split[1]
766 version = identifier_split[2]
767 else:
768 raise Exception('No "version" found in ' + configuration)
769 else:
Clément Béra43b5cf02023-11-06 14:14:52 +0100770 if configuration_format_version == (200 if is_for_maven else 101):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200771 raise Exception('No "version" expected in ' + configuration)
772 # Disallow prerelease, as older R8 versions cannot parse it causing hard to
773 # understand errors.
774 check_basic_semver_version(version,
775 'in ' + configuration,
776 allowPrerelease=False)
777 return (name, version)
778
Søren Gjesse1e171532019-09-03 09:44:22 +0200779
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100780class SemanticVersion:
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100781
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200782 def __init__(self, major, minor, patch, prerelease):
783 self.major = major
784 self.minor = minor
785 self.patch = patch
786 self.prerelease = prerelease
787 # Build metadata currently not suppported
788
789 def larger_than(self, other):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200790 if self.major > other.major:
791 return True
792 if self.major == other.major and self.minor > other.minor:
793 return True
794 if self.patch:
795 return (self.major == other.major and self.minor == other.minor and
796 self.patch > other.patch)
Søren Gjesse749a0be2023-10-26 14:07:18 +0200797 if self.prerelease:
798 if other.prerelease:
799 return self.prerelease > other.prerelease
800 else:
801 return False
802
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200803 else:
804 return False
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100805
806
Søren Gjesse705a3b12022-03-17 11:37:30 +0100807# Check that the passed string is formatted as a basic semver version (x.y.z or x.y.z-prerelease
808# depending on the value of allowPrerelease).
809# See https://semver.org/. The regexp parts used are not all complient with what is suggested
810# on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string.
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200811def check_basic_semver_version(version,
812 error_context='',
813 components=3,
814 allowPrerelease=False):
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100815 regexp = '^'
816 for x in range(components):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200817 regexp += '([0-9]+)'
818 if x < components - 1:
819 regexp += '\\.'
Søren Gjesse705a3b12022-03-17 11:37:30 +0100820 if allowPrerelease:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200821 # This part is from
822 # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
823 regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?'
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100824 regexp += '$'
825 reg = re.compile(regexp)
826 match = reg.match(version)
827 if not match:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200828 raise Exception("Invalid version '" + version + "'" +
829 (' ' + error_context) if len(error_context) > 0 else '')
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100830 if components == 2:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200831 return SemanticVersion(int(match.group(1)), int(match.group(2)), None,
832 None)
Søren Gjesse705a3b12022-03-17 11:37:30 +0100833 elif components == 3 and not allowPrerelease:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200834 return SemanticVersion(int(match.group(1)), int(match.group(2)),
835 int(match.group(3)), None)
Søren Gjesse705a3b12022-03-17 11:37:30 +0100836 elif components == 3 and allowPrerelease:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200837 return SemanticVersion(int(match.group(1)), int(match.group(2)),
838 int(match.group(3)), match.group('prerelease'))
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100839 else:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200840 raise Exception('Argument "components" must be 2 or 3')