blob: c6b01261278674f63eb0cfda1990e698dcfa81c8 [file] [log] [blame]
Mads Ager418d1ca2017-05-22 09:35:49 +02001# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
2# for details. All rights reserved. Use of this source code is governed by a
3# BSD-style license that can be found in the LICENSE file.
4
5# Different utility functions used accross scripts
6
7import hashlib
Clément Béra3718ad02023-09-05 14:12:48 +02008import jdk
Søren Gjesse6e5e5842019-09-03 08:48:30 +02009import json
Mads Ager418d1ca2017-05-22 09:35:49 +020010import os
Tamas Kenez82efeb52017-06-12 13:56:22 +020011import re
Mads Ager418d1ca2017-05-22 09:35:49 +020012import shutil
13import subprocess
14import sys
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +020015import tarfile
Mads Ager418d1ca2017-05-22 09:35:49 +020016import tempfile
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +010017import zipfile
Mads Ager418d1ca2017-05-22 09:35:49 +020018
Ian Zerny37097652019-04-11 13:13:27 +020019import defines
Christoffer Quist Adamsen65ef2982023-08-24 08:45:39 +020020from thread_utils import print_thread
Ian Zerny37097652019-04-11 13:13:27 +020021
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +010022ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}'
23ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar')
Ian Zerny3f54e222019-02-12 10:51:17 +010024TOOLS_DIR = defines.TOOLS_DIR
25REPO_ROOT = defines.REPO_ROOT
26THIRD_PARTY = defines.THIRD_PARTY
Christoffer Quist Adamsen4d9fc512022-08-11 19:59:44 +020027BUNDLETOOL_JAR_DIR = os.path.join(THIRD_PARTY, 'bundletool/bundletool-1.11.0')
28BUNDLETOOL_JAR = os.path.join(BUNDLETOOL_JAR_DIR, 'bundletool-all-1.11.0.jar')
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +010029ANDROID_SDK = os.path.join(THIRD_PARTY, 'android_sdk')
Tamas Kenezfc34cd82017-07-13 12:43:57 +020030MEMORY_USE_TMP_FILE = 'memory_use.tmp'
Tamas Kenez02bff032017-07-18 12:13:58 +020031DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)')
Mads Ager12a56bc2017-11-27 11:51:25 +010032BUILD = os.path.join(REPO_ROOT, 'build')
Ian Zerny0f5fc732018-11-15 14:34:41 +010033BUILD_DEPS_DIR = os.path.join(BUILD, 'deps')
34BUILD_MAIN_DIR = os.path.join(BUILD, 'classes', 'main')
Morten Krogh-Jespersen51db2b02020-11-11 12:49:26 +010035BUILD_JAVA_MAIN_DIR = os.path.join(BUILD, 'classes', 'java', 'main')
Ian Zerny0f5fc732018-11-15 14:34:41 +010036BUILD_TEST_DIR = os.path.join(BUILD, 'classes', 'test')
Mads Ager12a56bc2017-11-27 11:51:25 +010037LIBS = os.path.join(BUILD, 'libs')
Clément Béra3718ad02023-09-05 14:12:48 +020038CUSTOM_CONVERSION_DIR = os.path.join(
Clément Béradb79e5d2023-09-06 14:48:35 +020039 THIRD_PARTY, 'openjdk', 'custom_conversion')
Mads Ager12a56bc2017-11-27 11:51:25 +010040GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense')
Mads Agera4911eb2017-11-22 13:19:36 +010041SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java')
Søren Gjesse6e5e5842019-09-03 08:48:30 +020042TEST_ROOT = os.path.join(REPO_ROOT, 'src', 'test', 'java')
Ian Zerny59dfa4c2019-10-25 10:34:36 +020043REPO_SOURCE = 'https://r8.googlesource.com/r8'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020044
45D8 = 'd8'
46R8 = 'r8'
Tamas Kenez03ab76f2018-12-07 14:33:25 +010047R8LIB = 'r8lib'
Morten Krogh-Jespersene28db462019-01-09 13:32:15 +010048R8LIB_NO_DEPS = 'r8LibNoDeps'
Morten Krogh-Jespersen98ee89a2021-10-25 20:59:02 +020049R8RETRACE = 'R8Retrace'
50R8RETRACE_NO_DEPS = 'R8RetraceNoDeps'
Mads Agerb10c07f2017-11-27 13:25:52 +010051R8_SRC = 'sourceJar'
Clément Bérab43073c2023-09-06 13:48:24 +020052LIBRARY_DESUGAR_CONVERSIONS =\
53 'download_deps_third_party_openjdk_custom_conversion'
Ian Zerny161ff742022-01-20 12:39:40 +010054R8_TESTS_TARGET = 'TestJar'
55R8_TESTS_DEPS_TARGET = 'RepackageTestDeps'
56R8LIB_TESTS_TARGET = 'configureTestForR8Lib'
57R8LIB_TESTS_DEPS_TARGET = R8_TESTS_DEPS_TARGET
Ian Zernyf13d18f2023-05-24 12:50:37 +020058KEEPANNO_ANNOTATIONS_TARGET = 'keepAnnoJar'
Søren Gjessedc9d8a22017-10-12 12:40:59 +020059
Morten Krogh-Jespersen51db2b02020-11-11 12:49:26 +010060ALL_DEPS_JAR = os.path.join(LIBS, 'deps_all.jar')
Rico Wind74fab302017-10-02 07:25:33 +020061R8_JAR = os.path.join(LIBS, 'r8.jar')
Ian Zerny161ff742022-01-20 12:39:40 +010062R8_WITH_RELOCATED_DEPS_JAR = os.path.join(LIBS, 'r8_with_relocated_deps.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010063R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar')
Rico Wind158ef9f2022-05-19 11:08:30 +020064R8LIB_MAP = '%s.map' % R8LIB_JAR
Mads Agerb10c07f2017-11-27 13:25:52 +010065R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')
Tamas Kenez03ab76f2018-12-07 14:33:25 +010066R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar')
Tamas Kenez180be092018-12-05 15:23:06 +010067R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar')
Morten Krogh-Jespersen98ee89a2021-10-25 20:59:02 +020068R8RETRACE_JAR = os.path.join(LIBS, 'r8retrace.jar')
69R8RETRACE_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8retrace-exclude-deps.jar')
Ian Zerny161ff742022-01-20 12:39:40 +010070R8_TESTS_JAR = os.path.join(LIBS, 'r8tests.jar')
71R8LIB_TESTS_JAR = os.path.join(LIBS, 'r8libtestdeps-cf.jar')
72R8_TESTS_DEPS_JAR = os.path.join(LIBS, 'test_deps_all.jar')
73R8LIB_TESTS_DEPS_JAR = R8_TESTS_DEPS_JAR
Rico Wind8fc8bfa2019-03-22 09:57:36 +010074MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip')
Clément Béra3718ad02023-09-05 14:12:48 +020075LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP = os.path.join(
76 CUSTOM_CONVERSION_DIR, 'library_desugar_conversions_legacy.jar')
77LIBRARY_DESUGAR_CONVERSIONS_ZIP = os.path.join(
78 CUSTOM_CONVERSION_DIR, 'library_desugar_conversions.jar')
Ian Zernyf13d18f2023-05-24 12:50:37 +020079KEEPANNO_ANNOTATIONS_JAR = os.path.join(LIBS, 'keepanno-annotations.jar')
Søren Gjesse17fc67d2019-12-04 14:50:17 +010080
Søren Gjesse6e5e5842019-09-03 08:48:30 +020081DESUGAR_CONFIGURATION = os.path.join(
Søren Gjesse927a92e2019-12-04 15:18:06 +010082 'src', 'library_desugar', 'desugar_jdk_libs.json')
Søren Gjesseee086b22020-10-30 11:46:39 +010083DESUGAR_IMPLEMENTATION = os.path.join(
Søren Gjesse3dc207b2021-02-15 09:45:30 +010084 'third_party', 'openjdk', 'desugar_jdk_libs', 'desugar_jdk_libs.jar')
Søren Gjesse705a3b12022-03-17 11:37:30 +010085DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join(
86 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
Søren Gjesse2b047692022-08-19 16:34:38 +020087DESUGAR_CONFIGURATION_JDK11_MINIMAL = os.path.join(
88 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json')
89DESUGAR_CONFIGURATION_JDK11 = os.path.join(
90 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs.json')
91DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join(
92 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_nio.json')
Søren Gjesse705a3b12022-03-17 11:37:30 +010093DESUGAR_IMPLEMENTATION_JDK11 = os.path.join(
94 'third_party', 'openjdk', 'desugar_jdk_libs_11', 'desugar_jdk_libs.jar')
Søren Gjesse6e5e5842019-09-03 08:48:30 +020095DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
96 LIBS, 'desugar_jdk_libs_configuration.zip')
Søren Gjessee18fa6e2022-06-24 15:14:53 +020097DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP = os.path.join(
98 LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip')
Søren Gjesse2b047692022-08-19 16:34:38 +020099DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP = os.path.join(
100 LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip')
101DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP = os.path.join(
102 LIBS, 'desugar_jdk_libs_configuration_jdk11.zip')
103DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP = os.path.join(
104 LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip')
Mads Ager12a56bc2017-11-27 11:51:25 +0100105GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
Mathias Rav3fb4a3a2018-05-29 15:41:36 +0200106RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
Mathias Ravb46dc002018-06-06 09:37:11 +0200107R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +0100108CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100109PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar')
110PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar')
Ian Zernyfbb1f7a2019-05-02 14:34:13 +0200111SAMPLE_LIBRARIES_SHA_FILE = os.path.join(
112 THIRD_PARTY, 'sample_libraries.tar.gz.sha1')
Morten Krogh-Jespersen45d7a7b2020-11-02 08:31:09 +0100113OPENSOURCE_DUMPS_DIR = os.path.join(THIRD_PARTY, 'opensource-apps')
Morten Krogh-Jespersen86222742021-03-02 11:13:33 +0100114INTERNAL_DUMPS_DIR = os.path.join(THIRD_PARTY, 'internal-apps')
Søren Gjesse1c115b52019-08-14 12:43:57 +0200115BAZEL_SHA_FILE = os.path.join(THIRD_PARTY, 'bazel.tar.gz.sha1')
116BAZEL_TOOL = os.path.join(THIRD_PARTY, 'bazel')
Søren Gjesse699f6362019-10-09 14:56:33 +0200117JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86.tar.gz.sha1')
Søren Gjesseef195772021-03-11 16:04:42 +0100118JAVA11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11', 'linux.tar.gz.sha1')
Søren Gjesseeddc6612022-09-02 15:38:39 +0200119DESUGAR_JDK_LIBS_11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'desugar_jdk_libs_11.tar.gz.sha1')
Christoffer Quist Adamsen1ca046c2021-02-21 11:25:16 +0100120IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test', 'ignorewarnings.rules')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100121
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100122ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME"
123ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION"
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100124USER_HOME = os.path.expanduser('~')
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100125
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200126R8_TEST_RESULTS_BUCKET = 'r8-test-results'
Rico Wind635b2de2022-04-25 10:35:14 +0200127R8_INTERNAL_TEST_RESULTS_BUCKET = 'r8-internal-test-results'
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200128
129def archive_file(name, gs_dir, src_file):
130 gs_file = '%s/%s' % (gs_dir, name)
Rico Wind03424282022-04-26 15:09:51 +0200131 upload_file_to_cloud_storage(src_file, gs_file)
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200132
133def archive_value(name, gs_dir, value):
134 with TempDir() as temp:
135 tempfile = os.path.join(temp, name);
136 with open(tempfile, 'w') as f:
137 f.write(str(value))
138 archive_file(name, gs_dir, tempfile)
139
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200140def find_cloud_storage_file_from_options(name, options, orElse=None):
141 # Import archive on-demand since archive depends on utils.
142 from archive import GetUploadDestination
143 hash_or_version = find_hash_or_version_from_options(options)
144 if not hash_or_version:
145 return orElse
146 is_hash = options.commit_hash is not None
147 download_path = GetUploadDestination(hash_or_version, name, is_hash)
148 if file_exists_on_cloud_storage(download_path):
149 out = tempfile.NamedTemporaryFile().name
150 download_file_from_cloud_storage(download_path, out)
151 return out
152 else:
153 raise Exception('Could not find file {} from hash/version: {}.'
154 .format(name, hash_or_version))
155
156def find_r8_jar_from_options(options):
157 return find_cloud_storage_file_from_options('r8.jar', options)
158
159def find_r8_lib_jar_from_options(options):
160 return find_cloud_storage_file_from_options('r8lib.jar', options)
161
162def find_hash_or_version_from_options(options):
163 if options.tag:
164 return find_hash_or_version_from_tag(options.tag)
165 else:
166 return options.commit_hash or options.version
167
168def find_hash_or_version_from_tag(tag_or_hash):
Rico Windfd186372022-02-28 08:55:48 +0100169 info = subprocess.check_output([
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200170 'git',
171 'show',
172 tag_or_hash,
173 '-s',
Rico Windfd186372022-02-28 08:55:48 +0100174 '--format=oneline']).decode('utf-8').splitlines()[-1].split()
Christoffer Quist Adamsen4d38d032021-04-20 12:31:31 +0200175 # The info should be on the following form [hash,"Version",version]
176 if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version":
177 return info[2]
178 return None
179
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100180def getAndroidHome():
181 return os.environ.get(
182 ANDROID_HOME_ENVIROMENT_NAME, os.path.join(USER_HOME, 'Android', 'Sdk'))
183
184def getAndroidBuildTools():
Christoffer Quist Adamsen8c803b42022-05-31 10:36:17 +0200185 if ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME in os.environ:
186 version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME)
187 build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
188 assert os.path.exists(build_tools_dir)
189 return build_tools_dir
190 else:
Morten Krogh-Jespersen9c1e7022023-02-07 13:56:35 +0100191 versions = ['33.0.1', '32.0.0']
Christoffer Quist Adamsen8c803b42022-05-31 10:36:17 +0200192 for version in versions:
193 build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
194 if os.path.exists(build_tools_dir):
195 return build_tools_dir
196 raise Exception('Unable to find Android build-tools')
Morten Krogh-Jespersenc8efedd2019-01-28 11:36:17 +0100197
Christoffer Quist Adamsen5c9ded12021-01-14 14:29:37 +0100198def is_python3():
199 return sys.version_info.major == 3
200
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100201def Print(s, quiet=False):
202 if quiet:
203 return
204 print(s)
205
206def Warn(message):
207 CRED = '\033[91m'
208 CEND = '\033[0m'
209 print(CRED + message + CEND)
210
Christoffer Quist Adamsen65ef2982023-08-24 08:45:39 +0200211def PrintCmd(cmd, env=None, quiet=False, worker_id=None):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100212 if quiet:
213 return
214 if type(cmd) is list:
215 cmd = ' '.join(cmd)
216 if env:
217 env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
Christoffer Quist Adamsen65ef2982023-08-24 08:45:39 +0200218 print_thread('Running: {} {}'.format(env, cmd), worker_id)
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100219 else:
Christoffer Quist Adamsen65ef2982023-08-24 08:45:39 +0200220 print_thread('Running: {}'.format(cmd), worker_id)
Mads Ager418d1ca2017-05-22 09:35:49 +0200221 # I know this will hit os on windows eventually if we don't do this.
222 sys.stdout.flush()
223
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100224class ProgressLogger(object):
225 CLEAR_LINE = '\033[K'
226 UP = '\033[F'
227
228 def __init__(self, quiet=False):
229 self._count = 0
230 self._has_printed = False
231 self._quiet = quiet
232
233 def log(self, text):
Christoffer Quist Adamsen7607ebe2022-06-28 11:52:46 +0200234 if len(text.strip()) == 0:
235 return
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100236 if self._quiet:
237 if self._has_printed:
238 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
239 if len(text) > 140:
240 text = text[0:140] + '...'
241 print(text)
242 self._has_printed = True
243
244 def done(self):
245 if self._quiet and self._has_printed:
246 sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
247 print('')
248 sys.stdout.write(ProgressLogger.UP)
249
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100250def RunCmd(cmd, env_vars=None, quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100251 PrintCmd(cmd, env=env_vars, quiet=quiet)
252 env = os.environ.copy()
253 if env_vars:
254 env.update(env_vars)
255 process = subprocess.Popen(
256 cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
257 stdout = []
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100258 logger = ProgressLogger(quiet=quiet) if logging else None
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100259 failed = False
260 while True:
Rico Wind744ba752021-01-22 06:24:49 +0100261 line = process.stdout.readline().decode('utf-8')
262 if line != '':
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100263 stripped = line.rstrip()
264 stdout.append(stripped)
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100265 if logger:
266 logger.log(stripped)
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100267 # TODO(christofferqa): r8 should fail with non-zero exit code.
Morten Krogh-Jespersen121c47b2019-01-25 09:57:21 +0100268 if ('AssertionError:' in stripped
269 or 'CompilationError:' in stripped
270 or 'CompilationFailedException:' in stripped
Morten Krogh-Jespersen5d02a6b2019-10-29 14:48:56 +0100271 or 'Compilation failed' in stripped
272 or 'FAILURE:' in stripped
273 or 'org.gradle.api.ProjectConfigurationException' in stripped
274 or 'BUILD FAILED' in stripped):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100275 failed = True
276 else:
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100277 if logger:
278 logger.done()
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100279 exit_code = process.poll()
280 if exit_code or failed:
281 for line in stdout:
282 Warn(line)
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100283 if fail:
284 raise subprocess.CalledProcessError(
285 exit_code or -1, cmd, output='\n'.join(stdout))
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100286 return stdout
287
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100288def RunGradlew(
289 args, clean=True, stacktrace=True, use_daemon=False, env_vars=None,
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100290 quiet=False, fail=True, logging=True):
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100291 cmd = ['./gradlew']
292 if clean:
293 assert 'clean' not in args
294 cmd.append('clean')
295 if stacktrace:
296 assert '--stacktrace' not in args
297 cmd.append('--stacktrace')
298 if not use_daemon:
299 assert '--no-daemon' not in args
300 cmd.append('--no-daemon')
301 cmd.extend(args)
Morten Krogh-Jespersen7cdd3a72019-03-13 14:58:25 +0100302 return RunCmd(cmd, env_vars=env_vars, quiet=quiet, fail=fail, logging=logging)
Christoffer Quist Adamsen7cf4c562019-03-07 10:57:33 +0100303
Rico Windf80f5a22017-06-16 09:15:57 +0200304def IsWindows():
Ian Zerny3f54e222019-02-12 10:51:17 +0100305 return defines.IsWindows()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100306
307def IsLinux():
Ian Zerny3f54e222019-02-12 10:51:17 +0100308 return defines.IsLinux()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100309
310def IsOsX():
Ian Zerny3f54e222019-02-12 10:51:17 +0100311 return defines.IsOsX()
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100312
313def EnsureDepFromGoogleCloudStorage(dep, tgz, sha1, msg):
314 if not os.path.exists(dep) or os.path.getmtime(tgz) < os.path.getmtime(sha1):
315 DownloadFromGoogleCloudStorage(sha1)
316 # Update the mtime of the tar file to make sure we do not run again unless
317 # there is an update.
318 os.utime(tgz, None)
319 else:
Rico Wind3d369b42021-01-12 10:26:24 +0100320 print('Ensure cloud dependency:', msg, 'present')
Rico Windf80f5a22017-06-16 09:15:57 +0200321
Jean-Marie Henaffe4e36d12018-04-05 10:33:50 +0200322def DownloadFromX20(sha1_file):
323 download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
324 cmd = [download_script, sha1_file]
325 PrintCmd(cmd)
326 subprocess.check_call(cmd)
327
Rico Wind59593922021-03-03 09:12:36 +0100328def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps', auth=False,
329 quiet=False):
Rico Windf80f5a22017-06-16 09:15:57 +0200330 suffix = '.bat' if IsWindows() else ''
331 download_script = 'download_from_google_storage%s' % suffix
Rico Wind533e3ce2019-04-04 10:26:12 +0200332 cmd = [download_script]
333 if not auth:
334 cmd.append('-n')
335 cmd.extend(['-b', bucket, '-u', '-s', sha1_file])
Rico Wind59593922021-03-03 09:12:36 +0100336 if not quiet:
337 PrintCmd(cmd)
338 subprocess.check_call(cmd)
339 else:
340 subprocess.check_output(cmd)
Mads Ager418d1ca2017-05-22 09:35:49 +0200341
342def get_sha1(filename):
343 sha1 = hashlib.sha1()
344 with open(filename, 'rb') as f:
345 while True:
346 chunk = f.read(1024*1024)
347 if not chunk:
348 break
349 sha1.update(chunk)
350 return sha1.hexdigest()
351
Rico Wind1b52acf2021-03-21 12:36:55 +0100352def is_main():
Rico Windfd186372022-02-28 08:55:48 +0100353 remotes = subprocess.check_output(['git', 'branch', '-r', '--contains',
354 'HEAD']).decode('utf-8')
Rico Wind1b52acf2021-03-21 12:36:55 +0100355 return 'origin/main' in remotes
Rico Wind1b09c562019-01-17 08:53:09 +0100356
Ian Zernyc2de7b72023-09-06 20:52:16 +0200357def get_HEAD_branch():
358 result = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8')
359 return result.strip()
360
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200361def get_HEAD_sha1():
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100362 return get_HEAD_sha1_for_checkout(REPO_ROOT)
363
Ian Zerny9f0345d2023-09-07 11:15:00 +0200364def get_HEAD_diff_stat():
365 return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8')
366
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100367def get_HEAD_sha1_for_checkout(checkout):
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200368 cmd = ['git', 'rev-parse', 'HEAD']
369 PrintCmd(cmd)
Morten Krogh-Jespersene0ce6a32019-02-07 11:44:45 +0100370 with ChangedWorkingDirectory(checkout):
Christoffer Quist Adamsen50930e42021-01-20 11:55:12 +0100371 return subprocess.check_output(cmd).decode('utf-8').strip()
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200372
Tamas Kenez971eec62017-05-24 11:08:40 +0200373def makedirs_if_needed(path):
374 try:
375 os.makedirs(path)
376 except OSError:
377 if not os.path.isdir(path):
378 raise
379
Rico Windef7420c2021-10-14 13:16:01 +0200380def get_gsutil():
381 return 'gsutil.py' if os.name != 'nt' else 'gsutil.py.bat'
382
Rico Wind03424282022-04-26 15:09:51 +0200383def upload_dir_to_cloud_storage(directory, destination, is_html=False):
Rico Winda94f01c2017-06-27 10:32:34 +0200384 # Upload and make the content encoding right for viewing directly
Rico Windef7420c2021-10-14 13:16:01 +0200385 cmd = [get_gsutil(), '-m', 'cp']
Rico Windd0d88cf2018-02-09 09:46:11 +0100386 if is_html:
387 cmd += ['-z', 'html']
Rico Wind800fd712018-09-24 11:29:33 +0200388 cmd += ['-R', directory, destination]
Rico Winda94f01c2017-06-27 10:32:34 +0200389 PrintCmd(cmd)
390 subprocess.check_call(cmd)
391
Rico Wind03424282022-04-26 15:09:51 +0200392def upload_file_to_cloud_storage(source, destination):
Rico Windef7420c2021-10-14 13:16:01 +0200393 cmd = [get_gsutil(), 'cp']
Rico Wind800fd712018-09-24 11:29:33 +0200394 cmd += [source, destination]
Rico Windb4621c12017-08-28 12:48:53 +0200395 PrintCmd(cmd)
396 subprocess.check_call(cmd)
397
Rico Wind139eece2018-09-25 09:42:09 +0200398def delete_file_from_cloud_storage(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200399 cmd = [get_gsutil(), 'rm', destination]
Rico Wind139eece2018-09-25 09:42:09 +0200400 PrintCmd(cmd)
401 subprocess.check_call(cmd)
402
Rico Wind4fd2dda2018-09-26 17:41:45 +0200403def ls_files_on_cloud_storage(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200404 cmd = [get_gsutil(), 'ls', destination]
Rico Wind4fd2dda2018-09-26 17:41:45 +0200405 PrintCmd(cmd)
Rico Windfd186372022-02-28 08:55:48 +0100406 return subprocess.check_output(cmd).decode('utf-8')
Rico Wind4fd2dda2018-09-26 17:41:45 +0200407
Rico Wind139eece2018-09-25 09:42:09 +0200408def cat_file_on_cloud_storage(destination, ignore_errors=False):
Rico Windef7420c2021-10-14 13:16:01 +0200409 cmd = [get_gsutil(), 'cat', destination]
Rico Wind139eece2018-09-25 09:42:09 +0200410 PrintCmd(cmd)
411 try:
Rico Wind8dadb312022-02-28 08:40:20 +0100412 return subprocess.check_output(cmd).decode('utf-8').strip()
Rico Wind139eece2018-09-25 09:42:09 +0200413 except subprocess.CalledProcessError as e:
414 if ignore_errors:
415 return ''
416 else:
417 raise e
418
419def file_exists_on_cloud_storage(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200420 cmd = [get_gsutil(), 'ls', destination]
Rico Wind139eece2018-09-25 09:42:09 +0200421 PrintCmd(cmd)
422 return subprocess.call(cmd) == 0
423
Christoffer Quist Adamsen870fa462020-12-15 10:50:54 +0100424def download_file_from_cloud_storage(source, destination, quiet=False):
Rico Windef7420c2021-10-14 13:16:01 +0200425 cmd = [get_gsutil(), 'cp', source, destination]
Christoffer Quist Adamsen870fa462020-12-15 10:50:54 +0100426 PrintCmd(cmd, quiet=quiet)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200427 subprocess.check_call(cmd)
428
Morten Krogh-Jespersen220e5702019-02-27 12:57:01 +0100429def create_archive(name, sources=None):
430 if not sources:
431 sources = [name]
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200432 tarname = '%s.tar.gz' % name
433 with tarfile.open(tarname, 'w:gz') as tar:
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100434 for source in sources:
435 tar.add(source)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200436 return tarname
437
438def extract_dir(filename):
439 return filename[0:len(filename) - len('.tar.gz')]
440
441def unpack_archive(filename):
442 dest_dir = extract_dir(filename)
443 if os.path.exists(dest_dir):
Rico Wind3d369b42021-01-12 10:26:24 +0100444 print('Deleting existing dir %s' % dest_dir)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200445 shutil.rmtree(dest_dir)
446 dirname = os.path.dirname(os.path.abspath(filename))
447 with tarfile.open(filename, 'r:gz') as tar:
448 tar.extractall(path=dirname)
449
Morten Krogh-Jespersenec3047b2020-08-18 13:09:06 +0200450def check_gcert():
Ian Zerny86c4cfd2022-01-17 13:43:37 +0100451 status = subprocess.call(['gcertstatus'])
452 if status != 0:
453 subprocess.check_call(['gcert'])
Morten Krogh-Jespersen54090862019-02-19 11:31:10 +0100454
Rico Wind1a29c4f2018-01-25 08:43:08 +0100455# Note that gcs is eventually consistent with regards to list operations.
456# This is not a problem in our case, but don't ever use this method
457# for synchronization.
458def cloud_storage_exists(destination):
Rico Windef7420c2021-10-14 13:16:01 +0200459 cmd = [get_gsutil(), 'ls', destination]
Rico Wind1a29c4f2018-01-25 08:43:08 +0100460 PrintCmd(cmd)
461 exit_code = subprocess.call(cmd)
462 return exit_code == 0
463
Mads Ager418d1ca2017-05-22 09:35:49 +0200464class TempDir(object):
Søren Gjessecbeae782019-05-21 14:14:25 +0200465 def __init__(self, prefix='', delete=True):
Mads Ager418d1ca2017-05-22 09:35:49 +0200466 self._temp_dir = None
467 self._prefix = prefix
Søren Gjessecbeae782019-05-21 14:14:25 +0200468 self._delete = delete
Mads Ager418d1ca2017-05-22 09:35:49 +0200469
470 def __enter__(self):
471 self._temp_dir = tempfile.mkdtemp(self._prefix)
472 return self._temp_dir
473
474 def __exit__(self, *_):
Søren Gjessecbeae782019-05-21 14:14:25 +0200475 if self._delete:
476 shutil.rmtree(self._temp_dir, ignore_errors=True)
Mads Ager418d1ca2017-05-22 09:35:49 +0200477
478class ChangedWorkingDirectory(object):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100479 def __init__(self, working_directory, quiet=False):
480 self._quiet = quiet
Mads Ager418d1ca2017-05-22 09:35:49 +0200481 self._working_directory = working_directory
482
483 def __enter__(self):
484 self._old_cwd = os.getcwd()
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100485 if not self._quiet:
Rico Wind3d369b42021-01-12 10:26:24 +0100486 print('Enter directory:', self._working_directory)
Mads Ager418d1ca2017-05-22 09:35:49 +0200487 os.chdir(self._working_directory)
488
489 def __exit__(self, *_):
Christoffer Quist Adamsen1de3dde2019-01-24 13:17:46 +0100490 if not self._quiet:
Rico Wind3d369b42021-01-12 10:26:24 +0100491 print('Enter directory:', self._old_cwd)
Mads Ager418d1ca2017-05-22 09:35:49 +0200492 os.chdir(self._old_cwd)
Tamas Kenez82efeb52017-06-12 13:56:22 +0200493
494# Reading Android CTS test_result.xml
495
496class CtsModule(object):
497 def __init__(self, module_name):
498 self.name = module_name
499
500class CtsTestCase(object):
501 def __init__(self, test_case_name):
502 self.name = test_case_name
503
504class CtsTest(object):
505 def __init__(self, test_name, outcome):
506 self.name = test_name
507 self.outcome = outcome
508
509# Generator yielding CtsModule, CtsTestCase or CtsTest from
510# reading through a CTS test_result.xml file.
511def read_cts_test_result(file_xml):
512 re_module = re.compile('<Module name="([^"]*)"')
513 re_test_case = re.compile('<TestCase name="([^"]*)"')
514 re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
515 with open(file_xml) as f:
516 for line in f:
517 m = re_module.search(line)
518 if m:
519 yield CtsModule(m.groups()[0])
520 continue
521 m = re_test_case.search(line)
522 if m:
523 yield CtsTestCase(m.groups()[0])
524 continue
525 m = re_test.search(line)
526 if m:
527 outcome = m.groups()[0]
Rico Windf80f5a22017-06-16 09:15:57 +0200528 assert outcome in ['fail', 'pass']
Tamas Kenez82efeb52017-06-12 13:56:22 +0200529 yield CtsTest(m.groups()[1], outcome == 'pass')
Tamas Kenezfc34cd82017-07-13 12:43:57 +0200530
531def grep_memoryuse(logfile):
532 re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
533 result = None
534 with open(logfile) as f:
535 for line in f:
536 m = re_vmhwm.search(line)
537 if m:
538 groups = m.groups()
539 s = len(groups)
540 if s >= 1:
541 result = int(groups[0])
542 if s >= 2:
543 unit = groups[1]
544 if unit == 'kB':
545 result *= 1024
546 elif unit != '':
547 raise Exception('Unrecognized unit in memory usage log: {}'
548 .format(unit))
549 if result is None:
550 raise Exception('No memory usage found in log: {}'.format(logfile))
Tamas Kenez02bff032017-07-18 12:13:58 +0200551 return result
552
553# Return a dictionary: {segment_name -> segments_size}
554def getDexSegmentSizes(dex_files):
555 assert len(dex_files) > 0
Ian Zerny3f54e222019-02-12 10:51:17 +0100556 cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments']
Tamas Kenez02bff032017-07-18 12:13:58 +0200557 cmd.extend(dex_files)
558 PrintCmd(cmd)
Morten Krogh-Jespersenf54dd782021-02-09 09:06:08 +0100559 output = subprocess.check_output(cmd).decode('utf-8')
Tamas Kenez02bff032017-07-18 12:13:58 +0200560
561 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
562
563 if matches is None or len(matches) == 0:
564 raise Exception('DexSegments failed to return any output for' \
565 ' these files: {}'.format(dex_files))
566
567 result = {}
568
569 for match in matches:
570 result[match[0]] = int(match[1])
571
572 return result
573
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100574# Return a dictionary: {segment_name -> segments_size}
575def getCfSegmentSizes(cfFile):
Ian Zerny3f54e222019-02-12 10:51:17 +0100576 cmd = [jdk.GetJavaExecutable(),
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100577 '-cp',
Morten Krogh-Jespersen480784d2019-02-05 08:10:46 +0100578 CF_SEGMENTS_TOOL,
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100579 'com.android.tools.r8.cf_segments.MeasureLib',
580 cfFile]
581 PrintCmd(cmd)
Rico Windfd186372022-02-28 08:55:48 +0100582 output = subprocess.check_output(cmd).decode('utf-8')
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100583
584 matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
585
586 if matches is None or len(matches) == 0:
587 raise Exception('CfSegments failed to return any output for' \
588 ' the file: ' + cfFile)
589
590 result = {}
591
592 for match in matches:
593 result[match[0]] = int(match[1])
594
595 return result
596
Søren Gjesse1c115b52019-08-14 12:43:57 +0200597def get_maven_path(artifact, version):
598 return os.path.join('com', 'android', 'tools', artifact, version)
Rico Windc0b16382018-05-17 13:23:43 +0200599
Morten Krogh-Jespersen38c7ca02019-02-04 10:39:57 +0100600def print_cfsegments(prefix, cf_files):
601 for cf_file in cf_files:
602 for segment_name, size in getCfSegmentSizes(cf_file).items():
603 print('{}-{}(CodeSize): {}'
604 .format(prefix, segment_name, size))
605
Christoffer Quist Adamsenbe6661d2023-08-24 11:01:12 +0200606def print_dexsegments(prefix, dex_files, worker_id=None):
Tamas Kenez02bff032017-07-18 12:13:58 +0200607 for segment_name, size in getDexSegmentSizes(dex_files).items():
Christoffer Quist Adamsenbe6661d2023-08-24 11:01:12 +0200608 print_thread(
609 '{}-{}(CodeSize): {}'.format(prefix, segment_name, size),
610 worker_id)
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200611
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100612# Ensure that we are not benchmarking with a google jvm.
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200613def check_java_version():
Ian Zerny3f54e222019-02-12 10:51:17 +0100614 cmd= [jdk.GetJavaExecutable(), '-version']
Rico Windfd186372022-02-28 08:55:48 +0100615 output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
616 m = re.search('openjdk version "([^"]*)"', output)
Tamas Kenez2cf47cf2017-07-25 10:22:52 +0200617 if m is None:
618 raise Exception("Can't check java version: no version string in output"
619 " of 'java -version': '{}'".format(output))
620 version = m.groups(0)[0]
Mads Agerbc7b2ce2018-02-05 11:28:47 +0100621 m = re.search('google', version)
622 if m is not None:
623 raise Exception("Do not use google JVM for benchmarking: " + version)
Tamas Kenez0cad51c2017-08-21 14:42:01 +0200624
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100625def get_android_jar_dir(api):
626 return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api))
627
Rico Wind9d70f612018-08-31 09:17:43 +0200628def get_android_jar(api):
629 return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
Rico Windda6836e2018-12-07 12:32:03 +0100630
Christoffer Quist Adamsen17879c12019-01-22 16:13:54 +0100631def get_android_optional_jars(api):
632 android_optional_jars_dir = os.path.join(get_android_jar_dir(api), 'optional')
633 android_optional_jars = [
634 os.path.join(android_optional_jars_dir, 'android.test.base.jar'),
635 os.path.join(android_optional_jars_dir, 'android.test.mock.jar'),
636 os.path.join(android_optional_jars_dir, 'android.test.runner.jar'),
637 os.path.join(android_optional_jars_dir, 'org.apache.http.legacy.jar')
638 ]
639 return [
640 android_optional_jar for android_optional_jar in android_optional_jars
641 if os.path.isfile(android_optional_jar)]
642
Rico Windfaaac012019-02-25 11:24:05 +0100643def is_bot():
Rico Winde11f4392019-03-18 07:59:37 +0100644 return 'SWARMING_BOT_ID' in os.environ
Morten Krogh-Jespersen16e925d2019-01-25 14:40:38 +0100645
646def uncompressed_size(path):
647 return sum(z.file_size for z in zipfile.ZipFile(path).infolist())
Morten Krogh-Jespersen0de13732019-03-01 08:56:39 +0100648
649def getR8Version(path):
650 cmd = [jdk.GetJavaExecutable(), '-cp', path, 'com.android.tools.r8.R8',
651 '--version']
Rico Windfd186372022-02-28 08:55:48 +0100652 output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
Jinseong Jeona2394232019-11-26 22:17:55 -0800653 # output is of the form 'R8 <version> (with additional info)'
654 # so we split on '('; clean up tailing spaces; and strip off 'R8 '.
655 return output.split('(')[0].strip()[3:]
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200656
Søren Gjesse2b047692022-08-19 16:34:38 +0200657def desugar_configuration_name_and_version(configuration, is_for_maven):
658 name = 'desugar_jdk_libs_configuration'
Søren Gjesse705a3b12022-03-17 11:37:30 +0100659 with open(configuration, 'r') as f:
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200660 configuration_json = json.loads(f.read())
661 configuration_format_version = \
662 configuration_json.get('configuration_format_version')
Søren Gjesse2b047692022-08-19 16:34:38 +0200663 if (not configuration_format_version):
664 raise Exception(
665 'No "configuration_format_version" found in ' + configuration)
666 if (configuration_format_version != 3
667 and configuration_format_version != 5
668 and configuration_format_version != (200 if is_for_maven else 100)):
669 raise Exception(
670 'Unsupported "configuration_format_version" "%s" found in %s'
671 % (configuration_format_version, configuration))
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200672 version = configuration_json.get('version')
673 if not version:
Søren Gjesse2b047692022-08-19 16:34:38 +0200674 if configuration_format_version == (200 if is_for_maven else 100):
675 identifier = configuration_json.get('identifier')
676 if not identifier:
677 raise Exception(
678 'No "identifier" found in ' + configuration)
679 identifier_split = identifier.split(':')
680 if (len(identifier_split) != 3):
681 raise Exception('Invalid "identifier" found in ' + configuration)
682 if (identifier_split[0] != 'com.tools.android'):
683 raise Exception('Invalid "identifier" found in ' + configuration)
684 if not identifier_split[1].startswith('desugar_jdk_libs_configuration'):
685 raise Exception('Invalid "identifier" found in ' + configuration)
686 name = identifier_split[1]
687 version = identifier_split[2]
688 else:
689 raise Exception(
690 'No "version" found in ' + configuration)
691 else:
692 if configuration_format_version == (200 if is_for_maven else 100):
693 raise Exception(
694 'No "version" expected in ' + configuration)
695 # Disallow prerelease, as older R8 versions cannot parse it causing hard to
696 # understand errors.
697 check_basic_semver_version(version, 'in ' + configuration, allowPrerelease = False)
698 return (name, version)
Søren Gjesse1e171532019-09-03 09:44:22 +0200699
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100700class SemanticVersion:
Søren Gjesse705a3b12022-03-17 11:37:30 +0100701 def __init__(self, major, minor, patch, prerelease):
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100702 self.major = major
703 self.minor = minor
704 self.patch = patch
Søren Gjesse705a3b12022-03-17 11:37:30 +0100705 self.prerelease = prerelease
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100706 # Build metadata currently not suppported
707
708 def larger_than(self, other):
Søren Gjesse29d108a2022-04-07 10:49:49 +0200709 if self.prerelease or other.prerelease:
Søren Gjesse705a3b12022-03-17 11:37:30 +0100710 raise Exception("Comparison with prerelease not implemented")
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100711 if self.major > other.major:
712 return True
713 if self.major == other.major and self.minor > other.minor:
714 return True
715 if self.patch:
716 return (self.major == other.major
717 and self.minor == other.minor
718 and self.patch > other.patch)
719 else:
720 return False
721
722
Søren Gjesse705a3b12022-03-17 11:37:30 +0100723# Check that the passed string is formatted as a basic semver version (x.y.z or x.y.z-prerelease
724# depending on the value of allowPrerelease).
725# See https://semver.org/. The regexp parts used are not all complient with what is suggested
726# on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string.
727def check_basic_semver_version(version, error_context = '', components = 3, allowPrerelease = False):
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100728 regexp = '^'
729 for x in range(components):
730 regexp += '([0-9]+)'
731 if x < components - 1:
732 regexp += '\\.'
Søren Gjesse705a3b12022-03-17 11:37:30 +0100733 if allowPrerelease:
734 # This part is from
735 # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
736 regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?'
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100737 regexp += '$'
738 reg = re.compile(regexp)
739 match = reg.match(version)
740 if not match:
Søren Gjesse1e171532019-09-03 09:44:22 +0200741 raise Exception("Invalid version '"
742 + version
743 + "'"
744 + (' ' + error_context) if len(error_context) > 0 else '')
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100745 if components == 2:
Søren Gjesse705a3b12022-03-17 11:37:30 +0100746 return SemanticVersion(int(match.group(1)), int(match.group(2)), None, None)
747 elif components == 3 and not allowPrerelease:
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100748 return SemanticVersion(
Søren Gjesse705a3b12022-03-17 11:37:30 +0100749 int(match.group(1)), int(match.group(2)), int(match.group(3)), None)
750 elif components == 3 and allowPrerelease:
751 return SemanticVersion(
752 int(match.group(1)), int(match.group(2)), int(match.group(3)), match.group('prerelease'))
Søren Gjesse4e5c6fe2019-11-05 17:17:43 +0100753 else:
754 raise Exception('Argument "components" must be 2 or 3')