| # Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file | |
| # for details. All rights reserved. Use of this source code is governed by a | |
| # BSD-style license that can be found in the LICENSE file. | |
| # Different utility functions used across scripts | |
| import hashlib | |
| import jdk | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tarfile | |
| import tempfile | |
| import zipfile | |
| import defines | |
| import historic_run | |
| from thread_utils import print_thread | |
| ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}' | |
| ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar') | |
| TOOLS_DIR = defines.TOOLS_DIR | |
| REPO_ROOT = defines.REPO_ROOT | |
| THIRD_PARTY = defines.THIRD_PARTY | |
| BUNDLETOOL_JAR_DIR = os.path.join(THIRD_PARTY, 'bundletool/bundletool-1.11.0') | |
| BUNDLETOOL_JAR = os.path.join(BUNDLETOOL_JAR_DIR, 'bundletool-all-1.11.0.jar') | |
| MEMORY_USE_TMP_FILE = 'memory_use.tmp' | |
| DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)') | |
| DEPENDENCIES_DIR = os.path.join(THIRD_PARTY, 'dependencies') | |
| BUILD = os.path.join(REPO_ROOT, 'build') | |
| BUILD_JAVA_MAIN_CLASSPATH = os.path.join(REPO_ROOT, 'd8_r8', 'main', 'build', | |
| 'classes', 'java', 'main') | |
| LIBS = os.path.join(BUILD, 'libs') | |
| CUSTOM_CONVERSION_DIR = os.path.join(THIRD_PARTY, 'openjdk', | |
| 'custom_conversion') | |
| GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense') | |
| SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java') | |
| REPO_SOURCE = 'https://r8.googlesource.com/r8' | |
| GRADLE_TASK_CLEAN_TEST = ':test:cleanTest' | |
| GRADLE_TASK_CONSOLIDATED_LICENSE = ':dist:consolidatedLicense' | |
| GRADLE_TASK_DOWNLOAD_DEPS = ':third_party:downloadDeps' | |
| GRADLE_TASK_KEEP_ANNO_JAR = ':keepanno:keepAnnoAnnotationsJar' | |
| GRADLE_TASK_KEEP_ANNO_LEGACY_JAR = ':keepanno:keepAnnoLegacyAnnotationsJar' | |
| GRADLE_TASK_KEEP_ANNO_ANDROIDX_JAR = ':keepanno:keepAnnoAndroidXAnnotationsJar' | |
| GRADLE_TASK_KEEP_ANNO_DOC = ':keepanno:keepAnnoAnnotationsDoc' | |
| GRADLE_TASK_MAIN_COMPILE = ':main:compileJava' | |
| GRADLE_TASK_PROCESS_KEEP_RULES_LIB_JAR = ':dist:processKeepRulesLibWithRelocatedDeps' | |
| GRADLE_TASK_R8 = ':dist:r8WithRelocatedDeps' | |
| GRADLE_TASK_R8_ASSISTANT = ':dist:r8AssistantLib' | |
| GRADLE_TASK_R8LIB = ':test:assembleR8LibWithRelocatedDeps' | |
| GRADLE_TASK_R8LIB_NO_DEPS = ':test:assembleR8LibNoDeps' | |
| GRADLE_TASK_THREADING_MODULE_BLOCKING = ':dist:threadingModuleBlockingJar' | |
| GRADLE_TASK_THREADING_MODULE_SINGLE_THREADED = ':dist:threadingModuleSingleThreadedJar' | |
| GRADLE_TASK_SOURCE_JAR = ':test:packageSources' | |
| GRADLE_TASK_SWISS_ARMY_KNIFE = ':swissarmyknife:jar' | |
| GRADLE_TASK_KEEPANNOTOOLS = ':dist:keepAnnoToolsWithRelocatedDeps' | |
| GRADLE_TASK_KEEPANNOTOOLSLIB = 'test:keepAnnoToolsLib' | |
| GRADLE_TASK_TEST = ':test:test' | |
| GRADLE_TASK_ALL_TESTS_WITH_APPLY_MAPPING_JAR = ':test:rewriteTestsForR8LibWithRelocatedDeps' | |
| GRADLE_TASK_TESTBASE_WITH_APPLY_MAPPING_JAR = ':test:rewriteTestBaseForR8LibWithRelocatedDeps' | |
| GRADLE_TASK_TEST_DEPS_JAR = ':test:packageTestDeps' | |
| GRADLE_TASK_TEST_BASE_JAR = ':test:relocateTestBaseForR8LibWithRelocatedDeps' | |
| GRADLE_TASK_TEST_JAR = ':test:relocateTestsForR8LibWithRelocatedDeps' | |
| GRADLE_TASK_TEST_UNZIP_TESTBASE = ':test:unzipTestBase' | |
| GRADLE_TASK_PREPARE_TEST_ARTIFACTS = ':test:prepareTestArtifactsForR8LibWithRelocatedDeps' | |
| GRADLE_TASK_PREPARE_TEST_ARTIFACTS_NO_DEPS = ':test:prepareTestArtifactsForR8LibNoDeps' | |
| R8 = 'r8' | |
| R8LIB = 'r8lib' | |
| ALL_DEPS_JAR = os.path.join(LIBS, 'deps_all.jar') | |
| PROCESS_KEEP_RULES_LIB_JAR = os.path.join(LIBS, 'processkeepruleslib.jar') | |
| R8_JAR = os.path.join(LIBS, 'r8.jar') | |
| R8_ASSISTANT_LIB_JAR = os.path.join(LIBS, 'r8-assistantlib.jar') | |
| TOOLS_JAR = os.path.join(REPO_ROOT, 'd8_r8', 'tools', 'build', 'libs', | |
| 'tools.jar') | |
| R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar') | |
| R8LIB_MAP = '%s.map' % R8LIB_JAR | |
| R8LIB_PARTITION_MAP = '%s_map.zip' % R8LIB_JAR | |
| R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar') | |
| R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar') | |
| R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar') | |
| THREADING_MODULE_BLOCKING_JAR = os.path.join(LIBS, | |
| 'threading-module-blocking.jar') | |
| THREADING_MODULE_SINGLE_THREADED_JAR = os.path.join( | |
| LIBS, 'threading-module-single-threaded.jar') | |
| KEEPANNOTOOLS_JAR = os.path.join(LIBS, 'keepanno-tools.jar') | |
| KEEPANNOTOOLSLIB_JAR = os.path.join(LIBS, 'keepanno-toolslib.jar') | |
| KEEPANNOTOOLSLIB_MAP = '%s.map' % R8LIB_JAR | |
| KEEPANNOTOOLS_SRC_JAR = os.path.join(LIBS, 'r8-src.jar') | |
| R8_TESTS_JAR = os.path.join(LIBS, 'r8tests.jar') | |
| R8_TESTBASE_JAR = os.path.join(LIBS, 'r8test_base.jar') | |
| R8LIB_TESTBASE_JAR = os.path.join(LIBS, 'r8libtestbase-cf.jar') | |
| R8LIB_TESTS_JAR = os.path.join(LIBS, 'r8libtestdeps-cf.jar') | |
| R8_TESTS_DEPS_JAR = os.path.join(LIBS, 'test_deps_all.jar') | |
| R8LIB_TESTS_DEPS_JAR = R8_TESTS_DEPS_JAR | |
| MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip') | |
| KEEPANNNO_TOOLS_MAVEN_ZIP_LIB = os.path.join(LIBS, 'keepanno-toolslib.zip') | |
| LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP = os.path.join( | |
| CUSTOM_CONVERSION_DIR, 'library_desugar_conversions_legacy.jar') | |
| LIBRARY_DESUGAR_CONVERSIONS_ZIP = os.path.join( | |
| CUSTOM_CONVERSION_DIR, 'library_desugar_conversions.jar') | |
| KEEPANNO_ANNOTATIONS_JAR = os.path.join(LIBS, 'keepanno-annotations.jar') | |
| KEEPANNO_ANNOTATIONS_DOC = os.path.join('d8_r8', 'keepanno', 'build', 'docs', | |
| 'javadoc') | |
| KEEPANNO_LEGACY_ANNOTATIONS_JAR = os.path.join( | |
| LIBS, 'keepanno-annotations-legacy.jar') | |
| KEEPANNO_ANDROIDX_ANNOTATIONS_JAR = os.path.join( | |
| LIBS, 'keepanno-annotations-androidx.jar') | |
| DESUGAR_CONFIGURATION = os.path.join('src', 'library_desugar', | |
| 'desugar_jdk_libs.json') | |
| DESUGAR_IMPLEMENTATION = os.path.join('third_party', 'openjdk', | |
| 'desugar_jdk_libs', | |
| 'desugar_jdk_libs.jar') | |
| DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join( | |
| 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json') | |
| DESUGAR_CONFIGURATION_JDK11_MINIMAL = os.path.join( | |
| 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json') | |
| DESUGAR_CONFIGURATION_JDK11 = os.path.join('src', 'library_desugar', 'jdk11', | |
| 'desugar_jdk_libs.json') | |
| DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join('src', 'library_desugar', | |
| 'jdk11', | |
| 'desugar_jdk_libs_nio.json') | |
| DESUGAR_IMPLEMENTATION_JDK11 = os.path.join('third_party', 'openjdk', | |
| 'desugar_jdk_libs_11', | |
| 'desugar_jdk_libs.jar') | |
| DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join( | |
| LIBS, 'desugar_jdk_libs_configuration.zip') | |
| DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP = os.path.join( | |
| LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip') | |
| DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP = os.path.join( | |
| LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip') | |
| DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP = os.path.join( | |
| LIBS, 'desugar_jdk_libs_configuration_jdk11.zip') | |
| DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP = os.path.join( | |
| LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip') | |
| GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE') | |
| RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar') | |
| R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt') | |
| CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments') | |
| PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar') | |
| PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar') | |
| OPENSOURCE_DUMPS_DIR = os.path.join(THIRD_PARTY, 'opensource-apps') | |
| INTERNAL_DUMPS_DIR = os.path.join(THIRD_PARTY, 'internal-apps') | |
| BAZEL_DIR = os.path.join(THIRD_PARTY, 'bazel') | |
| JAVA8_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86') | |
| JAVA11_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11', 'linux') | |
| JAVA17_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-17', 'linux') | |
| JAVA21_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-21', 'linux') | |
| DESUGAR_JDK_LIBS_11_DIR = os.path.join(THIRD_PARTY, 'openjdk', | |
| 'desugar_jdk_libs_11') | |
| IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test', | |
| 'ignorewarnings.rules') | |
| ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME" | |
| ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION" | |
| USER_HOME = os.path.expanduser('~') | |
| CLOUD_STORAGE_URL = 'https://storage.googleapis.com' | |
| R8_DEPS_URL = '%s/r8-deps' % CLOUD_STORAGE_URL | |
| R8_INTERNAL_DEPS_BUCKET = 'gs://r8-internal-deps' | |
| R8_INTERNAL_TEST_RESULTS_BUCKET = 'r8-internal-test-results' | |
| ARCHIVE_DETAILS = os.path.join(BUILD, 'archive_details.json') | |
| def archive_file(name, gs_dir, src_file): | |
| gs_file = '%s/%s' % (gs_dir, name) | |
| upload_file_to_cloud_storage(src_file, gs_file) | |
| def archive_value(name, gs_dir, value): | |
| with TempDir() as temp: | |
| temparchive = os.path.join(temp, name) | |
| with open(temparchive, 'w') as f: | |
| f.write(str(value)) | |
| archive_file(name, gs_dir, temparchive) | |
| def find_cloud_storage_file_from_options(name, options, orElse=None): | |
| # Import archive on-demand since archive depends on utils. | |
| from archive import get_upload_destination | |
| hash_or_version = find_hash_or_version_from_options(options) | |
| if not hash_or_version: | |
| return orElse | |
| is_hash = options.commit_hash is not None | |
| download_path = get_upload_destination(hash_or_version, name, is_hash) | |
| if file_exists_on_cloud_storage(download_path): | |
| out = tempfile.NamedTemporaryFile().name | |
| download_file_from_cloud_storage(download_path, out) | |
| return out | |
| else: | |
| raise Exception('Could not find file {} from hash/version: {}.'.format( | |
| name, hash_or_version)) | |
| def find_r8_jar_from_options(options): | |
| return find_cloud_storage_file_from_options('r8.jar', options) | |
| def find_hash_or_version_from_options(options): | |
| if options.tag: | |
| return find_hash_or_version_from_tag(options.tag) | |
| else: | |
| return options.commit_hash or options.version | |
| def find_hash_or_version_from_tag(tag_or_hash): | |
| info = subprocess.check_output( | |
| ['git', 'show', tag_or_hash, '-s', | |
| '--format=oneline']).decode('utf-8').splitlines()[-1].split() | |
| # The info should be on the following form [hash,"Version",version] | |
| if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version": | |
| return info[2] | |
| return None | |
| def getAndroidHome(): | |
| return os.environ.get(ANDROID_HOME_ENVIROMENT_NAME, | |
| os.path.join(USER_HOME, 'Android', 'Sdk')) | |
| def getAndroidBuildTools(): | |
| if ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME in os.environ: | |
| version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME) | |
| build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version) | |
| assert os.path.exists(build_tools_dir) | |
| return build_tools_dir | |
| else: | |
| versions = ['35.0.1', '34.0.0', '33.0.1', '32.0.0'] | |
| for version in versions: | |
| build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', | |
| version) | |
| if os.path.exists(build_tools_dir): | |
| return build_tools_dir | |
| raise Exception('Unable to find Android build-tools') | |
| def is_python3(): | |
| return sys.version_info.major == 3 | |
| def Print(s, quiet=False): | |
| if quiet: | |
| return | |
| print(s) | |
| def Warn(message): | |
| CRED = '\033[91m' | |
| CEND = '\033[0m' | |
| print(CRED + message + CEND) | |
| def PrintCmd(cmd, env=None, quiet=False, worker_id=None): | |
| if quiet: | |
| return | |
| if type(cmd) is list: | |
| cmd = ' '.join(cmd) | |
| if env: | |
| env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()]) | |
| print_thread('Running: {} {}'.format(env, cmd), worker_id) | |
| else: | |
| print_thread('Running: {}'.format(cmd), worker_id) | |
| # I know this will hit os on windows eventually if we don't do this. | |
| sys.stdout.flush() | |
| class ProgressLogger(object): | |
| CLEAR_LINE = '\033[K' | |
| UP = '\033[F' | |
| def __init__(self, quiet=False): | |
| self._count = 0 | |
| self._has_printed = False | |
| self._quiet = quiet | |
| def log(self, text): | |
| if len(text.strip()) == 0: | |
| return | |
| if self._quiet: | |
| if self._has_printed: | |
| sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE) | |
| if len(text) > 140: | |
| text = text[0:140] + '...' | |
| print(text) | |
| self._has_printed = True | |
| def done(self): | |
| if self._quiet and self._has_printed: | |
| sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE) | |
| print('') | |
| sys.stdout.write(ProgressLogger.UP) | |
| def RunCmd(cmd, | |
| env_vars=None, | |
| quiet=False, | |
| fail=True, | |
| logging=True, | |
| shell=False): | |
| PrintCmd(cmd, env=env_vars, quiet=quiet) | |
| env = os.environ.copy() | |
| if env_vars: | |
| env.update(env_vars) | |
| process = subprocess.Popen(cmd, | |
| env=env, | |
| shell=shell, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT) | |
| stdout = [] | |
| logger = ProgressLogger(quiet=quiet) if logging else None | |
| failed = False | |
| while True: | |
| line = process.stdout.readline().decode('utf-8') | |
| if line != '': | |
| stripped = line.rstrip() | |
| stdout.append(stripped) | |
| if logger: | |
| logger.log(stripped) | |
| # TODO(christofferqa): r8 should fail with non-zero exit code. | |
| if ('AssertionError:' in stripped or | |
| 'CompilationError:' in stripped or | |
| 'CompilationFailedException:' in stripped or | |
| 'Compilation failed' in stripped or | |
| 'FAILURE:' in stripped or | |
| 'org.gradle.api.ProjectConfigurationException' in stripped | |
| or 'BUILD FAILED' in stripped): | |
| failed = True | |
| else: | |
| if logger: | |
| logger.done() | |
| exit_code = process.poll() | |
| if exit_code or failed: | |
| for line in stdout: | |
| Warn(line) | |
| if fail: | |
| raise subprocess.CalledProcessError( | |
| exit_code or -1, cmd, output='\n'.join(stdout)) | |
| return stdout | |
| def IsWindows(): | |
| return defines.IsWindows() | |
| GRADLE_STOPPED = False | |
| def stop_gradle(): | |
| global GRADLE_STOPPED | |
| if GRADLE_STOPPED: | |
| return | |
| try: | |
| java_home = jdk.GetDefaultJdkHome() | |
| except Exception as _: | |
| # No Gradle daemon is running if the JDK home can't be found. | |
| return | |
| gradle_dir = os.path.join(THIRD_PARTY, 'gradle') | |
| if IsWindows(): | |
| gradle = os.path.join(gradle_dir, 'bin', 'gradle.bat') | |
| else: | |
| gradle = os.path.join(gradle_dir, 'bin', 'gradle') | |
| if not os.path.exists(gradle): | |
| # No Gradle daemon is running if the Gradle executable can't be found. | |
| return | |
| try: | |
| env = os.environ.copy() | |
| env['JAVA_HOME'] = java_home | |
| env['PATH'] = env['PATH'] + os.pathsep + os.path.join(java_home, 'bin') | |
| env['GRADLE_OPTS'] = '-Xmx1g' | |
| cmd = [gradle, '--stop'] | |
| PrintCmd(cmd) | |
| subprocess.call(cmd, env=env) | |
| GRADLE_STOPPED = True | |
| except Exception as e: | |
| Warn(f"Warning: Failed to stop gradle daemon: {e}") | |
| GRADLE_STOPPED = True | |
| def should_download(output_dir, tgz_file, sha1, success_file): | |
| if (os.path.exists(output_dir) and os.path.exists(tgz_file) and | |
| os.path.exists(success_file) and | |
| os.path.getmtime(sha1) <= os.path.getmtime(success_file)): | |
| return False | |
| return True | |
| def run_download(sha1, output_dir, success_file, internal, quiet=False): | |
| if os.path.exists(success_file): | |
| os.remove(success_file) | |
| if os.path.exists(output_dir): | |
| stop_gradle() | |
| shutil.rmtree(output_dir) | |
| if internal: | |
| download_script = os.path.join(REPO_ROOT, 'tools', | |
| 'download_from_x20.py') | |
| else: | |
| download_script = os.path.join(REPO_ROOT, 'tools', | |
| 'download_from_gcs.py') | |
| cmd = [sys.executable, download_script, sha1] | |
| if not quiet: | |
| PrintCmd(cmd) | |
| subprocess.check_call(cmd) | |
| else: | |
| subprocess.check_call(cmd, stdout=subprocess.DEVNULL) | |
| with open(success_file, 'w') as _: | |
| pass | |
| def get_main_repo_root(worktree_root=None): | |
| wt_root = worktree_root or REPO_ROOT | |
| git_path = os.path.join(wt_root, '.git') | |
| if not os.path.isfile(git_path): | |
| return None | |
| try: | |
| with open(git_path, 'r') as f: | |
| line = f.readline().strip() | |
| if not line.startswith('gitdir:'): | |
| return None | |
| gitdir = line[len('gitdir:'):].strip() | |
| if not os.path.isabs(gitdir): | |
| gitdir = os.path.abspath(os.path.join(wt_root, gitdir)) | |
| commondir_file = os.path.join(gitdir, 'commondir') | |
| if os.path.isfile(commondir_file): | |
| with open(commondir_file, 'r') as f: | |
| commondir = f.readline().strip() | |
| common_git_dir = os.path.abspath(os.path.join(gitdir, commondir)) | |
| else: | |
| common_git_dir = os.path.dirname(os.path.dirname(gitdir)) | |
| main_repo = os.path.dirname(common_git_dir) | |
| if (os.path.isdir(main_repo) and | |
| os.path.abspath(main_repo) != os.path.abspath(wt_root)): | |
| return main_repo | |
| except Exception: | |
| pass | |
| return None | |
| def link_or_copy(src, dst): | |
| if os.path.lexists(dst): | |
| try: | |
| if os.path.samefile(src, dst): | |
| return | |
| except OSError: | |
| pass | |
| os.remove(dst) | |
| try: | |
| os.link(src, dst) | |
| except OSError: | |
| shutil.copy2(src, dst) | |
| def link_tree(src, dst, ignore=None): | |
| shutil.copytree(src, | |
| dst, | |
| symlinks=True, | |
| copy_function=link_or_copy, | |
| ignore=ignore, | |
| dirs_exist_ok=True) | |
| def try_link_download_from_main_repo(path, worktree_root=None): | |
| wt_root = worktree_root or REPO_ROOT | |
| output_dir = path | |
| tar_gz_file = path + '.tar.gz' | |
| sha1_file = tar_gz_file + '.sha1' | |
| success_file = path + '.success' | |
| if not should_download(output_dir, tar_gz_file, sha1_file, success_file): | |
| return False | |
| main_repo = get_main_repo_root(wt_root) | |
| if not main_repo: | |
| return False | |
| try: | |
| rel_path = os.path.relpath(path, wt_root) | |
| except ValueError: | |
| return False | |
| if rel_path.startswith('..'): | |
| return False | |
| main_output_dir = os.path.join(main_repo, rel_path) | |
| main_tar_gz_file = main_output_dir + '.tar.gz' | |
| main_sha1_file = main_tar_gz_file + '.sha1' | |
| main_success_file = main_output_dir + '.success' | |
| if not os.path.exists(main_sha1_file) or not os.path.exists(sha1_file): | |
| return False | |
| if should_download(main_output_dir, main_tar_gz_file, main_sha1_file, | |
| main_success_file): | |
| return False | |
| if get_sha1(sha1_file) != get_sha1(main_sha1_file): | |
| return False | |
| try: | |
| if os.path.exists(success_file): | |
| os.remove(success_file) | |
| if os.path.exists(output_dir): | |
| shutil.rmtree(output_dir) | |
| if os.path.exists(tar_gz_file): | |
| os.remove(tar_gz_file) | |
| link_tree(main_output_dir, output_dir) | |
| link_or_copy(main_tar_gz_file, tar_gz_file) | |
| with open(success_file, 'w') as _: | |
| pass | |
| return True | |
| except Exception as e: | |
| Warn( | |
| f'Warning: Failed to link dependency from main repo {main_output_dir}: {e}' | |
| ) | |
| return False | |
| def ensure_worktree_deps(quiet=False, worktree_root=None): | |
| wt_root = worktree_root or REPO_ROOT | |
| main_repo = get_main_repo_root(wt_root) | |
| if not main_repo: | |
| return | |
| linked_deps = 0 | |
| try: | |
| sha1_files = subprocess.check_output( | |
| ['git', 'ls-files', '*.tar.gz.sha1'], cwd=wt_root, | |
| text=True).splitlines() | |
| for rel_sha1 in sha1_files: | |
| path = os.path.join(wt_root, rel_sha1[:-len('.tar.gz.sha1')]) | |
| if try_link_download_from_main_repo(path, worktree_root=wt_root): | |
| linked_deps += 1 | |
| except Exception as e: | |
| Warn(f'Warning: Failed to scan worktree dependencies: {e}') | |
| if linked_deps > 0 and not quiet: | |
| print( | |
| f'Linked {linked_deps} pre-downloaded dependencies from {main_repo}' | |
| ) | |
| ensure_worktree_intellij_config(worktree_root=wt_root) | |
| def ensure_worktree_intellij_config(worktree_root=None): | |
| wt_root = worktree_root or REPO_ROOT | |
| main_repo = get_main_repo_root(wt_root) | |
| if not main_repo: | |
| return | |
| wt_idea = os.path.join(wt_root, '.idea') | |
| main_idea = os.path.join(main_repo, '.idea') | |
| if not os.path.isdir(main_idea): | |
| return | |
| wt_gradle_xml = os.path.join(wt_idea, 'gradle.xml') | |
| wt_misc_xml = os.path.join(wt_idea, 'misc.xml') | |
| def file_contains(path, substring): | |
| if not os.path.isfile(path): | |
| return False | |
| try: | |
| with open(path, 'r') as f: | |
| return substring in f.read() | |
| except Exception: | |
| return False | |
| gradle_configured = file_contains(wt_gradle_xml, 'myGradleHome') | |
| misc_configured = file_contains(wt_misc_xml, 'ProjectRootManager') | |
| if gradle_configured and misc_configured: | |
| return | |
| try: | |
| os.makedirs(wt_idea, exist_ok=True) | |
| shared_files = [ | |
| 'gradle.xml', 'misc.xml', 'vcs.xml', 'compiler.xml', 'kotlinc.xml', | |
| '.gitignore' | |
| ] | |
| for fname in shared_files: | |
| src = os.path.join(main_idea, fname) | |
| dst = os.path.join(wt_idea, fname) | |
| if not os.path.isfile(src): | |
| continue | |
| if fname == 'gradle.xml' and not gradle_configured: | |
| with open(src, 'r') as f: | |
| content = f.read() | |
| main_gradle_dir = os.path.join(main_repo, 'third_party', | |
| 'gradle') | |
| user_home_abs = os.path.abspath(USER_HOME) | |
| main_gradle_abs = os.path.abspath(main_gradle_dir) | |
| if main_gradle_abs.startswith(user_home_abs + os.sep): | |
| rel_gradle = os.path.relpath(main_gradle_abs, | |
| user_home_abs).replace( | |
| '\\', '/') | |
| content = content.replace( | |
| f'$USER_HOME$/{rel_gradle}', | |
| '$PROJECT_DIR$/third_party/gradle') | |
| content = content.replace(main_gradle_dir.replace('\\', '/'), | |
| '$PROJECT_DIR$/third_party/gradle') | |
| content = content.replace(main_gradle_dir, | |
| '$PROJECT_DIR$/third_party/gradle') | |
| with open(dst, 'w') as f: | |
| f.write(content) | |
| elif fname == 'misc.xml' and not misc_configured: | |
| shutil.copy2(src, dst) | |
| elif not os.path.exists(dst): | |
| shutil.copy2(src, dst) | |
| except Exception as e: | |
| Warn( | |
| f'Warning: Failed to configure IntelliJ settings for worktree: {e}') | |
| def seed_gradle_user_home(gradle_user_home, worktree_root=None): | |
| seeded_marker = os.path.join(gradle_user_home, '.worktree_seeded') | |
| if os.path.exists(seeded_marker): | |
| return | |
| wt_root = worktree_root or REPO_ROOT | |
| main_repo = get_main_repo_root(wt_root) | |
| candidates = [os.path.expanduser(os.path.join('~', '.gradle', 'caches'))] | |
| if main_repo: | |
| candidates.append(os.path.join(main_repo, '.gradle_user_home', | |
| 'caches')) | |
| dst_caches = os.path.join(gradle_user_home, 'caches') | |
| dst_caches_abs = os.path.abspath(dst_caches) | |
| def ignore_locks_and_mutable(dir_path, names): | |
| return [ | |
| n for n in names | |
| if n.endswith('.lock') or n in ('journal-1', 'fileHashes', | |
| 'fileContent', 'gc.properties', | |
| 'cc-keystore', 'javaCompile', | |
| 'build-cache-1') | |
| ] | |
| seeded_any = False | |
| for src_caches in candidates: | |
| if os.path.isdir(src_caches) and os.path.abspath( | |
| src_caches) != dst_caches_abs: | |
| os.makedirs(dst_caches, exist_ok=True) | |
| try: | |
| link_tree(src_caches, | |
| dst_caches, | |
| ignore=ignore_locks_and_mutable) | |
| seeded_any = True | |
| except Exception as e: | |
| Warn( | |
| f'Warning: Failed to seed worktree Gradle caches from {src_caches}: {e}' | |
| ) | |
| if seeded_any: | |
| try: | |
| os.makedirs(gradle_user_home, exist_ok=True) | |
| with open(seeded_marker, 'w') as _: | |
| pass | |
| except Exception: | |
| pass | |
| def ensure_download(path, internal, quiet=False): | |
| output_dir = path | |
| tar_gz_file = path + '.tar.gz' | |
| sha1_file = tar_gz_file + '.sha1' | |
| success_file = path + '.success' | |
| if not os.path.exists(sha1_file): | |
| raise Exception(f"Missing sha1 file: {sha1_file}") | |
| if should_download(output_dir, tar_gz_file, sha1_file, success_file): | |
| if try_link_download_from_main_repo(path): | |
| return | |
| run_download(sha1_file, output_dir, success_file, internal, quiet) | |
| def ensure_google_download(path, quiet=False): | |
| ensure_download(path, False, quiet=quiet) | |
| def ensure_x20_download(path, quiet=False): | |
| ensure_download(path, True, quiet=quiet) | |
| def get_nth_sha1_from_revision(n, revision): | |
| result = subprocess.check_output([ | |
| 'git', 'log', '--first-parent', revision, f'--skip={n}', | |
| '--max-count=1', '--pretty=format:%H' | |
| ]).decode('utf-8') | |
| return result.strip() | |
| def get_sha1(filename): | |
| sha1 = hashlib.sha1() | |
| with open(filename, 'rb') as f: | |
| while True: | |
| chunk = f.read(1024 * 1024) | |
| if not chunk: | |
| break | |
| sha1.update(chunk) | |
| return sha1.hexdigest() | |
| def get_sha1_from_revision(revision): | |
| cmd = ['git', 'rev-parse', revision] | |
| PrintCmd(cmd) | |
| with ChangedWorkingDirectory(REPO_ROOT): | |
| return subprocess.check_output(cmd).decode('utf-8').strip() | |
| def get_HEAD_branch(): | |
| result = subprocess.check_output( | |
| ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8') | |
| return result.strip() | |
| def get_HEAD_commit(): | |
| return historic_run.git_commit_from_hash(get_HEAD_sha1()) | |
| def get_HEAD_sha1(): | |
| return get_sha1_from_revision('HEAD') | |
| def get_HEAD_diff_stat(): | |
| return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8') | |
| def makedirs_if_needed(path): | |
| try: | |
| os.makedirs(path) | |
| except OSError: | |
| if not os.path.isdir(path): | |
| raise | |
| # TODO(b/542096534): Remove gsutil fallback once all bots use gcloud. | |
| def get_gsutil(): | |
| return 'gsutil.py.bat' if IsWindows() else 'gsutil.py' | |
| def get_gcloud(): | |
| return 'gcloud.cmd' if IsWindows() else 'gcloud' | |
| def use_gcloud(): | |
| return 'SWARMING_BOT_ID' not in os.environ and shutil.which( | |
| get_gcloud()) is not None | |
| def upload_file_to_cloud_storage(source, destination, header=None): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'cp'] | |
| if header: | |
| if header.lower().startswith('cache-control:'): | |
| cmd.append('--cache-control=' + header.split(':', 1)[1].strip()) | |
| else: | |
| cmd.extend(['--custom-metadata=' + header.replace(':', '=')]) | |
| cmd.extend([source, destination]) | |
| else: | |
| cmd = [get_gsutil()] | |
| if header: | |
| cmd.extend(['-h', header]) | |
| cmd.extend(['cp', source, destination]) | |
| PrintCmd(cmd) | |
| subprocess.check_call(cmd) | |
| def check_dir_args(source, destination): | |
| # We require that the dirname of the paths coincide, e.g., src/dirname and dst/dirname | |
| # The target is then stripped so the upload command will be: cp -R src/dirname dst/ | |
| (destination_parent, destination_file) = os.path.split(destination) | |
| if os.path.basename(source) != destination_file: | |
| raise Exception( | |
| 'Attempt to upload directory with non-matching directory name: ' + | |
| f'{source} and {destination}') | |
| if len(destination_parent.strip()) == 0: | |
| raise Exception( | |
| 'Attempt to upload directory to empty destination directory: ' + | |
| destination) | |
| return destination_parent | |
| def upload_directory_to_cloud_storage(source, destination, parallel=True): | |
| destination_parent = check_dir_args(source, destination) | |
| if use_gcloud(): | |
| cmd = [ | |
| get_gcloud(), 'storage', 'cp', '--recursive', source, | |
| destination_parent + '/' | |
| ] | |
| else: | |
| cmd = [get_gsutil()] | |
| if parallel: | |
| cmd += ['-m'] | |
| cmd += ['cp', '-R'] | |
| cmd += [source, destination_parent + '/'] | |
| PrintCmd(cmd) | |
| subprocess.check_call(cmd) | |
| def rsync_directory_to_cloud_storage(source, destination, parallel=True): | |
| check_dir_args(source, destination) | |
| if use_gcloud(): | |
| cmd = [ | |
| get_gcloud(), 'storage', 'rsync', '--recursive', source, destination | |
| ] | |
| else: | |
| cmd = [get_gsutil()] | |
| if parallel: | |
| cmd += ['-m'] | |
| cmd += ['rsync', '-R'] | |
| cmd += [source, destination] | |
| PrintCmd(cmd) | |
| subprocess.check_call(cmd) | |
| def delete_file_from_cloud_storage(destination): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'rm', destination] | |
| else: | |
| cmd = [get_gsutil(), 'rm', destination] | |
| PrintCmd(cmd) | |
| subprocess.check_call(cmd) | |
| def ls_files_on_cloud_storage(destination): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'ls', destination] | |
| else: | |
| cmd = [get_gsutil(), 'ls', destination] | |
| PrintCmd(cmd) | |
| return subprocess.check_output(cmd).decode('utf-8') | |
| def cat_file_on_cloud_storage(destination, ignore_errors=False): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'cat', destination] | |
| else: | |
| cmd = [get_gsutil(), 'cat', destination] | |
| PrintCmd(cmd) | |
| try: | |
| return subprocess.check_output(cmd).decode('utf-8').strip() | |
| except subprocess.CalledProcessError as e: | |
| if ignore_errors: | |
| return '' | |
| else: | |
| raise e | |
| def file_exists_on_cloud_storage(destination): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'ls', destination] | |
| else: | |
| cmd = [get_gsutil(), 'ls', destination] | |
| PrintCmd(cmd) | |
| return subprocess.call(cmd) == 0 | |
| def download_file_from_cloud_storage(source, | |
| destination, | |
| concurrent=False, | |
| flags=None, | |
| printcmd=True, | |
| quiet=False): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'cp'] | |
| if quiet: | |
| cmd.extend(['--quiet', '--no-user-output-enabled']) | |
| if flags: | |
| cmd.extend(flags) | |
| cmd.extend([source, destination]) | |
| else: | |
| cmd = [get_gsutil()] | |
| if concurrent: | |
| cmd.append('-m') | |
| if quiet: | |
| cmd.append('-q') | |
| cmd.append('cp') | |
| if flags: | |
| cmd.extend(flags) | |
| cmd.extend([source, destination]) | |
| PrintCmd(cmd, quiet=not printcmd) | |
| subprocess.check_call(cmd) | |
| def create_archive(name, sources=None): | |
| if not sources: | |
| sources = [name] | |
| tarname = '%s.tar.gz' % name | |
| with tarfile.open(tarname, 'w:gz') as tar: | |
| for source in sources: | |
| tar.add(source) | |
| return tarname | |
| def extract_dir(filename): | |
| return filename[0:len(filename) - len('.tar.gz')] | |
| def unpack_archive(filename): | |
| dest_dir = extract_dir(filename) | |
| if os.path.exists(dest_dir): | |
| print('Deleting existing dir %s' % dest_dir) | |
| stop_gradle() | |
| shutil.rmtree(dest_dir) | |
| dirname = os.path.dirname(os.path.abspath(filename)) | |
| with tarfile.open(filename, 'r:gz') as tar: | |
| tar.extractall(path=dirname) | |
| def ensure_sha1_archive(sha1_file, download_fn): | |
| dest = sha1_file[:-5] | |
| print('Ensuring %s' % dest) | |
| with open(sha1_file, 'r') as input_sha: | |
| sha1 = input_sha.readline().strip() | |
| dest_dir = extract_dir(dest) | |
| if os.path.exists(dest) and get_sha1(dest) == sha1: | |
| print('sha1 matches, not downloading') | |
| if os.path.exists(dest_dir): | |
| print('destination directory exists, no extraction') | |
| else: | |
| unpack_archive(dest) | |
| return | |
| if os.path.exists(dest): | |
| os.remove(dest) | |
| if os.path.exists(dest_dir): | |
| shutil.rmtree(dest_dir) | |
| download_fn(sha1, dest) | |
| actual_sha1 = get_sha1(dest) | |
| if actual_sha1 != sha1: | |
| os.remove(dest) | |
| raise Exception('SHA1 mismatch for %s: expected %s, got %s' % | |
| (dest, sha1, actual_sha1)) | |
| unpack_archive(dest) | |
| def check_gcert(): | |
| status = subprocess.call(['gcertstatus']) | |
| if status != 0: | |
| subprocess.check_call(['gcert']) | |
| # Note that gcs is eventually consistent with regards to list operations. | |
| # This is not a problem in our case, but don't ever use this method | |
| # for synchronization. | |
| def cloud_storage_exists(destination): | |
| if use_gcloud(): | |
| cmd = [get_gcloud(), 'storage', 'ls', destination] | |
| else: | |
| cmd = [get_gsutil(), 'ls', destination] | |
| PrintCmd(cmd) | |
| exit_code = subprocess.call(cmd) | |
| return exit_code == 0 | |
| class TempDir(object): | |
| def __init__(self, prefix='r8-tmp-', delete=True): | |
| self._temp_dir = None | |
| self._prefix = prefix | |
| self._delete = delete | |
| def __enter__(self): | |
| self._temp_dir = tempfile.mkdtemp(self._prefix) | |
| return self._temp_dir | |
| def __exit__(self, *_): | |
| if self._delete: | |
| shutil.rmtree(self._temp_dir, ignore_errors=True) | |
| class ChangedWorkingDirectory(object): | |
| def __init__(self, working_directory, quiet=False): | |
| self._quiet = quiet | |
| self._working_directory = working_directory | |
| def __enter__(self): | |
| self._old_cwd = os.getcwd() | |
| if not self._quiet: | |
| print('Enter directory:', self._working_directory) | |
| os.chdir(self._working_directory) | |
| def __exit__(self, *_): | |
| if not self._quiet: | |
| print('Enter directory:', self._old_cwd) | |
| os.chdir(self._old_cwd) | |
| # Reading Android CTS test_result.xml | |
| class CtsModule(object): | |
| def __init__(self, module_name): | |
| self.name = module_name | |
| class CtsTestCase(object): | |
| def __init__(self, test_case_name): | |
| self.name = test_case_name | |
| class CtsTest(object): | |
| def __init__(self, test_name, outcome): | |
| self.name = test_name | |
| self.outcome = outcome | |
| # Generator yielding CtsModule, CtsTestCase or CtsTest from | |
| # reading through a CTS test_result.xml file. | |
| def read_cts_test_result(file_xml): | |
| re_module = re.compile('<Module name="([^"]*)"') | |
| re_test_case = re.compile('<TestCase name="([^"]*)"') | |
| re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"') | |
| with open(file_xml) as f: | |
| for line in f: | |
| m = re_module.search(line) | |
| if m: | |
| yield CtsModule(m.groups()[0]) | |
| continue | |
| m = re_test_case.search(line) | |
| if m: | |
| yield CtsTestCase(m.groups()[0]) | |
| continue | |
| m = re_test.search(line) | |
| if m: | |
| outcome = m.groups()[0] | |
| assert outcome in ['fail', 'pass'] | |
| yield CtsTest(m.groups()[1], outcome == 'pass') | |
| def grep_memoryuse(logfile): | |
| re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)') | |
| result = None | |
| with open(logfile) as f: | |
| for line in f: | |
| m = re_vmhwm.search(line) | |
| if m: | |
| groups = m.groups() | |
| s = len(groups) | |
| if s >= 1: | |
| result = int(groups[0]) | |
| if s >= 2: | |
| unit = groups[1] | |
| if unit == 'kB': | |
| result *= 1024 | |
| elif unit != '': | |
| raise Exception( | |
| 'Unrecognized unit in memory usage log: {}'. | |
| format(unit)) | |
| if result is None: | |
| raise Exception('No memory usage found in log: {}'.format(logfile)) | |
| return result | |
| # Return a dictionary: {segment_name -> segments_size} | |
| def getDexSegmentSizes(dex_files): | |
| assert len(dex_files) > 0 | |
| cmd = [ | |
| jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-jar', R8_JAR, | |
| 'dexsegments' | |
| ] | |
| cmd.extend(dex_files) | |
| PrintCmd(cmd) | |
| output = subprocess.check_output(cmd).decode('utf-8') | |
| matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output) | |
| if matches is None or len(matches) == 0: | |
| raise Exception('DexSegments failed to return any output for' \ | |
| ' these files: {}'.format(dex_files)) | |
| result = {} | |
| for match in matches: | |
| result[match[0]] = int(match[1]) | |
| return result | |
| # Return a dictionary: {segment_name -> segments_size} | |
| def getCfSegmentSizes(cfFile): | |
| cmd = [ | |
| jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-cp', CF_SEGMENTS_TOOL, | |
| 'com.android.tools.r8.cf_segments.MeasureLib', cfFile | |
| ] | |
| PrintCmd(cmd) | |
| output = subprocess.check_output(cmd).decode('utf-8') | |
| matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output) | |
| if matches is None or len(matches) == 0: | |
| raise Exception('CfSegments failed to return any output for' \ | |
| ' the file: ' + cfFile) | |
| result = {} | |
| for match in matches: | |
| result[match[0]] = int(match[1]) | |
| return result | |
| def get_maven_path(artifact, version): | |
| return os.path.join('com', 'android', 'tools', artifact, version) | |
| def print_cfsegments(prefix, cf_files): | |
| for cf_file in cf_files: | |
| for segment_name, size in getCfSegmentSizes(cf_file).items(): | |
| print('{}-{}(CodeSize): {}'.format(prefix, segment_name, size)) | |
| def print_dexsegments(prefix, dex_files, worker_id=None): | |
| for segment_name, size in getDexSegmentSizes(dex_files).items(): | |
| print_thread('{}-{}(CodeSize): {}'.format(prefix, segment_name, size), | |
| worker_id) | |
| # Ensure that we are not benchmarking with a google jvm. | |
| def check_java_version(): | |
| cmd = [jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-version'] | |
| output = subprocess.check_output(cmd, | |
| stderr=subprocess.STDOUT).decode('utf-8') | |
| m = re.search('openjdk version "([^"]*)"', output) | |
| if m is None: | |
| raise Exception("Can't check java version: no version string in output" | |
| " of 'java -version': '{}'".format(output)) | |
| version = m.groups(0)[0] | |
| m = re.search('google', version) | |
| if m is not None: | |
| raise Exception("Do not use google JVM for benchmarking: " + version) | |
| def api_str(api_level_major, api_level_minor): | |
| api = str(api_level_major) | |
| if api_level_minor > 0: | |
| api = api + '.' + str(api_level_minor) | |
| return api | |
| def get_android_jar_dir(api_level_major, api_level_minor): | |
| return os.path.join( | |
| REPO_ROOT, | |
| ANDROID_JAR_DIR.format(api=api_str(api_level_major, api_level_minor))) | |
| def get_android_jar(api_level_major, api_level_minor): | |
| return os.path.join( | |
| REPO_ROOT, | |
| ANDROID_JAR.format(api=api_str(api_level_major, api_level_minor))) | |
| def is_bot(): | |
| return 'SWARMING_BOT_ID' in os.environ or 'KOKORO_BUILD_ID' in os.environ | |
| def uncompressed_size(path): | |
| return sum(z.file_size for z in zipfile.ZipFile(path).infolist()) | |
| def desugar_configuration_name_and_version(configuration, is_for_maven): | |
| name = 'desugar_jdk_libs_configuration' | |
| with open(configuration, 'r') as f: | |
| configuration_json = json.loads(f.read()) | |
| configuration_format_version = \ | |
| configuration_json.get('configuration_format_version') | |
| if (not configuration_format_version): | |
| raise Exception('No "configuration_format_version" found in ' + | |
| configuration) | |
| if (configuration_format_version != 3 and | |
| configuration_format_version != 5 and | |
| configuration_format_version != (200 if is_for_maven else 101)): | |
| raise Exception( | |
| 'Unsupported "configuration_format_version" "%s" found in %s' % | |
| (configuration_format_version, configuration)) | |
| version = configuration_json.get('version') | |
| if not version: | |
| if configuration_format_version == (200 if is_for_maven else 101): | |
| identifier = configuration_json.get('identifier') | |
| if not identifier: | |
| raise Exception('No "identifier" found in ' + configuration) | |
| identifier_split = identifier.split(':') | |
| if (len(identifier_split) != 3): | |
| raise Exception('Invalid "identifier" found in ' + | |
| configuration) | |
| if (identifier_split[0] != 'com.tools.android'): | |
| raise Exception('Invalid "identifier" found in ' + | |
| configuration) | |
| if not identifier_split[1].startswith( | |
| 'desugar_jdk_libs_configuration'): | |
| raise Exception('Invalid "identifier" found in ' + | |
| configuration) | |
| name = identifier_split[1] | |
| version = identifier_split[2] | |
| else: | |
| raise Exception('No "version" found in ' + configuration) | |
| else: | |
| if configuration_format_version == (200 if is_for_maven else 101): | |
| raise Exception('No "version" expected in ' + configuration) | |
| # Disallow prerelease, as older R8 versions cannot parse it causing hard to | |
| # understand errors. | |
| check_basic_semver_version(version, | |
| 'in ' + configuration, | |
| allowPrerelease=False) | |
| return (name, version) | |
| class SemanticVersion: | |
| def __init__(self, major, minor, patch, prerelease): | |
| self.major = major | |
| self.minor = minor | |
| self.patch = patch | |
| self.prerelease = prerelease | |
| # Build metadata currently not suppported | |
| def larger_than(self, other): | |
| if self.major != other.major: | |
| return self.major > other.major | |
| if self.minor != other.minor: | |
| return self.minor > other.minor | |
| if self.patch is not None or other.patch is not None: | |
| if self.patch != other.patch: | |
| return self.patch > other.patch | |
| if self.prerelease or other.prerelease: | |
| if not self.prerelease: | |
| return True | |
| if not other.prerelease: | |
| return False | |
| if (self.prerelease.startswith('alpha') and | |
| other.prerelease.startswith('alpha') and | |
| self.prerelease[5:].isdigit() and | |
| other.prerelease[5:].isdigit()): | |
| return int(self.prerelease[5:]) > int(other.prerelease[5:]) | |
| return self.prerelease > other.prerelease | |
| return False | |
| def equals_to(self, other): | |
| return (self.major == other.major and self.minor == other.minor and | |
| self.patch == other.patch and | |
| self.prerelease == other.prerelease) | |
| def __str__(self): | |
| return str(self.major) + '.' + str(self.minor) + '.' + str( | |
| self.patch) + (('-' + self.prerelease) if self.prerelease else '') | |
| # Check that the passed string is formatted as a basic semver version (x.y.z or x.y.z-prerelease | |
| # depending on the value of allowPrerelease). | |
| # See https://semver.org/. The regexp parts used are not all complient with what is suggested | |
| # on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. | |
| def check_basic_semver_version(version, | |
| error_context='', | |
| components=3, | |
| allowPrerelease=False): | |
| regexp = '^' | |
| for x in range(components): | |
| regexp += '([0-9]+)' | |
| if x < components - 1: | |
| regexp += '\\.' | |
| if allowPrerelease: | |
| # This part is from | |
| # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string | |
| regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?' | |
| regexp += '$' | |
| reg = re.compile(regexp) | |
| match = reg.match(version) | |
| if not match: | |
| raise Exception("Invalid version '" + version + "'" + ( | |
| (' ' + error_context) if len(error_context) > 0 else '')) | |
| if components == 2: | |
| return SemanticVersion(int(match.group(1)), int(match.group(2)), None, | |
| None) | |
| elif components == 3 and not allowPrerelease: | |
| return SemanticVersion(int(match.group(1)), int(match.group(2)), | |
| int(match.group(3)), None) | |
| elif components == 3 and allowPrerelease: | |
| return SemanticVersion(int(match.group(1)), int(match.group(2)), | |
| int(match.group(3)), match.group('prerelease')) | |
| else: | |
| raise Exception('Argument "components" must be 2 or 3') | |
| def is_git_worktree(checkout_dir): | |
| """ | |
| Checks if the given path is the main Git worktree or a linked worktree. | |
| """ | |
| git_path = os.path.join(checkout_dir, ".git") | |
| if not os.path.exists(git_path): | |
| print("Checkout path does not look like a Git repository.") | |
| return False | |
| # In the main worktree, .git is a directory containing the repository data and | |
| # in a linked worktree, .git is a plain text file with a gitdir reference. | |
| if os.path.isdir(git_path): | |
| return False | |
| elif os.path.isfile(git_path): | |
| return True | |
| else: | |
| print("Checkout path does not look like a Git repository.") | |
| return False | |
| def append_gradle_user_home_for_worktree(force_worktree, with_no_daemon, args): | |
| if force_worktree or is_git_worktree(REPO_ROOT): | |
| if not force_worktree: | |
| print( | |
| 'git worktree detected, using worktree local Gradle User Home') | |
| gradle_user_home = os.path.join(REPO_ROOT, ".gradle_user_home") | |
| seed_gradle_user_home(gradle_user_home) | |
| args.append('--gradle-user-home=' + gradle_user_home) | |
| if with_no_daemon: | |
| args.append('--no-daemon') | |
| def compute_dex_size_in_zip(zip_path): | |
| if not os.path.isfile(zip_path): | |
| return 0 | |
| with zipfile.ZipFile(zip_path, 'r') as z: | |
| return sum(info.file_size | |
| for info in z.infolist() | |
| if info.filename.endswith('.dex')) |