# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# Different utility functions used across scripts

import hashlib
import jdk
import json
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import zipfile

import defines
import historic_run
from thread_utils import print_thread

ANDROID_JAR_DIR = 'third_party/android_jar/lib-v{api}'
ANDROID_JAR = os.path.join(ANDROID_JAR_DIR, 'android.jar')
TOOLS_DIR = defines.TOOLS_DIR
REPO_ROOT = defines.REPO_ROOT
THIRD_PARTY = defines.THIRD_PARTY
BUNDLETOOL_JAR_DIR = os.path.join(THIRD_PARTY, 'bundletool/bundletool-1.11.0')
BUNDLETOOL_JAR = os.path.join(BUNDLETOOL_JAR_DIR, 'bundletool-all-1.11.0.jar')
MEMORY_USE_TMP_FILE = 'memory_use.tmp'
DEX_SEGMENTS_RESULT_PATTERN = re.compile('- ([^:]+): ([0-9]+)')

DEPENDENCIES_DIR = os.path.join(THIRD_PARTY, 'dependencies')

BUILD = os.path.join(REPO_ROOT, 'build')
BUILD_JAVA_MAIN_CLASSPATH = os.path.join(REPO_ROOT, 'd8_r8', 'main', 'build',
                                         'classes', 'java', 'main')
LIBS = os.path.join(BUILD, 'libs')
CUSTOM_CONVERSION_DIR = os.path.join(THIRD_PARTY, 'openjdk',
                                     'custom_conversion')
GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense')
SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java')
REPO_SOURCE = 'https://r8.googlesource.com/r8'

GRADLE_TASK_CLEAN_TEST = ':test:cleanTest'
GRADLE_TASK_CONSOLIDATED_LICENSE = ':dist:consolidatedLicense'
GRADLE_TASK_DOWNLOAD_DEPS = ':third_party:downloadDeps'
GRADLE_TASK_KEEP_ANNO_JAR = ':keepanno:keepAnnoAnnotationsJar'
GRADLE_TASK_KEEP_ANNO_LEGACY_JAR = ':keepanno:keepAnnoLegacyAnnotationsJar'
GRADLE_TASK_KEEP_ANNO_ANDROIDX_JAR = ':keepanno:keepAnnoAndroidXAnnotationsJar'
GRADLE_TASK_KEEP_ANNO_DOC = ':keepanno:keepAnnoAnnotationsDoc'
GRADLE_TASK_MAIN_COMPILE = ':main:compileJava'
GRADLE_TASK_PROCESS_KEEP_RULES_LIB_JAR = ':dist:processKeepRulesLibWithRelocatedDeps'
GRADLE_TASK_R8 = ':dist:r8WithRelocatedDeps'
GRADLE_TASK_R8LIB = ':test:assembleR8LibWithRelocatedDeps'
GRADLE_TASK_R8LIB_NO_DEPS = ':test:assembleR8LibNoDeps'
GRADLE_TASK_THREADING_MODULE_BLOCKING = ':dist:threadingModuleBlockingJar'
GRADLE_TASK_THREADING_MODULE_SINGLE_THREADED = ':dist:threadingModuleSingleThreadedJar'
GRADLE_TASK_SOURCE_JAR = ':test:packageSources'
GRADLE_TASK_SWISS_ARMY_KNIFE = ':swissarmyknife:jar'
GRADLE_TASK_KEEPANNOTOOLS = ':dist:keepAnnoToolsWithRelocatedDeps'
GRADLE_TASK_KEEPANNOTOOLSLIB = 'test:keepAnnoToolsLib'
GRADLE_TASK_TEST = ':test:test'
GRADLE_TASK_ALL_TESTS_WITH_APPLY_MAPPING_JAR = ':test:rewriteTestsForR8LibWithRelocatedDeps'
GRADLE_TASK_TESTBASE_WITH_APPLY_MAPPING_JAR = ':test:rewriteTestBaseForR8LibWithRelocatedDeps'
GRADLE_TASK_TEST_DEPS_JAR = ':test:packageTestDeps'
GRADLE_TASK_TEST_BASE_JAR = ':test:relocateTestBaseForR8LibWithRelocatedDeps'
GRADLE_TASK_TEST_JAR = ':test:relocateTestsForR8LibWithRelocatedDeps'
GRADLE_TASK_TEST_UNZIP_TESTBASE = ':test:unzipTestBase'
GRADLE_TASK_PREPARE_TEST_ARTIFACTS = ':test:prepareTestArtifactsForR8LibWithRelocatedDeps'
GRADLE_TASK_PREPARE_TEST_ARTIFACTS_NO_DEPS = ':test:prepareTestArtifactsForR8LibNoDeps'

R8 = 'r8'
R8LIB = 'r8lib'

ALL_DEPS_JAR = os.path.join(LIBS, 'deps_all.jar')
PROCESS_KEEP_RULES_LIB_JAR = os.path.join(LIBS, 'processkeepruleslib.jar')
R8_JAR = os.path.join(LIBS, 'r8.jar')
TOOLS_JAR = os.path.join(REPO_ROOT, 'd8_r8', 'tools', 'build', 'libs',
                         'tools.jar')
R8LIB_JAR = os.path.join(LIBS, 'r8lib.jar')
R8LIB_MAP = '%s.map' % R8LIB_JAR
R8LIB_PARTITION_MAP = '%s_map.zip' % R8LIB_JAR
R8_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')
R8LIB_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8lib-exclude-deps.jar')
R8_FULL_EXCLUDE_DEPS_JAR = os.path.join(LIBS, 'r8-full-exclude-deps.jar')
THREADING_MODULE_BLOCKING_JAR = os.path.join(LIBS,
                                             'threading-module-blocking.jar')
THREADING_MODULE_SINGLE_THREADED_JAR = os.path.join(
    LIBS, 'threading-module-single-threaded.jar')
KEEPANNOTOOLS_JAR = os.path.join(LIBS, 'keepanno-tools.jar')
KEEPANNOTOOLSLIB_JAR = os.path.join(LIBS, 'keepanno-toolslib.jar')
KEEPANNOTOOLSLIB_MAP = '%s.map' % R8LIB_JAR
KEEPANNOTOOLS_SRC_JAR = os.path.join(LIBS, 'r8-src.jar')

R8_TESTS_JAR = os.path.join(LIBS, 'r8tests.jar')
R8_TESTBASE_JAR = os.path.join(LIBS, 'r8test_base.jar')
R8LIB_TESTBASE_JAR = os.path.join(LIBS, 'r8libtestbase-cf.jar')
R8LIB_TESTS_JAR = os.path.join(LIBS, 'r8libtestdeps-cf.jar')
R8_TESTS_DEPS_JAR = os.path.join(LIBS, 'test_deps_all.jar')
R8LIB_TESTS_DEPS_JAR = R8_TESTS_DEPS_JAR
MAVEN_ZIP_LIB = os.path.join(LIBS, 'r8lib.zip')
KEEPANNNO_TOOLS_MAVEN_ZIP_LIB = os.path.join(LIBS, 'keepanno-toolslib.zip')
LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP = os.path.join(
    CUSTOM_CONVERSION_DIR, 'library_desugar_conversions_legacy.jar')
LIBRARY_DESUGAR_CONVERSIONS_ZIP = os.path.join(
    CUSTOM_CONVERSION_DIR, 'library_desugar_conversions.jar')
KEEPANNO_ANNOTATIONS_JAR = os.path.join(LIBS, 'keepanno-annotations.jar')
KEEPANNO_ANNOTATIONS_DOC = os.path.join('d8_r8', 'keepanno', 'build', 'docs',
                                        'javadoc')
KEEPANNO_LEGACY_ANNOTATIONS_JAR = os.path.join(
    LIBS, 'keepanno-annotations-legacy.jar')
KEEPANNO_ANDROIDX_ANNOTATIONS_JAR = os.path.join(
    LIBS, 'keepanno-annotations-androidx.jar')

DESUGAR_CONFIGURATION = os.path.join('src', 'library_desugar',
                                     'desugar_jdk_libs.json')
DESUGAR_IMPLEMENTATION = os.path.join('third_party', 'openjdk',
                                      'desugar_jdk_libs',
                                      'desugar_jdk_libs.jar')
DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join(
    'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
DESUGAR_CONFIGURATION_JDK11_MINIMAL = os.path.join(
    'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json')
DESUGAR_CONFIGURATION_JDK11 = os.path.join('src', 'library_desugar', 'jdk11',
                                           'desugar_jdk_libs.json')
DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join('src', 'library_desugar',
                                               'jdk11',
                                               'desugar_jdk_libs_nio.json')
DESUGAR_IMPLEMENTATION_JDK11 = os.path.join('third_party', 'openjdk',
                                            'desugar_jdk_libs_11',
                                            'desugar_jdk_libs.jar')
DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
    LIBS, 'desugar_jdk_libs_configuration.zip')
DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP = os.path.join(
    LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip')
DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP = os.path.join(
    LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip')
DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP = os.path.join(
    LIBS, 'desugar_jdk_libs_configuration_jdk11.zip')
DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP = os.path.join(
    LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip')
GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
CF_SEGMENTS_TOOL = os.path.join(THIRD_PARTY, 'cf_segments')
PINNED_R8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8.jar')
PINNED_PGR8_JAR = os.path.join(REPO_ROOT, 'third_party/r8/r8-pg6.0.1.jar')

OPENSOURCE_DUMPS_DIR = os.path.join(THIRD_PARTY, 'opensource-apps')
INTERNAL_DUMPS_DIR = os.path.join(THIRD_PARTY, 'internal-apps')
BAZEL_DIR = os.path.join(THIRD_PARTY, 'bazel')
JAVA8_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86')
JAVA11_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11', 'linux')
JAVA17_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-17', 'linux')
JAVA21_DIR = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-21', 'linux')
DESUGAR_JDK_LIBS_11_DIR = os.path.join(THIRD_PARTY, 'openjdk',
                                       'desugar_jdk_libs_11')
IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test',
                                     'ignorewarnings.rules')
ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME"
ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION"
USER_HOME = os.path.expanduser('~')

R8_TEST_RESULTS_BUCKET = 'r8-test-results'
R8_INTERNAL_TEST_RESULTS_BUCKET = 'r8-internal-test-results'

ARCHIVE_DETAILS = os.path.join(BUILD, 'archive_details.json')


def archive_file(name, gs_dir, src_file):
    gs_file = '%s/%s' % (gs_dir, name)
    upload_file_to_cloud_storage(src_file, gs_file)


def archive_value(name, gs_dir, value):
    with TempDir() as temp:
        temparchive = os.path.join(temp, name)
        with open(temparchive, 'w') as f:
            f.write(str(value))
        archive_file(name, gs_dir, temparchive)


def find_cloud_storage_file_from_options(name, options, orElse=None):
    # Import archive on-demand since archive depends on utils.
    from archive import get_upload_destination
    hash_or_version = find_hash_or_version_from_options(options)
    if not hash_or_version:
        return orElse
    is_hash = options.commit_hash is not None
    download_path = get_upload_destination(hash_or_version, name, is_hash)
    if file_exists_on_cloud_storage(download_path):
        out = tempfile.NamedTemporaryFile().name
        download_file_from_cloud_storage(download_path, out)
        return out
    else:
        raise Exception('Could not find file {} from hash/version: {}.'.format(
            name, hash_or_version))


def find_r8_jar_from_options(options):
    return find_cloud_storage_file_from_options('r8.jar', options)


def find_hash_or_version_from_options(options):
    if options.tag:
        return find_hash_or_version_from_tag(options.tag)
    else:
        return options.commit_hash or options.version


def find_hash_or_version_from_tag(tag_or_hash):
    info = subprocess.check_output(
        ['git', 'show', tag_or_hash, '-s',
         '--format=oneline']).decode('utf-8').splitlines()[-1].split()
    # The info should be on the following form [hash,"Version",version]
    if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version":
        return info[2]
    return None


def getAndroidHome():
    return os.environ.get(ANDROID_HOME_ENVIROMENT_NAME,
                          os.path.join(USER_HOME, 'Android', 'Sdk'))


def getAndroidBuildTools():
    if ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME in os.environ:
        version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME)
        build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
        assert os.path.exists(build_tools_dir)
        return build_tools_dir
    else:
        versions = ['35.0.1', '34.0.0', '33.0.1', '32.0.0']
        for version in versions:
            build_tools_dir = os.path.join(getAndroidHome(), 'build-tools',
                                           version)
            if os.path.exists(build_tools_dir):
                return build_tools_dir
    raise Exception('Unable to find Android build-tools')


def is_python3():
    return sys.version_info.major == 3


def Print(s, quiet=False):
    if quiet:
        return
    print(s)


def Warn(message):
    CRED = '\033[91m'
    CEND = '\033[0m'
    print(CRED + message + CEND)


def PrintCmd(cmd, env=None, quiet=False, worker_id=None):
    if quiet:
        return
    if type(cmd) is list:
        cmd = ' '.join(cmd)
    if env:
        env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
        print_thread('Running: {} {}'.format(env, cmd), worker_id)
    else:
        print_thread('Running: {}'.format(cmd), worker_id)
    # I know this will hit os on windows eventually if we don't do this.
    sys.stdout.flush()


class ProgressLogger(object):
    CLEAR_LINE = '\033[K'
    UP = '\033[F'

    def __init__(self, quiet=False):
        self._count = 0
        self._has_printed = False
        self._quiet = quiet

    def log(self, text):
        if len(text.strip()) == 0:
            return
        if self._quiet:
            if self._has_printed:
                sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
            if len(text) > 140:
                text = text[0:140] + '...'
        print(text)
        self._has_printed = True

    def done(self):
        if self._quiet and self._has_printed:
            sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
            print('')
            sys.stdout.write(ProgressLogger.UP)


def RunCmd(cmd,
           env_vars=None,
           quiet=False,
           fail=True,
           logging=True,
           shell=False):
    PrintCmd(cmd, env=env_vars, quiet=quiet)
    env = os.environ.copy()
    if env_vars:
        env.update(env_vars)
    process = subprocess.Popen(cmd,
                               env=env,
                               shell=shell,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT)
    stdout = []
    logger = ProgressLogger(quiet=quiet) if logging else None
    failed = False
    while True:
        line = process.stdout.readline().decode('utf-8')
        if line != '':
            stripped = line.rstrip()
            stdout.append(stripped)
            if logger:
                logger.log(stripped)
            # TODO(christofferqa): r8 should fail with non-zero exit code.
            if ('AssertionError:' in stripped or
                    'CompilationError:' in stripped or
                    'CompilationFailedException:' in stripped or
                    'Compilation failed' in stripped or
                    'FAILURE:' in stripped or
                    'org.gradle.api.ProjectConfigurationException' in stripped
                    or 'BUILD FAILED' in stripped):
                failed = True
        else:
            if logger:
                logger.done()
            exit_code = process.poll()
            if exit_code or failed:
                for line in stdout:
                    Warn(line)
                if fail:
                    raise subprocess.CalledProcessError(
                        exit_code or -1, cmd, output='\n'.join(stdout))
            return stdout


def IsWindows():
    return defines.IsWindows()


GRADLE_STOPPED = False


def stop_gradle():
    global GRADLE_STOPPED
    if GRADLE_STOPPED:
        return

    try:
        java_home = jdk.GetDefaultJdkHome()
    except Exception as _:
        # No Gradle daemon is running if the JDK home can't be found.
        return

    gradle_dir = os.path.join(THIRD_PARTY, 'gradle')
    if IsWindows():
        gradle = os.path.join(gradle_dir, 'bin', 'gradle.bat')
    else:
        gradle = os.path.join(gradle_dir, 'bin', 'gradle')

    if not os.path.exists(gradle):
        # No Gradle daemon is running if the Gradle executable can't be found.
        return

    try:
        env = os.environ.copy()
        env['JAVA_HOME'] = java_home
        env['PATH'] = env['PATH'] + os.pathsep + os.path.join(java_home, 'bin')
        env['GRADLE_OPTS'] = '-Xmx1g'

        cmd = [gradle, '--stop']
        PrintCmd(cmd)
        subprocess.call(cmd, env=env)
        GRADLE_STOPPED = True
    except Exception as e:
        Warn(f"Warning: Failed to stop gradle daemon: {e}")
        GRADLE_STOPPED = True


def should_download(output_dir, tgz_file, sha1, success_file):
    if (os.path.exists(output_dir) and os.path.exists(tgz_file) and
            os.path.exists(success_file) and
            os.path.getmtime(sha1) <= os.path.getmtime(success_file)):
        return False
    return True


def run_download(sha1, output_dir, success_file, internal, quiet=False):
    if os.path.exists(success_file):
        os.remove(success_file)

    if os.path.exists(output_dir):
        stop_gradle()
        shutil.rmtree(output_dir)

    if internal:
        download_script = os.path.join(REPO_ROOT, 'tools',
                                       'download_from_x20.py')
        cmd = [download_script, sha1]
    else:
        suffix = '.bat' if IsWindows() else ''
        script = 'download_from_google_storage%s' % suffix
        cmd = [script]
        cmd.extend(['-b', 'r8-deps', '-u', '-s', sha1])

    if not quiet:
        PrintCmd(cmd)
        subprocess.check_call(cmd)
    else:
        subprocess.check_call(cmd, stdout=subprocess.DEVNULL)

    with open(success_file, 'w') as _:
        pass


def get_main_repo_root(worktree_root=None):
    wt_root = worktree_root or REPO_ROOT
    git_path = os.path.join(wt_root, '.git')
    if not os.path.isfile(git_path):
        return None
    try:
        with open(git_path, 'r') as f:
            line = f.readline().strip()
        if not line.startswith('gitdir:'):
            return None
        gitdir = line[len('gitdir:'):].strip()
        if not os.path.isabs(gitdir):
            gitdir = os.path.abspath(os.path.join(wt_root, gitdir))
        commondir_file = os.path.join(gitdir, 'commondir')
        if os.path.isfile(commondir_file):
            with open(commondir_file, 'r') as f:
                commondir = f.readline().strip()
            common_git_dir = os.path.abspath(os.path.join(gitdir, commondir))
        else:
            common_git_dir = os.path.dirname(os.path.dirname(gitdir))
        main_repo = os.path.dirname(common_git_dir)
        if (os.path.isdir(main_repo) and
                os.path.abspath(main_repo) != os.path.abspath(wt_root)):
            return main_repo
    except Exception:
        pass
    return None


def link_or_copy(src, dst):
    if os.path.lexists(dst):
        try:
            if os.path.samefile(src, dst):
                return
        except OSError:
            pass
        os.remove(dst)
    try:
        os.link(src, dst)
    except OSError:
        shutil.copy2(src, dst)


def link_tree(src, dst, ignore=None):
    shutil.copytree(src,
                    dst,
                    symlinks=True,
                    copy_function=link_or_copy,
                    ignore=ignore,
                    dirs_exist_ok=True)


def try_link_download_from_main_repo(path, worktree_root=None):
    wt_root = worktree_root or REPO_ROOT
    output_dir = path
    tar_gz_file = path + '.tar.gz'
    sha1_file = tar_gz_file + '.sha1'
    success_file = path + '.success'
    if not should_download(output_dir, tar_gz_file, sha1_file, success_file):
        return False

    main_repo = get_main_repo_root(wt_root)
    if not main_repo:
        return False
    try:
        rel_path = os.path.relpath(path, wt_root)
    except ValueError:
        return False
    if rel_path.startswith('..'):
        return False

    main_output_dir = os.path.join(main_repo, rel_path)
    main_tar_gz_file = main_output_dir + '.tar.gz'
    main_sha1_file = main_tar_gz_file + '.sha1'
    main_success_file = main_output_dir + '.success'

    if not os.path.exists(main_sha1_file) or not os.path.exists(sha1_file):
        return False
    if should_download(main_output_dir, main_tar_gz_file, main_sha1_file,
                       main_success_file):
        return False
    if get_sha1(sha1_file) != get_sha1(main_sha1_file):
        return False

    try:
        if os.path.exists(success_file):
            os.remove(success_file)
        if os.path.exists(output_dir):
            shutil.rmtree(output_dir)
        if os.path.exists(tar_gz_file):
            os.remove(tar_gz_file)
        link_tree(main_output_dir, output_dir)
        link_or_copy(main_tar_gz_file, tar_gz_file)
        with open(success_file, 'w') as _:
            pass
        return True
    except Exception as e:
        Warn(
            f'Warning: Failed to link dependency from main repo {main_output_dir}: {e}'
        )
        return False


def ensure_worktree_deps(quiet=False, worktree_root=None):
    wt_root = worktree_root or REPO_ROOT
    main_repo = get_main_repo_root(wt_root)
    if not main_repo:
        return

    linked_deps = 0
    try:
        sha1_files = subprocess.check_output(
            ['git', 'ls-files', '*.tar.gz.sha1'], cwd=wt_root,
            text=True).splitlines()
        for rel_sha1 in sha1_files:
            path = os.path.join(wt_root, rel_sha1[:-len('.tar.gz.sha1')])
            if try_link_download_from_main_repo(path, worktree_root=wt_root):
                linked_deps += 1
    except Exception as e:
        Warn(f'Warning: Failed to scan worktree dependencies: {e}')

    if linked_deps > 0 and not quiet:
        print(
            f'Linked {linked_deps} pre-downloaded dependencies from {main_repo}'
        )


def seed_gradle_user_home(gradle_user_home, worktree_root=None):
    seeded_marker = os.path.join(gradle_user_home, '.worktree_seeded')
    if os.path.exists(seeded_marker):
        return

    wt_root = worktree_root or REPO_ROOT
    main_repo = get_main_repo_root(wt_root)
    candidates = [os.path.expanduser(os.path.join('~', '.gradle', 'caches'))]
    if main_repo:
        candidates.append(os.path.join(main_repo, '.gradle_user_home',
                                       'caches'))

    dst_caches = os.path.join(gradle_user_home, 'caches')
    dst_caches_abs = os.path.abspath(dst_caches)

    def ignore_locks_and_mutable(dir_path, names):
        return [
            n for n in names
            if n.endswith('.lock') or n in ('journal-1', 'fileHashes',
                                            'fileContent', 'gc.properties',
                                            'cc-keystore', 'javaCompile',
                                            'build-cache-1')
        ]

    seeded_any = False
    for src_caches in candidates:
        if os.path.isdir(src_caches) and os.path.abspath(
                src_caches) != dst_caches_abs:
            os.makedirs(dst_caches, exist_ok=True)
            try:
                link_tree(src_caches,
                          dst_caches,
                          ignore=ignore_locks_and_mutable)
                seeded_any = True
            except Exception as e:
                Warn(
                    f'Warning: Failed to seed worktree Gradle caches from {src_caches}: {e}'
                )

    if seeded_any:
        try:
            os.makedirs(gradle_user_home, exist_ok=True)
            with open(seeded_marker, 'w') as _:
                pass
        except Exception:
            pass


def ensure_download(path, internal, quiet=False):
    output_dir = path
    tar_gz_file = path + '.tar.gz'
    sha1_file = tar_gz_file + '.sha1'
    success_file = path + '.success'
    if not os.path.exists(sha1_file):
        raise Exception(f"Missing sha1 file: {sha1_file}")

    if should_download(output_dir, tar_gz_file, sha1_file, success_file):
        if try_link_download_from_main_repo(path):
            return
        run_download(sha1_file, output_dir, success_file, internal, quiet)


def ensure_google_download(path, quiet=False):
    ensure_download(path, False, quiet=quiet)


def ensure_x20_download(path, quiet=False):
    ensure_download(path, True, quiet=quiet)


def get_nth_sha1_from_revision(n, revision):
    result = subprocess.check_output([
        'git', 'log', revision, f'--skip={n}', '--max-count=1',
        '--pretty=format:%H'
    ]).decode('utf-8')
    return result.strip()


def get_sha1(filename):
    sha1 = hashlib.sha1()
    with open(filename, 'rb') as f:
        while True:
            chunk = f.read(1024 * 1024)
            if not chunk:
                break
            sha1.update(chunk)
    return sha1.hexdigest()


def get_sha1_from_revision(revision):
    cmd = ['git', 'rev-parse', revision]
    PrintCmd(cmd)
    with ChangedWorkingDirectory(REPO_ROOT):
        return subprocess.check_output(cmd).decode('utf-8').strip()


def get_HEAD_branch():
    result = subprocess.check_output(
        ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8')
    return result.strip()


def get_HEAD_commit():
    return historic_run.git_commit_from_hash(get_HEAD_sha1())


def get_HEAD_sha1():
    return get_sha1_from_revision('HEAD')


def get_HEAD_diff_stat():
    return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8')


def makedirs_if_needed(path):
    try:
        os.makedirs(path)
    except OSError:
        if not os.path.isdir(path):
            raise


def get_gsutil():
    return 'gsutil.py' if os.name != 'nt' else 'gsutil.py.bat'


def upload_file_to_cloud_storage(source, destination, header=None):
    cmd = [get_gsutil()]
    if header:
        cmd.extend(['-h', header])
    cmd.extend(['cp', source, destination])
    PrintCmd(cmd)
    subprocess.check_call(cmd)


def check_dir_args(source, destination):
    # We require that the dirname of the paths coincide, e.g., src/dirname and dst/dirname
    # The target is then stripped so the upload command will be: cp -R src/dirname dst/
    (destination_parent, destination_file) = os.path.split(destination)
    if os.path.basename(source) != destination_file:
        raise Exception(
            'Attempt to upload directory with non-matching directory name: ' +
            f'{source} and {destination}')
    if len(destination_parent.strip()) == 0:
        raise Exception(
            'Attempt to upload directory to empty destination directory: ' +
            destination)
    return destination_parent


def upload_directory_to_cloud_storage(source, destination, parallel=True):
    destination_parent = check_dir_args(source, destination)
    cmd = [get_gsutil()]
    if parallel:
        cmd += ['-m']
    cmd += ['cp', '-R']
    cmd += [source, destination_parent + '/']
    PrintCmd(cmd)
    subprocess.check_call(cmd)


def rsync_directory_to_cloud_storage(source, destination, parallel=True):
    check_dir_args(source, destination)
    cmd = [get_gsutil()]
    if parallel:
        cmd += ['-m']
    cmd += ['rsync', '-R']
    cmd += [source, destination]
    PrintCmd(cmd)
    subprocess.check_call(cmd)


def delete_file_from_cloud_storage(destination):
    cmd = [get_gsutil(), 'rm', destination]
    PrintCmd(cmd)
    subprocess.check_call(cmd)


def ls_files_on_cloud_storage(destination):
    cmd = [get_gsutil(), 'ls', destination]
    PrintCmd(cmd)
    return subprocess.check_output(cmd).decode('utf-8')


def cat_file_on_cloud_storage(destination, ignore_errors=False):
    cmd = [get_gsutil(), 'cat', destination]
    PrintCmd(cmd)
    try:
        return subprocess.check_output(cmd).decode('utf-8').strip()
    except subprocess.CalledProcessError as e:
        if ignore_errors:
            return ''
        else:
            raise e


def file_exists_on_cloud_storage(destination):
    cmd = [get_gsutil(), 'ls', destination]
    PrintCmd(cmd)
    return subprocess.call(cmd) == 0


def download_file_from_cloud_storage(source,
                                     destination,
                                     concurrent=False,
                                     flags=None,
                                     printcmd=True,
                                     quiet=False):
    cmd = [get_gsutil()]
    if concurrent:
        cmd.append('-m')
    if quiet:
        cmd.append('-q')
    cmd.append('cp')
    if flags:
        cmd.extend(flags)
    cmd.extend([source, destination])
    PrintCmd(cmd, quiet=not printcmd)
    subprocess.check_call(cmd)


def create_archive(name, sources=None):
    if not sources:
        sources = [name]
    tarname = '%s.tar.gz' % name
    with tarfile.open(tarname, 'w:gz') as tar:
        for source in sources:
            tar.add(source)
    return tarname


def extract_dir(filename):
    return filename[0:len(filename) - len('.tar.gz')]


def unpack_archive(filename):
    dest_dir = extract_dir(filename)
    if os.path.exists(dest_dir):
        print('Deleting existing dir %s' % dest_dir)
        stop_gradle()
        shutil.rmtree(dest_dir)
    dirname = os.path.dirname(os.path.abspath(filename))
    with tarfile.open(filename, 'r:gz') as tar:
        tar.extractall(path=dirname)


def check_gcert():
    status = subprocess.call(['gcertstatus'])
    if status != 0:
        subprocess.check_call(['gcert'])


# Note that gcs is eventually consistent with regards to list operations.
# This is not a problem in our case, but don't ever use this method
# for synchronization.
def cloud_storage_exists(destination):
    cmd = [get_gsutil(), 'ls', destination]
    PrintCmd(cmd)
    exit_code = subprocess.call(cmd)
    return exit_code == 0


class TempDir(object):

    def __init__(self, prefix='r8-tmp-', delete=True):
        self._temp_dir = None
        self._prefix = prefix
        self._delete = delete

    def __enter__(self):
        self._temp_dir = tempfile.mkdtemp(self._prefix)
        return self._temp_dir

    def __exit__(self, *_):
        if self._delete:
            shutil.rmtree(self._temp_dir, ignore_errors=True)


class ChangedWorkingDirectory(object):

    def __init__(self, working_directory, quiet=False):
        self._quiet = quiet
        self._working_directory = working_directory

    def __enter__(self):
        self._old_cwd = os.getcwd()
        if not self._quiet:
            print('Enter directory:', self._working_directory)
        os.chdir(self._working_directory)

    def __exit__(self, *_):
        if not self._quiet:
            print('Enter directory:', self._old_cwd)
        os.chdir(self._old_cwd)


# Reading Android CTS test_result.xml


class CtsModule(object):

    def __init__(self, module_name):
        self.name = module_name


class CtsTestCase(object):

    def __init__(self, test_case_name):
        self.name = test_case_name


class CtsTest(object):

    def __init__(self, test_name, outcome):
        self.name = test_name
        self.outcome = outcome


# Generator yielding CtsModule, CtsTestCase or CtsTest from
# reading through a CTS test_result.xml file.
def read_cts_test_result(file_xml):
    re_module = re.compile('<Module name="([^"]*)"')
    re_test_case = re.compile('<TestCase name="([^"]*)"')
    re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
    with open(file_xml) as f:
        for line in f:
            m = re_module.search(line)
            if m:
                yield CtsModule(m.groups()[0])
                continue
            m = re_test_case.search(line)
            if m:
                yield CtsTestCase(m.groups()[0])
                continue
            m = re_test.search(line)
            if m:
                outcome = m.groups()[0]
                assert outcome in ['fail', 'pass']
                yield CtsTest(m.groups()[1], outcome == 'pass')


def grep_memoryuse(logfile):
    re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
    result = None
    with open(logfile) as f:
        for line in f:
            m = re_vmhwm.search(line)
            if m:
                groups = m.groups()
                s = len(groups)
                if s >= 1:
                    result = int(groups[0])
                    if s >= 2:
                        unit = groups[1]
                        if unit == 'kB':
                            result *= 1024
                        elif unit != '':
                            raise Exception(
                                'Unrecognized unit in memory usage log: {}'.
                                format(unit))
    if result is None:
        raise Exception('No memory usage found in log: {}'.format(logfile))
    return result


# Return a dictionary: {segment_name -> segments_size}
def getDexSegmentSizes(dex_files):
    assert len(dex_files) > 0
    cmd = [
        jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-jar', R8_JAR,
        'dexsegments'
    ]
    cmd.extend(dex_files)
    PrintCmd(cmd)
    output = subprocess.check_output(cmd).decode('utf-8')

    matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)

    if matches is None or len(matches) == 0:
        raise Exception('DexSegments failed to return any output for' \
            ' these files: {}'.format(dex_files))

    result = {}

    for match in matches:
        result[match[0]] = int(match[1])

    return result


# Return a dictionary: {segment_name -> segments_size}
def getCfSegmentSizes(cfFile):
    cmd = [
        jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-cp', CF_SEGMENTS_TOOL,
        'com.android.tools.r8.cf_segments.MeasureLib', cfFile
    ]
    PrintCmd(cmd)
    output = subprocess.check_output(cmd).decode('utf-8')

    matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)

    if matches is None or len(matches) == 0:
        raise Exception('CfSegments failed to return any output for' \
                        ' the file: ' + cfFile)

    result = {}

    for match in matches:
        result[match[0]] = int(match[1])

    return result


def get_maven_path(artifact, version):
    return os.path.join('com', 'android', 'tools', artifact, version)


def print_cfsegments(prefix, cf_files):
    for cf_file in cf_files:
        for segment_name, size in getCfSegmentSizes(cf_file).items():
            print('{}-{}(CodeSize): {}'.format(prefix, segment_name, size))


def print_dexsegments(prefix, dex_files, worker_id=None):
    for segment_name, size in getDexSegmentSizes(dex_files).items():
        print_thread('{}-{}(CodeSize): {}'.format(prefix, segment_name, size),
                     worker_id)


# Ensure that we are not benchmarking with a google jvm.
def check_java_version():
    cmd = [jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-version']
    output = subprocess.check_output(cmd,
                                     stderr=subprocess.STDOUT).decode('utf-8')
    m = re.search('openjdk version "([^"]*)"', output)
    if m is None:
        raise Exception("Can't check java version: no version string in output"
                        " of 'java -version': '{}'".format(output))
    version = m.groups(0)[0]
    m = re.search('google', version)
    if m is not None:
        raise Exception("Do not use google JVM for benchmarking: " + version)


def api_str(api_level_major, api_level_minor):
    api = str(api_level_major)
    if api_level_minor > 0:
        api = api + '.' + str(api_level_minor)
    return api


def get_android_jar_dir(api_level_major, api_level_minor):
    return os.path.join(
        REPO_ROOT,
        ANDROID_JAR_DIR.format(api=api_str(api_level_major, api_level_minor)))


def get_android_jar(api_level_major, api_level_minor):
    return os.path.join(
        REPO_ROOT,
        ANDROID_JAR.format(api=api_str(api_level_major, api_level_minor)))


def is_bot():
    return 'SWARMING_BOT_ID' in os.environ


def uncompressed_size(path):
    return sum(z.file_size for z in zipfile.ZipFile(path).infolist())


def desugar_configuration_name_and_version(configuration, is_for_maven):
    name = 'desugar_jdk_libs_configuration'
    with open(configuration, 'r') as f:
        configuration_json = json.loads(f.read())
        configuration_format_version = \
            configuration_json.get('configuration_format_version')
        if (not configuration_format_version):
            raise Exception('No "configuration_format_version" found in ' +
                            configuration)
        if (configuration_format_version != 3 and
                configuration_format_version != 5 and
                configuration_format_version != (200 if is_for_maven else 101)):
            raise Exception(
                'Unsupported "configuration_format_version" "%s" found in %s' %
                (configuration_format_version, configuration))
        version = configuration_json.get('version')
        if not version:
            if configuration_format_version == (200 if is_for_maven else 101):
                identifier = configuration_json.get('identifier')
                if not identifier:
                    raise Exception('No "identifier" found in ' + configuration)
                identifier_split = identifier.split(':')
                if (len(identifier_split) != 3):
                    raise Exception('Invalid "identifier" found in ' +
                                    configuration)
                if (identifier_split[0] != 'com.tools.android'):
                    raise Exception('Invalid "identifier" found in ' +
                                    configuration)
                if not identifier_split[1].startswith(
                        'desugar_jdk_libs_configuration'):
                    raise Exception('Invalid "identifier" found in ' +
                                    configuration)
                name = identifier_split[1]
                version = identifier_split[2]
            else:
                raise Exception('No "version" found in ' + configuration)
        else:
            if configuration_format_version == (200 if is_for_maven else 101):
                raise Exception('No "version" expected in ' + configuration)
        # Disallow prerelease, as older R8 versions cannot parse it causing hard to
        # understand errors.
        check_basic_semver_version(version,
                                   'in ' + configuration,
                                   allowPrerelease=False)
        return (name, version)


class SemanticVersion:

    def __init__(self, major, minor, patch, prerelease):
        self.major = major
        self.minor = minor
        self.patch = patch
        self.prerelease = prerelease
        # Build metadata currently not suppported

    def larger_than(self, other):
        if self.major > other.major:
            return True
        if self.major == other.major and self.minor > other.minor:
            return True
        if self.patch:
            return (self.major == other.major and self.minor == other.minor and
                    self.patch > other.patch)
        if self.prerelease:
            if other.prerelease:
                return self.prerelease > other.prerelease
            else:
                return False

        else:
            return False

    def equals_to(self, other):
        return (self.major == other.major and self.minor == other.minor and
                self.patch == other.patch and
                self.prerelease == other.prerelease)

    def __str__(self):
        return str(self.major) + '.' + str(self.minor) + '.' + str(
            self.patch) + (('-' + self.prerelease) if self.prerelease else '')


# Check that the passed string is formatted as a basic semver version (x.y.z or x.y.z-prerelease
# depending on the value of allowPrerelease).
# See https://semver.org/. The regexp parts used are not all complient with what is suggested
# on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string.
def check_basic_semver_version(version,
                               error_context='',
                               components=3,
                               allowPrerelease=False):
    regexp = '^'
    for x in range(components):
        regexp += '([0-9]+)'
        if x < components - 1:
            regexp += '\\.'
    if allowPrerelease:
        # This part is from
        # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
        regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?'
    regexp += '$'
    reg = re.compile(regexp)
    match = reg.match(version)
    if not match:
        raise Exception("Invalid version '" + version + "'" + (
            (' ' + error_context) if len(error_context) > 0 else ''))
    if components == 2:
        return SemanticVersion(int(match.group(1)), int(match.group(2)), None,
                               None)
    elif components == 3 and not allowPrerelease:
        return SemanticVersion(int(match.group(1)), int(match.group(2)),
                               int(match.group(3)), None)
    elif components == 3 and allowPrerelease:
        return SemanticVersion(int(match.group(1)), int(match.group(2)),
                               int(match.group(3)), match.group('prerelease'))
    else:
        raise Exception('Argument "components" must be 2 or 3')


def is_git_worktree(checkout_dir):
    """
    Checks if the given path is the main Git worktree or a linked worktree.
    """
    git_path = os.path.join(checkout_dir, ".git")
    if not os.path.exists(git_path):
        print("Checkout path does not look like a Git repository.")
        return False
    # In the main worktree, .git is a directory containing the repository data and
    # in a linked worktree, .git is a plain text file with a gitdir reference.
    if os.path.isdir(git_path):
        return False
    elif os.path.isfile(git_path):
        return True
    else:
        print("Checkout path does not look like a Git repository.")
        return False


def append_gradle_user_home_for_worktree(force_worktree, with_no_daemon, args):
    if force_worktree or is_git_worktree(REPO_ROOT):
        if not force_worktree:
            print(
                'git worktree detected, using worktree local Gradle User Home')
        gradle_user_home = os.path.join(REPO_ROOT, ".gradle_user_home")
        seed_gradle_user_home(gradle_user_home)
        args.append('--gradle-user-home=' + gradle_user_home)
        if with_no_daemon:
            args.append('--no-daemon')


def compute_dex_size_in_zip(zip_path):
    if not os.path.isfile(zip_path):
        return 0
    with zipfile.ZipFile(zip_path, 'r') as z:
        return sum(info.file_size
                   for info in z.infolist()
                   if info.filename.endswith('.dex'))