tools/archive_desugar_jdk_libs.py

#!/usr/bin/env python3
# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# This script is designed to run on a buildbot to build from the source
# of https://github.com/google/desugar_jdk_libs and publish to the
# r8-release Cloud Storage Bucket.
#
# These files are uploaded:
#
#   raw/desugar_jdk_libs/<VERSION>/desugar_jdk_libs.jar
#   raw/desugar_jdk_libs/<VERSION>/desugar_jdk_libs.zip
#   raw/com/android/tools/desugar_jdk_libs/<VERSION>/desugar_jdk_libs-<VERSION>.jar
#
# The first two are the raw jar file and the maven compatible zip file. The
# third is the raw jar file placed and named so that the URL
# https://storage.googleapis.com/r8-releases/raw can be treated as a maven
# repository to fetch the artifact com.android.tools:desugar_jdk_libs:1.0.0

import archive
import defines
import git_utils
import gradle
import hashlib
import jdk
import optparse
import os
import re
import shutil
import subprocess
import sys
import utils
import zipfile

VERSION_FILE_JDK8 = 'VERSION.txt'
VERSION_FILE_JDK11_LEGACY = 'VERSION_JDK11_LEGACY.txt'
VERSION_FILE_JDK11_MINIMAL = 'VERSION_JDK11_MINIMAL.txt'
VERSION_FILE_JDK11 = 'VERSION_JDK11.txt'
VERSION_FILE_JDK11_NIO = 'VERSION_JDK11_NIO.txt'

VERSION_MAP = {
    'jdk8': VERSION_FILE_JDK8,
    'jdk11_legacy': VERSION_FILE_JDK11_LEGACY,
    'jdk11_minimal': VERSION_FILE_JDK11_MINIMAL,
    'jdk11': VERSION_FILE_JDK11,
    'jdk11_nio': VERSION_FILE_JDK11_NIO
}

GITHUB_REPRO = 'desugar_jdk_libs'

BASE_LIBRARY_NAME = 'desugar_jdk_libs'

LIBRARY_NAME_MAP = {
    'jdk8': BASE_LIBRARY_NAME,
    'jdk11_legacy': BASE_LIBRARY_NAME,
    'jdk11_minimal': BASE_LIBRARY_NAME + '_minimal',
    'jdk11': BASE_LIBRARY_NAME,
    'jdk11_nio': BASE_LIBRARY_NAME + '_nio'
}

MAVEN_RELEASE_TARGET_MAP = {
    'jdk8': 'maven_release',
    'jdk11_legacy': 'maven_release_jdk11_legacy',
    'jdk11_minimal': 'maven_release_jdk11_minimal',
    'jdk11': 'maven_release_jdk11',
    'jdk11_nio': 'maven_release_jdk11_nio'
}

MAVEN_RELEASE_ZIP = {
    'jdk8': BASE_LIBRARY_NAME + '.zip',
    'jdk11_legacy': BASE_LIBRARY_NAME + '_jdk11_legacy.zip',
    'jdk11_minimal': BASE_LIBRARY_NAME + '_jdk11_minimal.zip',
    'jdk11': BASE_LIBRARY_NAME + '_jdk11.zip',
    'jdk11_nio': BASE_LIBRARY_NAME + '_jdk11_nio.zip'
}

DESUGAR_JDK_LIBS_HASH_FILE = os.path.join(defines.THIRD_PARTY, 'openjdk',
                                          'desugar_jdk_libs_11',
                                          'desugar_jdk_libs_hash')


def ParseOptions(argv):
    result = optparse.OptionParser()
    result.add_option(
        '--variant',
        help="Variant(s) to build",
        metavar=('<variants(s)>'),
        choices=['jdk8', 'jdk11_legacy', 'jdk11_minimal', 'jdk11', 'jdk11_nio'],
        default=[],
        action='append')
    result.add_option('--dry-run',
                      '--dry_run',
                      help='Running on bot, use third_party dependency.',
                      default=False,
                      action='store_true')
    result.add_option('--dry-run-output',
                      '--dry_run_output',
                      help='Output directory for dry run.',
                      type="string",
                      action="store")
    result.add_option('--github-account',
                      '--github_account',
                      help='GitHub account to clone from.',
                      default="google",
                      type="string",
                      action="store")
    result.add_option('--build_only',
                      '--build-only',
                      help='Build desugared library without archiving.',
                      type="string",
                      action="store")
    (options, args) = result.parse_args(argv)
    return (options, args)


def GetVersion(version_file_name):
    with open(version_file_name, 'r') as version_file:
        lines = [line.strip() for line in version_file.readlines()]
        lines = [line for line in lines if not line.startswith('#')]
        if len(lines) != 1:
            raise Exception('Version file ' + version_file +
                            ' is expected to have exactly one line')
        version = lines[0].strip()
        utils.check_basic_semver_version(version,
                                         'in version file ' + version_file_name,
                                         allowPrerelease=True)
        return version


def Upload(options, file_name, storage_path, destination, is_main):
    print('Uploading %s to %s' % (file_name, destination))
    if options.dry_run:
        if options.dry_run_output:
            dry_run_destination = \
                os.path.join(options.dry_run_output, os.path.basename(file_name))
            print('Dry run, not actually uploading. Copying to ' +
                  dry_run_destination)
            shutil.copyfile(file_name, dry_run_destination)
        else:
            print('Dry run, not actually uploading')
    else:
        utils.upload_file_to_cloud_storage(file_name, destination)
        print(
            'File available at: %s' %
            destination.replace('gs://', 'https://storage.googleapis.com/', 1))


def CloneDesugaredLibrary(github_account, checkout_dir, desugar_jdk_libs_hash):
    git_utils.GitClone(
        'https://github.com/' + github_account + '/' + GITHUB_REPRO,
        checkout_dir)
    git_utils.GitCheckout(desugar_jdk_libs_hash, checkout_dir)


def GetJavaEnv(androidHomeTemp):
    java_env = dict(os.environ, JAVA_HOME=jdk.GetJdk11Home())
    java_env['PATH'] = java_env['PATH'] + os.pathsep + os.path.join(
        jdk.GetJdk11Home(), 'bin')
    java_env['GRADLE_OPTS'] = '-Xmx1g'
    java_env['ANDROID_HOME'] = androidHomeTemp
    return java_env


def setUpFakeAndroidHome(androidHomeTemp):
    # Bazel will check if 30 is present then extract android.jar from 32.
    # We copy android.jar from third_party to mimic repository structure.
    subpath = os.path.join(androidHomeTemp, "build-tools")
    cmd = ["mkdir", subpath]
    subprocess.check_call(cmd)
    subpath = os.path.join(subpath, "32.0.0")
    cmd = ["mkdir", subpath]
    subprocess.check_call(cmd)
    subpath = os.path.join(androidHomeTemp, "platforms")
    cmd = ["mkdir", subpath]
    subprocess.check_call(cmd)
    subpath30 = os.path.join(subpath, "android-30")
    cmd = ["mkdir", subpath30]
    subprocess.check_call(cmd)
    subpath = os.path.join(subpath, "android-32")
    cmd = ["mkdir", subpath]
    subprocess.check_call(cmd)
    dest = os.path.join(subpath, "android.jar")
    sha = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32.tar.gz.sha1")
    utils.DownloadFromGoogleCloudStorage(sha)
    src = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32",
                       "android.jar")
    cmd = ["cp", src, dest]
    subprocess.check_call(cmd)


def BuildDesugaredLibrary(checkout_dir, variant, version=None):
    if not variant in MAVEN_RELEASE_TARGET_MAP:
        raise Exception('Variant ' + variant + ' is not supported')
    if variant != 'jdk8' and variant != 'jdk11_legacy' and version is None:
        raise Exception('Variant ' + variant +
                        ' require version for undesugaring')
    if variant != 'jdk8':
        # Hack to workaround b/256723819.
        os.remove(
            os.path.join(checkout_dir, "jdk11", "src", "java.base", "share",
                         "classes", "java", "time", "format",
                         "DesugarDateTimeFormatterBuilder.java"))
    with utils.ChangedWorkingDirectory(checkout_dir):
        with utils.TempDir() as androidHomeTemp:
            setUpFakeAndroidHome(androidHomeTemp)
            javaEnv = GetJavaEnv(androidHomeTemp)
            bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin',
                                 'bazel')
            cmd = [
                bazel, '--bazelrc=/dev/null', 'build', '--spawn_strategy=local',
                '--verbose_failures', MAVEN_RELEASE_TARGET_MAP[variant]
            ]
            utils.PrintCmd(cmd)
            subprocess.check_call(cmd, env=javaEnv)
            cmd = [bazel, 'shutdown']
            utils.PrintCmd(cmd)
            subprocess.check_call(cmd, env=javaEnv)

        # Locate the library jar and the maven zip with the jar from the
        # bazel build.
        if variant == 'jdk8':
            library_jar = os.path.join(checkout_dir, 'bazel-bin', 'src',
                                       'share', 'classes', 'java',
                                       'libjava.jar')
        else:
            # All JDK11 variants use the same library code.
            library_jar = os.path.join(checkout_dir, 'bazel-bin', 'jdk11',
                                       'src',
                                       'd8_java_base_selected_with_addon.jar')
        maven_zip = os.path.join(checkout_dir, 'bazel-bin',
                                 MAVEN_RELEASE_ZIP[variant])

        if variant != 'jdk8' and variant != 'jdk11_legacy':
            # The undesugaring is temporary...
            undesugared_maven_zip = os.path.join(checkout_dir,
                                                 'undesugared_maven')
            Undesugar(variant, maven_zip, version, undesugared_maven_zip)
            undesugared_maven_zip = os.path.join(checkout_dir,
                                                 'undesugared_maven.zip')
            return (library_jar, undesugared_maven_zip)
        else:
            return (library_jar, maven_zip)


def hash_for(file, hash):
    with open(file, 'rb') as f:
        while True:
            # Read chunks of 1MB
            chunk = f.read(2**20)
            if not chunk:
                break
            hash.update(chunk)
    return hash.hexdigest()


def write_md5_for(file):
    hexdigest = hash_for(file, hashlib.md5())
    with (open(file + '.md5', 'w')) as file:
        file.write(hexdigest)


def write_sha1_for(file):
    hexdigest = hash_for(file, hashlib.sha1())
    with (open(file + '.sha1', 'w')) as file:
        file.write(hexdigest)


def Undesugar(variant, maven_zip, version, undesugared_maven_zip):
    gradle.RunGradle([
        utils.GRADLE_TASK_R8,
        utils.GRADLE_TASK_TEST_BASE_JAR,
        utils.GRADLE_TASK_TEST_JAR,
        utils.GRADLE_TASK_TEST_DEPS_JAR, '-Pno_internal'
    ])
    with utils.TempDir() as tmp:
        with zipfile.ZipFile(maven_zip, 'r') as zip_ref:
            zip_ref.extractall(tmp)
        desugar_jdk_libs_jar = os.path.join(
            tmp, 'com', 'android', 'tools', LIBRARY_NAME_MAP[variant], version,
            '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version))
        print(desugar_jdk_libs_jar)
        undesugared_jar = os.path.join(tmp, 'undesugared.jar')
        buildLibs = os.path.join(defines.REPO_ROOT, 'build', 'libs')
        cmd = [
            jdk.GetJavaExecutable(), '-cp',
            '%s:%s:%s:%s' %
            (utils.R8_JAR, utils.R8_TESTBASE_JAR, utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR),
            'com.android.tools.r8.desugar.desugaredlibrary.jdk11.DesugaredLibraryJDK11Undesugarer',
            desugar_jdk_libs_jar, undesugared_jar
        ]
        print(cmd)
        try:
            output = subprocess.check_output(
                cmd, stderr=subprocess.STDOUT).decode('utf-8')
        except subprocess.CalledProcessError as e:
            print(e)
            print(e.output)
            raise e
        print(output)
        # Copy the undesugared jar into place and update the checksums.
        shutil.copyfile(undesugared_jar, desugar_jdk_libs_jar)
        write_md5_for(desugar_jdk_libs_jar)
        write_sha1_for(desugar_jdk_libs_jar)
        shutil.make_archive(undesugared_maven_zip, 'zip', tmp)
        print(undesugared_maven_zip)
        output = subprocess.check_output(
            ['ls', '-l', os.path.dirname(undesugared_maven_zip)],
            stderr=subprocess.STDOUT).decode('utf-8')
        print(output)


def MustBeExistingDirectory(path):
    if (not os.path.exists(path) or not os.path.isdir(path)):
        raise Exception(path + ' does not exist or is not a directory')


def BuildAndUpload(options, variant):
    desugar_jdk_libs_hash = ''
    with open(DESUGAR_JDK_LIBS_HASH_FILE, 'r') as input_hash:
        desugar_jdk_libs_hash = input_hash.readline()
    if options.build_only:
        with utils.TempDir() as checkout_dir:
            CloneDesugaredLibrary(options.github_account, checkout_dir,
                                  desugar_jdk_libs_hash)
            (library_jar,
             maven_zip) = BuildDesugaredLibrary(checkout_dir, variant,
                                                desugar_jdk_libs_hash)
            shutil.copyfile(
                library_jar,
                os.path.join(options.build_only, os.path.basename(library_jar)))
            shutil.copyfile(
                maven_zip,
                os.path.join(options.build_only, os.path.basename(maven_zip)))
            return

    # Only handling versioned desugar_jdk_libs.
    is_main = False

    with utils.TempDir() as checkout_dir:
        CloneDesugaredLibrary(options.github_account, checkout_dir,
                              desugar_jdk_libs_hash)
        version = GetVersion(os.path.join(checkout_dir, VERSION_MAP[variant]))

        destination = archive.GetVersionDestination(
            'gs://', LIBRARY_NAME_MAP[variant] + '/' + version, is_main)
        if utils.cloud_storage_exists(destination) and not options.dry_run:
            raise Exception('Target archive directory %s already exists' %
                            destination)

        (library_jar,
         maven_zip) = BuildDesugaredLibrary(checkout_dir, variant, version)

        storage_path = LIBRARY_NAME_MAP[variant] + '/' + version
        # Upload the jar file with the library.
        destination = archive.GetUploadDestination(
            storage_path, LIBRARY_NAME_MAP[variant] + '.jar', is_main)
        Upload(options, library_jar, storage_path, destination, is_main)

        # Upload the maven zip file with the library.
        destination = archive.GetUploadDestination(storage_path,
                                                   MAVEN_RELEASE_ZIP[variant],
                                                   is_main)
        Upload(options, maven_zip, storage_path, destination, is_main)

        # Upload the jar file for accessing GCS as a maven repro.
        maven_destination = archive.GetUploadDestination(
            utils.get_maven_path(LIBRARY_NAME_MAP[variant], version),
            '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version), is_main)
        if options.dry_run:
            print('Dry run, not actually creating maven repo')
        else:
            utils.upload_file_to_cloud_storage(library_jar, maven_destination)
            print('Maven repo root available at: %s' %
                  archive.GetMavenUrl(is_main))


def Main(argv):
    (options, args) = ParseOptions(argv)
    if (len(args) > 0):
        raise Exception('Unsupported arguments')
    if not utils.is_bot() and not (options.dry_run or options.build_only):
        raise Exception('You are not a bot, don\'t archive builds. ' +
                        'Use --dry-run or --build-only to test locally')
    if options.dry_run_output:
        MustBeExistingDirectory(options.dry_run_output)
    if options.build_only:
        MustBeExistingDirectory(options.build_only)
    if utils.is_bot():
        archive.SetRLimitToMax()

    # Make sure bazel is extracted in third_party.
    utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
    utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
    utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
    utils.DownloadFromGoogleCloudStorage(utils.DESUGAR_JDK_LIBS_11_SHA_FILE)

    for v in options.variant:
        BuildAndUpload(options, v)


if __name__ == '__main__':
    sys.exit(Main(sys.argv[1:]))