Prepare release scripts for new JDK-11 desugared library variants
Bug: b/222647019
Change-Id: I87da0208803346413e09ef31fef228a6121ec486
diff --git a/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/jdk11/DesugaredLibraryJDK11Undesugarer.java b/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/jdk11/DesugaredLibraryJDK11Undesugarer.java
index c9c9a31..c4ef343 100644
--- a/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/jdk11/DesugaredLibraryJDK11Undesugarer.java
+++ b/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/jdk11/DesugaredLibraryJDK11Undesugarer.java
@@ -16,6 +16,7 @@
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Map;
import java.util.zip.ZipEntry;
@@ -38,6 +39,16 @@
.put("wrapper/adapter/HybridFileTypeDetector", "java/adapter/HybridFileTypeDetector")
.build();
+ public static void main(String[] args) {
+ if (!Files.exists(Paths.get(args[0]))) {
+ throw new RuntimeException("Undesugarer source not found");
+ }
+ if (Files.exists(Paths.get(args[1]))) {
+ throw new RuntimeException("Undesugarer destination already exists");
+ }
+ generateUndesugaredJar(Paths.get(args[0]), Paths.get(args[1]));
+ }
+
public static Path undesugaredJarJDK11(Path undesugarFolder, Path jdk11Jar) {
String fileName = jdk11Jar.getFileName().toString();
String newFileName = fileName.substring(0, fileName.length() - 4) + "_undesugared.jar";
diff --git a/tools/archive.py b/tools/archive.py
index 652fbe9..bfd3a1b 100755
--- a/tools/archive.py
+++ b/tools/archive.py
@@ -169,6 +169,23 @@
utils.DESUGAR_IMPLEMENTATION_JDK11,
utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
+ ### THIS IS MISSING CONVERTING OF THE JSON ###
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_NIO,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+
version = GetVersion()
is_main = IsMain(version)
if is_main:
diff --git a/tools/archive_desugar_jdk_libs.py b/tools/archive_desugar_jdk_libs.py
index 5298435..2aa841c 100755
--- a/tools/archive_desugar_jdk_libs.py
+++ b/tools/archive_desugar_jdk_libs.py
@@ -19,7 +19,10 @@
# repository to fetch the artifact com.android.tools:desugar_jdk_libs:1.0.0
import archive
+import defines
import git_utils
+import gradle
+import hashlib
import jdk
import optparse
import os
@@ -28,16 +31,56 @@
import subprocess
import sys
import utils
+import zipfile
VERSION_FILE_JDK8 = 'VERSION.txt'
+VERSION_FILE_JDK11_LEGACY = 'VERSION_JDK11_LEGACY.txt'
+VERSION_FILE_JDK11_MINIMAL = 'VERSION_JDK11_MINIMAL.txt'
VERSION_FILE_JDK11 = 'VERSION_JDK11.txt'
-LIBRARY_NAME = 'desugar_jdk_libs'
+VERSION_FILE_JDK11_NIO = 'VERSION_JDK11_NIO.txt'
+
+VERSION_MAP = {
+ 'jdk8': VERSION_FILE_JDK8,
+ 'jdk11_legacy': VERSION_FILE_JDK11_LEGACY,
+ 'jdk11_minimal': VERSION_FILE_JDK11_MINIMAL,
+ 'jdk11': VERSION_FILE_JDK11,
+ 'jdk11_nio': VERSION_FILE_JDK11_NIO
+}
+
+GITHUB_REPRO = 'desugar_jdk_libs'
+
+BASE_LIBRARY_NAME = 'desugar_jdk_libs'
+
+LIBRARY_NAME_MAP = {
+ 'jdk8': BASE_LIBRARY_NAME,
+ 'jdk11_legacy': BASE_LIBRARY_NAME,
+ 'jdk11_minimal': BASE_LIBRARY_NAME + '_minimal',
+ 'jdk11': BASE_LIBRARY_NAME,
+ 'jdk11_nio': BASE_LIBRARY_NAME + '_nio'
+}
+
+MAVEN_RELEASE_TARGET_MAP = {
+ 'jdk8': 'maven_release',
+ 'jdk11_legacy': 'maven_release_jdk11_legacy',
+ 'jdk11_minimal': 'maven_release_jdk11_minimal',
+ 'jdk11': 'maven_release_jdk11',
+ 'jdk11_nio': 'maven_release_jdk11_nio'
+}
+
+MAVEN_RELEASE_ZIP = {
+ 'jdk8': BASE_LIBRARY_NAME + '.zip',
+ 'jdk11_legacy': BASE_LIBRARY_NAME + '_jdk11_legacy.zip',
+ 'jdk11_minimal': BASE_LIBRARY_NAME + '_jdk11_minimal.zip',
+ 'jdk11': BASE_LIBRARY_NAME + '_jdk11.zip',
+ 'jdk11_nio': BASE_LIBRARY_NAME + '_jdk11_nio.zip'
+}
+
def ParseOptions(argv):
result = optparse.OptionParser()
result.add_option('--variant',
- help='.',
- choices = ['jdk8', 'jdk11'],
+ help="Variant to build",
+ choices=['jdk8', 'jdk11_legacy', 'jdk11_minimal', 'jdk11', 'jdk11_nio'],
default='jdk11')
result.add_option('--dry-run', '--dry_run',
help='Running on bot, use third_party dependency.',
@@ -89,7 +132,7 @@
def CloneDesugaredLibrary(github_account, checkout_dir):
git_utils.GitClone(
'https://github.com/'
- + github_account + '/' + LIBRARY_NAME, checkout_dir)
+ + github_account + '/' + GITHUB_REPRO, checkout_dir)
def GetJavaEnv():
java_env = dict(os.environ, JAVA_HOME = jdk.GetJdk11Home())
@@ -98,9 +141,11 @@
return java_env
-def BuildDesugaredLibrary(checkout_dir, variant):
- if (variant != 'jdk8' and variant != 'jdk11'):
- raise Exception('Variant ' + variant + 'is not supported')
+def BuildDesugaredLibrary(checkout_dir, variant, version = None):
+ if not variant in MAVEN_RELEASE_TARGET_MAP:
+ raise Exception('Variant ' + variant + ' is not supported')
+ if variant != 'jdk8' and variant != 'jdk11_legacy' and version is None:
+ raise Exception('Variant ' + variant + ' require version for undesugaring')
with utils.ChangedWorkingDirectory(checkout_dir):
bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel')
cmd = [
@@ -109,7 +154,7 @@
'build',
'--spawn_strategy=local',
'--verbose_failures',
- 'maven_release' + ('_jdk11' if variant == 'jdk11' else '')]
+ MAVEN_RELEASE_TARGET_MAP[variant]]
utils.PrintCmd(cmd)
subprocess.check_call(cmd, env=GetJavaEnv())
cmd = [bazel, 'shutdown']
@@ -122,14 +167,81 @@
library_jar = os.path.join(
checkout_dir, 'bazel-bin', 'src', 'share', 'classes', 'java', 'libjava.jar')
else:
+ # All JDK11 variants use the same library code.
library_jar = os.path.join(
checkout_dir, 'bazel-bin', 'jdk11', 'src', 'd8_java_base_selected_with_addon.jar')
maven_zip = os.path.join(
checkout_dir,
'bazel-bin',
- LIBRARY_NAME + ('_jdk11' if variant == 'jdk11' else '') +'.zip')
- return (library_jar, maven_zip)
+ MAVEN_RELEASE_ZIP[variant])
+ if variant != 'jdk8' and variant != 'jdk11_legacy':
+ # The undesugaring is temporary...
+ undesugared_maven_zip = os.path.join(checkout_dir, 'undesugared_maven')
+ Undesugar(variant, maven_zip, version, undesugared_maven_zip)
+ undesugared_maven_zip = os.path.join(checkout_dir, 'undesugared_maven.zip')
+ return (library_jar, undesugared_maven_zip)
+ else:
+ return (library_jar, maven_zip)
+
+def hash_for(file, hash):
+ with open(file, 'rb') as f:
+ while True:
+ # Read chunks of 1MB
+ chunk = f.read(2 ** 20)
+ if not chunk:
+ break
+ hash.update(chunk)
+ return hash.hexdigest()
+
+def write_md5_for(file):
+ hexdigest = hash_for(file, hashlib.md5())
+ with (open(file + '.md5', 'w')) as file:
+ file.write(hexdigest)
+
+def write_sha1_for(file):
+ hexdigest = hash_for(file, hashlib.sha1())
+ with (open(file + '.sha1', 'w')) as file:
+ file.write(hexdigest)
+
+def Undesugar(variant, maven_zip, version, undesugared_maven_zip):
+ gradle.RunGradle(['testJar', 'repackageTestDeps'])
+ with utils.TempDir() as tmp:
+ with zipfile.ZipFile(maven_zip, 'r') as zip_ref:
+ zip_ref.extractall(tmp)
+ desugar_jdk_libs_jar = os.path.join(
+ tmp,
+ 'com',
+ 'android',
+ 'tools',
+ LIBRARY_NAME_MAP[variant],
+ version,
+ '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version))
+ print(desugar_jdk_libs_jar)
+ undesugared_jar = os.path.join(tmp, 'undesugared.jar')
+ buildLibs = os.path.join(defines.REPO_ROOT, 'build', 'libs')
+ cmd = [jdk.GetJavaExecutable(),
+ '-cp',
+ '%s:%s:%s' % (os.path.join(buildLibs, 'r8_with_deps.jar'), os.path.join(buildLibs, 'r8tests.jar'), os.path.join(buildLibs, 'test_deps_all.jar')),
+ 'com.android.tools.r8.desugar.desugaredlibrary.jdk11.DesugaredLibraryJDK11Undesugarer',
+ desugar_jdk_libs_jar,
+ undesugared_jar]
+ print(cmd)
+ try:
+ output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
+ except subprocess.CalledProcessError as e:
+ print(e)
+ print(e.output)
+ raise e
+ print(output)
+ # Copy the undesugared jar into place and update the checksums.
+ shutil.copyfile(undesugared_jar, desugar_jdk_libs_jar)
+ write_md5_for(desugar_jdk_libs_jar)
+ write_sha1_for(desugar_jdk_libs_jar)
+ shutil.make_archive(undesugared_maven_zip, 'zip', tmp)
+ print(undesugared_maven_zip)
+ output = subprocess.check_output(['ls', '-l', os.path.dirname(undesugared_maven_zip)], stderr = subprocess.STDOUT).decode('utf-8')
+ print(output)
def MustBeExistingDirectory(path):
if (not os.path.exists(path) or not os.path.isdir(path)):
@@ -171,28 +283,25 @@
with utils.TempDir() as checkout_dir:
CloneDesugaredLibrary(options.github_account, checkout_dir)
- version = GetVersion(
- os.path.join(
- checkout_dir,
- VERSION_FILE_JDK11 if options.variant == 'jdk11' else VERSION_FILE_JDK8))
+ version = GetVersion(os.path.join(checkout_dir, VERSION_MAP[options.variant]))
destination = archive.GetVersionDestination(
- 'gs://', LIBRARY_NAME + '/' + version, is_main)
+ 'gs://', LIBRARY_NAME_MAP[options.variant] + '/' + version, is_main)
if utils.cloud_storage_exists(destination) and not options.dry_run:
raise Exception(
'Target archive directory %s already exists' % destination)
- (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, options.variant)
+ (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, options.variant, version)
- storage_path = LIBRARY_NAME + '/' + version
+ storage_path = LIBRARY_NAME_MAP[options.variant] + '/' + version
# Upload the jar file with the library.
destination = archive.GetUploadDestination(
- storage_path, LIBRARY_NAME + '.jar', is_main)
+ storage_path, LIBRARY_NAME_MAP[options.variant] + '.jar', is_main)
Upload(options, library_jar, storage_path, destination, is_main)
# Upload the maven zip file with the library.
destination = archive.GetUploadDestination(
- storage_path, LIBRARY_NAME + '.zip', is_main)
+ storage_path, MAVEN_RELEASE_ZIP[options.variant], is_main)
Upload(options, maven_zip, storage_path, destination, is_main)
# Upload the jar file for accessing GCS as a maven repro.
diff --git a/tools/test.py b/tools/test.py
index 27c8b6a..5586802 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -260,7 +260,7 @@
utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
- (library_jar, maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(checkout_dir, 'jdk11' if options.desugared_library_configuration == 'jdk11' else 'jdk8')
+ (library_jar, maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(checkout_dir, 'jdk11_legacy' if options.desugared_library_configuration == 'jdk11' else 'jdk8')
desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir, os.path.basename(library_jar))
shutil.copyfile(library_jar, desugar_jdk_libs)
print('Desugared library for test in ' + desugar_jdk_libs)
diff --git a/tools/utils.py b/tools/utils.py
index 9bd518f..07cfb3c 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -77,12 +77,24 @@
'third_party', 'openjdk', 'desugar_jdk_libs', 'desugar_jdk_libs.jar')
DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join(
'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
+DESUGAR_CONFIGURATION_JDK11_MINIMAL = os.path.join(
+ 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json')
+DESUGAR_CONFIGURATION_JDK11 = os.path.join(
+ 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs.json')
+DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join(
+ 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_nio.json')
DESUGAR_IMPLEMENTATION_JDK11 = os.path.join(
'third_party', 'openjdk', 'desugar_jdk_libs_11', 'desugar_jdk_libs.jar')
DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
LIBS, 'desugar_jdk_libs_configuration.zip')
DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP = os.path.join(
LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip')
+DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP = os.path.join(
+ LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip')
+DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP = os.path.join(
+ LIBS, 'desugar_jdk_libs_configuration_jdk11.zip')
+DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP = os.path.join(
+ LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip')
GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
