blob: ead0bdd1d0fee6d96a8a930f07651536bb967b8e [file] [log] [blame]
Ian Zernydcb172e2022-02-22 15:36:45 +01001#!/usr/bin/env python3
Søren Gjesse1c115b52019-08-14 12:43:57 +02002# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
3# for details. All rights reserved. Use of this source code is governed by a
4# BSD-style license that can be found in the LICENSE file.
5
6# This script is designed to run on a buildbot to build from the source
7# of https://github.com/google/desugar_jdk_libs and publish to the
8# r8-release Cloud Storage Bucket.
9#
10# These files are uploaded:
11#
12# raw/desugar_jdk_libs/<VERSION>/desugar_jdk_libs.jar
13# raw/desugar_jdk_libs/<VERSION>/desugar_jdk_libs.zip
14# raw/com/android/tools/desugar_jdk_libs/<VERSION>/desugar_jdk_libs-<VERSION>.jar
15#
16# The first two are the raw jar file and the maven compatible zip file. The
17# third is the raw jar file placed and named so that the URL
Rico Wind70d614f2020-01-31 08:45:21 +010018# https://storage.googleapis.com/r8-releases/raw can be treated as a maven
Søren Gjesse1c115b52019-08-14 12:43:57 +020019# repository to fetch the artifact com.android.tools:desugar_jdk_libs:1.0.0
20
21import archive
Søren Gjesse2b047692022-08-19 16:34:38 +020022import defines
Søren Gjesse1c115b52019-08-14 12:43:57 +020023import git_utils
Søren Gjesse2b047692022-08-19 16:34:38 +020024import gradle
25import hashlib
Clément Béra54696f72021-11-16 09:27:22 +000026import jdk
Søren Gjesse1c115b52019-08-14 12:43:57 +020027import optparse
28import os
29import re
30import shutil
31import subprocess
32import sys
33import utils
Søren Gjesse2b047692022-08-19 16:34:38 +020034import zipfile
Søren Gjesse1c115b52019-08-14 12:43:57 +020035
Søren Gjesse5614efc2022-06-24 12:25:33 +020036VERSION_FILE_JDK8 = 'VERSION.txt'
Søren Gjesse2b047692022-08-19 16:34:38 +020037VERSION_FILE_JDK11_LEGACY = 'VERSION_JDK11_LEGACY.txt'
38VERSION_FILE_JDK11_MINIMAL = 'VERSION_JDK11_MINIMAL.txt'
Søren Gjesse5614efc2022-06-24 12:25:33 +020039VERSION_FILE_JDK11 = 'VERSION_JDK11.txt'
Søren Gjesse2b047692022-08-19 16:34:38 +020040VERSION_FILE_JDK11_NIO = 'VERSION_JDK11_NIO.txt'
41
42VERSION_MAP = {
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020043 'jdk8': VERSION_FILE_JDK8,
44 'jdk11_legacy': VERSION_FILE_JDK11_LEGACY,
45 'jdk11_minimal': VERSION_FILE_JDK11_MINIMAL,
46 'jdk11': VERSION_FILE_JDK11,
47 'jdk11_nio': VERSION_FILE_JDK11_NIO
Søren Gjesse2b047692022-08-19 16:34:38 +020048}
49
50GITHUB_REPRO = 'desugar_jdk_libs'
51
52BASE_LIBRARY_NAME = 'desugar_jdk_libs'
53
54LIBRARY_NAME_MAP = {
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020055 'jdk8': BASE_LIBRARY_NAME,
56 'jdk11_legacy': BASE_LIBRARY_NAME,
57 'jdk11_minimal': BASE_LIBRARY_NAME + '_minimal',
58 'jdk11': BASE_LIBRARY_NAME,
59 'jdk11_nio': BASE_LIBRARY_NAME + '_nio'
Søren Gjesse2b047692022-08-19 16:34:38 +020060}
61
62MAVEN_RELEASE_TARGET_MAP = {
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020063 'jdk8': 'maven_release',
64 'jdk11_legacy': 'maven_release_jdk11_legacy',
65 'jdk11_minimal': 'maven_release_jdk11_minimal',
66 'jdk11': 'maven_release_jdk11',
67 'jdk11_nio': 'maven_release_jdk11_nio'
Søren Gjesse2b047692022-08-19 16:34:38 +020068}
69
70MAVEN_RELEASE_ZIP = {
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020071 'jdk8': BASE_LIBRARY_NAME + '.zip',
72 'jdk11_legacy': BASE_LIBRARY_NAME + '_jdk11_legacy.zip',
73 'jdk11_minimal': BASE_LIBRARY_NAME + '_jdk11_minimal.zip',
74 'jdk11': BASE_LIBRARY_NAME + '_jdk11.zip',
75 'jdk11_nio': BASE_LIBRARY_NAME + '_jdk11_nio.zip'
Søren Gjesse2b047692022-08-19 16:34:38 +020076}
77
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020078DESUGAR_JDK_LIBS_HASH_FILE = os.path.join(defines.THIRD_PARTY, 'openjdk',
79 'desugar_jdk_libs_11',
80 'desugar_jdk_libs_hash')
Søren Gjesse85c37922022-09-01 16:58:07 +020081
Søren Gjesse1c115b52019-08-14 12:43:57 +020082
83def ParseOptions(argv):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +020084 result = optparse.OptionParser()
85 result.add_option(
86 '--variant',
87 help="Variant(s) to build",
88 metavar=('<variants(s)>'),
89 choices=['jdk8', 'jdk11_legacy', 'jdk11_minimal', 'jdk11', 'jdk11_nio'],
90 default=[],
91 action='append')
92 result.add_option('--dry-run',
93 '--dry_run',
94 help='Running on bot, use third_party dependency.',
95 default=False,
96 action='store_true')
97 result.add_option('--dry-run-output',
98 '--dry_run_output',
99 help='Output directory for dry run.',
100 type="string",
101 action="store")
102 result.add_option('--github-account',
103 '--github_account',
104 help='GitHub account to clone from.',
105 default="google",
106 type="string",
107 action="store")
108 result.add_option('--build_only',
109 '--build-only',
110 help='Build desugared library without archiving.',
111 type="string",
112 action="store")
113 (options, args) = result.parse_args(argv)
114 return (options, args)
Søren Gjesse1c115b52019-08-14 12:43:57 +0200115
116
Søren Gjesse07b6aea2019-12-12 11:11:58 +0100117def GetVersion(version_file_name):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200118 with open(version_file_name, 'r') as version_file:
119 lines = [line.strip() for line in version_file.readlines()]
120 lines = [line for line in lines if not line.startswith('#')]
121 if len(lines) != 1:
122 raise Exception('Version file ' + version_file +
123 ' is expected to have exactly one line')
124 version = lines[0].strip()
125 utils.check_basic_semver_version(version,
126 'in version file ' + version_file_name,
127 allowPrerelease=True)
128 return version
Søren Gjesse1c115b52019-08-14 12:43:57 +0200129
130
Rico Wind1b52acf2021-03-21 12:36:55 +0100131def Upload(options, file_name, storage_path, destination, is_main):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200132 print('Uploading %s to %s' % (file_name, destination))
133 if options.dry_run:
134 if options.dry_run_output:
135 dry_run_destination = \
136 os.path.join(options.dry_run_output, os.path.basename(file_name))
137 print('Dry run, not actually uploading. Copying to ' +
138 dry_run_destination)
139 shutil.copyfile(file_name, dry_run_destination)
140 else:
141 print('Dry run, not actually uploading')
Søren Gjesse1c115b52019-08-14 12:43:57 +0200142 else:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200143 utils.upload_file_to_cloud_storage(file_name, destination)
144 print(
145 'File available at: %s' %
146 destination.replace('gs://', 'https://storage.googleapis.com/', 1))
147
Søren Gjesse1c115b52019-08-14 12:43:57 +0200148
Søren Gjesse85c37922022-09-01 16:58:07 +0200149def CloneDesugaredLibrary(github_account, checkout_dir, desugar_jdk_libs_hash):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200150 git_utils.GitClone(
151 'https://github.com/' + github_account + '/' + GITHUB_REPRO,
152 checkout_dir)
153 git_utils.GitCheckout(desugar_jdk_libs_hash, checkout_dir)
154
Søren Gjesse53d14482021-02-10 16:53:38 +0100155
Clément Béra2dc26b02022-11-14 09:50:57 +0100156def GetJavaEnv(androidHomeTemp):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200157 java_env = dict(os.environ, JAVA_HOME=jdk.GetJdk11Home())
158 java_env['PATH'] = java_env['PATH'] + os.pathsep + os.path.join(
159 jdk.GetJdk11Home(), 'bin')
160 java_env['GRADLE_OPTS'] = '-Xmx1g'
161 java_env['ANDROID_HOME'] = androidHomeTemp
162 return java_env
163
Clément Béra54696f72021-11-16 09:27:22 +0000164
Clément Béra2dc26b02022-11-14 09:50:57 +0100165def setUpFakeAndroidHome(androidHomeTemp):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200166 # Bazel will check if 30 is present then extract android.jar from 32.
167 # We copy android.jar from third_party to mimic repository structure.
168 subpath = os.path.join(androidHomeTemp, "build-tools")
169 cmd = ["mkdir", subpath]
170 subprocess.check_call(cmd)
171 subpath = os.path.join(subpath, "32.0.0")
172 cmd = ["mkdir", subpath]
173 subprocess.check_call(cmd)
174 subpath = os.path.join(androidHomeTemp, "platforms")
175 cmd = ["mkdir", subpath]
176 subprocess.check_call(cmd)
177 subpath30 = os.path.join(subpath, "android-30")
178 cmd = ["mkdir", subpath30]
179 subprocess.check_call(cmd)
180 subpath = os.path.join(subpath, "android-32")
181 cmd = ["mkdir", subpath]
182 subprocess.check_call(cmd)
183 dest = os.path.join(subpath, "android.jar")
184 sha = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32.tar.gz.sha1")
185 utils.DownloadFromGoogleCloudStorage(sha)
186 src = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32",
187 "android.jar")
188 cmd = ["cp", src, dest]
189 subprocess.check_call(cmd)
Clément Béra54696f72021-11-16 09:27:22 +0000190
Søren Gjesse53d14482021-02-10 16:53:38 +0100191
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200192def BuildDesugaredLibrary(checkout_dir, variant, version=None):
193 if not variant in MAVEN_RELEASE_TARGET_MAP:
194 raise Exception('Variant ' + variant + ' is not supported')
195 if variant != 'jdk8' and variant != 'jdk11_legacy' and version is None:
196 raise Exception('Variant ' + variant +
197 ' require version for undesugaring')
198 if variant != 'jdk8':
199 # Hack to workaround b/256723819.
200 os.remove(
201 os.path.join(checkout_dir, "jdk11", "src", "java.base", "share",
202 "classes", "java", "time", "format",
203 "DesugarDateTimeFormatterBuilder.java"))
204 with utils.ChangedWorkingDirectory(checkout_dir):
205 with utils.TempDir() as androidHomeTemp:
206 setUpFakeAndroidHome(androidHomeTemp)
207 javaEnv = GetJavaEnv(androidHomeTemp)
208 bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin',
209 'bazel')
210 cmd = [
211 bazel, '--bazelrc=/dev/null', 'build', '--spawn_strategy=local',
212 '--verbose_failures', MAVEN_RELEASE_TARGET_MAP[variant]
213 ]
214 utils.PrintCmd(cmd)
215 subprocess.check_call(cmd, env=javaEnv)
216 cmd = [bazel, 'shutdown']
217 utils.PrintCmd(cmd)
218 subprocess.check_call(cmd, env=javaEnv)
Søren Gjesse53d14482021-02-10 16:53:38 +0100219
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200220 # Locate the library jar and the maven zip with the jar from the
221 # bazel build.
222 if variant == 'jdk8':
223 library_jar = os.path.join(checkout_dir, 'bazel-bin', 'src',
224 'share', 'classes', 'java',
225 'libjava.jar')
226 else:
227 # All JDK11 variants use the same library code.
228 library_jar = os.path.join(checkout_dir, 'bazel-bin', 'jdk11',
229 'src',
230 'd8_java_base_selected_with_addon.jar')
231 maven_zip = os.path.join(checkout_dir, 'bazel-bin',
232 MAVEN_RELEASE_ZIP[variant])
233
234 if variant != 'jdk8' and variant != 'jdk11_legacy':
235 # The undesugaring is temporary...
236 undesugared_maven_zip = os.path.join(checkout_dir,
237 'undesugared_maven')
238 Undesugar(variant, maven_zip, version, undesugared_maven_zip)
239 undesugared_maven_zip = os.path.join(checkout_dir,
240 'undesugared_maven.zip')
241 return (library_jar, undesugared_maven_zip)
242 else:
243 return (library_jar, maven_zip)
244
Søren Gjesse2b047692022-08-19 16:34:38 +0200245
246def hash_for(file, hash):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200247 with open(file, 'rb') as f:
248 while True:
249 # Read chunks of 1MB
250 chunk = f.read(2**20)
251 if not chunk:
252 break
253 hash.update(chunk)
254 return hash.hexdigest()
255
Søren Gjesse2b047692022-08-19 16:34:38 +0200256
257def write_md5_for(file):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200258 hexdigest = hash_for(file, hashlib.md5())
259 with (open(file + '.md5', 'w')) as file:
260 file.write(hexdigest)
261
Søren Gjesse2b047692022-08-19 16:34:38 +0200262
263def write_sha1_for(file):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200264 hexdigest = hash_for(file, hashlib.sha1())
265 with (open(file + '.sha1', 'w')) as file:
266 file.write(hexdigest)
267
Søren Gjesse2b047692022-08-19 16:34:38 +0200268
269def Undesugar(variant, maven_zip, version, undesugared_maven_zip):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200270 gradle.RunGradle([
271 utils.GRADLE_TASK_R8, utils.GRADLE_TASK_TEST_JAR,
272 utils.GRADLE_TASK_TEST_DEPS_JAR, '-Pno_internal'
273 ])
274 with utils.TempDir() as tmp:
275 with zipfile.ZipFile(maven_zip, 'r') as zip_ref:
276 zip_ref.extractall(tmp)
277 desugar_jdk_libs_jar = os.path.join(
278 tmp, 'com', 'android', 'tools', LIBRARY_NAME_MAP[variant], version,
279 '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version))
280 print(desugar_jdk_libs_jar)
281 undesugared_jar = os.path.join(tmp, 'undesugared.jar')
282 buildLibs = os.path.join(defines.REPO_ROOT, 'build', 'libs')
283 cmd = [
284 jdk.GetJavaExecutable(), '-cp',
285 '%s:%s:%s' %
286 (utils.R8_JAR, utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR),
287 'com.android.tools.r8.desugar.desugaredlibrary.jdk11.DesugaredLibraryJDK11Undesugarer',
288 desugar_jdk_libs_jar, undesugared_jar
289 ]
290 print(cmd)
291 try:
292 output = subprocess.check_output(
293 cmd, stderr=subprocess.STDOUT).decode('utf-8')
294 except subprocess.CalledProcessError as e:
295 print(e)
296 print(e.output)
297 raise e
298 print(output)
299 # Copy the undesugared jar into place and update the checksums.
300 shutil.copyfile(undesugared_jar, desugar_jdk_libs_jar)
301 write_md5_for(desugar_jdk_libs_jar)
302 write_sha1_for(desugar_jdk_libs_jar)
303 shutil.make_archive(undesugared_maven_zip, 'zip', tmp)
304 print(undesugared_maven_zip)
305 output = subprocess.check_output(
306 ['ls', '-l', os.path.dirname(undesugared_maven_zip)],
307 stderr=subprocess.STDOUT).decode('utf-8')
308 print(output)
309
Søren Gjesse53d14482021-02-10 16:53:38 +0100310
311def MustBeExistingDirectory(path):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200312 if (not os.path.exists(path) or not os.path.isdir(path)):
313 raise Exception(path + ' does not exist or is not a directory')
314
Søren Gjesse1c115b52019-08-14 12:43:57 +0200315
Søren Gjesse2b047692022-08-19 16:34:38 +0200316def BuildAndUpload(options, variant):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200317 desugar_jdk_libs_hash = ''
318 with open(DESUGAR_JDK_LIBS_HASH_FILE, 'r') as input_hash:
319 desugar_jdk_libs_hash = input_hash.readline()
320 if options.build_only:
321 with utils.TempDir() as checkout_dir:
322 CloneDesugaredLibrary(options.github_account, checkout_dir,
323 desugar_jdk_libs_hash)
324 (library_jar,
325 maven_zip) = BuildDesugaredLibrary(checkout_dir, variant,
326 desugar_jdk_libs_hash)
327 shutil.copyfile(
328 library_jar,
329 os.path.join(options.build_only, os.path.basename(library_jar)))
330 shutil.copyfile(
331 maven_zip,
332 os.path.join(options.build_only, os.path.basename(maven_zip)))
333 return
334
335 # Only handling versioned desugar_jdk_libs.
336 is_main = False
337
Søren Gjesse2b047692022-08-19 16:34:38 +0200338 with utils.TempDir() as checkout_dir:
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200339 CloneDesugaredLibrary(options.github_account, checkout_dir,
340 desugar_jdk_libs_hash)
341 version = GetVersion(os.path.join(checkout_dir, VERSION_MAP[variant]))
Søren Gjesse2b047692022-08-19 16:34:38 +0200342
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200343 destination = archive.GetVersionDestination(
344 'gs://', LIBRARY_NAME_MAP[variant] + '/' + version, is_main)
345 if utils.cloud_storage_exists(destination) and not options.dry_run:
346 raise Exception('Target archive directory %s already exists' %
347 destination)
Søren Gjesse2b047692022-08-19 16:34:38 +0200348
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200349 (library_jar,
350 maven_zip) = BuildDesugaredLibrary(checkout_dir, variant, version)
Søren Gjesse2b047692022-08-19 16:34:38 +0200351
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200352 storage_path = LIBRARY_NAME_MAP[variant] + '/' + version
353 # Upload the jar file with the library.
354 destination = archive.GetUploadDestination(
355 storage_path, LIBRARY_NAME_MAP[variant] + '.jar', is_main)
356 Upload(options, library_jar, storage_path, destination, is_main)
Søren Gjesse2b047692022-08-19 16:34:38 +0200357
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200358 # Upload the maven zip file with the library.
359 destination = archive.GetUploadDestination(storage_path,
360 MAVEN_RELEASE_ZIP[variant],
361 is_main)
362 Upload(options, maven_zip, storage_path, destination, is_main)
Søren Gjesse2b047692022-08-19 16:34:38 +0200363
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200364 # Upload the jar file for accessing GCS as a maven repro.
365 maven_destination = archive.GetUploadDestination(
366 utils.get_maven_path(LIBRARY_NAME_MAP[variant], version),
367 '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version), is_main)
368 if options.dry_run:
369 print('Dry run, not actually creating maven repo')
370 else:
371 utils.upload_file_to_cloud_storage(library_jar, maven_destination)
372 print('Maven repo root available at: %s' %
373 archive.GetMavenUrl(is_main))
Søren Gjesse2b047692022-08-19 16:34:38 +0200374
Søren Gjesse2b047692022-08-19 16:34:38 +0200375
Søren Gjesse1c115b52019-08-14 12:43:57 +0200376def Main(argv):
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200377 (options, args) = ParseOptions(argv)
378 if (len(args) > 0):
379 raise Exception('Unsupported arguments')
380 if not utils.is_bot() and not (options.dry_run or options.build_only):
381 raise Exception('You are not a bot, don\'t archive builds. ' +
382 'Use --dry-run or --build-only to test locally')
383 if options.dry_run_output:
384 MustBeExistingDirectory(options.dry_run_output)
385 if options.build_only:
386 MustBeExistingDirectory(options.build_only)
387 if utils.is_bot():
388 archive.SetRLimitToMax()
Søren Gjesse1c115b52019-08-14 12:43:57 +0200389
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200390 # Make sure bazel is extracted in third_party.
391 utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
392 utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
393 utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
394 utils.DownloadFromGoogleCloudStorage(utils.DESUGAR_JDK_LIBS_11_SHA_FILE)
Søren Gjesse1c115b52019-08-14 12:43:57 +0200395
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200396 for v in options.variant:
397 BuildAndUpload(options, v)
398
Søren Gjesse1c115b52019-08-14 12:43:57 +0200399
400if __name__ == '__main__':
Christoffer Quist Adamsen2434a4d2023-10-16 11:29:03 +0200401 sys.exit(Main(sys.argv[1:]))