blob: 137f584266dd8b4ff2e88a7d40feb1d5d0bbd865 [file] [log] [blame]
Ian Zernydcb172e2022-02-22 15:36:45 +01001#!/usr/bin/env python3
Mads Agera9745612017-11-02 12:42:15 +01002# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
3# for details. All rights reserved. Use of this source code is governed by a
4# BSD-style license that can be found in the LICENSE file.
5
Mads Agera9745612017-11-02 12:42:15 +01006import argparse
Mads Agera4911eb2017-11-22 13:19:36 +01007import gradle
8import hashlib
Søren Gjessea70d3bd2019-09-24 15:07:00 +02009import jdk
Søren Gjesse6e5e5842019-09-03 08:48:30 +020010import json
Mads Agera9745612017-11-02 12:42:15 +010011from os import makedirs
Rico Windcdc39b62022-04-08 12:37:57 +020012from os.path import join, basename
Søren Gjesse6e5e5842019-09-03 08:48:30 +020013from shutil import copyfile, make_archive, move, rmtree
Mads Agera9745612017-11-02 12:42:15 +010014import subprocess
15import sys
16from string import Template
17import tempfile
Mads Ager0fab4912017-11-20 13:52:48 +010018import utils
Søren Gjesse17fc67d2019-12-04 14:50:17 +010019import zipfile
Mads Agera9745612017-11-02 12:42:15 +010020
Søren Gjessec425e6a2019-06-28 11:41:14 +020021LICENSETEMPLATE = Template(
22"""
23 <license>
24 <name>$name</name>
25 <url>$url</url>
26 <distribution>repo</distribution>
27 </license>""")
28
Søren Gjesse6e5e5842019-09-03 08:48:30 +020029R8_POMTEMPLATE = Template(
Mads Agera9745612017-11-02 12:42:15 +010030"""<project
31 xmlns="http://maven.apache.org/POM/4.0.0"
32 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
33 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
34 <modelVersion>4.0.0</modelVersion>
35 <groupId>com.android.tools</groupId>
36 <artifactId>r8</artifactId>
37 <version>$version</version>
38 <name>D8 dexer and R8 shrinker</name>
39 <description>
40 D8 dexer and R8 shrinker.
41 </description>
42 <url>http://r8.googlesource.com/r8</url>
43 <inceptionYear>2016</inceptionYear>
44 <licenses>
45 <license>
46 <name>BSD-3-Clause</name>
47 <url>https://opensource.org/licenses/BSD-3-Clause</url>
48 <distribution>repo</distribution>
Søren Gjessec425e6a2019-06-28 11:41:14 +020049 </license>$library_licenses
Mads Ager73460192017-11-08 10:02:50 +010050 </licenses>
Rico Windd6323ad2023-09-14 09:33:59 +020051 <dependencies>
Mads Agera4911eb2017-11-22 13:19:36 +010052 </dependencies>
Mads Agera9745612017-11-02 12:42:15 +010053 <developers>
54 <developer>
55 <name>The Android Open Source Project</name>
56 </developer>
57 </developers>
58 <scm>
59 <connection>
60 https://r8.googlesource.com/r8.git
61 </connection>
62 <url>
63 https://r8.googlesource.com/r8
64 </url>
65 </scm>
66</project>
67""")
68
Søren Gjesse6e5e5842019-09-03 08:48:30 +020069DESUGAR_CONFIGUATION_POMTEMPLATE = Template(
70"""<project
71 xmlns="http://maven.apache.org/POM/4.0.0"
72 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
73 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
74 <modelVersion>4.0.0</modelVersion>
75 <groupId>com.android.tools</groupId>
Søren Gjesse2b047692022-08-19 16:34:38 +020076 <artifactId>$artifactId</artifactId>
Søren Gjesse6e5e5842019-09-03 08:48:30 +020077 <version>$version</version>
78 <name>D8 configuration to desugar desugar_jdk_libs</name>
79 <description>
80 D8 configuration to desugar desugar_jdk_libs.
81 </description>
82 <url>http://r8.googlesource.com/r8</url>
83 <inceptionYear>2019</inceptionYear>
84 <licenses>
85 <license>
86 <name>BSD-3-Clause</name>
87 <url>https://opensource.org/licenses/BSD-3-Clause</url>
88 <distribution>repo</distribution>
89 </license>
90 </licenses>
Søren Gjesse6e5e5842019-09-03 08:48:30 +020091 <developers>
92 <developer>
93 <name>The Android Open Source Project</name>
94 </developer>
95 </developers>
96 <scm>
97 <connection>
98 https://r8.googlesource.com/r8.git
99 </connection>
100 <url>
101 https://r8.googlesource.com/r8
102 </url>
103 </scm>
104</project>
105""")
106
Mads Agera9745612017-11-02 12:42:15 +0100107def parse_options(argv):
108 result = argparse.ArgumentParser()
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100109 result.add_argument('--out', help='The zip file to output')
Søren Gjesse705a3b12022-03-17 11:37:30 +0100110 group = result.add_mutually_exclusive_group()
Søren Gjesse705a3b12022-03-17 11:37:30 +0100111 group.add_argument('--desugar-configuration', action='store_true',
112 help='Build desugar library configuration (original JDK-8)')
113 group.add_argument('--desugar-configuration-jdk8', action='store_true',
114 help='Build desugar library configuration (original JDK-8)')
115 group.add_argument('--desugar-configuration-jdk11-legacy', action='store_true',
116 help='Build desugar library configuration (JDK-11 legacy)')
Søren Gjesse1a8ccad2023-06-06 15:00:46 +0200117 group.add_argument('--desugar-configuration-jdk11-minimal', action='store_true',
118 help='Build desugar library configuration (JDK-11 minimal)')
119 group.add_argument('--desugar-configuration-jdk11', action='store_true',
120 help='Build desugar library configuration (JDK-11)')
121 group.add_argument('--desugar-configuration-jdk11-nio', action='store_true',
122 help='Build desugar library configuration (JDK-11 nio)')
Mads Agera9745612017-11-02 12:42:15 +0100123 return result.parse_args(argv)
124
Mads Agera4911eb2017-11-22 13:19:36 +0100125def determine_version():
126 version_file = join(
127 utils.SRC_ROOT, 'com', 'android', 'tools', 'r8', 'Version.java')
128 with open(version_file, 'r') as file:
129 for line in file:
130 if 'final String LABEL ' in line:
Rico Wind6feccf22018-05-08 11:18:22 +0200131 result = line[line.find('"') + 1:]
Mads Agera4911eb2017-11-22 13:19:36 +0100132 result = result[:result.find('"')]
133 return result
134 raise Exception('Unable to determine version.')
Mads Agera9745612017-11-02 12:42:15 +0100135
136def generate_library_licenses():
Søren Gjessec425e6a2019-06-28 11:41:14 +0200137 artifact_prefix = '- artifact: '
Mads Agera9745612017-11-02 12:42:15 +0100138 license_prefix = 'license: '
139 licenses = []
140 license_url_prefix = 'licenseUrl: '
141 license_urls = []
Søren Gjesseea678302019-08-27 14:41:03 +0200142 # The ./LIBRARY-LICENSE file is a simple yaml file, which for each dependency
143 # has the following information:
144 #
145 # - artifact: <maven artifact> // in the form <group-id>:<artifact-id>:+
146 # name: <name of dependency>
147 # copyrightHolder: <name of copyright holder>
148 # license: <license name>
149 # licenseUrl: <url to license test>
150 #
151 # E.g. for Guava:
152 #
153 # - artifact: com.google.guava:guava:+
154 # name: Guava Google Core Libraries for Java
155 # copyrightHolder: The Guava Authors
156 # license: The Apache Software License, Version 2.0
157 # licenseUrl: http://www.apache.org/licenses/LICENSE-2.0.txt
158 #
159 # This file should always be up to date as the build will fail if it
160 # is does not have information for all dependencies.
Mads Agera9745612017-11-02 12:42:15 +0100161 with open('LIBRARY-LICENSE', 'r') as file:
Søren Gjessec425e6a2019-06-28 11:41:14 +0200162 name = None
163 url = None
Mads Agera9745612017-11-02 12:42:15 +0100164 for line in file:
165 trimmed = line.strip()
Søren Gjessec425e6a2019-06-28 11:41:14 +0200166 # Collect license name and url for each artifact. They must come in
167 # pairs for each artifact.
168 if trimmed.startswith(artifact_prefix):
169 assert not name
170 assert not url
Mads Agera9745612017-11-02 12:42:15 +0100171 if trimmed.startswith(license_prefix):
Mads Agera9745612017-11-02 12:42:15 +0100172 name = trimmed[len(license_prefix):]
Mads Agera9745612017-11-02 12:42:15 +0100173 if trimmed.startswith(license_url_prefix):
174 url = trimmed[len(license_url_prefix):]
Søren Gjessec425e6a2019-06-28 11:41:14 +0200175 # Licenses come in name/url pairs. When both are present add pair
176 # to collected licenses if either name or url has not been recorded yet,
177 # as some licenses with slightly different names point to the same url.
178 if name and url:
179 if (not name in licenses) or (not url in license_urls):
180 licenses.append(name)
Mads Agera9745612017-11-02 12:42:15 +0100181 license_urls.append(url)
Søren Gjessec425e6a2019-06-28 11:41:14 +0200182 name = None
183 url = None
184 assert len(licenses) == len(license_urls)
Mads Agera9745612017-11-02 12:42:15 +0100185 result = ''
186 for i in range(len(licenses)):
187 name = licenses[i]
188 url = license_urls[i]
189 result += LICENSETEMPLATE.substitute(name=name, url=url)
190 return result
191
Rico Wind257044c2019-11-22 08:21:21 +0100192def write_default_r8_pom_file(pom_file, version):
Rico Winddf585e82023-09-14 09:16:49 +0200193 write_pom_file(R8_POMTEMPLATE, pom_file, version)
Rico Wind257044c2019-11-22 08:21:21 +0100194
Søren Gjesse2b047692022-08-19 16:34:38 +0200195def write_pom_file(
Rico Windd6323ad2023-09-14 09:33:59 +0200196 template, pom_file, version, artifact_id=None, library_licenses=''):
Søren Gjesse2b047692022-08-19 16:34:38 +0200197 version_pom = (
198 template.substitute(
199 artifactId=artifact_id,
200 version=version,
Søren Gjesse2b047692022-08-19 16:34:38 +0200201 library_licenses=library_licenses)
202 if artifact_id else
203 template.substitute(
Rico Windd6323ad2023-09-14 09:33:59 +0200204 version=version, library_licenses=library_licenses))
Mads Agera9745612017-11-02 12:42:15 +0100205 with open(pom_file, 'w') as file:
206 file.write(version_pom)
207
208def hash_for(file, hash):
209 with open(file, 'rb') as f:
210 while True:
211 # Read chunks of 1MB
212 chunk = f.read(2 ** 20)
213 if not chunk:
214 break
215 hash.update(chunk)
216 return hash.hexdigest()
217
218def write_md5_for(file):
219 hexdigest = hash_for(file, hashlib.md5())
220 with (open(file + '.md5', 'w')) as file:
221 file.write(hexdigest)
222
223def write_sha1_for(file):
224 hexdigest = hash_for(file, hashlib.sha1())
225 with (open(file + '.sha1', 'w')) as file:
226 file.write(hexdigest)
227
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200228def generate_maven_zip(name, version, pom_file, jar_file, out):
Mads Ager14d9b072017-11-20 13:42:55 +0100229 with utils.TempDir() as tmp_dir:
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200230 # Create the base maven version directory
231 version_dir = join(tmp_dir, utils.get_maven_path(name, version))
Mads Ager14d9b072017-11-20 13:42:55 +0100232 makedirs(version_dir)
233 # Write the pom file.
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200234 pom_file_location = join(version_dir, name + '-' + version + '.pom')
235 copyfile(pom_file, pom_file_location)
236 # Write the jar file.
237 jar_file_location = join(version_dir, name + '-' + version + '.jar')
238 copyfile(jar_file, jar_file_location)
Mads Ager14d9b072017-11-20 13:42:55 +0100239 # Create check sums.
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200240 write_md5_for(jar_file_location)
241 write_md5_for(pom_file_location)
242 write_sha1_for(jar_file_location)
243 write_sha1_for(pom_file_location)
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100244 # Zip it up - make_archive will append zip to the file, so remove.
245 assert out.endswith('.zip')
246 base_no_zip = out[0:len(out)-4]
247 make_archive(base_no_zip, 'zip', tmp_dir)
248
Rico Wind293d6e02023-09-14 15:15:47 +0200249def generate_r8_maven_zip(out, version_file=None, skip_gradle_build=False,
250 new_gradle=False):
Søren Gjesse1b035b32022-08-19 08:53:57 +0200251 if not skip_gradle_build:
Rico Wind293d6e02023-09-14 15:15:47 +0200252 if (new_gradle):
Rico Wind49f16fe2023-09-15 11:13:23 +0200253 gradle.RunGradle([':test:r8LibWithRelocatedDeps',
254 '-Pno_internal'], new_gradle=True)
Rico Wind293d6e02023-09-14 15:15:47 +0200255 else:
256 gradle.RunGradle([utils.R8LIB, '-Pno_internal'])
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200257
258 version = determine_version()
Rico Wind52ece1b2020-08-25 14:33:17 +0200259 with utils.TempDir() as tmp_dir:
Rico Windcdc39b62022-04-08 12:37:57 +0200260 file_copy = join(tmp_dir, 'copy_of_jar.jar')
Rico Windd6323ad2023-09-14 09:33:59 +0200261 copyfile(utils.R8LIB_JAR, file_copy)
Rico Windcdc39b62022-04-08 12:37:57 +0200262
263 if version_file:
264 with zipfile.ZipFile(file_copy, 'a') as zip:
265 zip.write(version_file, basename(version_file))
266
Rico Wind52ece1b2020-08-25 14:33:17 +0200267 # Generate the pom file.
268 pom_file = join(tmp_dir, 'r8.pom')
269 write_pom_file(
270 R8_POMTEMPLATE,
271 pom_file,
272 version,
Rico Windd6323ad2023-09-14 09:33:59 +0200273 library_licenses=generate_library_licenses())
Rico Wind52ece1b2020-08-25 14:33:17 +0200274 # Write the maven zip file.
275 generate_maven_zip(
276 'r8',
277 version,
278 pom_file,
Rico Windcdc39b62022-04-08 12:37:57 +0200279 file_copy,
Rico Wind52ece1b2020-08-25 14:33:17 +0200280 out)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200281
282# Write the desugaring configuration of a jar file with the following content:
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100283# java/
284# util/
285# <java.util conversions classes>
286# time/
287# <java.time conversions classes>
288# META-INF/
289# desugar/
290# d8/
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200291# desugar.json
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100292# lint/
293# <lint files>
Søren Gjessed98f24c2020-10-30 12:42:31 +0100294def generate_jar_with_desugar_configuration(
295 configuration, implementation, conversions, destination):
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200296 with utils.TempDir() as tmp_dir:
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100297 # Add conversion classes.
298 with zipfile.ZipFile(conversions, 'r') as conversions_zip:
299 conversions_zip.extractall(tmp_dir)
300
Søren Gjesse1a8ccad2023-06-06 15:00:46 +0200301 # Add configuration.
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200302 configuration_dir = join(tmp_dir, 'META-INF', 'desugar', 'd8')
303 makedirs(configuration_dir)
304 copyfile(configuration, join(configuration_dir, 'desugar.json'))
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200305
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100306 # Add lint configuartion.
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200307 lint_dir = join(configuration_dir, 'lint')
308 makedirs(lint_dir)
309 cmd = [
310 jdk.GetJavaExecutable(),
311 '-cp',
312 utils.R8_JAR,
Clément Béra9ef8f842023-05-01 09:16:50 +0200313 'com.android.tools.r8.ir.desugar.desugaredlibrary.lint.GenerateDesugaredLibraryLintFiles',
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200314 configuration,
Søren Gjessed98f24c2020-10-30 12:42:31 +0100315 implementation,
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200316 lint_dir]
317 utils.PrintCmd(cmd)
318 subprocess.check_call(cmd)
319
Søren Gjesse1a8ccad2023-06-06 15:00:46 +0200320 # Add LICENSE file.
321 copyfile(join(utils.REPO_ROOT, 'LICENSE'), join(tmp_dir, 'LICENSE'))
322
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200323 make_archive(destination, 'zip', tmp_dir)
324 move(destination + '.zip', destination)
325
Søren Gjesse79298102022-08-23 16:25:41 +0200326def convert_desugar_configuration(
327 configuration, conversions, implementation, machine_configuration):
Søren Gjesse2b047692022-08-19 16:34:38 +0200328 cmd = [jdk.GetJavaExecutable(),
329 '-cp',
330 utils.R8_JAR,
331 'com.android.tools.r8.ir.desugar.desugaredlibrary.specificationconversion.DesugaredLibraryConverter',
332 configuration,
Søren Gjesse79298102022-08-23 16:25:41 +0200333 implementation,
334 conversions,
Søren Gjesse2b047692022-08-19 16:34:38 +0200335 utils.get_android_jar(33),
336 machine_configuration]
337 subprocess.check_call(cmd)
338
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200339# Generate the maven zip for the configuration to desugar desugar_jdk_libs.
Søren Gjessee18fa6e2022-06-24 15:14:53 +0200340def generate_desugar_configuration_maven_zip(
341 out, configuration, implementation, conversions):
Rico Wind52ece1b2020-08-25 14:33:17 +0200342 with utils.TempDir() as tmp_dir:
Søren Gjesse2b047692022-08-19 16:34:38 +0200343 (name, version) = utils.desugar_configuration_name_and_version(configuration, False)
344
345 if (not version.startswith("1.")):
346 machine_configuration = join(tmp_dir, "machine.json")
Søren Gjesse79298102022-08-23 16:25:41 +0200347 convert_desugar_configuration(configuration, conversions, implementation, machine_configuration)
Søren Gjesse2b047692022-08-19 16:34:38 +0200348 configuration = machine_configuration
349
Rico Wind52ece1b2020-08-25 14:33:17 +0200350 # Generate the pom file.
351 pom_file = join(tmp_dir, 'desugar_configuration.pom')
Søren Gjesse2b047692022-08-19 16:34:38 +0200352 write_pom_file(DESUGAR_CONFIGUATION_POMTEMPLATE, pom_file, version, artifact_id=name)
Rico Wind52ece1b2020-08-25 14:33:17 +0200353 # Generate the jar with the configuration file.
354 jar_file = join(tmp_dir, 'desugar_configuration.jar')
355 generate_jar_with_desugar_configuration(
Søren Gjesse705a3b12022-03-17 11:37:30 +0100356 configuration,
357 implementation,
Søren Gjessee18fa6e2022-06-24 15:14:53 +0200358 conversions,
Rico Wind52ece1b2020-08-25 14:33:17 +0200359 jar_file)
360 # Write the maven zip file.
Søren Gjesse2b047692022-08-19 16:34:38 +0200361 generate_maven_zip(name, version, pom_file, jar_file, out)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200362
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100363def main(argv):
364 options = parse_options(argv)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200365 if options.out == None:
366 raise Exception(
367 'Need to supply output zip with --out.')
Søren Gjesse705a3b12022-03-17 11:37:30 +0100368 if options.desugar_configuration or options.desugar_configuration_jdk8:
369 generate_desugar_configuration_maven_zip(
Søren Gjesse1a8ccad2023-06-06 15:00:46 +0200370 options.out,
371 utils.DESUGAR_CONFIGURATION,
372 utils.DESUGAR_IMPLEMENTATION,
373 utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
Søren Gjesse705a3b12022-03-17 11:37:30 +0100374 elif options.desugar_configuration_jdk11_legacy:
375 generate_desugar_configuration_maven_zip(
Søren Gjesse1a8ccad2023-06-06 15:00:46 +0200376 options.out,
377 utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
378 utils.DESUGAR_IMPLEMENTATION_JDK11,
379 utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
380 elif options.desugar_configuration_jdk11_minimal:
381 generate_desugar_configuration_maven_zip(
382 options.out,
383 utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL,
384 utils.DESUGAR_IMPLEMENTATION_JDK11,
385 utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
386 elif options.desugar_configuration_jdk11:
387 generate_desugar_configuration_maven_zip(
388 options.out,
389 utils.DESUGAR_CONFIGURATION_JDK11,
390 utils.DESUGAR_IMPLEMENTATION_JDK11,
391 utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
392 elif options.desugar_configuration_jdk11_nio:
393 generate_desugar_configuration_maven_zip(
394 options.out,
395 utils.DESUGAR_CONFIGURATION_JDK11_NIO,
396 utils.DESUGAR_IMPLEMENTATION_JDK11,
397 utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200398 else:
Rico Windd6323ad2023-09-14 09:33:59 +0200399 generate_r8_maven_zip(options.out)
Mads Agera9745612017-11-02 12:42:15 +0100400
401if __name__ == "__main__":
402 exit(main(sys.argv[1:]))