blob: 7d73a93494917df648040a77b3113270946aabf7 [file] [log] [blame]
Ian Zernydcb172e2022-02-22 15:36:45 +01001#!/usr/bin/env python3
Mads Agera9745612017-11-02 12:42:15 +01002# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
3# for details. All rights reserved. Use of this source code is governed by a
4# BSD-style license that can be found in the LICENSE file.
5
Mads Agera9745612017-11-02 12:42:15 +01006import argparse
Mads Agera4911eb2017-11-22 13:19:36 +01007import gradle
8import hashlib
Søren Gjessea70d3bd2019-09-24 15:07:00 +02009import jdk
Søren Gjesse6e5e5842019-09-03 08:48:30 +020010import json
Mads Agera9745612017-11-02 12:42:15 +010011from os import makedirs
Rico Windcdc39b62022-04-08 12:37:57 +020012from os.path import join, basename
Søren Gjesse6e5e5842019-09-03 08:48:30 +020013from shutil import copyfile, make_archive, move, rmtree
Mads Agera9745612017-11-02 12:42:15 +010014import subprocess
15import sys
16from string import Template
17import tempfile
Mads Ager0fab4912017-11-20 13:52:48 +010018import utils
Søren Gjesse17fc67d2019-12-04 14:50:17 +010019import zipfile
Mads Agera9745612017-11-02 12:42:15 +010020
Mads Agera4911eb2017-11-22 13:19:36 +010021DEPENDENCYTEMPLATE = Template(
Mads Agera9745612017-11-02 12:42:15 +010022"""
Mads Agera4911eb2017-11-22 13:19:36 +010023 <dependency>
24 <groupId>$group</groupId>
25 <artifactId>$artifact</artifactId>
26 <version>$version</version>
27 </dependency>""")
Mads Agera9745612017-11-02 12:42:15 +010028
Søren Gjessec425e6a2019-06-28 11:41:14 +020029LICENSETEMPLATE = Template(
30"""
31 <license>
32 <name>$name</name>
33 <url>$url</url>
34 <distribution>repo</distribution>
35 </license>""")
36
Søren Gjesse6e5e5842019-09-03 08:48:30 +020037R8_POMTEMPLATE = Template(
Mads Agera9745612017-11-02 12:42:15 +010038"""<project
39 xmlns="http://maven.apache.org/POM/4.0.0"
40 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
41 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
42 <modelVersion>4.0.0</modelVersion>
43 <groupId>com.android.tools</groupId>
44 <artifactId>r8</artifactId>
45 <version>$version</version>
46 <name>D8 dexer and R8 shrinker</name>
47 <description>
48 D8 dexer and R8 shrinker.
49 </description>
50 <url>http://r8.googlesource.com/r8</url>
51 <inceptionYear>2016</inceptionYear>
52 <licenses>
53 <license>
54 <name>BSD-3-Clause</name>
55 <url>https://opensource.org/licenses/BSD-3-Clause</url>
56 <distribution>repo</distribution>
Søren Gjessec425e6a2019-06-28 11:41:14 +020057 </license>$library_licenses
Mads Ager73460192017-11-08 10:02:50 +010058 </licenses>
Mads Agera4911eb2017-11-22 13:19:36 +010059 <dependencies>$dependencies
60 </dependencies>
Mads Agera9745612017-11-02 12:42:15 +010061 <developers>
62 <developer>
63 <name>The Android Open Source Project</name>
64 </developer>
65 </developers>
66 <scm>
67 <connection>
68 https://r8.googlesource.com/r8.git
69 </connection>
70 <url>
71 https://r8.googlesource.com/r8
72 </url>
73 </scm>
74</project>
75""")
76
Søren Gjesse6e5e5842019-09-03 08:48:30 +020077DESUGAR_CONFIGUATION_POMTEMPLATE = Template(
78"""<project
79 xmlns="http://maven.apache.org/POM/4.0.0"
80 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
81 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
82 <modelVersion>4.0.0</modelVersion>
83 <groupId>com.android.tools</groupId>
Søren Gjesse2b047692022-08-19 16:34:38 +020084 <artifactId>$artifactId</artifactId>
Søren Gjesse6e5e5842019-09-03 08:48:30 +020085 <version>$version</version>
86 <name>D8 configuration to desugar desugar_jdk_libs</name>
87 <description>
88 D8 configuration to desugar desugar_jdk_libs.
89 </description>
90 <url>http://r8.googlesource.com/r8</url>
91 <inceptionYear>2019</inceptionYear>
92 <licenses>
93 <license>
94 <name>BSD-3-Clause</name>
95 <url>https://opensource.org/licenses/BSD-3-Clause</url>
96 <distribution>repo</distribution>
97 </license>
98 </licenses>
Søren Gjesse6e5e5842019-09-03 08:48:30 +020099 <developers>
100 <developer>
101 <name>The Android Open Source Project</name>
102 </developer>
103 </developers>
104 <scm>
105 <connection>
106 https://r8.googlesource.com/r8.git
107 </connection>
108 <url>
109 https://r8.googlesource.com/r8
110 </url>
111 </scm>
112</project>
113""")
114
Mads Agera9745612017-11-02 12:42:15 +0100115def parse_options(argv):
116 result = argparse.ArgumentParser()
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100117 result.add_argument('--out', help='The zip file to output')
Søren Gjesse705a3b12022-03-17 11:37:30 +0100118 group = result.add_mutually_exclusive_group()
119 group.add_argument('--r8lib', action='store_true',
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100120 help='Build r8 with dependencies included shrunken')
Søren Gjesse705a3b12022-03-17 11:37:30 +0100121 group.add_argument('--desugar-configuration', action='store_true',
122 help='Build desugar library configuration (original JDK-8)')
123 group.add_argument('--desugar-configuration-jdk8', action='store_true',
124 help='Build desugar library configuration (original JDK-8)')
125 group.add_argument('--desugar-configuration-jdk11-legacy', action='store_true',
126 help='Build desugar library configuration (JDK-11 legacy)')
Mads Agera9745612017-11-02 12:42:15 +0100127 return result.parse_args(argv)
128
Mads Agera4911eb2017-11-22 13:19:36 +0100129def determine_version():
130 version_file = join(
131 utils.SRC_ROOT, 'com', 'android', 'tools', 'r8', 'Version.java')
132 with open(version_file, 'r') as file:
133 for line in file:
134 if 'final String LABEL ' in line:
Rico Wind6feccf22018-05-08 11:18:22 +0200135 result = line[line.find('"') + 1:]
Mads Agera4911eb2017-11-22 13:19:36 +0100136 result = result[:result.find('"')]
137 return result
138 raise Exception('Unable to determine version.')
Mads Agera9745612017-11-02 12:42:15 +0100139
140def generate_library_licenses():
Søren Gjessec425e6a2019-06-28 11:41:14 +0200141 artifact_prefix = '- artifact: '
Mads Agera9745612017-11-02 12:42:15 +0100142 license_prefix = 'license: '
143 licenses = []
144 license_url_prefix = 'licenseUrl: '
145 license_urls = []
Søren Gjesseea678302019-08-27 14:41:03 +0200146 # The ./LIBRARY-LICENSE file is a simple yaml file, which for each dependency
147 # has the following information:
148 #
149 # - artifact: <maven artifact> // in the form <group-id>:<artifact-id>:+
150 # name: <name of dependency>
151 # copyrightHolder: <name of copyright holder>
152 # license: <license name>
153 # licenseUrl: <url to license test>
154 #
155 # E.g. for Guava:
156 #
157 # - artifact: com.google.guava:guava:+
158 # name: Guava Google Core Libraries for Java
159 # copyrightHolder: The Guava Authors
160 # license: The Apache Software License, Version 2.0
161 # licenseUrl: http://www.apache.org/licenses/LICENSE-2.0.txt
162 #
163 # This file should always be up to date as the build will fail if it
164 # is does not have information for all dependencies.
Mads Agera9745612017-11-02 12:42:15 +0100165 with open('LIBRARY-LICENSE', 'r') as file:
Søren Gjessec425e6a2019-06-28 11:41:14 +0200166 name = None
167 url = None
Mads Agera9745612017-11-02 12:42:15 +0100168 for line in file:
169 trimmed = line.strip()
Søren Gjessec425e6a2019-06-28 11:41:14 +0200170 # Collect license name and url for each artifact. They must come in
171 # pairs for each artifact.
172 if trimmed.startswith(artifact_prefix):
173 assert not name
174 assert not url
Mads Agera9745612017-11-02 12:42:15 +0100175 if trimmed.startswith(license_prefix):
Mads Agera9745612017-11-02 12:42:15 +0100176 name = trimmed[len(license_prefix):]
Mads Agera9745612017-11-02 12:42:15 +0100177 if trimmed.startswith(license_url_prefix):
178 url = trimmed[len(license_url_prefix):]
Søren Gjessec425e6a2019-06-28 11:41:14 +0200179 # Licenses come in name/url pairs. When both are present add pair
180 # to collected licenses if either name or url has not been recorded yet,
181 # as some licenses with slightly different names point to the same url.
182 if name and url:
183 if (not name in licenses) or (not url in license_urls):
184 licenses.append(name)
Mads Agera9745612017-11-02 12:42:15 +0100185 license_urls.append(url)
Søren Gjessec425e6a2019-06-28 11:41:14 +0200186 name = None
187 url = None
188 assert len(licenses) == len(license_urls)
Mads Agera9745612017-11-02 12:42:15 +0100189 result = ''
190 for i in range(len(licenses)):
191 name = licenses[i]
192 url = license_urls[i]
193 result += LICENSETEMPLATE.substitute(name=name, url=url)
194 return result
195
Mads Agera4911eb2017-11-22 13:19:36 +0100196
197# Generate the dependencies block for the pom file.
198#
199# We ask gradle to list all dependencies. In that output
200# we locate the runtimeClasspath block for 'main' which
201# looks something like:
202#
203# runtimeClasspath - Runtime classpath of source set 'main'.
204# +--- net.sf.jopt-simple:jopt-simple:4.6
205# +--- com.googlecode.json-simple:json-simple:1.1
206# +--- com.google.guava:guava:23.0
207# +--- it.unimi.dsi:fastutil:7.2.0
208# +--- org.ow2.asm:asm:6.0
209# +--- org.ow2.asm:asm-commons:6.0
210# | \--- org.ow2.asm:asm-tree:6.0
211# | \--- org.ow2.asm:asm:6.0
212# +--- org.ow2.asm:asm-tree:6.0 (*)
213# +--- org.ow2.asm:asm-analysis:6.0
214# | \--- org.ow2.asm:asm-tree:6.0 (*)
215# \--- org.ow2.asm:asm-util:6.0
216# \--- org.ow2.asm:asm-tree:6.0 (*)
217#
218# We filter out the repeats that are marked by '(*)'.
219#
220# For each remaining line, we remove the junk at the start
221# in chunks. As an example:
222#
223# ' | \--- org.ow2.asm:asm-tree:6.0 ' --strip-->
224# '| \--- org.ow2.asm:asm-tree:6.0' -->
225# '\--- org.ow2.asm:asm-tree:6.0' -->
226# 'org.ow2.asm:asm-tree:6.0'
227#
228# The end result is the dependency we are looking for:
229#
230# groupId: org.ow2.asm
231# artifact: asm-tree
232# version: 6.0
233def generate_dependencies():
234 dependencies = gradle.RunGradleGetOutput(['dependencies'])
235 dependency_lines = []
236 collect = False
237 for line in dependencies.splitlines():
Rico Wind5f2d5a72022-02-22 14:23:37 +0100238 if line and 'runtimeClasspath' in line and "'main'" in line:
Mads Agera4911eb2017-11-22 13:19:36 +0100239 collect = True
240 continue
241 if collect:
242 if not len(line) == 0:
243 if not '(*)' in line:
244 trimmed = line.strip()
245 while trimmed.find(' ') != -1:
246 trimmed = trimmed[trimmed.find(' ') + 1:].strip()
247 if not trimmed in dependency_lines:
248 dependency_lines.append(trimmed)
249 else:
250 break
251 result = ''
252 for dep in dependency_lines:
253 components = dep.split(':')
254 assert len(components) == 3
255 group = components[0]
256 artifact = components[1]
257 version = components[2]
258 result += DEPENDENCYTEMPLATE.substitute(
259 group=group, artifact=artifact, version=version)
260 return result
261
Rico Wind257044c2019-11-22 08:21:21 +0100262def write_default_r8_pom_file(pom_file, version):
Søren Gjesse2b047692022-08-19 16:34:38 +0200263 write_pom_file(R8_POMTEMPLATE, pom_file, version, dependencies=generate_dependencies())
Rico Wind257044c2019-11-22 08:21:21 +0100264
Søren Gjesse2b047692022-08-19 16:34:38 +0200265def write_pom_file(
266 template, pom_file, version, artifact_id=None, dependencies='', library_licenses=''):
267 version_pom = (
268 template.substitute(
269 artifactId=artifact_id,
270 version=version,
271 dependencies=dependencies,
272 library_licenses=library_licenses)
273 if artifact_id else
274 template.substitute(
275 version=version, dependencies=dependencies, library_licenses=library_licenses))
Mads Agera9745612017-11-02 12:42:15 +0100276 with open(pom_file, 'w') as file:
277 file.write(version_pom)
278
279def hash_for(file, hash):
280 with open(file, 'rb') as f:
281 while True:
282 # Read chunks of 1MB
283 chunk = f.read(2 ** 20)
284 if not chunk:
285 break
286 hash.update(chunk)
287 return hash.hexdigest()
288
289def write_md5_for(file):
290 hexdigest = hash_for(file, hashlib.md5())
291 with (open(file + '.md5', 'w')) as file:
292 file.write(hexdigest)
293
294def write_sha1_for(file):
295 hexdigest = hash_for(file, hashlib.sha1())
296 with (open(file + '.sha1', 'w')) as file:
297 file.write(hexdigest)
298
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200299def generate_maven_zip(name, version, pom_file, jar_file, out):
Mads Ager14d9b072017-11-20 13:42:55 +0100300 with utils.TempDir() as tmp_dir:
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200301 # Create the base maven version directory
302 version_dir = join(tmp_dir, utils.get_maven_path(name, version))
Mads Ager14d9b072017-11-20 13:42:55 +0100303 makedirs(version_dir)
304 # Write the pom file.
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200305 pom_file_location = join(version_dir, name + '-' + version + '.pom')
306 copyfile(pom_file, pom_file_location)
307 # Write the jar file.
308 jar_file_location = join(version_dir, name + '-' + version + '.jar')
309 copyfile(jar_file, jar_file_location)
Mads Ager14d9b072017-11-20 13:42:55 +0100310 # Create check sums.
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200311 write_md5_for(jar_file_location)
312 write_md5_for(pom_file_location)
313 write_sha1_for(jar_file_location)
314 write_sha1_for(pom_file_location)
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100315 # Zip it up - make_archive will append zip to the file, so remove.
316 assert out.endswith('.zip')
317 base_no_zip = out[0:len(out)-4]
318 make_archive(base_no_zip, 'zip', tmp_dir)
319
Søren Gjesse1b035b32022-08-19 08:53:57 +0200320def generate_r8_maven_zip(out, is_r8lib=False, version_file=None, skip_gradle_build=False):
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200321 # Build the R8 no deps artifact.
Søren Gjesse1b035b32022-08-19 08:53:57 +0200322 if not skip_gradle_build:
323 if not is_r8lib:
324 gradle.RunGradleExcludeDeps([utils.R8])
325 else:
326 gradle.RunGradle([utils.R8LIB, '-Pno_internal'])
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200327
328 version = determine_version()
Rico Wind52ece1b2020-08-25 14:33:17 +0200329 with utils.TempDir() as tmp_dir:
Rico Windcdc39b62022-04-08 12:37:57 +0200330 file_copy = join(tmp_dir, 'copy_of_jar.jar')
331 copyfile(utils.R8LIB_JAR if is_r8lib else utils.R8_JAR, file_copy)
332
333 if version_file:
334 with zipfile.ZipFile(file_copy, 'a') as zip:
335 zip.write(version_file, basename(version_file))
336
Rico Wind52ece1b2020-08-25 14:33:17 +0200337 # Generate the pom file.
338 pom_file = join(tmp_dir, 'r8.pom')
339 write_pom_file(
340 R8_POMTEMPLATE,
341 pom_file,
342 version,
Søren Gjesse2b047692022-08-19 16:34:38 +0200343 dependencies='' if is_r8lib else generate_dependencies(),
344 library_licenses=generate_library_licenses() if is_r8lib else '')
Rico Wind52ece1b2020-08-25 14:33:17 +0200345 # Write the maven zip file.
346 generate_maven_zip(
347 'r8',
348 version,
349 pom_file,
Rico Windcdc39b62022-04-08 12:37:57 +0200350 file_copy,
Rico Wind52ece1b2020-08-25 14:33:17 +0200351 out)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200352
353# Write the desugaring configuration of a jar file with the following content:
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100354# java/
355# util/
356# <java.util conversions classes>
357# time/
358# <java.time conversions classes>
359# META-INF/
360# desugar/
361# d8/
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200362# desugar.json
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100363# lint/
364# <lint files>
Søren Gjessed98f24c2020-10-30 12:42:31 +0100365def generate_jar_with_desugar_configuration(
366 configuration, implementation, conversions, destination):
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200367 with utils.TempDir() as tmp_dir:
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100368 # Add conversion classes.
369 with zipfile.ZipFile(conversions, 'r') as conversions_zip:
370 conversions_zip.extractall(tmp_dir)
371
372 # Add configuration
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200373 configuration_dir = join(tmp_dir, 'META-INF', 'desugar', 'd8')
374 makedirs(configuration_dir)
375 copyfile(configuration, join(configuration_dir, 'desugar.json'))
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200376
Søren Gjesse17fc67d2019-12-04 14:50:17 +0100377 # Add lint configuartion.
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200378 lint_dir = join(configuration_dir, 'lint')
379 makedirs(lint_dir)
380 cmd = [
381 jdk.GetJavaExecutable(),
382 '-cp',
383 utils.R8_JAR,
Clément Béra5ebd53e2022-08-02 14:05:22 +0200384 'com.android.tools.r8.ir.desugar.desugaredlibrary.lint.GenerateLintFiles',
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200385 configuration,
Søren Gjessed98f24c2020-10-30 12:42:31 +0100386 implementation,
Søren Gjessea70d3bd2019-09-24 15:07:00 +0200387 lint_dir]
388 utils.PrintCmd(cmd)
389 subprocess.check_call(cmd)
390
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200391 make_archive(destination, 'zip', tmp_dir)
392 move(destination + '.zip', destination)
393
Søren Gjesse79298102022-08-23 16:25:41 +0200394def convert_desugar_configuration(
395 configuration, conversions, implementation, machine_configuration):
Søren Gjesse2b047692022-08-19 16:34:38 +0200396 cmd = [jdk.GetJavaExecutable(),
397 '-cp',
398 utils.R8_JAR,
399 'com.android.tools.r8.ir.desugar.desugaredlibrary.specificationconversion.DesugaredLibraryConverter',
400 configuration,
Søren Gjesse79298102022-08-23 16:25:41 +0200401 implementation,
402 conversions,
Søren Gjesse2b047692022-08-19 16:34:38 +0200403 utils.get_android_jar(33),
404 machine_configuration]
405 subprocess.check_call(cmd)
406
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200407# Generate the maven zip for the configuration to desugar desugar_jdk_libs.
Søren Gjessee18fa6e2022-06-24 15:14:53 +0200408def generate_desugar_configuration_maven_zip(
409 out, configuration, implementation, conversions):
Rico Wind52ece1b2020-08-25 14:33:17 +0200410 with utils.TempDir() as tmp_dir:
Søren Gjesse2b047692022-08-19 16:34:38 +0200411 (name, version) = utils.desugar_configuration_name_and_version(configuration, False)
412
413 if (not version.startswith("1.")):
414 machine_configuration = join(tmp_dir, "machine.json")
Søren Gjesse79298102022-08-23 16:25:41 +0200415 convert_desugar_configuration(configuration, conversions, implementation, machine_configuration)
Søren Gjesse2b047692022-08-19 16:34:38 +0200416 configuration = machine_configuration
417
Rico Wind52ece1b2020-08-25 14:33:17 +0200418 # Generate the pom file.
419 pom_file = join(tmp_dir, 'desugar_configuration.pom')
Søren Gjesse2b047692022-08-19 16:34:38 +0200420 write_pom_file(DESUGAR_CONFIGUATION_POMTEMPLATE, pom_file, version, artifact_id=name)
Rico Wind52ece1b2020-08-25 14:33:17 +0200421 # Generate the jar with the configuration file.
422 jar_file = join(tmp_dir, 'desugar_configuration.jar')
423 generate_jar_with_desugar_configuration(
Søren Gjesse705a3b12022-03-17 11:37:30 +0100424 configuration,
425 implementation,
Søren Gjessee18fa6e2022-06-24 15:14:53 +0200426 conversions,
Rico Wind52ece1b2020-08-25 14:33:17 +0200427 jar_file)
428 # Write the maven zip file.
Søren Gjesse2b047692022-08-19 16:34:38 +0200429 generate_maven_zip(name, version, pom_file, jar_file, out)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200430
Rico Wind8fc8bfa2019-03-22 09:57:36 +0100431def main(argv):
432 options = parse_options(argv)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200433 if options.out == None:
434 raise Exception(
435 'Need to supply output zip with --out.')
Søren Gjesse705a3b12022-03-17 11:37:30 +0100436 if options.desugar_configuration or options.desugar_configuration_jdk8:
437 generate_desugar_configuration_maven_zip(
438 options.out, utils.DESUGAR_CONFIGURATION, utils.DESUGAR_IMPLEMENTATION)
439 elif options.desugar_configuration_jdk11_legacy:
440 generate_desugar_configuration_maven_zip(
441 options.out, utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
442 utils.DESUGAR_IMPLEMENTATION_JDK11)
Søren Gjesse6e5e5842019-09-03 08:48:30 +0200443 else:
444 generate_r8_maven_zip(options.out, options.r8lib)
Mads Agera9745612017-11-02 12:42:15 +0100445
446if __name__ == "__main__":
447 exit(main(sys.argv[1:]))