| #!/usr/bin/env python3 | |
| # Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file | |
| # for details. All rights reserved. Use of this source code is governed by a | |
| # BSD-style license that can be found in the LICENSE file. | |
| import jdk | |
| import optparse | |
| import os | |
| import create_maven_release | |
| import gradle | |
| from timing import Timing | |
| try: | |
| import resource | |
| except ImportError: | |
| # Not a Unix system. Do what Gandalf tells you not to. | |
| pass | |
| import json | |
| import luci_utils | |
| import shutil | |
| import subprocess | |
| import sys | |
| import utils | |
| import zipfile | |
| ARCHIVE_BUCKET = 'r8-releases' | |
| def parse_options(): | |
| result = optparse.OptionParser() | |
| result.add_option('--dry-run', | |
| '--dry_run', | |
| help='Build only, no upload.', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--dry-run-output', | |
| '--dry_run_output', | |
| help='Output directory for \'build only, no upload\'.', | |
| type="string", | |
| action="store") | |
| result.add_option( | |
| '--skip-gradle-build', | |
| '--skip_gradle_build', | |
| help='Skip Gradle build. Can only be used for local testing.', | |
| default=False, | |
| action='store_true') | |
| result.add_option('--archive-details', | |
| '--archive_details', | |
| help='File to store JSON with archive details.', | |
| type="string", | |
| default=utils.ARCHIVE_DETAILS, | |
| action="store") | |
| return result.parse_args() | |
| def get_version(): | |
| output = subprocess.check_output([ | |
| jdk.GetJavaExecutable(jdk.GetDefaultJdkHome()), '-cp', utils.R8_JAR, | |
| 'com.android.tools.r8.R8', '--version' | |
| ]).decode('utf-8') | |
| r8_version = output.splitlines()[0].strip() | |
| return r8_version.split()[1] | |
| def get_git_branches(): | |
| return subprocess.check_output(['git', 'show', '-s', '--pretty=%d', 'HEAD']) | |
| def get_git_hash(): | |
| return subprocess.check_output(['git', 'rev-parse', | |
| 'HEAD']).decode('utf-8').strip() | |
| def check_main(version): | |
| branches = subprocess.check_output( | |
| ['git', 'branch', '-r', '--contains', 'HEAD']).decode('utf-8') | |
| # CL runs from gerrit does not have a branch, we always treat them as main | |
| # commits to archive these to the hash based location | |
| if len(branches) == 0: | |
| return True | |
| if not version == 'main': | |
| # Sanity check, we don't want to archive on top of release builds EVER | |
| # Note that even though we branch, we never push the bots to build the same | |
| # commit as main on a branch since we always change the version to | |
| # not be just 'main' (or we crash here :-)). | |
| if 'origin/main' in branches: | |
| raise Exception('We are seeing origin/main in a commit that ' | |
| 'don\'t have \'main\' as version') | |
| return False | |
| if not 'origin/main' in branches: | |
| raise Exception('We are not seeing origin/main ' | |
| 'in a commit that have \'main\' as version') | |
| return True | |
| def get_storage_path(version_or_path, is_main): | |
| archive_dir = 'raw/main' if is_main else 'raw' | |
| return '%s/%s' % (archive_dir, version_or_path) | |
| def get_storage_destination(storage_prefix, version_or_path, file_name, | |
| is_main): | |
| # We archive main commits under raw/main instead of directly under raw | |
| version_dir = get_version_destination(storage_prefix, version_or_path, | |
| is_main) | |
| return '%s/%s' % (version_dir, file_name) | |
| def get_version_destination(storage_prefix, version_or_path, is_main): | |
| storage_path = get_storage_path(version_or_path, is_main) | |
| return '%s%s/%s' % (storage_prefix, ARCHIVE_BUCKET, storage_path) | |
| def get_upload_destination(version_or_path, file_name, is_main): | |
| return get_storage_destination('gs://', version_or_path, file_name, is_main) | |
| def get_url(version_or_path, file_name, is_main): | |
| return get_storage_destination(utils.CLOUD_STORAGE_URL + '/', | |
| version_or_path, file_name, is_main) | |
| def get_maven_url(is_main): | |
| return get_version_destination(utils.CLOUD_STORAGE_URL + '/', '', is_main) | |
| def set_r_limit_to_max(): | |
| if not utils.is_bot() or utils.IsWindows(): | |
| return | |
| (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) | |
| resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) | |
| def print_resource_info(): | |
| if utils.IsWindows(): | |
| return | |
| (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) | |
| print('INFO: Open files soft limit: %s' % soft) | |
| print('INFO: Open files hard limit: %s' % hard) | |
| # Ensure all archived artifacts has been built before archiving. | |
| # The target tasks postfixed by 'lib' depend on the actual target task so | |
| # building it invokes the original task first. | |
| # The '-Pno_internal' flag is important because we generate the lib based on uses in tests. | |
| def run_gradle_build(options, timing): | |
| if options.skip_gradle_build: | |
| return | |
| timing.begin("Download deps") | |
| gradle.run_gradle([utils.GRADLE_TASK_DOWNLOAD_DEPS, '-Pno_internal']) | |
| timing.end() | |
| timing.begin("Build consolidated license") | |
| gradle.run_gradle([utils.GRADLE_TASK_CONSOLIDATED_LICENSE, '-Pno_internal']) | |
| timing.end() | |
| timing.begin("Build keep anno") | |
| gradle.run_gradle([ | |
| utils.GRADLE_TASK_KEEP_ANNO_JAR, | |
| utils.GRADLE_TASK_KEEP_ANNO_DOC, | |
| utils.GRADLE_TASK_KEEP_ANNO_LEGACY_JAR, | |
| utils.GRADLE_TASK_KEEP_ANNO_ANDROIDX_JAR, | |
| '-Pno_internal', | |
| ]) | |
| timing.end() | |
| timing.begin("Build process keep rules") | |
| gradle.run_gradle([ | |
| utils.GRADLE_TASK_PROCESS_KEEP_RULES_LIB_JAR, | |
| '-Pno_internal', | |
| ]) | |
| timing.end() | |
| timing.begin("Build R8") | |
| gradle.run_gradle([ | |
| utils.GRADLE_TASK_R8, | |
| utils.GRADLE_TASK_R8LIB, | |
| utils.GRADLE_TASK_R8LIB_NO_DEPS, | |
| utils.GRADLE_TASK_THREADING_MODULE_BLOCKING, | |
| utils.GRADLE_TASK_THREADING_MODULE_SINGLE_THREADED, | |
| '-Pno_internal', | |
| ]) | |
| timing.end() | |
| timing.begin("Build source jar") | |
| gradle.run_gradle([utils.GRADLE_TASK_SOURCE_JAR, '-Pno_internal']) | |
| timing.end() | |
| timing.begin("Build swiss army knife") | |
| gradle.run_gradle([utils.GRADLE_TASK_SWISS_ARMY_KNIFE, '-Pno_internal']) | |
| timing.end() | |
| timing.begin("Build keepanno tools") | |
| gradle.run_gradle([ | |
| utils.GRADLE_TASK_KEEPANNOTOOLS, | |
| utils.GRADLE_TASK_KEEPANNOTOOLSLIB, | |
| '-Pno_internal', | |
| ]) | |
| timing.end() | |
| def rsync_dir(src_dir, version_or_path, dst_dir, is_main, options): | |
| destination = get_upload_destination(version_or_path, dst_dir, is_main) | |
| print(f'RSyncing {src_dir} to {destination}') | |
| if options.dry_run: | |
| if options.dry_run_output: | |
| dry_run_destination = os.path.join(options.dry_run_output, | |
| version_or_path, dst_dir) | |
| print( | |
| f'Dry run, not actually syncing. Copying to {dry_run_destination}' | |
| ) | |
| os.makedirs(os.path.dirname(dry_run_destination), exist_ok=True) | |
| shutil.copytree(src_dir, dry_run_destination) | |
| else: | |
| print('Dry run, not actually uploading') | |
| else: | |
| utils.rsync_directory_to_cloud_storage(src_dir, destination) | |
| print( | |
| f'Directory available at: {get_url(version_or_path, dst_dir, is_main)}' | |
| ) | |
| def upload_dir(src_dir, version_or_path, dst_dir, is_main, options): | |
| destination = get_upload_destination(version_or_path, dst_dir, is_main) | |
| print(f'Uploading {src_dir} to {destination}') | |
| if options.dry_run: | |
| if options.dry_run_output: | |
| dry_run_destination = os.path.join(options.dry_run_output, | |
| version_or_path, dst_dir) | |
| print( | |
| f'Dry run, not actually uploading. Copying to {dry_run_destination}' | |
| ) | |
| os.makedirs(os.path.dirname(dry_run_destination), exist_ok=True) | |
| shutil.copytree(src_dir, dry_run_destination) | |
| else: | |
| print('Dry run, not actually uploading') | |
| else: | |
| utils.upload_directory_to_cloud_storage(src_dir, destination) | |
| print( | |
| f'Directory available at: {get_url(version_or_path, dst_dir, is_main)}' | |
| ) | |
| def upload_file(src_file, version_or_path, dst_file, is_main, options): | |
| destination = get_upload_destination(version_or_path, dst_file, is_main) | |
| print(f'Uploading {src_file} to {destination}') | |
| if options.dry_run: | |
| if options.dry_run_output: | |
| dry_run_destination = os.path.join(options.dry_run_output, | |
| version_or_path, dst_file) | |
| print( | |
| f'Dry run, not actually uploading. Copying to {dry_run_destination}' | |
| ) | |
| os.makedirs(os.path.dirname(dry_run_destination), exist_ok=True) | |
| shutil.copyfile(src_file, dry_run_destination) | |
| else: | |
| print('Dry run, not actually uploading') | |
| else: | |
| utils.upload_file_to_cloud_storage(src_file, destination) | |
| print( | |
| f'File available at: {get_url(version_or_path, dst_file, is_main)}') | |
| def main(): | |
| (options, args) = parse_options() | |
| run(options) | |
| def run(options): | |
| if not utils.is_bot() and not options.dry_run: | |
| raise Exception('You are not a bot, don\'t archive builds. ' + | |
| 'Use --dry-run to test locally') | |
| if (options.dry_run_output and | |
| (not os.path.exists(options.dry_run_output) or | |
| not os.path.isdir(options.dry_run_output))): | |
| raise Exception(options.dry_run_output + | |
| ' does not exist or is not a directory') | |
| if options.skip_gradle_build and not options.dry_run: | |
| raise Exception( | |
| 'Using --skip-gradle-build only supported with --dry-run') | |
| timing = Timing() | |
| set_r_limit_to_max() | |
| print_resource_info() | |
| run_gradle_build(options, timing) | |
| version = get_version() | |
| is_main = check_main(version) | |
| if is_main: | |
| # On main we use the git hash to archive with | |
| print('On main, using git hash for archiving') | |
| version = get_git_hash() | |
| with utils.TempDir() as temp: | |
| # Extract the LUCI invocation_id. | |
| luci_invocation_id = luci_utils.get_luci_invocation_id( | |
| force_invocation_id='invocations/fake-12345678' if ( | |
| options.dry_run or 'KOKORO_BUILD_ID' in os.environ) else None) | |
| timing.begin("Generate r8 maven zip") | |
| version_file = os.path.join(temp, 'r8-version.properties') | |
| with open(version_file, 'w') as version_writer: | |
| version_writer.write('version.sha=' + get_git_hash() + '\n') | |
| version_writer.write('version.version=' + version + '\n') | |
| bot_id = os.environ.get('SWARMING_BOT_ID') or os.environ.get( | |
| 'KOKORO_BUILD_ID') | |
| if not bot_id and not options.dry_run: | |
| raise Exception( | |
| 'Environment variable SWARMING_BOT_ID or KOKORO_BUILD_ID not set' | |
| ) | |
| releaser = ("releaser=<local developer build>\n" if | |
| (options.dry_run and not bot_id) else | |
| 'releaser=go/r8bot (' + (bot_id or 'foo') + ')\n') | |
| version_writer.write(releaser) | |
| version_writer.write('luci_invocation_id=' + luci_invocation_id + | |
| '\n') | |
| version_writer.write('version-file.version.code=1\n') | |
| luci_invocation_id_file = os.path.join('luci_invocation_id') | |
| with open(luci_invocation_id_file, 'w') as luci_invocation_id_writer: | |
| luci_invocation_id_writer.write(luci_invocation_id + '\n') | |
| create_maven_release.generate_r8_maven_zip( | |
| utils.MAVEN_ZIP_LIB, | |
| version_file=version_file, | |
| skip_gradle_build=options.skip_gradle_build) | |
| create_maven_release.generate_keepanno_tools_maven_zip( | |
| utils.KEEPANNNO_TOOLS_MAVEN_ZIP_LIB, | |
| version_file=version_file, | |
| skip_gradle_build=options.skip_gradle_build) | |
| timing.end() | |
| # Create maven release of the desugar_jdk_libs configuration. This requires | |
| # an r8.jar with dependencies to have been built. | |
| timing.begin("Generate desugar configuration maven zip") | |
| create_maven_release.generate_desugar_configuration_maven_zip( | |
| utils.DESUGAR_CONFIGURATION_MAVEN_ZIP, utils.DESUGAR_CONFIGURATION, | |
| utils.DESUGAR_IMPLEMENTATION, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP) | |
| create_maven_release.generate_desugar_configuration_maven_zip( | |
| utils.DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_LEGACY, | |
| utils.DESUGAR_IMPLEMENTATION_JDK11, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP) | |
| create_maven_release.generate_desugar_configuration_maven_zip( | |
| utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL, | |
| utils.DESUGAR_IMPLEMENTATION_JDK11, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP) | |
| create_maven_release.generate_desugar_configuration_maven_zip( | |
| utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11, | |
| utils.DESUGAR_IMPLEMENTATION_JDK11, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP) | |
| create_maven_release.generate_desugar_configuration_maven_zip( | |
| utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_NIO, | |
| utils.DESUGAR_IMPLEMENTATION_JDK11, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP) | |
| timing.end() | |
| destination = get_version_destination('gs://', version, is_main) | |
| if utils.cloud_storage_exists(destination) and not options.dry_run: | |
| raise Exception('Target archive directory %s already exists' % | |
| destination) | |
| # Create pom file for our maven repository that we build for testing. | |
| timing.begin("Create pom file") | |
| default_pom_file = os.path.join(temp, 'r8.pom') | |
| create_maven_release.write_default_r8_pom_file(default_pom_file, | |
| version) | |
| gradle.run_gradle([ | |
| ':dist:spdxSbom', '-PspdxVersion=' + version, | |
| '-PspdxRevision=' + get_git_hash() | |
| ]) | |
| timing.end() | |
| # Upload keep-anno javadoc to a fixed "docs" location. | |
| timing.begin("Uploade keep-anno Javadoc") | |
| if is_main: | |
| version_or_path = 'docs' | |
| dst_dir = 'keepanno/javadoc' | |
| rsync_dir(utils.KEEPANNO_ANNOTATIONS_DOC, version_or_path, dst_dir, | |
| is_main, options) | |
| timing.end() | |
| # Upload directories. | |
| timing.begin("Upload directories") | |
| dirs_for_archiving = [ | |
| (utils.KEEPANNO_ANNOTATIONS_DOC, 'keepanno/javadoc'), | |
| ] | |
| for (src_dir, dst_dir) in dirs_for_archiving: | |
| upload_dir(src_dir, version, dst_dir, is_main, options) | |
| timing.end() | |
| # Upload files. | |
| timing.begin("Upload files") | |
| lib_jars = [ | |
| utils.KEEPANNOTOOLSLIB_JAR, | |
| utils.PROCESS_KEEP_RULES_LIB_JAR, | |
| utils.R8LIB_JAR, | |
| utils.R8LIB_EXCLUDE_DEPS_JAR, | |
| ] | |
| for_archiving = [] | |
| for lib_jar in lib_jars: | |
| for_archiving.append(lib_jar) | |
| for_archiving.append(lib_jar + '.map') | |
| for_archiving.append(lib_jar + '_map.zip') | |
| for_archiving.extend([ | |
| utils.R8_JAR, utils.R8_FULL_EXCLUDE_DEPS_JAR, utils.MAVEN_ZIP_LIB, | |
| utils.KEEPANNNO_TOOLS_MAVEN_ZIP_LIB, | |
| utils.THREADING_MODULE_BLOCKING_JAR, | |
| utils.THREADING_MODULE_SINGLE_THREADED_JAR, utils.KEEPANNOTOOLS_JAR, | |
| utils.DESUGAR_CONFIGURATION, utils.DESUGAR_CONFIGURATION_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_LEGACY, | |
| utils.DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP, | |
| utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP, utils.R8_SRC_JAR, | |
| utils.KEEPANNO_ANNOTATIONS_JAR, | |
| utils.KEEPANNO_LEGACY_ANNOTATIONS_JAR, | |
| utils.KEEPANNO_ANDROIDX_ANNOTATIONS_JAR, utils.GENERATED_LICENSE, | |
| 'd8_r8/dist/build/spdx/r8.spdx.json', | |
| 'src/keepradius/proto/keepradius.proto', | |
| 'src/keepradius/proto/keepradiussummary.proto', | |
| 'src/libanalyzer/proto/libraryanalyzerresult.proto', | |
| 'luci_invocation_id' | |
| ]) | |
| for file in for_archiving: | |
| file_name = os.path.basename(file) | |
| tagged_jar = os.path.join(temp, file_name) | |
| shutil.copyfile(file, tagged_jar) | |
| if file_name.endswith( | |
| '.jar') and not file_name.endswith('-src.jar'): | |
| with zipfile.ZipFile(tagged_jar, 'a') as f: | |
| f.write(version_file, os.path.basename(version_file)) | |
| upload_file(tagged_jar, version, file_name, is_main, options) | |
| # Upload R8 to a maven compatible location. | |
| if file == utils.R8_JAR: | |
| maven_path = utils.get_maven_path('r8', version) | |
| upload_file(tagged_jar, maven_path, 'r8-%s.jar' % version, | |
| is_main, options) | |
| upload_file(default_pom_file, maven_path, 'r8-%s.pom' % version, | |
| is_main, options) | |
| if not options.dry_run: | |
| print('Maven repo root available at: %s' % | |
| get_maven_url(is_main)) | |
| # Upload R8LIB to latest on dev channel, this is used by godbolt. | |
| if ((version.endswith('-dev') or '-alpha' in version) and | |
| (file == utils.R8LIB_JAR or | |
| file == utils.KEEPANNO_ANNOTATIONS_JAR)): | |
| upload_file(tagged_jar, 'latest-dev', file_name, is_main, | |
| options) | |
| # Upload desugar_jdk_libs configuration to a maven compatible location. | |
| if file == utils.DESUGAR_CONFIGURATION: | |
| jar_basename = 'desugar_jdk_libs_configuration.jar' | |
| jar_version_name = 'desugar_jdk_libs_configuration-%s.jar' % version | |
| maven_path = utils.get_maven_path( | |
| 'desugar_jdk_libs_configuration', version) | |
| with utils.TempDir() as tmp_dir: | |
| desugar_jdk_libs_configuration_jar = os.path.join( | |
| tmp_dir, jar_version_name) | |
| create_maven_release.generate_jar_with_desugar_configuration( | |
| utils.DESUGAR_CONFIGURATION, | |
| utils.DESUGAR_IMPLEMENTATION, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP, | |
| desugar_jdk_libs_configuration_jar) | |
| upload_file(desugar_jdk_libs_configuration_jar, maven_path, | |
| jar_version_name, is_main, options) | |
| if not options.dry_run: | |
| print('Maven repo root available at: %s' % | |
| get_maven_url(is_main)) | |
| # Also archive the jar as non maven destination for Google3 | |
| upload_file(desugar_jdk_libs_configuration_jar, version, | |
| jar_basename, is_main, options) | |
| # TODO(b/237636871): Refactor this to avoid the duplication of what is above. | |
| # Upload desugar_jdk_libs JDK-11 legacyconfiguration to a maven compatible location. | |
| if file == utils.DESUGAR_CONFIGURATION_JDK11_LEGACY: | |
| jar_basename = 'desugar_jdk_libs_configuration.jar' | |
| jar_version_name = 'desugar_jdk_libs_configuration-%s-jdk11-legacy.jar' % version | |
| maven_path = utils.get_maven_path( | |
| 'desugar_jdk_libs_configuration', version) | |
| with utils.TempDir() as tmp_dir: | |
| desugar_jdk_libs_configuration_jar = os.path.join( | |
| tmp_dir, jar_version_name) | |
| create_maven_release.generate_jar_with_desugar_configuration( | |
| utils.DESUGAR_CONFIGURATION_JDK11_LEGACY, | |
| utils.DESUGAR_IMPLEMENTATION_JDK11, | |
| utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP, | |
| desugar_jdk_libs_configuration_jar) | |
| upload_file(desugar_jdk_libs_configuration_jar, maven_path, | |
| jar_version_name, is_main, options) | |
| if not options.dry_run: | |
| print('Maven repo root available at: %s' % | |
| get_maven_url(is_main)) | |
| # Also archive the jar as non maven destination for Google3 | |
| upload_file(desugar_jdk_libs_configuration_jar, version, | |
| jar_basename, is_main, options) | |
| # Write the archive details JSON (for the LUCI recipie to read). | |
| archive_details = { | |
| 'is_main': is_main, | |
| 'version': version, | |
| 'bucket': ARCHIVE_BUCKET, | |
| 'bucket_path': get_storage_path(version, is_main), | |
| 'luci_invocation_id': luci_invocation_id | |
| } | |
| with open(options.archive_details, "w") as f: | |
| json.dump(archive_details, f, indent=4) | |
| # Cleanup. | |
| os.remove('luci_invocation_id') | |
| timing.end() | |
| timing.report() | |
| if __name__ == '__main__': | |
| sys.exit(main()) |