blob: 8a7f4aa1ab584deb582f65e56fdfe8e81608df23 [file] [log] [blame] [edit]
#!/usr/bin/env python3
# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import argparse
import gradle
import fnmatch
import os
import re
import sys
import utils
import xml.etree.ElementTree as ET
CLOUD_LOCATION = 'gs://r8-deps/maven_mirror'
GRADLE_CACHE = os.path.join(utils.USER_HOME, '.gradle', 'caches')
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument('--check_mirror',
help = 'Checks that all dependencies are mirrored, '
'returns non-zero if not.',
default = False,
action = 'store_true')
return parser.parse_args()
def xml_get(element, tag):
# The tags are prefixed with like: {http://maven.apache.org/POM/4.0.0}parent
for child in element.iter():
if child.tag.endswith(tag):
yield child
def xml_get_single(element, tag):
elements = list(xml_get(element, tag))
if len(elements) == 0:
return None
assert len(elements) == 1
return elements[0]
def find_pom_in_gradle_cache(pom, cached_poms):
cached = [p for p in cached_poms if p.endswith(pom)]
assert len(cached) == 1, 'did not find %s in %s' % (pom, cached_poms)
return cached[0]
# This is a hack, gradle does not provide access to parent poms, which
# we need to mirror.
def get_parent(pom_file, gradle_cached_poms):
tree = ET.parse(pom_file)
root = tree.getroot()
ns = {'pom': 'http://maven.apache.org/POM/4.0.0' }
parent = root.find('pom:parent', ns)
if parent is not None:
group = parent.find('pom:groupId', ns).text
artifact = parent.find('pom:artifactId', ns).text
version = parent.find('pom:version', ns).text
filename = '%s-%s.pom' % (artifact, version)
parent_path = find_pom_in_gradle_cache(filename, gradle_cached_poms)
return Entry(group, artifact, version, parent_path)
# Returns a tuple: (group, artifact, version)
def parse_descriptor(descriptor):
# Descriptor like org.ow2.asm:asm:6.2.1
split = descriptor.split(':')
return (split[0], split[1], split[2])
class Entry(object):
def __init__(self, group, artifact, version, path):
self.group = group
self.artifact = artifact
self.version = version
self.path = path
assert os.path.exists(self.path)
self.jar = None
def set_jar(self, jar):
self.jar = jar
assert os.path.exists(jar)
def get_cloud_dir(self):
return os.path.join(CLOUD_LOCATION, '/'.join(self.group.split('.')))
def get_name(self):
return '%s-%s' % (self.artifact, self.version)
def get_cloud_destination(self):
return os.path.join(self.get_cloud_dir(), self.artifact, self.version,
self.get_name())
def get_cloud_jar_location(self):
assert self.jar is not None
suffix = self.jar[len(self.jar)-4:]
return self.get_cloud_destination() + suffix
def get_cloud_pom_location(self):
return self.get_cloud_destination() + '.pom'
def read_gradle_cache_pom_files():
pom_files = []
for root, dirnames, filenames in os.walk(GRADLE_CACHE):
for filename in fnmatch.filter(filenames, '*.pom'):
pom_files.append(os.path.join(root, filename))
return pom_files
# We set the name to be group:artifact:version, same as gradle prints
def get_descriptor_from_path(entry):
# Example
# /usr.../org.ow2.asm/asm/6.2.1/3bc91be104d9292ff1dcc3dbf1002b7c320e767d/asm-6.2.1.pom
basename = os.path.basename(entry)
dirname = os.path.dirname(os.path.dirname(entry))
version = os.path.basename(dirname)
dirname = os.path.dirname(dirname)
artifact = os.path.basename(dirname)
dirname = os.path.dirname(dirname)
group = os.path.basename(dirname)
# Sanity, filename is artifact-version.{pom,jar}
assert '%s-%s' % (artifact, version) == basename[0:len(basename)-4]
return '%s:%s:%s' % (group, artifact, version)
def Main():
args = parse_arguments()
# Ensure that everything is downloaded before generating the pom list
gradle.RunGradle(['-stop'])
gradle_deps = gradle.RunGradleGetOutput(
['printMavenDeps', '-Pupdatemavendeps']).splitlines()
gradle_poms = read_gradle_cache_pom_files()
# Example output lines:
# POM: /usr.../org.ow2.asm/asm/6.2.1/3bc91be104d9292ff1dcc3dbf1002b7c320e767d/asm-6.2.1.pom org.ow2.asm:asm:6.2.1
# JAR: /usr.../com.google.code.gson/gson/2.7/751f548c85fa49f330cecbb1875893f971b33c4e/gson-2.7.jar
poms = [l[5:] for l in gradle_deps if l.startswith('POM: ')]
jars = [l[5:] for l in gradle_deps if l.startswith('JAR: ')]
descriptor_to_entry = {}
parents = []
for pom in poms:
split = pom.split(' ')
filepath = split[0]
gradle_descriptor = split[1]
descriptor = get_descriptor_from_path(filepath)
assert descriptor == gradle_descriptor
(group, artifact, version) = parse_descriptor(gradle_descriptor)
descriptor_to_entry[descriptor] = Entry(group, artifact, version, filepath)
parent = get_parent(filepath, gradle_poms)
while parent:
descriptor = get_descriptor_from_path(parent.path)
descriptor_to_entry[descriptor] = parent
parent = get_parent(parent.path, gradle_poms)
for jar in jars:
if jar.startswith(utils.REPO_ROOT):
continue
descriptor = get_descriptor_from_path(jar)
assert descriptor in descriptor_to_entry
descriptor_to_entry[descriptor].set_jar(jar)
has_missing = False
for descriptor in descriptor_to_entry:
entry = descriptor_to_entry[descriptor]
if not utils.file_exists_on_cloud_storage(entry.get_cloud_pom_location()):
if args.check_mirror:
has_missing = True
print 'Missing dependency for: ' + descriptor
else:
print 'Uploading: %s' % entry.path
utils.upload_file_to_cloud_storage(entry.path, entry.get_cloud_pom_location())
if entry.jar:
if not utils.file_exists_on_cloud_storage(entry.get_cloud_jar_location()):
if args.check_mirror:
has_missing = True
print 'Missing dependency for: ' + descriptor
else:
print 'Uploading: %s' % entry.jar
utils.upload_file_to_cloud_storage(entry.jar, entry.get_cloud_jar_location())
if args.check_mirror:
if has_missing:
print('The maven mirror has missing dependencies, please run'
'tools/maven_mirror.py')
return 1
else:
print('Mirror is up to date with all dependencies')
if __name__ == '__main__':
sys.exit(Main())