Merge "Script for uploading opensource apps to x20"
diff --git a/tools/run_on_as_app.py b/tools/run_on_as_app.py
index 4373ce6..64fa1be 100755
--- a/tools/run_on_as_app.py
+++ b/tools/run_on_as_app.py
@@ -27,8 +27,7 @@
WORKING_DIR = os.environ['R8_BENCHMARK_DIR']
# For running on Golem all APPS are bundled as an x20-dependency and then copied
-# to WORKING_DIR. To make it easier to update the app-bundle, remove the folder
-# WORKING_DIR and then run run_on_as_app.py --download-only.
+# to WORKING_DIR. To update the app-bundle use 'run_on_as_app_x20_packager.py'.
APPS = {
# 'app-name': {
# 'git_repo': ...
@@ -193,14 +192,14 @@
def IsTrackedByGit(file):
return subprocess.check_output(['git', 'ls-files', file]).strip() != ''
-def GitClone(git_url, revision, checkout_dir, options):
+def GitClone(git_url, revision, checkout_dir, quiet):
result = subprocess.check_output(
['git', 'clone', git_url, checkout_dir]).strip()
head_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir)
if revision == head_rev:
return result
warn('Target revision is not head in {}.'.format(checkout_dir))
- with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet):
+ with utils.ChangedWorkingDirectory(checkout_dir, quiet=quiet):
subprocess.check_output(['git', 'reset', '--hard', revision])
return result
@@ -268,7 +267,8 @@
if not os.path.exists(checkout_dir) and not options.golem:
with utils.ChangedWorkingDirectory(WORKING_DIR, quiet=options.quiet):
- GitClone(config['git_repo'], config['revision'], checkout_dir, options)
+ GitClone(
+ config['git_repo'], config['revision'], checkout_dir, options.quiet)
checkout_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir)
if config['revision'] != checkout_rev:
@@ -719,13 +719,13 @@
assert shrinker in SHRINKERS
return (options, args)
-def download_apps(options):
+def download_apps(quiet):
# Download apps and place in build
with utils.ChangedWorkingDirectory(WORKING_DIR):
for app, config in APPS.iteritems():
app_dir = os.path.join(WORKING_DIR, app)
if not os.path.exists(app_dir):
- GitClone(config['git_repo'], config['revision'], app_dir, options)
+ GitClone(config['git_repo'], config['revision'], app_dir, quiet)
def main(argv):
@@ -743,7 +743,7 @@
os.makedirs(WORKING_DIR)
if options.download_only:
- download_apps(options)
+ download_apps(options.quiet)
return
with utils.TempDir() as temp_dir:
diff --git a/tools/run_on_as_app_x20_packager.py b/tools/run_on_as_app_x20_packager.py
new file mode 100755
index 0000000..5c84626
--- /dev/null
+++ b/tools/run_on_as_app_x20_packager.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import os
+import run_on_as_app
+import shutil
+import sys
+import upload_to_x20
+import utils
+
+def main():
+ # We need prodaccess to upload to x20
+ utils.check_prodacces()
+
+ working_dir = run_on_as_app.WORKING_DIR
+
+ print 'Removing directories that do not match checked out revision'
+ with utils.ChangedWorkingDirectory(working_dir):
+ for app, config in run_on_as_app.APPS.iteritems():
+ app_dir = os.path.join(working_dir, app)
+ if os.path.exists(app_dir) \
+ and utils.get_HEAD_sha1_for_checkout(app_dir) != config['revision']:
+ print 'Removing %s' % app_dir
+ shutil.rmtree(app_dir)
+
+ print 'Downloading all missing apps'
+ run_on_as_app.download_apps(quiet=False)
+
+ # Package all files as x20 dependency
+ parent_dir = os.path.dirname(working_dir)
+ with utils.ChangedWorkingDirectory(parent_dir):
+ print 'Creating archive for opensource_apps (this may take some time)'
+ working_dir_name = os.path.basename(working_dir)
+ app_dirs = [working_dir_name + '/' + name
+ for name in run_on_as_app.APPS.keys()]
+ filename = utils.create_archive("opensource_apps", app_dirs)
+ sha1 = utils.get_sha1(filename)
+ dest = os.path.join(upload_to_x20.GMSCORE_DEPS, sha1)
+ upload_to_x20.uploadFile(filename, dest)
+ sha1_file = '%s.sha1' % filename
+ with open(sha1_file, 'w') as output:
+ output.write(sha1)
+ shutil.move(sha1_file,
+ os.path.join(utils.THIRD_PARTY, 'opensource_apps.tar.gz.sha1'))
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools/utils.py b/tools/utils.py
index 892fe7d..f6e0a70 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -271,9 +271,13 @@
subprocess.check_call(cmd)
def create_archive(name):
+ return create_archive(name, [name])
+
+def create_archive(name, sources):
tarname = '%s.tar.gz' % name
with tarfile.open(tarname, 'w:gz') as tar:
- tar.add(name)
+ for source in sources:
+ tar.add(source)
return tarname
def extract_dir(filename):
@@ -288,6 +292,9 @@
with tarfile.open(filename, 'r:gz') as tar:
tar.extractall(path=dirname)
+def check_prodacces():
+ subprocess.check_call(['prodaccess'])
+
# Note that gcs is eventually consistent with regards to list operations.
# This is not a problem in our case, but don't ever use this method
# for synchronization.
