Add rsync upload and make directory upload target parent directory
Change-Id: I36fc3cd5d79d451abbbecb56516504e09881fc07
diff --git a/tools/archive.py b/tools/archive.py
index a906a82..d9f8335 100755
--- a/tools/archive.py
+++ b/tools/archive.py
@@ -122,6 +122,20 @@
print('INFO: Open files hard limit: %s' % hard)
+def RSyncDir(src_dir, version_or_path, dst_dir, is_main, options):
+ destination = GetUploadDestination(version_or_path, dst_dir, is_main)
+ print(f'RSyncing {src_dir} to {destination}')
+ if options.dry_run:
+ if options.dry_run_output:
+ dry_run_destination = os.path.join(options.dry_run_output, version_or_path, dst_dir)
+ print(f'Dry run, not actually syncing. Copying to {dry_run_destination}')
+ shutil.copytree(src_dir, dry_run_destination)
+ else:
+ print('Dry run, not actually uploading')
+ else:
+ utils.rsync_directory_to_cloud_storage(src_dir, destination)
+ print(f'Directory available at: {GetUrl(version_or_path, dst_dir, is_main)}')
+
def UploadDir(src_dir, version_or_path, dst_dir, is_main, options):
destination = GetUploadDestination(version_or_path, dst_dir, is_main)
print(f'Uploading {src_dir} to {destination}')
@@ -250,7 +264,7 @@
if is_main:
version_or_path = 'docs'
dst_dir = 'keepanno/javadoc'
- UploadDir(utils.KEEPANNO_ANNOTATIONS_DOC, version_or_path, dst_dir, is_main, options)
+ RSyncDir(utils.KEEPANNO_ANNOTATIONS_DOC, version_or_path, dst_dir, is_main, options)
# Upload directories.
dirs_for_archiving = [
diff --git a/tools/utils.py b/tools/utils.py
index e859748..1df6d83 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -393,11 +393,36 @@
PrintCmd(cmd)
subprocess.check_call(cmd)
+def check_dir_args(source, destination):
+ # We require that the dirname of the paths coincide, e.g., src/dirname and dst/dirname
+ # The target is then stripped so the upload command will be: cp -R src/dirname dst/
+ (destination_parent, destination_file) = os.path.split(destination)
+ if os.path.basename(source) != destination_file:
+ raise Exception(
+ 'Attempt to upload directory with non-matching directory name: ' +
+ f'{source} and {destination}')
+ if len(destination_parent.strip()) == 0:
+ raise Exception(
+ 'Attempt to upload directory to empty destination directory: '
+ + destination)
+ return destination_parent
+
def upload_directory_to_cloud_storage(source, destination, parallel=True):
+ destination_parent = check_dir_args(source, destination)
cmd = [get_gsutil()]
if parallel:
cmd += ['-m']
cmd += ['cp', '-R']
+ cmd += [source, destination_parent + '/']
+ PrintCmd(cmd)
+ subprocess.check_call(cmd)
+
+def rsync_directory_to_cloud_storage(source, destination, parallel=True):
+ check_dir_args(source, destination)
+ cmd = [get_gsutil()]
+ if parallel:
+ cmd += ['-m']
+ cmd += ['rsync', '-R']
cmd += [source, destination]
PrintCmd(cmd)
subprocess.check_call(cmd)
