blob: 182ccf8e3c418684667b71a56aaa70ce87df4bbb [file] [log] [blame]
Rico Wind800fd712018-09-24 11:29:33 +02001#!/usr/bin/env python
2# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
3# for details. All rights reserved. Use of this source code is governed by a
4# BSD-style license that can be found in the LICENSE file.
5
6# Run all internal tests, archive result to cloud storage.
Rico Wind139eece2018-09-25 09:42:09 +02007# In the continuous operation flow we have a tester continuously checking
8# a specific cloud storage location for a file with a git hash.
9# If the file is there, the tester will remove the file, and add another
10# file stating that this is now being run. After successfully running,
11# the tester will add yet another file, and remove the last one.
12# Complete flow with states:
13# 1:
14# BOT:
15# Add file READY_FOR_TESTING (contains git hash)
16# Wait until file TESTING_COMPLETE exists (contains git hash)
17# Timeout if no progress for RUN_TIMEOUT
18# Cleanup READY_FOR_TESTING and TESTING
19# 2:
20# TESTER:
21# Replace file READY_FOR_TESTING by TESTING (contains git hash)
22# Run tests for git hash
23# Upload commit specific logs if failures
24# Upload git specific overall status file (failed or succeeded)
25# Replace file TESTING by TESTING_COMPLETE (contains git hash)
26# 3:
27# BOT:
28# Read overall status
29# Delete TESTING_COMPLETE
30# Exit based on status
Rico Wind800fd712018-09-24 11:29:33 +020031
Rico Windba63dc82019-03-29 14:33:47 +010032import gradle
Rico Wind800fd712018-09-24 11:29:33 +020033import optparse
34import os
35import subprocess
36import sys
37import time
38import utils
Morten Krogh-Jespersenf2412302019-10-22 10:18:04 +020039import run_on_app
Rico Wind800fd712018-09-24 11:29:33 +020040
Rico Wind139eece2018-09-25 09:42:09 +020041# How often the bot/tester should check state
42PULL_DELAY = 30
Rico Wind800fd712018-09-24 11:29:33 +020043TEST_RESULT_DIR = 'internal'
44
Rico Wind139eece2018-09-25 09:42:09 +020045# Magic files
46READY_FOR_TESTING = 'READY_FOR_TESTING'
47TESTING = 'TESTING'
48TESTING_COMPLETE = 'TESTING_COMPLETE'
49
50ALL_MAGIC = [READY_FOR_TESTING, TESTING, TESTING_COMPLETE]
51
52# Log file names
53STDERR = 'stderr'
54STDOUT = 'stdout'
55EXITCODE = 'exitcode'
56TIMED_OUT = 'timed_out'
57
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020058BENCHMARK_APPS = [
59 {
60 'app': 'r8',
61 'version': 'cf',
62 'find-xmx-min': 128,
63 'find-xmx-max': 400,
64 'find-xmx-range': 16,
Morten Krogh-Jespersen3793dc92019-10-09 14:48:48 +020065 'oom-threshold': 247,
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020066 },
67 {
68 'app': 'chrome',
69 'version': '180917',
70 'find-xmx-min': 256,
71 'find-xmx-max': 450,
72 'find-xmx-range': 16,
Morten Krogh-Jespersen50d48522019-10-11 21:49:46 +020073 'oom-threshold': 426,
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020074 },
75 {
76 'app': 'youtube',
77 'version': '12.22',
Morten Krogh-Jespersen50d48522019-10-11 21:49:46 +020078 'find-xmx-min': 800,
79 'find-xmx-max': 1200,
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020080 'find-xmx-range': 32,
Morten Krogh-Jespersen3e7e3772019-10-15 10:18:27 +020081 'oom-threshold': 1037,
Morten Krogh-Jespersenb042c972019-11-04 08:57:59 +010082 # TODO(b/143431825): Youtube can OOM randomly in memory configurations
83 # that should work.
84 'skip-find-xmx-max': True,
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020085 },
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020086 {
87 'app': 'iosched',
88 'version': '2019',
89 'find-xmx-min': 128,
90 'find-xmx-max': 1024,
91 'find-xmx-range': 16,
Morten Krogh-Jespersen50d48522019-10-11 21:49:46 +020092 'oom-threshold': 267,
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020093 },
94]
95
96def find_min_xmx_command(record):
Jinseong Jeond28cb582019-10-15 00:01:40 -070097 assert record['find-xmx-min'] < record['find-xmx-max']
98 assert record['find-xmx-range'] < record['find-xmx-max'] - record['find-xmx-min']
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +020099 return [
100 'tools/run_on_app.py',
101 '--compiler=r8',
102 '--compiler-build=lib',
103 '--app=%s' % record['app'],
104 '--version=%s' % record['version'],
105 '--no-debug',
106 '--no-build',
107 '--find-min-xmx',
108 '--find-min-xmx-min-memory=%s' % record['find-xmx-min'],
109 '--find-min-xmx-max-memory=%s' % record['find-xmx-max'],
110 '--find-min-xmx-range-size=%s' % record['find-xmx-range'],
111 '--find-min-xmx-archive']
112
Morten Krogh-Jespersen3793dc92019-10-09 14:48:48 +0200113def compile_with_memory_max_command(record):
Morten Krogh-Jespersenb042c972019-11-04 08:57:59 +0100114 return [] if 'skip-find-xmx-max' in record else [
Morten Krogh-Jespersen3793dc92019-10-09 14:48:48 +0200115 'tools/run_on_app.py',
116 '--compiler=r8',
117 '--compiler-build=lib',
118 '--app=%s' % record['app'],
119 '--version=%s' % record['version'],
120 '--no-debug',
121 '--no-build',
Morten Krogh-Jespersen092e9712019-10-24 14:06:32 +0200122 '--max-memory=%s' % int(record['oom-threshold'] * 1.15)
Morten Krogh-Jespersen3793dc92019-10-09 14:48:48 +0200123 ]
124
125def compile_with_memory_min_command(record):
126 return [
127 'tools/run_on_app.py',
128 '--compiler=r8',
129 '--compiler-build=lib',
130 '--app=%s' % record['app'],
131 '--version=%s' % record['version'],
132 '--no-debug',
133 '--no-build',
134 '--expect-oom',
Morten Krogh-Jespersen092e9712019-10-24 14:06:32 +0200135 '--max-memory=%s' % int(record['oom-threshold'] * 0.85)
Morten Krogh-Jespersen3793dc92019-10-09 14:48:48 +0200136 ]
137
Rico Wind6847d132018-09-26 08:18:48 +0200138TEST_COMMANDS = [
Morten Krogh-Jespersen2243b162019-01-14 08:40:53 +0100139 # Run test.py internal testing.
Rico Winde623bf32019-08-28 09:28:00 +0200140 ['tools/test.py', '--only_internal', '--slow_tests',
Rico Wind97b0a992019-08-30 11:09:15 +0200141 '--java_max_memory_size=8G'],
Morten Krogh-Jespersen2243b162019-01-14 08:40:53 +0100142 # Ensure that all internal apps compile.
Morten Krogh-Jespersen82bde822019-10-09 14:50:35 +0200143 ['tools/run_on_app.py', '--run-all', '--out=out'],
Morten Krogh-Jespersen6cd9f1d2019-10-09 14:01:04 +0200144 # Find min xmx for selected benchmark apps
145 ['tools/gradle.py', 'r8lib'],
Morten Krogh-Jespersen561a2e92019-10-10 11:11:30 +0200146] + (map(find_min_xmx_command, BENCHMARK_APPS)
147 + map(compile_with_memory_max_command, BENCHMARK_APPS)
148 + map(compile_with_memory_min_command, BENCHMARK_APPS))
Morten Krogh-Jespersen3793dc92019-10-09 14:48:48 +0200149
Rico Wind6847d132018-09-26 08:18:48 +0200150# Command timeout, in seconds.
Rico Windf77c5b92019-06-11 08:50:41 +0200151RUN_TIMEOUT = 3600 * 6
Rico Windf021d832018-12-13 11:29:22 +0100152BOT_RUN_TIMEOUT = RUN_TIMEOUT * len(TEST_COMMANDS)
Rico Wind6847d132018-09-26 08:18:48 +0200153
Rico Wind1200f512018-09-26 08:48:37 +0200154def log(str):
Rico Windffccab12018-09-26 12:39:42 +0200155 print("%s: %s" % (time.strftime("%c"), str))
Rico Wind1b09c562019-01-17 08:53:09 +0100156 sys.stdout.flush()
Rico Wind1200f512018-09-26 08:48:37 +0200157
Rico Wind800fd712018-09-24 11:29:33 +0200158def ParseOptions():
159 result = optparse.OptionParser()
160 result.add_option('--continuous',
161 help='Continuously run internal tests and post results to GCS.',
162 default=False, action='store_true')
Rico Wind4fd2dda2018-09-26 17:41:45 +0200163 result.add_option('--print_logs',
164 help='Fetch logs from gcs and print them, takes the commit to print for.',
165 default=None)
Rico Wind139eece2018-09-25 09:42:09 +0200166 result.add_option('--bot',
167 help='Run in bot mode, i.e., scheduling runs.',
168 default=False, action='store_true')
Rico Wind800fd712018-09-24 11:29:33 +0200169 result.add_option('--archive',
170 help='Post result to GCS, implied by --continuous',
171 default=False, action='store_true')
172 return result.parse_args()
173
174def get_own_file_content():
175 with open(sys.argv[0], 'r') as us:
176 return us.read()
177
178def restart_if_new_version(original_content):
179 new_content = get_own_file_content()
Rico Wind1b09c562019-01-17 08:53:09 +0100180 log('Lengths %s %s' % (len(original_content), len(new_content)))
181 log('is master %s ' % utils.is_master())
182 # Restart if the script got updated.
Rico Wind800fd712018-09-24 11:29:33 +0200183 if new_content != original_content:
Rico Wind1200f512018-09-26 08:48:37 +0200184 log('Restarting tools/internal_test.py, content changed')
Rico Wind800fd712018-09-24 11:29:33 +0200185 os.execv(sys.argv[0], sys.argv)
186
Rico Wind139eece2018-09-25 09:42:09 +0200187def ensure_git_clean():
Rico Wind800fd712018-09-24 11:29:33 +0200188 # Ensure clean git repo.
189 diff = subprocess.check_output(['git', 'diff'])
190 if len(diff) > 0:
Rico Wind1200f512018-09-26 08:48:37 +0200191 log('Local modifications to the git repo, exiting')
Rico Wind800fd712018-09-24 11:29:33 +0200192 sys.exit(1)
Rico Wind139eece2018-09-25 09:42:09 +0200193
194def git_pull():
195 ensure_git_clean()
Rico Wind2a19d932018-09-25 16:48:56 +0200196 subprocess.check_call(['git', 'checkout', 'master'])
Rico Wind800fd712018-09-24 11:29:33 +0200197 subprocess.check_call(['git', 'pull'])
198 return utils.get_HEAD_sha1()
199
Rico Wind139eece2018-09-25 09:42:09 +0200200def git_checkout(git_hash):
201 ensure_git_clean()
202 # Ensure that we are up to date to get the commit.
203 git_pull()
Rico Windd7d91062019-04-29 09:24:10 +0200204 exitcode = subprocess.call(['git', 'checkout', git_hash])
205 if exitcode != 0:
206 return None
Rico Wind139eece2018-09-25 09:42:09 +0200207 return utils.get_HEAD_sha1()
208
209def get_test_result_dir():
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200210 return os.path.join(utils.R8_TEST_RESULTS_BUCKET, TEST_RESULT_DIR)
Rico Wind139eece2018-09-25 09:42:09 +0200211
Rico Wind800fd712018-09-24 11:29:33 +0200212def get_sha_destination(sha):
Rico Wind139eece2018-09-25 09:42:09 +0200213 return os.path.join(get_test_result_dir(), sha)
Rico Wind800fd712018-09-24 11:29:33 +0200214
215def archive_status(failed):
216 gs_destination = 'gs://%s' % get_sha_destination(utils.get_HEAD_sha1())
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200217 utils.archive_value('status', gs_destination, failed)
Rico Wind800fd712018-09-24 11:29:33 +0200218
Rico Wind139eece2018-09-25 09:42:09 +0200219def get_status(sha):
220 gs_destination = 'gs://%s/status' % get_sha_destination(sha)
221 return utils.cat_file_on_cloud_storage(gs_destination)
222
Rico Wind800fd712018-09-24 11:29:33 +0200223def archive_log(stdout, stderr, exitcode, timed_out, cmd):
224 sha = utils.get_HEAD_sha1()
Rico Wind139eece2018-09-25 09:42:09 +0200225 cmd_dir = cmd.replace(' ', '_').replace('/', '_')
Rico Wind800fd712018-09-24 11:29:33 +0200226 destination = os.path.join(get_sha_destination(sha), cmd_dir)
227 gs_destination = 'gs://%s' % destination
228 url = 'https://storage.cloud.google.com/%s' % destination
Rico Wind1200f512018-09-26 08:48:37 +0200229 log('Archiving logs to: %s' % gs_destination)
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200230 utils.archive_value(EXITCODE, gs_destination, exitcode)
231 utils.archive_value(TIMED_OUT, gs_destination, timed_out)
232 utils.archive_file(STDOUT, gs_destination, stdout)
233 utils.archive_file(STDERR, gs_destination, stderr)
Rico Wind1200f512018-09-26 08:48:37 +0200234 log('Logs available at: %s' % url)
Rico Wind800fd712018-09-24 11:29:33 +0200235
Rico Wind139eece2018-09-25 09:42:09 +0200236def get_magic_file_base_path():
237 return 'gs://%s/magic' % get_test_result_dir()
238
239def get_magic_file_gs_path(name):
240 return '%s/%s' % (get_magic_file_base_path(), name)
241
242def get_magic_file_exists(name):
243 return utils.file_exists_on_cloud_storage(get_magic_file_gs_path(name))
244
245def delete_magic_file(name):
246 utils.delete_file_from_cloud_storage(get_magic_file_gs_path(name))
247
248def put_magic_file(name, sha):
Morten Krogh-Jespersen0981b722019-10-09 10:00:33 +0200249 utils.archive_value(name, get_magic_file_base_path(), sha)
Rico Wind139eece2018-09-25 09:42:09 +0200250
251def get_magic_file_content(name, ignore_errors=False):
252 return utils.cat_file_on_cloud_storage(get_magic_file_gs_path(name),
253 ignore_errors=ignore_errors)
254
255def print_magic_file_state():
Rico Wind1200f512018-09-26 08:48:37 +0200256 log('Magic file status:')
Rico Wind139eece2018-09-25 09:42:09 +0200257 for magic in ALL_MAGIC:
258 if get_magic_file_exists(magic):
259 content = get_magic_file_content(magic, ignore_errors=True)
Rico Wind1200f512018-09-26 08:48:37 +0200260 log('%s content: %s' % (magic, content))
Rico Wind139eece2018-09-25 09:42:09 +0200261
Rico Wind4fd2dda2018-09-26 17:41:45 +0200262def fetch_and_print_logs(hash):
263 gs_base = 'gs://%s' % get_sha_destination(hash)
264 listing = utils.ls_files_on_cloud_storage(gs_base).strip().split('\n')
265 for entry in listing:
266 if not entry.endswith('/status'): # Ignore the overall status file
267 for to_print in [EXITCODE, TIMED_OUT, STDERR, STDOUT]:
268 gs_location = '%s%s' % (entry, to_print)
269 value = utils.cat_file_on_cloud_storage(gs_location)
270 print('\n\n%s had value:\n%s' % (to_print, value))
Morten Krogh-Jespersenf2412302019-10-22 10:18:04 +0200271 print("\n\nPrinting find-min-xmx ranges for apps")
272 run_on_app.print_min_xmx_ranges_for_hash(hash, 'r8', 'lib')
Rico Wind4fd2dda2018-09-26 17:41:45 +0200273
Rico Wind139eece2018-09-25 09:42:09 +0200274def run_bot():
275 print_magic_file_state()
276 # Ensure that there is nothing currently scheduled (broken/stopped run)
277 for magic in ALL_MAGIC:
278 if get_magic_file_exists(magic):
Rico Wind1200f512018-09-26 08:48:37 +0200279 log('ERROR: Synchronizing file %s exists, cleaning up' % magic)
Rico Wind139eece2018-09-25 09:42:09 +0200280 delete_magic_file(magic)
281 print_magic_file_state()
282 assert not get_magic_file_exists(READY_FOR_TESTING)
283 git_hash = utils.get_HEAD_sha1()
284 put_magic_file(READY_FOR_TESTING, git_hash)
285 begin = time.time()
286 while True:
287 if time.time() - begin > BOT_RUN_TIMEOUT:
Rico Wind1200f512018-09-26 08:48:37 +0200288 log('Timeout exceeded: http://go/internal-r8-doc')
Rico Wind139eece2018-09-25 09:42:09 +0200289 raise Exception('Bot timeout')
290 if get_magic_file_exists(TESTING_COMPLETE):
291 if get_magic_file_content(TESTING_COMPLETE) == git_hash:
292 break
293 else:
294 raise Exception('Non matching git hashes %s and %s' % (
295 get_magic_file_content(TESTING_COMPLETE), git_hash))
Rico Wind1200f512018-09-26 08:48:37 +0200296 log('Still waiting for test result')
Rico Wind139eece2018-09-25 09:42:09 +0200297 print_magic_file_state()
298 time.sleep(PULL_DELAY)
299 total_time = time.time()-begin
Rico Wind1200f512018-09-26 08:48:37 +0200300 log('Done running test for %s in %ss' % (git_hash, total_time))
Rico Wind139eece2018-09-25 09:42:09 +0200301 test_status = get_status(git_hash)
302 delete_magic_file(TESTING_COMPLETE)
Rico Wind1200f512018-09-26 08:48:37 +0200303 log('Test status is: %s' % test_status)
Rico Wind139eece2018-09-25 09:42:09 +0200304 if test_status != '0':
Rico Wind9e9449e2019-04-04 14:42:29 +0200305 print('Tests failed, you can print the logs by running(googlers only):')
Rico Wind1cb65e12019-04-26 08:54:17 +0200306 print(' tools/internal_test.py --print_logs %s' % git_hash)
Rico Wind139eece2018-09-25 09:42:09 +0200307 return 1
308
Rico Wind800fd712018-09-24 11:29:33 +0200309def run_continuously():
310 # If this script changes, we will restart ourselves
311 own_content = get_own_file_content()
Rico Wind800fd712018-09-24 11:29:33 +0200312 while True:
313 restart_if_new_version(own_content)
Rico Wind139eece2018-09-25 09:42:09 +0200314 print_magic_file_state()
315 if get_magic_file_exists(READY_FOR_TESTING):
316 git_hash = get_magic_file_content(READY_FOR_TESTING)
317 checked_out = git_checkout(git_hash)
Rico Windd7d91062019-04-29 09:24:10 +0200318 if not checked_out:
319 # Gerrit change, we don't run these on internal.
320 archive_status(0)
321 put_magic_file(TESTING_COMPLETE, git_hash)
322 delete_magic_file(READY_FOR_TESTING)
323 continue
Rico Wind9519a152019-01-23 13:34:20 +0100324 # If the script changed, we need to restart now to get correct commands
325 # Note that we have not removed the READY_FOR_TESTING yet, so if we
326 # execv we will pick up the same version.
327 restart_if_new_version(own_content)
Rico Wind139eece2018-09-25 09:42:09 +0200328 # Sanity check, if this does not succeed stop.
329 if checked_out != git_hash:
Rico Wind1200f512018-09-26 08:48:37 +0200330 log('Inconsistent state: %s %s' % (git_hash, checked_out))
Rico Wind139eece2018-09-25 09:42:09 +0200331 sys.exit(1)
332 put_magic_file(TESTING, git_hash)
333 delete_magic_file(READY_FOR_TESTING)
Rico Wind1200f512018-09-26 08:48:37 +0200334 log('Running with hash: %s' % git_hash)
Rico Wind139eece2018-09-25 09:42:09 +0200335 exitcode = run_once(archive=True)
Rico Wind1200f512018-09-26 08:48:37 +0200336 log('Running finished with exit code %s' % exitcode)
Rico Wind139eece2018-09-25 09:42:09 +0200337 put_magic_file(TESTING_COMPLETE, git_hash)
338 delete_magic_file(TESTING)
339 time.sleep(PULL_DELAY)
Rico Wind800fd712018-09-24 11:29:33 +0200340
341def handle_output(archive, stderr, stdout, exitcode, timed_out, cmd):
342 if archive:
343 archive_log(stdout, stderr, exitcode, timed_out, cmd)
344 else:
345 print 'Execution of %s resulted in:' % cmd
346 print 'exit code: %s ' % exitcode
347 print 'timeout: %s ' % timed_out
348 with open(stderr, 'r') as f:
349 print 'stderr: %s' % f.read()
350 with open(stdout, 'r') as f:
351 print 'stdout: %s' % f.read()
352
Rico Wind6e2205d2018-10-25 13:27:13 +0200353def execute(cmd, archive, env=None):
Morten Krogh-Jespersenb042c972019-11-04 08:57:59 +0100354 if cmd == []:
355 return
356
Rico Wind800fd712018-09-24 11:29:33 +0200357 utils.PrintCmd(cmd)
358 with utils.TempDir() as temp:
359 try:
360 stderr_fd = None
361 stdout_fd = None
362 exitcode = 0
363 stderr = os.path.join(temp, 'stderr')
364 stderr_fd = open(stderr, 'w')
365 stdout = os.path.join(temp, 'stdout')
366 stdout_fd = open(stdout, 'w')
367 popen = subprocess.Popen(cmd,
368 bufsize=1024*1024*10,
369 stdout=stdout_fd,
Rico Wind6e2205d2018-10-25 13:27:13 +0200370 stderr=stderr_fd,
371 env=env)
Rico Wind800fd712018-09-24 11:29:33 +0200372 begin = time.time()
373 timed_out = False
374 while popen.poll() == None:
375 if time.time() - begin > RUN_TIMEOUT:
376 popen.terminate()
377 timed_out = True
378 time.sleep(2)
379 exitcode = popen.returncode
380 finally:
381 if stderr_fd:
382 stderr_fd.close()
383 if stdout_fd:
384 stdout_fd.close()
Morten Krogh-Jespersen4df02ad2019-10-08 14:58:39 +0000385 if exitcode != 0:
386 handle_output(archive, stderr, stdout, popen.returncode,
387 timed_out, ' '.join(cmd))
Rico Wind800fd712018-09-24 11:29:33 +0200388 return exitcode
389
390def run_once(archive):
391 failed = False
392 git_hash = utils.get_HEAD_sha1()
Rico Wind1200f512018-09-26 08:48:37 +0200393 log('Running once with hash %s' % git_hash)
Rico Wind6e2205d2018-10-25 13:27:13 +0200394 env = os.environ.copy()
395 # Bot does not have a lot of memory.
Rico Wind3defc8d2019-03-27 08:07:31 +0100396 env['R8_GRADLE_CORES_PER_FORK'] = '16'
Morten Krogh-Jespersen2243b162019-01-14 08:40:53 +0100397 failed = any([execute(cmd, archive, env) for cmd in TEST_COMMANDS])
Rico Windba63dc82019-03-29 14:33:47 +0100398 # Gradle daemon occasionally leaks memory, stop it.
Rico Wind74233002019-04-04 08:28:58 +0200399 gradle.RunGradle(['--stop'])
Rico Wind800fd712018-09-24 11:29:33 +0200400 archive_status(1 if failed else 0)
Rico Wind139eece2018-09-25 09:42:09 +0200401 return failed
Rico Wind800fd712018-09-24 11:29:33 +0200402
403def Main():
404 (options, args) = ParseOptions()
405 if options.continuous:
406 run_continuously()
Rico Wind139eece2018-09-25 09:42:09 +0200407 elif options.bot:
408 return run_bot()
Rico Wind4fd2dda2018-09-26 17:41:45 +0200409 elif options.print_logs:
410 return fetch_and_print_logs(options.print_logs)
Rico Wind800fd712018-09-24 11:29:33 +0200411 else:
Rico Wind139eece2018-09-25 09:42:09 +0200412 return run_once(options.archive)
Rico Wind800fd712018-09-24 11:29:33 +0200413
414if __name__ == '__main__':
415 sys.exit(Main())