Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file |
| 3 | # for details. All rights reserved. Use of this source code is governed by a |
| 4 | # BSD-style license that can be found in the LICENSE file. |
| 5 | |
| 6 | # Run all internal tests, archive result to cloud storage. |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 7 | # In the continuous operation flow we have a tester continuously checking |
| 8 | # a specific cloud storage location for a file with a git hash. |
| 9 | # If the file is there, the tester will remove the file, and add another |
| 10 | # file stating that this is now being run. After successfully running, |
| 11 | # the tester will add yet another file, and remove the last one. |
| 12 | # Complete flow with states: |
| 13 | # 1: |
| 14 | # BOT: |
| 15 | # Add file READY_FOR_TESTING (contains git hash) |
| 16 | # Wait until file TESTING_COMPLETE exists (contains git hash) |
| 17 | # Timeout if no progress for RUN_TIMEOUT |
| 18 | # Cleanup READY_FOR_TESTING and TESTING |
| 19 | # 2: |
| 20 | # TESTER: |
| 21 | # Replace file READY_FOR_TESTING by TESTING (contains git hash) |
| 22 | # Run tests for git hash |
| 23 | # Upload commit specific logs if failures |
| 24 | # Upload git specific overall status file (failed or succeeded) |
| 25 | # Replace file TESTING by TESTING_COMPLETE (contains git hash) |
| 26 | # 3: |
| 27 | # BOT: |
| 28 | # Read overall status |
| 29 | # Delete TESTING_COMPLETE |
| 30 | # Exit based on status |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 31 | |
Rico Wind | ba63dc8 | 2019-03-29 14:33:47 +0100 | [diff] [blame] | 32 | import gradle |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 33 | import optparse |
| 34 | import os |
| 35 | import subprocess |
| 36 | import sys |
| 37 | import time |
| 38 | import utils |
Morten Krogh-Jespersen | f241230 | 2019-10-22 10:18:04 +0200 | [diff] [blame] | 39 | import run_on_app |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 40 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 41 | # How often the bot/tester should check state |
| 42 | PULL_DELAY = 30 |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 43 | TEST_RESULT_DIR = 'internal' |
| 44 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 45 | # Magic files |
| 46 | READY_FOR_TESTING = 'READY_FOR_TESTING' |
| 47 | TESTING = 'TESTING' |
| 48 | TESTING_COMPLETE = 'TESTING_COMPLETE' |
| 49 | |
| 50 | ALL_MAGIC = [READY_FOR_TESTING, TESTING, TESTING_COMPLETE] |
| 51 | |
| 52 | # Log file names |
| 53 | STDERR = 'stderr' |
| 54 | STDOUT = 'stdout' |
| 55 | EXITCODE = 'exitcode' |
| 56 | TIMED_OUT = 'timed_out' |
| 57 | |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 58 | BENCHMARK_APPS = [ |
| 59 | { |
| 60 | 'app': 'r8', |
| 61 | 'version': 'cf', |
| 62 | 'find-xmx-min': 128, |
| 63 | 'find-xmx-max': 400, |
| 64 | 'find-xmx-range': 16, |
Morten Krogh-Jespersen | 3793dc9 | 2019-10-09 14:48:48 +0200 | [diff] [blame] | 65 | 'oom-threshold': 247, |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 66 | }, |
| 67 | { |
| 68 | 'app': 'chrome', |
| 69 | 'version': '180917', |
| 70 | 'find-xmx-min': 256, |
| 71 | 'find-xmx-max': 450, |
| 72 | 'find-xmx-range': 16, |
Morten Krogh-Jespersen | 50d4852 | 2019-10-11 21:49:46 +0200 | [diff] [blame] | 73 | 'oom-threshold': 426, |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 74 | }, |
| 75 | { |
| 76 | 'app': 'youtube', |
| 77 | 'version': '12.22', |
Morten Krogh-Jespersen | 50d4852 | 2019-10-11 21:49:46 +0200 | [diff] [blame] | 78 | 'find-xmx-min': 800, |
| 79 | 'find-xmx-max': 1200, |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 80 | 'find-xmx-range': 32, |
Morten Krogh-Jespersen | 3e7e377 | 2019-10-15 10:18:27 +0200 | [diff] [blame] | 81 | 'oom-threshold': 1037, |
Morten Krogh-Jespersen | b042c97 | 2019-11-04 08:57:59 +0100 | [diff] [blame] | 82 | # TODO(b/143431825): Youtube can OOM randomly in memory configurations |
| 83 | # that should work. |
| 84 | 'skip-find-xmx-max': True, |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 85 | }, |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 86 | { |
| 87 | 'app': 'iosched', |
| 88 | 'version': '2019', |
| 89 | 'find-xmx-min': 128, |
| 90 | 'find-xmx-max': 1024, |
| 91 | 'find-xmx-range': 16, |
Morten Krogh-Jespersen | 50d4852 | 2019-10-11 21:49:46 +0200 | [diff] [blame] | 92 | 'oom-threshold': 267, |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 93 | }, |
| 94 | ] |
| 95 | |
| 96 | def find_min_xmx_command(record): |
Jinseong Jeon | d28cb58 | 2019-10-15 00:01:40 -0700 | [diff] [blame] | 97 | assert record['find-xmx-min'] < record['find-xmx-max'] |
| 98 | assert record['find-xmx-range'] < record['find-xmx-max'] - record['find-xmx-min'] |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 99 | return [ |
| 100 | 'tools/run_on_app.py', |
| 101 | '--compiler=r8', |
| 102 | '--compiler-build=lib', |
| 103 | '--app=%s' % record['app'], |
| 104 | '--version=%s' % record['version'], |
| 105 | '--no-debug', |
| 106 | '--no-build', |
| 107 | '--find-min-xmx', |
| 108 | '--find-min-xmx-min-memory=%s' % record['find-xmx-min'], |
| 109 | '--find-min-xmx-max-memory=%s' % record['find-xmx-max'], |
| 110 | '--find-min-xmx-range-size=%s' % record['find-xmx-range'], |
| 111 | '--find-min-xmx-archive'] |
| 112 | |
Morten Krogh-Jespersen | 3793dc9 | 2019-10-09 14:48:48 +0200 | [diff] [blame] | 113 | def compile_with_memory_max_command(record): |
Morten Krogh-Jespersen | b042c97 | 2019-11-04 08:57:59 +0100 | [diff] [blame] | 114 | return [] if 'skip-find-xmx-max' in record else [ |
Morten Krogh-Jespersen | 3793dc9 | 2019-10-09 14:48:48 +0200 | [diff] [blame] | 115 | 'tools/run_on_app.py', |
| 116 | '--compiler=r8', |
| 117 | '--compiler-build=lib', |
| 118 | '--app=%s' % record['app'], |
| 119 | '--version=%s' % record['version'], |
| 120 | '--no-debug', |
| 121 | '--no-build', |
Morten Krogh-Jespersen | 092e971 | 2019-10-24 14:06:32 +0200 | [diff] [blame] | 122 | '--max-memory=%s' % int(record['oom-threshold'] * 1.15) |
Morten Krogh-Jespersen | 3793dc9 | 2019-10-09 14:48:48 +0200 | [diff] [blame] | 123 | ] |
| 124 | |
| 125 | def compile_with_memory_min_command(record): |
| 126 | return [ |
| 127 | 'tools/run_on_app.py', |
| 128 | '--compiler=r8', |
| 129 | '--compiler-build=lib', |
| 130 | '--app=%s' % record['app'], |
| 131 | '--version=%s' % record['version'], |
| 132 | '--no-debug', |
| 133 | '--no-build', |
| 134 | '--expect-oom', |
Morten Krogh-Jespersen | 092e971 | 2019-10-24 14:06:32 +0200 | [diff] [blame] | 135 | '--max-memory=%s' % int(record['oom-threshold'] * 0.85) |
Morten Krogh-Jespersen | 3793dc9 | 2019-10-09 14:48:48 +0200 | [diff] [blame] | 136 | ] |
| 137 | |
Rico Wind | 6847d13 | 2018-09-26 08:18:48 +0200 | [diff] [blame] | 138 | TEST_COMMANDS = [ |
Morten Krogh-Jespersen | 2243b16 | 2019-01-14 08:40:53 +0100 | [diff] [blame] | 139 | # Run test.py internal testing. |
Rico Wind | e623bf3 | 2019-08-28 09:28:00 +0200 | [diff] [blame] | 140 | ['tools/test.py', '--only_internal', '--slow_tests', |
Rico Wind | 97b0a99 | 2019-08-30 11:09:15 +0200 | [diff] [blame] | 141 | '--java_max_memory_size=8G'], |
Morten Krogh-Jespersen | 2243b16 | 2019-01-14 08:40:53 +0100 | [diff] [blame] | 142 | # Ensure that all internal apps compile. |
Morten Krogh-Jespersen | 82bde82 | 2019-10-09 14:50:35 +0200 | [diff] [blame] | 143 | ['tools/run_on_app.py', '--run-all', '--out=out'], |
Morten Krogh-Jespersen | 6cd9f1d | 2019-10-09 14:01:04 +0200 | [diff] [blame] | 144 | # Find min xmx for selected benchmark apps |
| 145 | ['tools/gradle.py', 'r8lib'], |
Morten Krogh-Jespersen | 561a2e9 | 2019-10-10 11:11:30 +0200 | [diff] [blame] | 146 | ] + (map(find_min_xmx_command, BENCHMARK_APPS) |
| 147 | + map(compile_with_memory_max_command, BENCHMARK_APPS) |
| 148 | + map(compile_with_memory_min_command, BENCHMARK_APPS)) |
Morten Krogh-Jespersen | 3793dc9 | 2019-10-09 14:48:48 +0200 | [diff] [blame] | 149 | |
Rico Wind | 6847d13 | 2018-09-26 08:18:48 +0200 | [diff] [blame] | 150 | # Command timeout, in seconds. |
Rico Wind | f77c5b9 | 2019-06-11 08:50:41 +0200 | [diff] [blame] | 151 | RUN_TIMEOUT = 3600 * 6 |
Rico Wind | f021d83 | 2018-12-13 11:29:22 +0100 | [diff] [blame] | 152 | BOT_RUN_TIMEOUT = RUN_TIMEOUT * len(TEST_COMMANDS) |
Rico Wind | 6847d13 | 2018-09-26 08:18:48 +0200 | [diff] [blame] | 153 | |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 154 | def log(str): |
Rico Wind | ffccab1 | 2018-09-26 12:39:42 +0200 | [diff] [blame] | 155 | print("%s: %s" % (time.strftime("%c"), str)) |
Rico Wind | 1b09c56 | 2019-01-17 08:53:09 +0100 | [diff] [blame] | 156 | sys.stdout.flush() |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 157 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 158 | def ParseOptions(): |
| 159 | result = optparse.OptionParser() |
| 160 | result.add_option('--continuous', |
| 161 | help='Continuously run internal tests and post results to GCS.', |
| 162 | default=False, action='store_true') |
Rico Wind | 4fd2dda | 2018-09-26 17:41:45 +0200 | [diff] [blame] | 163 | result.add_option('--print_logs', |
| 164 | help='Fetch logs from gcs and print them, takes the commit to print for.', |
| 165 | default=None) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 166 | result.add_option('--bot', |
| 167 | help='Run in bot mode, i.e., scheduling runs.', |
| 168 | default=False, action='store_true') |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 169 | result.add_option('--archive', |
| 170 | help='Post result to GCS, implied by --continuous', |
| 171 | default=False, action='store_true') |
| 172 | return result.parse_args() |
| 173 | |
| 174 | def get_own_file_content(): |
| 175 | with open(sys.argv[0], 'r') as us: |
| 176 | return us.read() |
| 177 | |
| 178 | def restart_if_new_version(original_content): |
| 179 | new_content = get_own_file_content() |
Rico Wind | 1b09c56 | 2019-01-17 08:53:09 +0100 | [diff] [blame] | 180 | log('Lengths %s %s' % (len(original_content), len(new_content))) |
| 181 | log('is master %s ' % utils.is_master()) |
| 182 | # Restart if the script got updated. |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 183 | if new_content != original_content: |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 184 | log('Restarting tools/internal_test.py, content changed') |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 185 | os.execv(sys.argv[0], sys.argv) |
| 186 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 187 | def ensure_git_clean(): |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 188 | # Ensure clean git repo. |
| 189 | diff = subprocess.check_output(['git', 'diff']) |
| 190 | if len(diff) > 0: |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 191 | log('Local modifications to the git repo, exiting') |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 192 | sys.exit(1) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 193 | |
| 194 | def git_pull(): |
| 195 | ensure_git_clean() |
Rico Wind | 2a19d93 | 2018-09-25 16:48:56 +0200 | [diff] [blame] | 196 | subprocess.check_call(['git', 'checkout', 'master']) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 197 | subprocess.check_call(['git', 'pull']) |
| 198 | return utils.get_HEAD_sha1() |
| 199 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 200 | def git_checkout(git_hash): |
| 201 | ensure_git_clean() |
| 202 | # Ensure that we are up to date to get the commit. |
| 203 | git_pull() |
Rico Wind | d7d9106 | 2019-04-29 09:24:10 +0200 | [diff] [blame] | 204 | exitcode = subprocess.call(['git', 'checkout', git_hash]) |
| 205 | if exitcode != 0: |
| 206 | return None |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 207 | return utils.get_HEAD_sha1() |
| 208 | |
| 209 | def get_test_result_dir(): |
Morten Krogh-Jespersen | 0981b72 | 2019-10-09 10:00:33 +0200 | [diff] [blame] | 210 | return os.path.join(utils.R8_TEST_RESULTS_BUCKET, TEST_RESULT_DIR) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 211 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 212 | def get_sha_destination(sha): |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 213 | return os.path.join(get_test_result_dir(), sha) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 214 | |
| 215 | def archive_status(failed): |
| 216 | gs_destination = 'gs://%s' % get_sha_destination(utils.get_HEAD_sha1()) |
Morten Krogh-Jespersen | 0981b72 | 2019-10-09 10:00:33 +0200 | [diff] [blame] | 217 | utils.archive_value('status', gs_destination, failed) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 218 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 219 | def get_status(sha): |
| 220 | gs_destination = 'gs://%s/status' % get_sha_destination(sha) |
| 221 | return utils.cat_file_on_cloud_storage(gs_destination) |
| 222 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 223 | def archive_log(stdout, stderr, exitcode, timed_out, cmd): |
| 224 | sha = utils.get_HEAD_sha1() |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 225 | cmd_dir = cmd.replace(' ', '_').replace('/', '_') |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 226 | destination = os.path.join(get_sha_destination(sha), cmd_dir) |
| 227 | gs_destination = 'gs://%s' % destination |
| 228 | url = 'https://storage.cloud.google.com/%s' % destination |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 229 | log('Archiving logs to: %s' % gs_destination) |
Morten Krogh-Jespersen | 0981b72 | 2019-10-09 10:00:33 +0200 | [diff] [blame] | 230 | utils.archive_value(EXITCODE, gs_destination, exitcode) |
| 231 | utils.archive_value(TIMED_OUT, gs_destination, timed_out) |
| 232 | utils.archive_file(STDOUT, gs_destination, stdout) |
| 233 | utils.archive_file(STDERR, gs_destination, stderr) |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 234 | log('Logs available at: %s' % url) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 235 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 236 | def get_magic_file_base_path(): |
| 237 | return 'gs://%s/magic' % get_test_result_dir() |
| 238 | |
| 239 | def get_magic_file_gs_path(name): |
| 240 | return '%s/%s' % (get_magic_file_base_path(), name) |
| 241 | |
| 242 | def get_magic_file_exists(name): |
| 243 | return utils.file_exists_on_cloud_storage(get_magic_file_gs_path(name)) |
| 244 | |
| 245 | def delete_magic_file(name): |
| 246 | utils.delete_file_from_cloud_storage(get_magic_file_gs_path(name)) |
| 247 | |
| 248 | def put_magic_file(name, sha): |
Morten Krogh-Jespersen | 0981b72 | 2019-10-09 10:00:33 +0200 | [diff] [blame] | 249 | utils.archive_value(name, get_magic_file_base_path(), sha) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 250 | |
| 251 | def get_magic_file_content(name, ignore_errors=False): |
| 252 | return utils.cat_file_on_cloud_storage(get_magic_file_gs_path(name), |
| 253 | ignore_errors=ignore_errors) |
| 254 | |
| 255 | def print_magic_file_state(): |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 256 | log('Magic file status:') |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 257 | for magic in ALL_MAGIC: |
| 258 | if get_magic_file_exists(magic): |
| 259 | content = get_magic_file_content(magic, ignore_errors=True) |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 260 | log('%s content: %s' % (magic, content)) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 261 | |
Rico Wind | 4fd2dda | 2018-09-26 17:41:45 +0200 | [diff] [blame] | 262 | def fetch_and_print_logs(hash): |
| 263 | gs_base = 'gs://%s' % get_sha_destination(hash) |
| 264 | listing = utils.ls_files_on_cloud_storage(gs_base).strip().split('\n') |
| 265 | for entry in listing: |
| 266 | if not entry.endswith('/status'): # Ignore the overall status file |
| 267 | for to_print in [EXITCODE, TIMED_OUT, STDERR, STDOUT]: |
| 268 | gs_location = '%s%s' % (entry, to_print) |
| 269 | value = utils.cat_file_on_cloud_storage(gs_location) |
| 270 | print('\n\n%s had value:\n%s' % (to_print, value)) |
Morten Krogh-Jespersen | f241230 | 2019-10-22 10:18:04 +0200 | [diff] [blame] | 271 | print("\n\nPrinting find-min-xmx ranges for apps") |
| 272 | run_on_app.print_min_xmx_ranges_for_hash(hash, 'r8', 'lib') |
Rico Wind | 4fd2dda | 2018-09-26 17:41:45 +0200 | [diff] [blame] | 273 | |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 274 | def run_bot(): |
| 275 | print_magic_file_state() |
| 276 | # Ensure that there is nothing currently scheduled (broken/stopped run) |
| 277 | for magic in ALL_MAGIC: |
| 278 | if get_magic_file_exists(magic): |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 279 | log('ERROR: Synchronizing file %s exists, cleaning up' % magic) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 280 | delete_magic_file(magic) |
| 281 | print_magic_file_state() |
| 282 | assert not get_magic_file_exists(READY_FOR_TESTING) |
| 283 | git_hash = utils.get_HEAD_sha1() |
| 284 | put_magic_file(READY_FOR_TESTING, git_hash) |
| 285 | begin = time.time() |
| 286 | while True: |
| 287 | if time.time() - begin > BOT_RUN_TIMEOUT: |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 288 | log('Timeout exceeded: http://go/internal-r8-doc') |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 289 | raise Exception('Bot timeout') |
| 290 | if get_magic_file_exists(TESTING_COMPLETE): |
| 291 | if get_magic_file_content(TESTING_COMPLETE) == git_hash: |
| 292 | break |
| 293 | else: |
| 294 | raise Exception('Non matching git hashes %s and %s' % ( |
| 295 | get_magic_file_content(TESTING_COMPLETE), git_hash)) |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 296 | log('Still waiting for test result') |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 297 | print_magic_file_state() |
| 298 | time.sleep(PULL_DELAY) |
| 299 | total_time = time.time()-begin |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 300 | log('Done running test for %s in %ss' % (git_hash, total_time)) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 301 | test_status = get_status(git_hash) |
| 302 | delete_magic_file(TESTING_COMPLETE) |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 303 | log('Test status is: %s' % test_status) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 304 | if test_status != '0': |
Rico Wind | 9e9449e | 2019-04-04 14:42:29 +0200 | [diff] [blame] | 305 | print('Tests failed, you can print the logs by running(googlers only):') |
Rico Wind | 1cb65e1 | 2019-04-26 08:54:17 +0200 | [diff] [blame] | 306 | print(' tools/internal_test.py --print_logs %s' % git_hash) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 307 | return 1 |
| 308 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 309 | def run_continuously(): |
| 310 | # If this script changes, we will restart ourselves |
| 311 | own_content = get_own_file_content() |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 312 | while True: |
| 313 | restart_if_new_version(own_content) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 314 | print_magic_file_state() |
| 315 | if get_magic_file_exists(READY_FOR_TESTING): |
| 316 | git_hash = get_magic_file_content(READY_FOR_TESTING) |
| 317 | checked_out = git_checkout(git_hash) |
Rico Wind | d7d9106 | 2019-04-29 09:24:10 +0200 | [diff] [blame] | 318 | if not checked_out: |
| 319 | # Gerrit change, we don't run these on internal. |
| 320 | archive_status(0) |
| 321 | put_magic_file(TESTING_COMPLETE, git_hash) |
| 322 | delete_magic_file(READY_FOR_TESTING) |
| 323 | continue |
Rico Wind | 9519a15 | 2019-01-23 13:34:20 +0100 | [diff] [blame] | 324 | # If the script changed, we need to restart now to get correct commands |
| 325 | # Note that we have not removed the READY_FOR_TESTING yet, so if we |
| 326 | # execv we will pick up the same version. |
| 327 | restart_if_new_version(own_content) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 328 | # Sanity check, if this does not succeed stop. |
| 329 | if checked_out != git_hash: |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 330 | log('Inconsistent state: %s %s' % (git_hash, checked_out)) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 331 | sys.exit(1) |
| 332 | put_magic_file(TESTING, git_hash) |
| 333 | delete_magic_file(READY_FOR_TESTING) |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 334 | log('Running with hash: %s' % git_hash) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 335 | exitcode = run_once(archive=True) |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 336 | log('Running finished with exit code %s' % exitcode) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 337 | put_magic_file(TESTING_COMPLETE, git_hash) |
| 338 | delete_magic_file(TESTING) |
| 339 | time.sleep(PULL_DELAY) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 340 | |
| 341 | def handle_output(archive, stderr, stdout, exitcode, timed_out, cmd): |
| 342 | if archive: |
| 343 | archive_log(stdout, stderr, exitcode, timed_out, cmd) |
| 344 | else: |
| 345 | print 'Execution of %s resulted in:' % cmd |
| 346 | print 'exit code: %s ' % exitcode |
| 347 | print 'timeout: %s ' % timed_out |
| 348 | with open(stderr, 'r') as f: |
| 349 | print 'stderr: %s' % f.read() |
| 350 | with open(stdout, 'r') as f: |
| 351 | print 'stdout: %s' % f.read() |
| 352 | |
Rico Wind | 6e2205d | 2018-10-25 13:27:13 +0200 | [diff] [blame] | 353 | def execute(cmd, archive, env=None): |
Morten Krogh-Jespersen | b042c97 | 2019-11-04 08:57:59 +0100 | [diff] [blame] | 354 | if cmd == []: |
| 355 | return |
| 356 | |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 357 | utils.PrintCmd(cmd) |
| 358 | with utils.TempDir() as temp: |
| 359 | try: |
| 360 | stderr_fd = None |
| 361 | stdout_fd = None |
| 362 | exitcode = 0 |
| 363 | stderr = os.path.join(temp, 'stderr') |
| 364 | stderr_fd = open(stderr, 'w') |
| 365 | stdout = os.path.join(temp, 'stdout') |
| 366 | stdout_fd = open(stdout, 'w') |
| 367 | popen = subprocess.Popen(cmd, |
| 368 | bufsize=1024*1024*10, |
| 369 | stdout=stdout_fd, |
Rico Wind | 6e2205d | 2018-10-25 13:27:13 +0200 | [diff] [blame] | 370 | stderr=stderr_fd, |
| 371 | env=env) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 372 | begin = time.time() |
| 373 | timed_out = False |
| 374 | while popen.poll() == None: |
| 375 | if time.time() - begin > RUN_TIMEOUT: |
| 376 | popen.terminate() |
| 377 | timed_out = True |
| 378 | time.sleep(2) |
| 379 | exitcode = popen.returncode |
| 380 | finally: |
| 381 | if stderr_fd: |
| 382 | stderr_fd.close() |
| 383 | if stdout_fd: |
| 384 | stdout_fd.close() |
Morten Krogh-Jespersen | 4df02ad | 2019-10-08 14:58:39 +0000 | [diff] [blame] | 385 | if exitcode != 0: |
| 386 | handle_output(archive, stderr, stdout, popen.returncode, |
| 387 | timed_out, ' '.join(cmd)) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 388 | return exitcode |
| 389 | |
| 390 | def run_once(archive): |
| 391 | failed = False |
| 392 | git_hash = utils.get_HEAD_sha1() |
Rico Wind | 1200f51 | 2018-09-26 08:48:37 +0200 | [diff] [blame] | 393 | log('Running once with hash %s' % git_hash) |
Rico Wind | 6e2205d | 2018-10-25 13:27:13 +0200 | [diff] [blame] | 394 | env = os.environ.copy() |
| 395 | # Bot does not have a lot of memory. |
Rico Wind | 3defc8d | 2019-03-27 08:07:31 +0100 | [diff] [blame] | 396 | env['R8_GRADLE_CORES_PER_FORK'] = '16' |
Morten Krogh-Jespersen | 2243b16 | 2019-01-14 08:40:53 +0100 | [diff] [blame] | 397 | failed = any([execute(cmd, archive, env) for cmd in TEST_COMMANDS]) |
Rico Wind | ba63dc8 | 2019-03-29 14:33:47 +0100 | [diff] [blame] | 398 | # Gradle daemon occasionally leaks memory, stop it. |
Rico Wind | 7423300 | 2019-04-04 08:28:58 +0200 | [diff] [blame] | 399 | gradle.RunGradle(['--stop']) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 400 | archive_status(1 if failed else 0) |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 401 | return failed |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 402 | |
| 403 | def Main(): |
| 404 | (options, args) = ParseOptions() |
| 405 | if options.continuous: |
| 406 | run_continuously() |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 407 | elif options.bot: |
| 408 | return run_bot() |
Rico Wind | 4fd2dda | 2018-09-26 17:41:45 +0200 | [diff] [blame] | 409 | elif options.print_logs: |
| 410 | return fetch_and_print_logs(options.print_logs) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 411 | else: |
Rico Wind | 139eece | 2018-09-25 09:42:09 +0200 | [diff] [blame] | 412 | return run_once(options.archive) |
Rico Wind | 800fd71 | 2018-09-24 11:29:33 +0200 | [diff] [blame] | 413 | |
| 414 | if __name__ == '__main__': |
| 415 | sys.exit(Main()) |