blob: 2cab0e1aea3a3ff54c64d4481831593014ae9f7f [file] [log] [blame]
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +00001#!/usr/bin/env python
Mads Ager418d1ca2017-05-22 09:35:49 +02002# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
3# for details. All rights reserved. Use of this source code is governed by a
4# BSD-style license that can be found in the LICENSE file.
5
6# Convenience script for running tests. If no argument is given run all tests,
7# if an argument is given, run only tests with that pattern. This script will
8# force the tests to run, even if no input changed.
9
Ian Zerny019e7782021-04-20 15:34:01 +020010import archive_desugar_jdk_libs
Morten Krogh-Jespersenc69a5542021-09-23 08:10:23 +020011import download_kotlin_dev
Ian Zerny019e7782021-04-20 15:34:01 +020012import notify
Mads Ager418d1ca2017-05-22 09:35:49 +020013import optparse
Ian Zerny5fffb0a2019-02-11 13:54:22 +010014import os
Ian Zerny24398bc2019-02-22 11:59:18 +010015import shutil
Rico Windf65a1d62017-06-30 09:41:56 +020016import subprocess
Mads Ager418d1ca2017-05-22 09:35:49 +020017import sys
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +000018import thread
Rico Windda6836e2018-12-07 12:32:03 +010019import time
Rico Wind06487ac2018-12-10 09:09:19 +010020import uuid
Ian Zerny5fffb0a2019-02-11 13:54:22 +010021
22import gradle
Ian Zerny5fffb0a2019-02-11 13:54:22 +010023import utils
Jean-Marie Henaffce162f32017-10-04 10:39:27 +020024
Søren Gjessefe7c0112018-12-03 12:33:12 +010025ALL_ART_VMS = [
26 "default",
Søren Gjesse953f7c42021-08-18 16:42:06 +020027 "12.0.0",
clementbera97d5cce2019-11-22 15:09:27 +010028 "10.0.0",
Søren Gjessefe7c0112018-12-03 12:33:12 +010029 "9.0.0",
30 "8.1.0",
31 "7.0.0",
32 "6.0.1",
33 "5.1.1",
34 "4.4.4",
35 "4.0.4"]
Mads Ager418d1ca2017-05-22 09:35:49 +020036
Rico Windda6836e2018-12-07 12:32:03 +010037# How often do we check for progress on the bots:
38# Should be long enough that a normal run would always have med progress
39# Should be short enough that we ensure that two calls are close enough
40# to happen before bot times out.
41# A false positiv, i.e., printing the stacks of non hanging processes
42# is not a problem, no harm done except some logging in the stdout.
43TIMEOUT_HANDLER_PERIOD = 60 * 18
44
Rico Wind06487ac2018-12-10 09:09:19 +010045BUCKET = 'r8-test-results'
46
Ian Zerny24398bc2019-02-22 11:59:18 +010047NUMBER_OF_TEST_REPORTS = 5
48REPORTS_PATH = os.path.join(utils.BUILD, 'reports')
49REPORT_INDEX = ['tests', 'test', 'index.html']
Ian Zernybcbd6c52019-10-29 15:27:04 +010050VALID_RUNTIMES = [
51 'none',
52 'jdk8',
53 'jdk9',
54 'jdk11',
55] + [ 'dex-%s' % dexvm for dexvm in ALL_ART_VMS ]
Ian Zerny24398bc2019-02-22 11:59:18 +010056
Mads Ager418d1ca2017-05-22 09:35:49 +020057def ParseOptions():
58 result = optparse.OptionParser()
Søren Gjesse77527982018-10-05 12:58:49 +020059 result.add_option('--no-internal', '--no_internal',
Mads Ager418d1ca2017-05-22 09:35:49 +020060 help='Do not run Google internal tests.',
61 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +020062 result.add_option('--archive-failures', '--archive_failures',
Rico Winda94f01c2017-06-27 10:32:34 +020063 help='Upload test results to cloud storage on failure.',
64 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +020065 result.add_option('--only-internal', '--only_internal',
Mads Ager418d1ca2017-05-22 09:35:49 +020066 help='Only run Google internal tests.',
67 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +020068 result.add_option('--all-tests', '--all_tests',
Mads Ager418d1ca2017-05-22 09:35:49 +020069 help='Run tests in all configurations.',
70 default=False, action='store_true')
Christoffer Quist Adamsen748e4662019-08-23 14:53:49 +020071 result.add_option('--slow-tests', '--slow_tests',
72 help='Also run slow tests.',
73 default=False, action='store_true')
Mads Ager418d1ca2017-05-22 09:35:49 +020074 result.add_option('-v', '--verbose',
75 help='Print test stdout to, well, stdout.',
76 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +020077 result.add_option('--dex-vm', '--dex_vm',
Mads Ager418d1ca2017-05-22 09:35:49 +020078 help='The android version of the vm to use. "all" will run the tests on '
79 'all art vm versions (stopping after first failed execution)',
80 default="default",
81 choices=ALL_ART_VMS + ["all"])
Søren Gjesse77527982018-10-05 12:58:49 +020082 result.add_option('--dex-vm-kind', '--dex_vm_kind',
Jean-Marie Henaffce162f32017-10-04 10:39:27 +020083 help='Whether to use host or target version of runtime',
84 default="host",
85 nargs=1,
86 choices=["host", "target"])
Søren Gjesse77527982018-10-05 12:58:49 +020087 result.add_option('--one-line-per-test', '--one_line_per_test',
Mads Ager418d1ca2017-05-22 09:35:49 +020088 help='Print a line before a tests starts and after it ends to stdout.',
89 default=False, action='store_true')
90 result.add_option('--tool',
Tamas Kenezcfb2c052018-10-12 11:03:57 +020091 help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
92 ' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
93 default=None, choices=["r8", "d8", "r8cf"])
Mads Ager418d1ca2017-05-22 09:35:49 +020094 result.add_option('--jctf',
Tamas Kenezcfb2c052018-10-12 11:03:57 +020095 help='Run JCTF tests with: "r8" (default) or "d8" or "r8cf".',
Mads Ager418d1ca2017-05-22 09:35:49 +020096 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +020097 result.add_option('--only-jctf', '--only_jctf',
Tamas Kenezcfb2c052018-10-12 11:03:57 +020098 help='Run only JCTF tests with: "r8" (default) or "d8" or "r8cf".',
Mads Ager418d1ca2017-05-22 09:35:49 +020099 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +0200100 result.add_option('--jctf-compile-only', '--jctf_compile_only',
Mads Ager418d1ca2017-05-22 09:35:49 +0200101 help="Don't run, only compile JCTF tests.",
102 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +0200103 result.add_option('--disable-assertions', '--disable_assertions',
Tamas Kenezb77b7d82017-08-17 14:05:16 +0200104 help='Disable assertions when running tests.',
Søren Gjesseaf1c5e22017-06-15 12:24:03 +0200105 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +0200106 result.add_option('--with-code-coverage', '--with_code_coverage',
Tamas Kenezb77b7d82017-08-17 14:05:16 +0200107 help='Enable code coverage with Jacoco.',
Sebastien Hertze2687b62017-07-25 11:16:04 +0200108 default=False, action='store_true')
Søren Gjesse77527982018-10-05 12:58:49 +0200109 result.add_option('--test-dir', '--test_dir',
Tamas Kenezb77b7d82017-08-17 14:05:16 +0200110 help='Use a custom directory for the test artifacts instead of a'
111 ' temporary (which is automatically removed after the test).'
112 ' Note that the directory will not be cleared before the test.')
Søren Gjesse77527982018-10-05 12:58:49 +0200113 result.add_option('--java-home', '--java_home',
Mikaël Peltier5c0a3232017-10-18 09:14:40 +0200114 help='Use a custom java version to run tests.')
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100115 result.add_option('--java-max-memory-size', '--java_max_memory_size',
Rico Wind97b0a992019-08-30 11:09:15 +0200116 help='Set memory for running tests, default 4G',
117 default='4G')
Rico Windba151112020-10-01 08:16:33 +0200118 result.add_option('--test-namespace', '--test_namespace',
119 help='Only run tests in this namespace. The namespace is relative to '
120 'com/android/tools/r8, e.g., desugar/desugaredlibrary',
121 default=None)
Rico Wind8e2f7e42019-02-21 10:13:21 +0100122 result.add_option('--shard-count', '--shard_count',
123 help='We are running this many shards.')
124 result.add_option('--shard-number', '--shard_number',
125 help='We are running this shard.')
Søren Gjesse77527982018-10-05 12:58:49 +0200126 result.add_option('--generate-golden-files-to', '--generate_golden_files_to',
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200127 help='Store dex files produced by tests in the specified directory.'
128 ' It is aimed to be read on platforms with no host runtime available'
129 ' for comparison.')
Søren Gjesse77527982018-10-05 12:58:49 +0200130 result.add_option('--use-golden-files-in', '--use_golden_files_in',
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200131 help='Download golden files hierarchy for this commit in the specified'
132 ' location and use them instead of executing on host runtime.')
Søren Gjesse1956bdb2019-02-19 08:37:52 +0100133 result.add_option('--no-r8lib', '--no_r8lib',
134 default=False, action='store_true',
Morten Krogh-Jespersen2243b162019-01-14 08:40:53 +0100135 help='Run the tests on R8 full with relocated dependencies.')
Søren Gjesse1956bdb2019-02-19 08:37:52 +0100136 result.add_option('--r8lib-no-deps', '--r8lib_no_deps',
137 default=False, action='store_true',
Morten Krogh-Jespersen807b15f2018-12-17 14:24:22 +0100138 help='Run the tests on r8lib without relocated dependencies.')
Ian Zerny24398bc2019-02-22 11:59:18 +0100139 result.add_option('--failed',
140 default=False, action='store_true',
141 help='Run the tests that failed last execution.')
142 result.add_option('--fail-fast', '--fail_fast',
143 default=False, action='store_true',
144 help='Stop on first failure. Passes --fail-fast to gradle test runner.')
Morten Krogh-Jespersen89d005b2019-10-15 13:32:23 +0200145 result.add_option('--worktree',
146 default=False, action='store_true',
147 help='Tests are run in worktree and should not use gradle user home.')
Ian Zernybcbd6c52019-10-29 15:27:04 +0100148 result.add_option('--runtimes',
149 default=None,
150 help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
151 ' Special values include: all (for all runtimes)'
152 ' and empty (for no runtimes).')
Morten Krogh-Jespersendcb8d452020-07-09 15:07:50 +0200153 result.add_option('--print-hanging-stacks', '--print_hanging_stacks',
154 default=-1, type="int", help='Print hanging stacks after timeout in seconds')
Christoffer Quist Adamsen7bf60342020-11-09 14:00:27 +0100155 result.add_option('--print-full-stacktraces', '--print_full_stacktraces',
156 default=False, action='store_true',
157 help='Print the full stacktraces without any filtering applied')
158 result.add_option(
159 '--print-obfuscated-stacktraces', '--print_obfuscated_stacktraces',
160 default=False, action='store_true',
161 help='Print the obfuscated stacktraces')
Morten Krogh-Jespersen2d3aca62020-11-30 12:48:11 +0100162 result.add_option(
163 '--debug-agent', '--debug_agent',
164 help='Enable Java debug agent and suspend compilation (default disabled)',
165 default=False,
166 action='store_true')
Rico Winde1393092021-03-21 12:57:01 +0100167 result.add_option('--desugared-library-configuration',
168 '--desugared_library-configuration',
Søren Gjesseef195772021-03-11 16:04:42 +0100169 help='Use alternative desugared library configuration.')
Søren Gjesse4a45f9b2021-02-11 14:05:29 +0100170 result.add_option('--desugared-library', '--desugared_library',
171 help='Build and use desugared library from GitHub.')
Rico Windf2f4c292021-04-23 07:06:13 +0200172 result.add_option('--print-times', '--print_times',
173 help='Print the execution time of the slowest tests..',
174 default=False, action='store_true')
Ian Zerny9a0e96a2021-04-28 12:35:49 +0200175 result.add_option(
Ian Zernycae764d2021-08-16 08:25:15 +0200176 '--testing-state-name',
177 help='Set an explict name for the testing state '
178 '(used in conjunction with --with/reset-testing-state).')
179 result.add_option(
Ian Zerny27ea4c72021-04-29 22:35:49 +0200180 '--with-testing-state',
181 help='Run/resume tests using testing state.',
182 default=False, action='store_true')
183 result.add_option(
184 '--reset-testing-state',
185 help='Clean the testing state and rerun tests (implies --with-testing-state).',
Ian Zerny9a0e96a2021-04-28 12:35:49 +0200186 default=False, action='store_true')
187 result.add_option(
188 '--stacktrace',
189 help='Pass --stacktrace to the gradle run',
190 default=False, action='store_true')
Morten Krogh-Jespersenc616fcd2021-09-23 08:25:09 +0200191 result.add_option('--kotlin-compiler-dev',
Morten Krogh-Jespersenc69a5542021-09-23 08:10:23 +0200192 help='Specify to download a kotlin dev compiler and run '
193 'tests with that',
194 default=False, action='store_true')
Mads Ager418d1ca2017-05-22 09:35:49 +0200195 return result.parse_args()
196
Rico Wind06487ac2018-12-10 09:09:19 +0100197def archive_failures():
198 upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
199 u_dir = uuid.uuid4()
200 destination = 'gs://%s/%s' % (BUCKET, u_dir)
201 utils.upload_dir_to_cloud_storage(upload_dir, destination, is_html=True)
Rico Wind70d614f2020-01-31 08:45:21 +0100202 url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET, u_dir)
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000203 print 'Test results available at: %s' % url
204 print '@@@STEP_LINK@Test failures@%s@@@' % url
Rico Wind06487ac2018-12-10 09:09:19 +0100205
Mads Ager418d1ca2017-05-22 09:35:49 +0200206def Main():
207 (options, args) = ParseOptions()
Rico Wind3f82c202019-11-25 07:19:23 +0100208
Rico Windda6836e2018-12-07 12:32:03 +0100209 if utils.is_bot():
Rico Wind22707fc2019-03-15 13:19:57 +0100210 gradle.RunGradle(['--no-daemon', 'clean'])
Sebastien Hertze2687b62017-07-25 11:16:04 +0200211
Søren Gjesseef195772021-03-11 16:04:42 +0100212 desugar_jdk_json_dir = None
213 if options.desugared_library_configuration:
214 if options.desugared_library_configuration != 'jdk11':
215 print("Only value supported for --desugared-library is 'jdk11'")
216 exit(1)
217 desugar_jdk_json_dir = 'src/library_desugar/jdk11'
218
Søren Gjesse4a45f9b2021-02-11 14:05:29 +0100219 desugar_jdk_libs = None
220 if options.desugared_library:
221 if options.desugared_library != 'HEAD':
222 print("Only value supported for --desugared-library is 'HEAD'")
223 exit(1)
224 desugar_jdk_libs_dir = 'build/desugar_jdk_libs'
225 shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True)
226 os.makedirs(desugar_jdk_libs_dir)
227 print('Building desugared library.')
228 with utils.TempDir() as checkout_dir:
229 archive_desugar_jdk_libs.CloneDesugaredLibrary('google', checkout_dir)
Rico Winde4b43a32021-03-02 09:29:52 +0100230 # Make sure bazel is extracted in third_party.
231 utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
232 utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
Søren Gjesseef195772021-03-11 16:04:42 +0100233 utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
234 (library_jar, maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(checkout_dir, 'jdk11' if options.desugared_library_configuration == 'jdk11' else 'jdk8')
Søren Gjesse4a45f9b2021-02-11 14:05:29 +0100235 desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir, os.path.basename(library_jar))
236 shutil.copyfile(library_jar, desugar_jdk_libs)
237 print('Desugared library for test in ' + desugar_jdk_libs)
238
Ian Zerny9a0e96a2021-04-28 12:35:49 +0200239 gradle_args = []
240
241 if options.stacktrace or utils.is_bot():
242 gradle_args.append('--stacktrace')
243
Rico Wind22707fc2019-03-15 13:19:57 +0100244 if utils.is_bot():
Rico Wind24777b92020-06-19 22:44:14 +0200245 # Bots don't like dangling processes.
Rico Wind22707fc2019-03-15 13:19:57 +0100246 gradle_args.append('--no-daemon')
247
Sebastien Hertze2687b62017-07-25 11:16:04 +0200248 # Set all necessary Gradle properties and options first.
Rico Wind8e2f7e42019-02-21 10:13:21 +0100249 if options.shard_count:
250 assert options.shard_number
251 gradle_args.append('-Pshard_count=%s' % options.shard_count)
252 gradle_args.append('-Pshard_number=%s' % options.shard_number)
Mads Ager418d1ca2017-05-22 09:35:49 +0200253 if options.verbose:
254 gradle_args.append('-Pprint_test_stdout')
255 if options.no_internal:
256 gradle_args.append('-Pno_internal')
257 if options.only_internal:
258 gradle_args.append('-Ponly_internal')
259 if options.all_tests:
260 gradle_args.append('-Pall_tests')
Christoffer Quist Adamsen748e4662019-08-23 14:53:49 +0200261 if options.slow_tests:
262 gradle_args.append('-Pslow_tests=1')
Mads Ager418d1ca2017-05-22 09:35:49 +0200263 if options.tool:
264 gradle_args.append('-Ptool=%s' % options.tool)
265 if options.one_line_per_test:
266 gradle_args.append('-Pone_line_per_test')
267 if options.jctf:
268 gradle_args.append('-Pjctf')
269 if options.only_jctf:
270 gradle_args.append('-Ponly_jctf')
Rico Windba151112020-10-01 08:16:33 +0200271 if options.test_namespace:
272 gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
Mads Ager418d1ca2017-05-22 09:35:49 +0200273 if options.jctf_compile_only:
274 gradle_args.append('-Pjctf_compile_only')
Søren Gjesseaf1c5e22017-06-15 12:24:03 +0200275 if options.disable_assertions:
276 gradle_args.append('-Pdisable_assertions')
Sebastien Hertze2687b62017-07-25 11:16:04 +0200277 if options.with_code_coverage:
278 gradle_args.append('-Pwith_code_coverage')
Christoffer Quist Adamsen7bf60342020-11-09 14:00:27 +0100279 if options.print_full_stacktraces:
280 gradle_args.append('-Pprint_full_stacktraces')
281 if options.print_obfuscated_stacktraces:
282 gradle_args.append('-Pprint_obfuscated_stacktraces')
Morten Krogh-Jespersenc69a5542021-09-23 08:10:23 +0200283 if options.kotlin_dev_compiler:
Morten Krogh-Jespersenc616fcd2021-09-23 08:25:09 +0200284 gradle_args.append('-Dcom.android.tools.r8.kotlincompilerdev=1')
Morten Krogh-Jespersenc69a5542021-09-23 08:10:23 +0200285 download_kotlin_dev.download_newest()
Jean-Marie Henaff7b424e92017-06-15 11:02:56 +0200286 if os.name == 'nt':
287 # temporary hack
288 gradle_args.append('-Pno_internal')
289 gradle_args.append('-x')
290 gradle_args.append('createJctfTests')
291 gradle_args.append('-x')
292 gradle_args.append('jctfCommonJar')
293 gradle_args.append('-x')
294 gradle_args.append('jctfTestsClasses')
Tamas Kenezb77b7d82017-08-17 14:05:16 +0200295 if options.test_dir:
296 gradle_args.append('-Ptest_dir=' + options.test_dir)
297 if not os.path.exists(options.test_dir):
298 os.makedirs(options.test_dir)
Mikaël Peltier5c0a3232017-10-18 09:14:40 +0200299 if options.java_home:
300 gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
Ian Zerny5fffb0a2019-02-11 13:54:22 +0100301 if options.java_max_memory_size:
Rico Wind97b0a992019-08-30 11:09:15 +0200302 gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200303 if options.generate_golden_files_to:
304 gradle_args.append('-Pgenerate_golden_files_to=' + options.generate_golden_files_to)
305 if not os.path.exists(options.generate_golden_files_to):
306 os.makedirs(options.generate_golden_files_to)
307 gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
308 if options.use_golden_files_in:
309 gradle_args.append('-Puse_golden_files_in=' + options.use_golden_files_in)
310 if not os.path.exists(options.use_golden_files_in):
311 os.makedirs(options.use_golden_files_in)
312 gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
Morten Krogh-Jespersen432dd912019-01-14 13:26:35 +0100313 if (not options.no_r8lib) and options.r8lib_no_deps:
Morten Krogh-Jespersen2243b162019-01-14 08:40:53 +0100314 print('Cannot run tests on r8lib with and without deps. R8lib is now default target.')
Morten Krogh-Jespersen807b15f2018-12-17 14:24:22 +0100315 exit(1)
Morten Krogh-Jespersen2243b162019-01-14 08:40:53 +0100316 if not options.no_r8lib:
Morten Krogh-Jespersen807b15f2018-12-17 14:24:22 +0100317 gradle_args.append('-Pr8lib')
Morten Krogh-Jespersen54f196e2019-01-14 16:10:08 +0100318 # Force gradle to build a version of r8lib without dependencies for
319 # BootstrapCurrentEqualityTest.
320 gradle_args.append('R8LibNoDeps')
Morten Krogh-Jespersen0d7a96f2021-01-11 17:48:12 +0100321 gradle_args.append('R8Retrace')
Morten Krogh-Jespersen807b15f2018-12-17 14:24:22 +0100322 if options.r8lib_no_deps:
323 gradle_args.append('-Pr8lib_no_deps')
Morten Krogh-Jespersen89d005b2019-10-15 13:32:23 +0200324 if options.worktree:
325 gradle_args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
Ian Zernya97ccc42020-02-10 14:18:07 +0100326 gradle_args.append('--no-daemon')
Morten Krogh-Jespersen2d3aca62020-11-30 12:48:11 +0100327 if options.debug_agent:
328 gradle_args.append('--no-daemon')
Søren Gjesseef195772021-03-11 16:04:42 +0100329 if desugar_jdk_json_dir:
330 gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir)
Søren Gjesse4a45f9b2021-02-11 14:05:29 +0100331 if desugar_jdk_libs:
332 gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
Ian Zerny27ea4c72021-04-29 22:35:49 +0200333 if options.reset_testing_state:
334 gradle_args.append('-Ptesting-state')
335 gradle_args.append('-Preset-testing-state')
336 elif options.with_testing_state:
337 gradle_args.append('-Ptesting-state')
Ian Zernycae764d2021-08-16 08:25:15 +0200338 if options.testing_state_name:
339 gradle_args.append('-Ptesting-state-name=' + options.testing_state_name)
Morten Krogh-Jespersen807b15f2018-12-17 14:24:22 +0100340
Morten Krogh-Jespersen54f196e2019-01-14 16:10:08 +0100341 # Build an R8 with dependencies for bootstrapping tests before adding test sources.
Morten Krogh-Jespersen47393d92020-05-01 12:39:38 +0200342 gradle_args.append('r8WithDeps')
343 gradle_args.append('r8WithDeps11')
Morten Krogh-Jespersen54f196e2019-01-14 16:10:08 +0100344 gradle_args.append('r8WithRelocatedDeps')
clementbera55e84822019-06-06 16:08:11 +0200345 gradle_args.append('r8WithRelocatedDeps11')
Morten Krogh-Jespersen54f196e2019-01-14 16:10:08 +0100346
Sebastien Hertze2687b62017-07-25 11:16:04 +0200347 # Add Gradle tasks
348 gradle_args.append('cleanTest')
349 gradle_args.append('test')
Morten Krogh-Jespersen2d3aca62020-11-30 12:48:11 +0100350 if options.debug_agent:
351 gradle_args.append('--debug-jvm')
Ian Zerny24398bc2019-02-22 11:59:18 +0100352 if options.fail_fast:
353 gradle_args.append('--fail-fast')
354 if options.failed:
355 args = compute_failed_tests(args)
356 if args is None:
357 return 1
358 if len(args) == 0:
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000359 print "No failing tests"
Ian Zerny24398bc2019-02-22 11:59:18 +0100360 return 0
Sebastien Hertz0f4e7fb2017-10-02 11:33:45 +0200361 # Test filtering. Must always follow the 'test' task.
362 for testFilter in args:
Sebastien Hertze2687b62017-07-25 11:16:04 +0200363 gradle_args.append('--tests')
Sebastien Hertz0f4e7fb2017-10-02 11:33:45 +0200364 gradle_args.append(testFilter)
Sebastien Hertze2687b62017-07-25 11:16:04 +0200365 if options.with_code_coverage:
366 # Create Jacoco report after tests.
367 gradle_args.append('jacocoTestReport')
368
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200369 if options.use_golden_files_in:
370 sha1 = '%s' % utils.get_HEAD_sha1()
371 with utils.ChangedWorkingDirectory(options.use_golden_files_in):
372 utils.download_file_from_cloud_storage(
373 'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
374 '%s.tar.gz' % sha1)
375 utils.unpack_archive('%s.tar.gz' % sha1)
376
Morten Krogh-Jespersendcb8d452020-07-09 15:07:50 +0200377 print_stacks_timeout = options.print_hanging_stacks
378 if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
Rico Windda6836e2018-12-07 12:32:03 +0100379 timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
380 if os.path.exists(timestamp_file):
381 os.remove(timestamp_file)
382 gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
Morten Krogh-Jespersendcb8d452020-07-09 15:07:50 +0200383 print_stacks_timeout = (print_stacks_timeout
384 if print_stacks_timeout != -1
385 else TIMEOUT_HANDLER_PERIOD)
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000386 thread.start_new_thread(timeout_handler, (timestamp_file, print_stacks_timeout,))
Ian Zerny24398bc2019-02-22 11:59:18 +0100387 rotate_test_reports()
388
Ian Zerny324d7612019-03-20 10:52:28 +0100389 if options.only_jctf:
390 # Note: not setting -Pruntimes will run with all available runtimes.
391 return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
Ian Zerny17561362019-05-27 15:16:26 +0200392 return archive_and_return(return_code, options)
Ian Zerny324d7612019-03-20 10:52:28 +0100393
Sebastien Hertze2687b62017-07-25 11:16:04 +0200394 # Now run tests on selected runtime(s).
Ian Zernybcbd6c52019-10-29 15:27:04 +0100395 if options.runtimes:
396 if options.dex_vm != 'default':
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000397 print 'Unexpected runtimes and dex_vm argument: ' + options.dex_vm
Ian Zernybcbd6c52019-10-29 15:27:04 +0100398 sys.exit(1)
399 if options.runtimes == 'empty':
400 # Set runtimes with no content will configure no runtimes.
401 gradle_args.append('-Pruntimes=')
402 elif options.runtimes == 'all':
403 # An unset runtimes will configure all runtimes
404 pass
405 else:
406 prefixes = [prefix.strip() for prefix in options.runtimes.split(':')]
407 runtimes = []
408 for prefix in prefixes:
409 matches = [ rt for rt in VALID_RUNTIMES if rt.startswith(prefix) ]
410 if len(matches) == 0:
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000411 print "Invalid runtime prefix '%s'." % prefix
412 print "Must be just 'all', 'empty'," \
413 " or a prefix of %s" % ', '.join(VALID_RUNTIMES)
Ian Zernybcbd6c52019-10-29 15:27:04 +0100414 sys.exit(1)
415 runtimes.extend(matches)
416 gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
417
418 return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
419 return archive_and_return(return_code, options)
420
421 # Legacy testing populates the runtimes based on dex_vm.
Mads Ager418d1ca2017-05-22 09:35:49 +0200422 vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
Rico Wind124b2712019-02-28 13:49:25 +0100423
Rico Windf2f4c292021-04-23 07:06:13 +0200424 if options.print_times:
425 gradle_args.append('-Pprint_times=true')
Mads Ager418d1ca2017-05-22 09:35:49 +0200426 for art_vm in vms_to_test:
Ian Zerny4dfd5a52019-03-12 07:56:11 +0100427 vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
Ian Zerny324d7612019-03-20 10:52:28 +0100428 runtimes = ['dex-' + art_vm]
Ian Zerny019e7782021-04-20 15:34:01 +0200429 # Append the "none" runtime and default JVM if running the "default" DEX VM.
Ian Zerny324d7612019-03-20 10:52:28 +0100430 if art_vm == "default":
Morten Krogh-Jespersen69358942020-10-09 11:07:09 +0200431 runtimes.extend(['jdk11', 'none'])
Ian Zerny4dfd5a52019-03-12 07:56:11 +0100432 return_code = gradle.RunGradle(
433 gradle_args + [
434 '-Pdex_vm=%s' % art_vm + vm_suffix,
Ian Zerny324d7612019-03-20 10:52:28 +0100435 '-Pruntimes=%s' % ':'.join(runtimes),
436 ],
Ian Zerny4dfd5a52019-03-12 07:56:11 +0100437 throw_on_failure=False)
Jean-Marie Henaff7a64eec2018-05-31 15:30:35 +0200438 if options.generate_golden_files_to:
439 sha1 = '%s' % utils.get_HEAD_sha1()
440 with utils.ChangedWorkingDirectory(options.generate_golden_files_to):
441 archive = utils.create_archive(sha1)
442 utils.upload_file_to_cloud_storage(archive,
443 'gs://r8-test-results/golden-files/' + archive)
444
Rico Winda94f01c2017-06-27 10:32:34 +0200445 if return_code != 0:
Ian Zerny17561362019-05-27 15:16:26 +0200446 return archive_and_return(return_code, options)
Mads Ager418d1ca2017-05-22 09:35:49 +0200447
Jinseong Jeon9749d172017-09-19 00:25:01 -0700448 return 0
449
Rico Windda6836e2018-12-07 12:32:03 +0100450
Ian Zerny17561362019-05-27 15:16:26 +0200451def archive_and_return(return_code, options):
Rico Windc58a20e2019-05-23 09:43:19 +0200452 if return_code != 0:
453 if options.archive_failures and os.name != 'nt':
454 archive_failures()
455 return return_code
456
Rico Windda6836e2018-12-07 12:32:03 +0100457def print_jstacks():
458 processes = subprocess.check_output(['ps', 'aux'])
459 for l in processes.splitlines():
460 if 'java' in l and 'openjdk' in l:
461 # Example line:
462 # ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
463 columns = l.split()
464 pid = columns[1]
465 return_value = subprocess.call(['jstack', pid])
466 if return_value:
467 print('Could not jstack %s' % l)
Rico Windda6836e2018-12-07 12:32:03 +0100468
469def get_time_from_file(timestamp_file):
470 if os.path.exists(timestamp_file):
471 timestamp = os.stat(timestamp_file).st_mtime
472 print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
Rico Windda6836e2018-12-07 12:32:03 +0100473 sys.stdout.flush()
474 return timestamp
475 else:
476 print('TIMEOUT HANDLER no timestamp file yet')
Rico Windda6836e2018-12-07 12:32:03 +0100477 sys.stdout.flush()
478 return None
479
Morten Krogh-Jespersendcb8d452020-07-09 15:07:50 +0200480def timeout_handler(timestamp_file, timeout_handler_period):
Rico Windda6836e2018-12-07 12:32:03 +0100481 last_timestamp = None
482 while True:
Morten Krogh-Jespersendcb8d452020-07-09 15:07:50 +0200483 time.sleep(timeout_handler_period)
Rico Windda6836e2018-12-07 12:32:03 +0100484 new_timestamp = get_time_from_file(timestamp_file)
485 if last_timestamp and new_timestamp == last_timestamp:
486 print_jstacks()
487 last_timestamp = new_timestamp
488
Ian Zerny24398bc2019-02-22 11:59:18 +0100489def report_dir_path(index):
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000490 if index is 0:
Ian Zerny24398bc2019-02-22 11:59:18 +0100491 return REPORTS_PATH
492 return '%s%d' % (REPORTS_PATH, index)
493
494def report_index_path(index):
495 return os.path.join(report_dir_path(index), *REPORT_INDEX)
496
497# Rotate test results so previous results are still accessible.
498def rotate_test_reports():
499 if not os.path.exists(report_dir_path(0)):
500 return
501 i = 1
502 while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
503 i += 1
504 if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
505 shutil.rmtree(report_dir_path(i))
506 while i > 0:
507 shutil.move(report_dir_path(i - 1), report_dir_path(i))
508 i -= 1
509
510def compute_failed_tests(args):
511 if len(args) > 1:
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000512 print "Running with --failed can take an optional path to a report index (or report number)."
Ian Zerny24398bc2019-02-22 11:59:18 +0100513 return None
514 report = report_index_path(0)
515 # If the default report does not exist, fall back to the previous report as it may be a failed
516 # gradle run which has already moved the report to report1, but did not produce a new report.
517 if not os.path.exists(report):
518 report1 = report_index_path(1)
519 if os.path.exists(report1):
520 report = report1
521 if len(args) == 1:
522 try:
523 # try to parse the arg as a report index.
524 index = int(args[0])
525 report = report_index_path(index)
526 except ValueError:
527 # if integer parsing failed assume it is a report file path.
528 report = args[0]
529 if not os.path.exists(report):
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000530 print "Can't re-run failing, no report at:", report
Ian Zerny24398bc2019-02-22 11:59:18 +0100531 return None
Morten Krogh-Jespersenefc67e02021-07-08 13:31:55 +0000532 print "Reading failed tests in", report
Ian Zerny24398bc2019-02-22 11:59:18 +0100533 failing = set()
534 inFailedSection = False
535 for line in file(report):
536 l = line.strip()
537 if l == "<h2>Failed tests</h2>":
538 inFailedSection = True
539 elif l.startswith("<h2>"):
540 inFailedSection = False
541 prefix = '<a href="classes/'
542 if inFailedSection and l.startswith(prefix):
543 href = l[len(prefix):l.index('">')]
544 # Ignore enties ending with .html which are test classes, not test methods.
545 if not href.endswith('.html'):
546 # Remove the .html and anchor separateor, also, a classMethod test is the static
547 # setup failing so rerun the full class of tests.
548 test = href.replace('.html','').replace('#', '.').replace('.classMethod', '')
549 failing.add(test)
550 return list(failing)
Mads Ager418d1ca2017-05-22 09:35:49 +0200551if __name__ == '__main__':
Stephan Herhutd24b1b72017-08-24 15:09:36 +0200552 return_code = Main()
553 if return_code != 0:
554 notify.notify("Tests failed.")
555 else:
556 notify.notify("Tests passed.")
557 sys.exit(return_code)