blob: 95766cac48478257eabff3d9f4bacb25584bcb61 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Convenience script for running tests. If no argument is given run all tests,
# if an argument is given, run only tests with that pattern. This script will
# force the tests to run, even if no input changed.
import optparse
import os
import shutil
import subprocess
import sys
import thread
import time
import uuid
import gradle
import notify
import utils
ALL_ART_VMS = [
"default",
"10.0.0",
"9.0.0",
"8.1.0",
"7.0.0",
"6.0.1",
"5.1.1",
"4.4.4",
"4.0.4"]
# How often do we check for progress on the bots:
# Should be long enough that a normal run would always have med progress
# Should be short enough that we ensure that two calls are close enough
# to happen before bot times out.
# A false positiv, i.e., printing the stacks of non hanging processes
# is not a problem, no harm done except some logging in the stdout.
TIMEOUT_HANDLER_PERIOD = 60 * 18
BUCKET = 'r8-test-results'
NUMBER_OF_TEST_REPORTS = 5
REPORTS_PATH = os.path.join(utils.BUILD, 'reports')
REPORT_INDEX = ['tests', 'test', 'index.html']
VALID_RUNTIMES = [
'none',
'jdk8',
'jdk9',
'jdk11',
] + [ 'dex-%s' % dexvm for dexvm in ALL_ART_VMS ]
def ParseOptions():
result = optparse.OptionParser()
result.add_option('--no-internal', '--no_internal',
help='Do not run Google internal tests.',
default=False, action='store_true')
result.add_option('--archive-failures', '--archive_failures',
help='Upload test results to cloud storage on failure.',
default=False, action='store_true')
result.add_option('--only-internal', '--only_internal',
help='Only run Google internal tests.',
default=False, action='store_true')
result.add_option('--all-tests', '--all_tests',
help='Run tests in all configurations.',
default=False, action='store_true')
result.add_option('--slow-tests', '--slow_tests',
help='Also run slow tests.',
default=False, action='store_true')
result.add_option('-v', '--verbose',
help='Print test stdout to, well, stdout.',
default=False, action='store_true')
result.add_option('--dex-vm', '--dex_vm',
help='The android version of the vm to use. "all" will run the tests on '
'all art vm versions (stopping after first failed execution)',
default="default",
choices=ALL_ART_VMS + ["all"])
result.add_option('--dex-vm-kind', '--dex_vm_kind',
help='Whether to use host or target version of runtime',
default="host",
nargs=1,
choices=["host", "target"])
result.add_option('--one-line-per-test', '--one_line_per_test',
help='Print a line before a tests starts and after it ends to stdout.',
default=False, action='store_true')
result.add_option('--tool',
help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
default=None, choices=["r8", "d8", "r8cf"])
result.add_option('--jctf',
help='Run JCTF tests with: "r8" (default) or "d8" or "r8cf".',
default=False, action='store_true')
result.add_option('--only-jctf', '--only_jctf',
help='Run only JCTF tests with: "r8" (default) or "d8" or "r8cf".',
default=False, action='store_true')
result.add_option('--jctf-compile-only', '--jctf_compile_only',
help="Don't run, only compile JCTF tests.",
default=False, action='store_true')
result.add_option('--disable-assertions', '--disable_assertions',
help='Disable assertions when running tests.',
default=False, action='store_true')
result.add_option('--with-code-coverage', '--with_code_coverage',
help='Enable code coverage with Jacoco.',
default=False, action='store_true')
result.add_option('--test-dir', '--test_dir',
help='Use a custom directory for the test artifacts instead of a'
' temporary (which is automatically removed after the test).'
' Note that the directory will not be cleared before the test.')
result.add_option('--java-home', '--java_home',
help='Use a custom java version to run tests.')
result.add_option('--java-max-memory-size', '--java_max_memory_size',
help='Set memory for running tests, default 4G',
default='4G')
result.add_option('--shard-count', '--shard_count',
help='We are running this many shards.')
result.add_option('--shard-number', '--shard_number',
help='We are running this shard.')
result.add_option('--generate-golden-files-to', '--generate_golden_files_to',
help='Store dex files produced by tests in the specified directory.'
' It is aimed to be read on platforms with no host runtime available'
' for comparison.')
result.add_option('--use-golden-files-in', '--use_golden_files_in',
help='Download golden files hierarchy for this commit in the specified'
' location and use them instead of executing on host runtime.')
result.add_option('--no-r8lib', '--no_r8lib',
default=False, action='store_true',
help='Run the tests on R8 full with relocated dependencies.')
result.add_option('--r8lib-no-deps', '--r8lib_no_deps',
default=False, action='store_true',
help='Run the tests on r8lib without relocated dependencies.')
result.add_option('--failed',
default=False, action='store_true',
help='Run the tests that failed last execution.')
result.add_option('--fail-fast', '--fail_fast',
default=False, action='store_true',
help='Stop on first failure. Passes --fail-fast to gradle test runner.')
result.add_option('--worktree',
default=False, action='store_true',
help='Tests are run in worktree and should not use gradle user home.')
result.add_option('--runtimes',
default=None,
help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
' Special values include: all (for all runtimes)'
' and empty (for no runtimes).')
return result.parse_args()
def archive_failures():
upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
u_dir = uuid.uuid4()
destination = 'gs://%s/%s' % (BUCKET, u_dir)
utils.upload_dir_to_cloud_storage(upload_dir, destination, is_html=True)
url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET, u_dir)
print 'Test results available at: %s' % url
print '@@@STEP_LINK@Test failures@%s@@@' % url
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
gradle_args = ['--stacktrace']
if utils.is_bot():
# Bots don't like dangling processes
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.slow_tests:
gradle_args.append('-Pslow_tests=1')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.jctf:
gradle_args.append('-Pjctf')
if options.only_jctf:
gradle_args.append('-Ponly_jctf')
if options.jctf_compile_only:
gradle_args.append('-Pjctf_compile_only')
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if os.name == 'nt':
# temporary hack
gradle_args.append('-Pno_internal')
gradle_args.append('-x')
gradle_args.append('createJctfTests')
gradle_args.append('-x')
gradle_args.append('jctfCommonJar')
gradle_args.append('-x')
gradle_args.append('jctfTestsClasses')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' + options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' + options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if (not options.no_r8lib) and options.r8lib_no_deps:
print('Cannot run tests on r8lib with and without deps. R8lib is now default target.')
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
if options.worktree:
gradle_args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
gradle_args.append('--no-daemon')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithDeps')
gradle_args.append('r8WithDeps11')
gradle_args.append('r8WithRelocatedDeps')
gradle_args.append('r8WithRelocatedDeps11')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print "No failing tests"
return 0
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
if utils.is_bot() and not utils.IsWindows():
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
thread.start_new_thread(timeout_handler, (timestamp_file,))
rotate_test_reports()
if options.only_jctf:
# Note: not setting -Pruntimes will run with all available runtimes.
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Now run tests on selected runtime(s).
if options.runtimes:
if options.dex_vm != 'default':
print 'Unexpected runtimes and dex_vm argument: ' + options.dex_vm
sys.exit(1)
if options.runtimes == 'empty':
# Set runtimes with no content will configure no runtimes.
gradle_args.append('-Pruntimes=')
elif options.runtimes == 'all':
# An unset runtimes will configure all runtimes
pass
else:
prefixes = [prefix.strip() for prefix in options.runtimes.split(':')]
runtimes = []
for prefix in prefixes:
matches = [ rt for rt in VALID_RUNTIMES if rt.startswith(prefix) ]
if len(matches) == 0:
print "Invalid runtime prefix '%s'." % prefix
print "Must be just 'all', 'empty'," \
" or a prefix of %s" % ', '.join(VALID_RUNTIMES)
sys.exit(1)
runtimes.extend(matches)
gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Legacy testing populates the runtimes based on dex_vm.
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Only append the "none" runtime and JVMs if running on the "default" DEX VM.
if art_vm == "default":
runtimes.extend(['jdk8', 'jdk9', 'jdk11', 'none'])
return_code = gradle.RunGradle(
gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(archive,
'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
return archive_and_return(return_code, options)
return 0
def archive_and_return(return_code, options):
if return_code != 0:
if options.archive_failures and os.name != 'nt':
archive_failures()
return return_code
def print_jstacks():
processes = subprocess.check_output(['ps', 'aux'])
for l in processes.splitlines():
if 'java' in l and 'openjdk' in l:
# Example line:
# ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
columns = l.split()
pid = columns[1]
return_value = subprocess.call(['jstack', pid])
if return_value:
print('Could not jstack %s' % l)
def get_time_from_file(timestamp_file):
if os.path.exists(timestamp_file):
timestamp = os.stat(timestamp_file).st_mtime
print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
sys.stdout.flush()
return timestamp
else:
print('TIMEOUT HANDLER no timestamp file yet')
sys.stdout.flush()
return None
def timeout_handler(timestamp_file):
last_timestamp = None
while True:
time.sleep(TIMEOUT_HANDLER_PERIOD)
new_timestamp = get_time_from_file(timestamp_file)
if last_timestamp and new_timestamp == last_timestamp:
print_jstacks()
last_timestamp = new_timestamp
def report_dir_path(index):
if index is 0:
return REPORTS_PATH
return '%s%d' % (REPORTS_PATH, index)
def report_index_path(index):
return os.path.join(report_dir_path(index), *REPORT_INDEX)
# Rotate test results so previous results are still accessible.
def rotate_test_reports():
if not os.path.exists(report_dir_path(0)):
return
i = 1
while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
i += 1
if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
shutil.rmtree(report_dir_path(i))
while i > 0:
shutil.move(report_dir_path(i - 1), report_dir_path(i))
i -= 1
def compute_failed_tests(args):
if len(args) > 1:
print "Running with --failed can take an optional path to a report index (or report number)."
return None
report = report_index_path(0)
# If the default report does not exist, fall back to the previous report as it may be a failed
# gradle run which has already moved the report to report1, but did not produce a new report.
if not os.path.exists(report):
report1 = report_index_path(1)
if os.path.exists(report1):
report = report1
if len(args) == 1:
try:
# try to parse the arg as a report index.
index = int(args[0])
report = report_index_path(index)
except ValueError:
# if integer parsing failed assume it is a report file path.
report = args[0]
if not os.path.exists(report):
print "Can't re-run failing, no report at:", report
return None
print "Reading failed tests in", report
failing = set()
inFailedSection = False
for line in file(report):
l = line.strip()
if l == "<h2>Failed tests</h2>":
inFailedSection = True
elif l.startswith("<h2>"):
inFailedSection = False
prefix = '<a href="classes/'
if inFailedSection and l.startswith(prefix):
href = l[len(prefix):l.index('">')]
# Ignore enties ending with .html which are test classes, not test methods.
if not href.endswith('.html'):
# Remove the .html and anchor separateor, also, a classMethod test is the static
# setup failing so rerun the full class of tests.
test = href.replace('.html','').replace('#', '.').replace('.classMethod', '')
failing.add(test)
return list(failing)
if __name__ == '__main__':
return_code = Main()
if return_code != 0:
notify.notify("Tests failed.")
else:
notify.notify("Tests passed.")
sys.exit(return_code)