blob: e67bea5fdb0e7f4c440151bbbfcb9200e23248b8 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Convenience script for running tests. If no argument is given run all tests,
# if an argument is given, run only tests with that pattern. This script will
# force the tests to run, even if no input changed.
import optparse
import os
import shutil
import subprocess
import sys
import time
import uuid
import archive_desugar_jdk_libs
import download_kotlin_dev
import gradle
import notify
import utils
if utils.is_python3():
import threading
else:
import thread
ALL_ART_VMS = [
"default",
"14.0.0",
"13.0.0",
"12.0.0",
"10.0.0",
"9.0.0",
"8.1.0",
"7.0.0",
"6.0.1",
"5.1.1",
"4.4.4",
"4.0.4"]
# How often do we check for progress on the bots:
# Should be long enough that a normal run would always have med progress
# Should be short enough that we ensure that two calls are close enough
# to happen before bot times out.
# A false positiv, i.e., printing the stacks of non hanging processes
# is not a problem, no harm done except some logging in stdout.
TIMEOUT_HANDLER_PERIOD = 60 * 18
BUCKET = 'r8-test-results'
NUMBER_OF_TEST_REPORTS = 5
REPORTS_PATH = os.path.join(utils.BUILD, 'reports')
REPORT_INDEX = ['tests', 'test', 'index.html']
VALID_RUNTIMES = [
'none',
'jdk8',
'jdk9',
'jdk11',
'jdk17',
'jdk20',
] + [ 'dex-%s' % dexvm for dexvm in ALL_ART_VMS ]
def ParseOptions():
result = optparse.OptionParser()
result.add_option('--no-internal', '--no_internal',
help='Do not run Google internal tests.',
default=False, action='store_true')
result.add_option('--archive-failures', '--archive_failures',
help='Upload test results to cloud storage on failure.',
default=False, action='store_true')
result.add_option('--archive-failures-file-name',
'--archive_failures_file_name',
help='Set file name for the archived failures file name',
default=uuid.uuid4())
result.add_option('--only-internal', '--only_internal',
help='Only run Google internal tests.',
default=False, action='store_true')
result.add_option('--all-tests', '--all_tests',
help='Run tests in all configurations.',
default=False, action='store_true')
result.add_option('--slow-tests', '--slow_tests',
help='Also run slow tests.',
default=False, action='store_true')
result.add_option('-v', '--verbose',
help='Print test stdout to, well, stdout.',
default=False, action='store_true')
result.add_option('--dex-vm', '--dex_vm',
help='The android version of the vm to use. "all" will run the tests on '
'all art vm versions (stopping after first failed execution)',
default="default",
choices=ALL_ART_VMS + ["all"])
result.add_option('--dex-vm-kind', '--dex_vm_kind',
help='Whether to use host or target version of runtime',
default="host",
nargs=1,
choices=["host", "target"])
result.add_option('--one-line-per-test', '--one_line_per_test',
help='Print a line before a tests starts and after it ends to stdout.',
default=False, action='store_true')
result.add_option('--tool',
help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
default=None, choices=["r8", "d8", "r8cf"])
result.add_option('--disable-assertions', '--disable_assertions',
help='Disable assertions when running tests.',
default=False, action='store_true')
result.add_option('--with-code-coverage', '--with_code_coverage',
help='Enable code coverage with Jacoco.',
default=False, action='store_true')
result.add_option('--test-dir', '--test_dir',
help='Use a custom directory for the test artifacts instead of a'
' temporary (which is automatically removed after the test).'
' Note that the directory will not be cleared before the test.')
result.add_option('--command-cache-dir', '--command_cache_dir',
help='Cache command invocations to this directory, speeds up test runs',
default=os.environ.get('R8_COMMAND_CACHE_DIR'))
result.add_option('--java-home', '--java_home',
help='Use a custom java version to run tests.')
result.add_option('--java-max-memory-size', '--java_max_memory_size',
help='Set memory for running tests, default 4G',
default=os.environ.get('R8_JAVA_MAX_MEMORY_SIZE', '4G'))
result.add_option('--test-namespace', '--test_namespace',
help='Only run tests in this namespace. The namespace is relative to '
'com/android/tools/r8, e.g., desugar/desugaredlibrary',
default=None)
result.add_option('--shard-count', '--shard_count',
help='We are running this many shards.')
result.add_option('--shard-number', '--shard_number',
help='We are running this shard.')
result.add_option('--generate-golden-files-to', '--generate_golden_files_to',
help='Store dex files produced by tests in the specified directory.'
' It is aimed to be read on platforms with no host runtime available'
' for comparison.')
result.add_option('--use-golden-files-in', '--use_golden_files_in',
help='Download golden files hierarchy for this commit in the specified'
' location and use them instead of executing on host runtime.')
result.add_option('--no-r8lib', '--no_r8lib',
default=False, action='store_true',
help='Run the tests on R8 full with relocated dependencies.')
result.add_option('--no-arttests', '--no_arttests',
default=False, action='store_true',
help='Do not run the art tests.')
result.add_option('--r8lib-no-deps', '--r8lib_no_deps',
default=False, action='store_true',
help='Run the tests on r8lib without relocated dependencies.')
result.add_option('--failed',
default=False, action='store_true',
help='Run the tests that failed last execution.')
result.add_option('--fail-fast', '--fail_fast',
default=False, action='store_true',
help='Stop on first failure. Passes --fail-fast to gradle test runner.')
result.add_option('--worktree',
default=False, action='store_true',
help='Tests are run in worktree and should not use gradle user home.')
result.add_option('--runtimes',
default=None,
help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
' Special values include: all (for all runtimes)'
' and empty (for no runtimes).')
result.add_option('--print-hanging-stacks', '--print_hanging_stacks',
default=-1, type="int", help='Print hanging stacks after timeout in seconds')
result.add_option('--print-full-stacktraces', '--print_full_stacktraces',
default=False, action='store_true',
help='Print the full stacktraces without any filtering applied')
result.add_option(
'--print-obfuscated-stacktraces', '--print_obfuscated_stacktraces',
default=False, action='store_true',
help='Print the obfuscated stacktraces')
result.add_option(
'--debug-agent', '--debug_agent',
help='Enable Java debug agent and suspend compilation (default disabled)',
default=False,
action='store_true')
result.add_option('--desugared-library-configuration',
'--desugared_library-configuration',
help='Use alternative desugared library configuration.')
result.add_option('--desugared-library', '--desugared_library',
help='Build and use desugared library from GitHub.')
result.add_option('--print-times', '--print_times',
help='Print the execution time of the slowest tests..',
default=False, action='store_true')
result.add_option(
'--testing-state-name',
help='Set an explict name for the testing state '
'(used in conjunction with --with/reset-testing-state).')
result.add_option(
'--with-testing-state',
help='Run/resume tests using testing state.',
default=False, action='store_true')
result.add_option(
'--reset-testing-state',
help='Clean the testing state and rerun tests (implies --with-testing-state).',
default=False, action='store_true')
result.add_option(
'--stacktrace',
help='Pass --stacktrace to the gradle run',
default=False, action='store_true')
result.add_option('--kotlin-compiler-dev',
help='Specify to download a kotlin dev compiler and run '
'tests with that',
default=False, action='store_true')
result.add_option('--kotlin-compiler-old',
help='Specify to run tests on older kotlin compilers',
default=False, action='store_true')
return result.parse_args()
def has_failures(classes_file):
with open(classes_file) as f:
contents = f.read()
# The report has a div tag with the percentage of tests that succeeded.
assert '<div class="percent">' in contents
return '<div class="percent">100%</div>' not in contents
def should_upload(filename, absolute_filename):
# filename is relative to REPO_ROOT/build/reports/tests
if filename.startswith('test/packages'):
# We don't upload the package overview
return False
if filename.startswith('test/classes'):
return has_failures(absolute_filename)
# Always upload index, css and js
return True
def archive_failures(options):
upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
file_name = options.archive_failures_file_name
destination_dir = 'gs://%s/%s/' % (BUCKET, file_name)
for (dir_path, dir_names, file_names) in os.walk(upload_dir):
for f in file_names:
absolute_file = os.path.join(dir_path, f)
relative_file = absolute_file[len(upload_dir)+1:]
if (should_upload(relative_file, absolute_file)):
utils.upload_file_to_cloud_storage(absolute_file,
destination_dir + relative_file)
url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET, file_name)
print('Test results available at: %s' % url)
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
print('Running with python ' + str(sys.version_info))
desugar_jdk_json_dir = None
if options.desugared_library_configuration:
if options.desugared_library_configuration != 'jdk11':
print("Only value supported for --desugared-library is 'jdk11'")
exit(1)
desugar_jdk_json_dir = 'src/library_desugar/jdk11'
desugar_jdk_libs = None
if options.desugared_library:
if options.desugared_library != 'HEAD':
print("Only value supported for --desugared-library is 'HEAD'")
exit(1)
desugar_jdk_libs_dir = 'build/desugar_jdk_libs'
shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True)
os.makedirs(desugar_jdk_libs_dir)
print('Building desugared library.')
with utils.TempDir() as checkout_dir:
archive_desugar_jdk_libs.CloneDesugaredLibrary('google', checkout_dir, 'HEAD')
# Make sure bazel is extracted in third_party.
utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
(library_jar, maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(checkout_dir, 'jdk11_legacy' if options.desugared_library_configuration == 'jdk11' else 'jdk8')
desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir, os.path.basename(library_jar))
shutil.copyfile(library_jar, desugar_jdk_libs)
print('Desugared library for test in ' + desugar_jdk_libs)
gradle_args = []
if options.stacktrace or utils.is_bot():
gradle_args.append('--stacktrace')
if utils.is_bot():
# Bots don't like dangling processes.
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.slow_tests:
gradle_args.append('-Pslow_tests=1')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.test_namespace:
gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if options.print_full_stacktraces:
gradle_args.append('-Pprint_full_stacktraces')
if options.print_obfuscated_stacktraces:
gradle_args.append('-Pprint_obfuscated_stacktraces')
if options.kotlin_compiler_old:
gradle_args.append('-Pkotlin_compiler_old')
if options.kotlin_compiler_dev:
gradle_args.append('-Pkotlin_compiler_dev')
download_kotlin_dev.download_newest()
if os.name == 'nt':
gradle_args.append('-Pno_internal')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.command_cache_dir:
gradle_args.append('-Pcommand_cache_dir=' + options.command_cache_dir)
if not os.path.exists(options.command_cache_dir):
os.makedirs(options.command_cache_dir)
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' + options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' + options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if (not options.no_r8lib) and options.r8lib_no_deps:
print('Cannot run tests on r8lib with and without deps. R8lib is now default target.')
exit(1)
if not options.no_r8lib:
gradle_args.append('-Pr8lib')
# Force gradle to build a version of r8lib without dependencies for
# BootstrapCurrentEqualityTest.
gradle_args.append('R8LibNoDeps')
gradle_args.append('R8Retrace')
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
if options.worktree:
gradle_args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
gradle_args.append('--no-daemon')
if options.debug_agent:
gradle_args.append('--no-daemon')
if desugar_jdk_json_dir:
gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir)
if desugar_jdk_libs:
gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
if options.no_arttests:
gradle_args.append('-Pno_arttests=true')
if options.reset_testing_state:
gradle_args.append('-Ptesting-state')
gradle_args.append('-Preset-testing-state')
elif options.with_testing_state:
gradle_args.append('-Ptesting-state')
if options.testing_state_name:
gradle_args.append('-Ptesting-state-name=' + options.testing_state_name)
# Enable completeness testing of ART profile rewriting.
gradle_args.append('-Part_profile_rewriting_completeness_check=true')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append('r8WithRelocatedDeps')
gradle_args.append('r8WithRelocatedDeps17')
# Add Gradle tasks
gradle_args.append('cleanTest')
gradle_args.append('test')
if options.debug_agent:
gradle_args.append('--debug-jvm')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print("No failing tests")
return 0
# Test filtering. Must always follow the 'test' task.
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
print_stacks_timeout = options.print_hanging_stacks
if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
print_stacks_timeout = (print_stacks_timeout
if print_stacks_timeout != -1
else TIMEOUT_HANDLER_PERIOD)
if utils.is_python3():
threading.Thread(
target=timeout_handler,
args=(timestamp_file, print_stacks_timeout),
daemon=True).start()
else:
thread.start_new_thread(
timeout_handler, (timestamp_file, print_stacks_timeout,))
rotate_test_reports()
# Now run tests on selected runtime(s).
if options.runtimes:
if options.dex_vm != 'default':
print('Unexpected runtimes and dex_vm argument: ' + options.dex_vm)
sys.exit(1)
if options.runtimes == 'empty':
# Set runtimes with no content will configure no runtimes.
gradle_args.append('-Pruntimes=')
elif options.runtimes == 'all':
# An unset runtimes will configure all runtimes
pass
else:
prefixes = [prefix.strip() for prefix in options.runtimes.split(':')]
runtimes = []
for prefix in prefixes:
matches = [ rt for rt in VALID_RUNTIMES if rt.startswith(prefix) ]
if len(matches) == 0:
print("Invalid runtime prefix '%s'." % prefix)
print("Must be just 'all', 'empty'," \
" or a prefix of %s" % ', '.join(VALID_RUNTIMES))
sys.exit(1)
runtimes.extend(matches)
gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Legacy testing populates the runtimes based on dex_vm.
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
if options.print_times:
gradle_args.append('-Pprint_times=true')
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Append the "none" runtime and default JVM if running the "default" DEX VM.
if art_vm == "default":
runtimes.extend(['jdk11', 'none'])
return_code = gradle.RunGradle(
gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(archive,
'gs://r8-test-results/golden-files/' + archive)
if return_code != 0:
return archive_and_return(return_code, options)
return 0
def archive_and_return(return_code, options):
if return_code != 0:
if options.archive_failures:
archive_failures(options)
return return_code
def print_jstacks():
processes = subprocess.check_output(['ps', 'aux']).decode('utf-8')
for l in processes.splitlines():
if 'art' in l or 'dalvik' in l:
print('Running art of dalvik process: \n%s' % l)
if 'java' in l and 'openjdk' in l:
print('Running jstack on process: \n%s' % l)
# Example line:
# ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
columns = l.split()
pid = columns[1]
return_value = subprocess.call(['jstack', pid])
if return_value:
print('Could not jstack %s' % l)
def get_time_from_file(timestamp_file):
if os.path.exists(timestamp_file):
timestamp = os.stat(timestamp_file).st_mtime
print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
sys.stdout.flush()
return timestamp
else:
print('TIMEOUT HANDLER no timestamp file yet')
sys.stdout.flush()
return None
def timeout_handler(timestamp_file, timeout_handler_period):
last_timestamp = None
while True:
time.sleep(timeout_handler_period)
new_timestamp = get_time_from_file(timestamp_file)
if last_timestamp and new_timestamp == last_timestamp:
print_jstacks()
last_timestamp = new_timestamp
def report_dir_path(index):
if index == 0:
return REPORTS_PATH
return '%s%d' % (REPORTS_PATH, index)
def report_index_path(index):
return os.path.join(report_dir_path(index), *REPORT_INDEX)
# Rotate test results so previous results are still accessible.
def rotate_test_reports():
if not os.path.exists(report_dir_path(0)):
return
i = 1
while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
i += 1
if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
shutil.rmtree(report_dir_path(i))
while i > 0:
shutil.move(report_dir_path(i - 1), report_dir_path(i))
i -= 1
def compute_failed_tests(args):
if len(args) > 1:
print("Running with --failed can take an optional path to a report index (or report number).")
return None
report = report_index_path(0)
# If the default report does not exist, fall back to the previous report as it may be a failed
# gradle run which has already moved the report to report1, but did not produce a new report.
if not os.path.exists(report):
report1 = report_index_path(1)
if os.path.exists(report1):
report = report1
if len(args) == 1:
try:
# try to parse the arg as a report index.
index = int(args[0])
report = report_index_path(index)
except ValueError:
# if integer parsing failed assume it is a report file path.
report = args[0]
if not os.path.exists(report):
print("Can't re-run failing, no report at:", report)
return None
print("Reading failed tests in", report)
failing = set()
inFailedSection = False
for line in open(report):
l = line.strip()
if l == "<h2>Failed tests</h2>":
inFailedSection = True
elif l.startswith("<h2>"):
inFailedSection = False
prefix = '<a href="classes/'
if inFailedSection and l.startswith(prefix):
href = l[len(prefix):l.index('">')]
# Ignore enties ending with .html which are test classes, not test methods.
if not href.endswith('.html'):
# Remove the .html and anchor separateor, also, a classMethod test is the static
# setup failing so rerun the full class of tests.
test = href.replace('.html','').replace('#', '.').replace('.classMethod', '')
failing.add(test)
return list(failing)
if __name__ == '__main__':
return_code = Main()
if return_code != 0:
notify.notify("Tests failed.")
else:
notify.notify("Tests passed.")
sys.exit(return_code)