blob: efbe5a1e175f45ced8f61500068973587aab5bb5 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Convenience script for running tests. If no argument is given run all tests,
# if an argument is given, run only tests with that pattern. This script will
# force the tests to run, even if no input changed.
import argparse
import os
import shutil
import subprocess
import sys
import time
import uuid
import archive_desugar_jdk_libs
import download_kotlin_dev
import gradle
import notify
import testing_state
import utils
if utils.is_python3():
import threading
else:
import thread
ALL_ART_VMS = [
"default", "14.0.0", "13.0.0", "12.0.0", "10.0.0", "9.0.0", "8.1.0",
"7.0.0", "6.0.1", "5.1.1", "4.4.4", "4.0.4"
]
# How often do we check for progress on the bots:
# Should be long enough that a normal run would always have med progress
# Should be short enough that we ensure that two calls are close enough
# to happen before bot times out.
# A false positiv, i.e., printing the stacks of non hanging processes
# is not a problem, no harm done except some logging in stdout.
TIMEOUT_HANDLER_PERIOD = 60 * 18
BUCKET = 'r8-test-results'
NUMBER_OF_TEST_REPORTS = 5
REPORTS_PATH = os.path.join(utils.BUILD, 'reports')
REPORT_INDEX = ['tests', 'test', 'index.html']
VALID_RUNTIMES = [
'none',
'jdk8',
'jdk9',
'jdk11',
'jdk17',
'jdk20',
] + ['dex-%s' % dexvm for dexvm in ALL_ART_VMS]
def ParseOptions():
result = argparse.ArgumentParser()
result.add_argument('--no-internal',
'--no_internal',
help='Do not run Google internal tests.',
default=False,
action='store_true')
result.add_argument('--archive-failures',
'--archive_failures',
help='Upload test results to cloud storage on failure.',
default=False,
action='store_true')
result.add_argument(
'--archive-failures-file-name',
'--archive_failures_file_name',
help='Set file name for the archived failures file name',
default=uuid.uuid4())
result.add_argument('--only-internal',
'--only_internal',
help='Only run Google internal tests.',
default=False,
action='store_true')
result.add_argument('--all-tests',
'--all_tests',
help='Run tests in all configurations.',
default=False,
action='store_true')
result.add_argument('--slow-tests',
'--slow_tests',
help='Also run slow tests.',
default=False,
action='store_true')
result.add_argument('-v',
'--verbose',
help='Print test stdout to, well, stdout.',
default=False,
action='store_true')
result.add_argument(
'--dex-vm',
'--dex_vm',
help='The android version of the vm to use. "all" will run the tests on '
'all art vm versions (stopping after first failed execution)',
default="default",
choices=ALL_ART_VMS + ["all"])
result.add_argument('--dex-vm-kind',
'--dex_vm_kind',
help='Whether to use host or target version of runtime',
default="host",
nargs=1,
choices=["host", "target"])
result.add_argument(
'--one-line-per-test',
'--one_line_per_test',
help='Print a line before a tests starts and after it ends to stdout.',
default=False,
action='store_true')
result.add_argument(
'--tool',
help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
default=None,
choices=["r8", "d8", "r8cf"])
result.add_argument(
'--disable-assertions',
'--disable_assertions',
'-da',
help='Disable Java assertions when running the compiler '
'(default enabled)',
default=False,
action='store_true')
result.add_argument('--with-code-coverage',
'--with_code_coverage',
help='Enable code coverage with Jacoco.',
default=False,
action='store_true')
result.add_argument(
'--test-dir',
'--test_dir',
help='Use a custom directory for the test artifacts instead of a'
' temporary (which is automatically removed after the test).'
' Note that the directory will not be cleared before the test.')
result.add_argument(
'--command-cache-dir',
'--command_cache_dir',
help='Cache command invocations to this directory, speeds up test runs',
default=os.environ.get('R8_COMMAND_CACHE_DIR'))
result.add_argument(
'--command-cache-stats',
'--command_cache_stats',
help='Collect and print statistics about the command cache.',
default=False,
action='store_true')
result.add_argument('--java-home',
'--java_home',
help='Use a custom java version to run tests.')
result.add_argument('--java-max-memory-size',
'--java_max_memory_size',
help='Set memory for running tests, default 4G',
default=os.environ.get('R8_JAVA_MAX_MEMORY_SIZE', '4G'))
result.add_argument(
'--test-namespace',
'--test_namespace',
help='Only run tests in this namespace. The namespace is relative to '
'com/android/tools/r8, e.g., desugar/desugaredlibrary',
default=None)
result.add_argument('--shard-count',
'--shard_count',
help='We are running this many shards.')
result.add_argument('--shard-number',
'--shard_number',
help='We are running this shard.')
result.add_argument(
'--generate-golden-files-to',
'--generate_golden_files_to',
help='Store dex files produced by tests in the specified directory.'
' It is aimed to be read on platforms with no host runtime available'
' for comparison.')
result.add_argument(
'--use-golden-files-in',
'--use_golden_files_in',
help='Download golden files hierarchy for this commit in the specified'
' location and use them instead of executing on host runtime.')
result.add_argument(
'--no-r8lib',
'--no_r8lib',
default=False,
action='store_true',
help='Run the tests on R8 full with relocated dependencies.')
result.add_argument('--no-arttests',
'--no_arttests',
default=False,
action='store_true',
help='Do not run the art tests.')
result.add_argument(
'--r8lib-no-deps',
'--r8lib_no_deps',
default=False,
action='store_true',
help='Run the tests on r8lib without relocated dependencies.')
result.add_argument('--failed',
default=False,
action='store_true',
help='Run the tests that failed last execution.')
result.add_argument(
'--fail-fast',
'--fail_fast',
default=False,
action='store_true',
help='Stop on first failure. Passes --fail-fast to gradle test runner.')
result.add_argument(
'--worktree',
default=False,
action='store_true',
help='Tests are run in worktree and should not use gradle user home.')
result.add_argument(
'--runtimes',
default=None,
help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
' Special values include: all (for all runtimes)'
' and empty (for no runtimes).')
result.add_argument('--print-hanging-stacks',
'--print_hanging_stacks',
default=-1,
type=int,
help='Print hanging stacks after timeout in seconds')
result.add_argument(
'--print-full-stacktraces',
'--print_full_stacktraces',
default=False,
action='store_true',
help='Print the full stacktraces without any filtering applied')
result.add_argument('--print-obfuscated-stacktraces',
'--print_obfuscated_stacktraces',
default=False,
action='store_true',
help='Print the obfuscated stacktraces')
result.add_argument(
'--debug-agent',
'--debug_agent',
help=
'Enable Java debug agent and suspend compilation (default disabled)',
default=False,
action='store_true')
result.add_argument('--desugared-library-configuration',
'--desugared_library-configuration',
help='Use alternative desugared library configuration.')
result.add_argument('--desugared-library',
'--desugared_library',
help='Build and use desugared library from GitHub.')
result.add_argument('--print-times',
'--print_times',
help='Print the execution time of the slowest tests..',
default=False,
action='store_true')
result.add_argument('--testing-state-dir',
help='Explicitly set the testing state directory '
'(defaults to build/test-state/<git-branch>).')
result.add_argument('--rerun',
help='Rerun tests (implicitly enables testing state).',
choices=testing_state.CHOICES)
result.add_argument('--stacktrace',
help='Pass --stacktrace to the gradle run',
default=False,
action='store_true')
result.add_argument(
'--kotlin-compiler-dev',
help='Specify to download a kotlin dev compiler and run '
'tests with that',
default=False,
action='store_true')
result.add_argument('--kotlin-compiler-old',
help='Specify to run tests on older kotlin compilers',
default=False,
action='store_true')
return result.parse_known_args()
def has_failures(classes_file):
with open(classes_file) as f:
contents = f.read()
# The report has a div tag with the percentage of tests that succeeded.
assert '<div class="percent">' in contents
return '<div class="percent">100%</div>' not in contents
def should_upload(filename, absolute_filename):
# filename is relative to REPO_ROOT/build/reports/tests
if filename.startswith('test/packages'):
# We don't upload the package overview
return False
if filename.startswith('test/classes'):
return has_failures(absolute_filename)
# Always upload index, css and js
return True
def archive_failures(options):
upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
file_name = options.archive_failures_file_name
destination_dir = 'gs://%s/%s/' % (BUCKET, file_name)
for (dir_path, dir_names, file_names) in os.walk(upload_dir):
for f in file_names:
absolute_file = os.path.join(dir_path, f)
relative_file = absolute_file[len(upload_dir) + 1:]
if (should_upload(relative_file, absolute_file)):
utils.upload_file_to_cloud_storage(
absolute_file, destination_dir + relative_file)
url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET,
file_name)
print('Test results available at: %s' % url)
def Main():
(options, args) = ParseOptions()
if utils.is_bot():
gradle.RunGradle(['--no-daemon', 'clean'])
print('Running with python ' + str(sys.version_info))
# Always print stats on bots if command cache is enabled
options.command_cache_stats = options.command_cache_dir is not None
desugar_jdk_json_dir = None
if options.desugared_library_configuration:
if options.desugared_library_configuration != 'jdk11':
print("Only value supported for --desugared-library is 'jdk11'")
exit(1)
desugar_jdk_json_dir = 'src/library_desugar/jdk11'
desugar_jdk_libs = None
if options.desugared_library:
if options.desugared_library != 'HEAD':
print("Only value supported for --desugared-library is 'HEAD'")
exit(1)
desugar_jdk_libs_dir = 'build/desugar_jdk_libs'
shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True)
os.makedirs(desugar_jdk_libs_dir)
print('Building desugared library.')
with utils.TempDir() as checkout_dir:
archive_desugar_jdk_libs.CloneDesugaredLibrary(
'google', checkout_dir, 'HEAD')
# Make sure bazel is extracted in third_party.
utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
(library_jar,
maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(
checkout_dir, 'jdk11_legacy' if
options.desugared_library_configuration == 'jdk11' else 'jdk8')
desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir,
os.path.basename(library_jar))
shutil.copyfile(library_jar, desugar_jdk_libs)
print('Desugared library for test in ' + desugar_jdk_libs)
gradle_args = []
if options.stacktrace or utils.is_bot():
gradle_args.append('--stacktrace')
if utils.is_bot():
# Bots don't like dangling processes.
gradle_args.append('--no-daemon')
# Set all necessary Gradle properties and options first.
if options.shard_count:
assert options.shard_number
gradle_args.append('-Pshard_count=%s' % options.shard_count)
gradle_args.append('-Pshard_number=%s' % options.shard_number)
if options.verbose:
gradle_args.append('-Pprint_test_stdout')
if options.no_internal:
gradle_args.append('-Pno_internal')
if options.only_internal:
gradle_args.append('-Ponly_internal')
if options.all_tests:
gradle_args.append('-Pall_tests')
if options.slow_tests:
gradle_args.append('-Pslow_tests=1')
if options.tool:
gradle_args.append('-Ptool=%s' % options.tool)
if options.one_line_per_test:
gradle_args.append('-Pone_line_per_test')
if options.test_namespace:
gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
if options.disable_assertions:
gradle_args.append('-Pdisable_assertions')
if options.with_code_coverage:
gradle_args.append('-Pwith_code_coverage')
if options.print_full_stacktraces:
gradle_args.append('-Pprint_full_stacktraces')
if options.print_obfuscated_stacktraces:
gradle_args.append('-Pprint_obfuscated_stacktraces')
if options.kotlin_compiler_old:
gradle_args.append('-Pkotlin_compiler_old')
if options.kotlin_compiler_dev:
gradle_args.append('-Pkotlin_compiler_dev')
download_kotlin_dev.download_newest()
if os.name == 'nt':
gradle_args.append('-Pno_internal')
if options.test_dir:
gradle_args.append('-Ptest_dir=' + options.test_dir)
if not os.path.exists(options.test_dir):
os.makedirs(options.test_dir)
if options.command_cache_dir:
gradle_args.append('-Pcommand_cache_dir=' + options.command_cache_dir)
if not os.path.exists(options.command_cache_dir):
os.makedirs(options.command_cache_dir)
if options.command_cache_stats:
stats_dir = os.path.join(options.command_cache_dir, 'stats')
gradle_args.append('-Pcommand_cache_stats_dir=' + stats_dir)
if not os.path.exists(stats_dir):
os.makedirs(stats_dir)
# Clean out old stats files
for (_, _, file_names) in os.walk(stats_dir):
for f in file_names:
os.remove(os.path.join(stats_dir, f))
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
if options.generate_golden_files_to:
gradle_args.append('-Pgenerate_golden_files_to=' +
options.generate_golden_files_to)
if not os.path.exists(options.generate_golden_files_to):
os.makedirs(options.generate_golden_files_to)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.use_golden_files_in:
gradle_args.append('-Puse_golden_files_in=' +
options.use_golden_files_in)
if not os.path.exists(options.use_golden_files_in):
os.makedirs(options.use_golden_files_in)
gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
if options.r8lib_no_deps and options.no_r8lib:
print(
'Inconsistent arguments: both --no-r8lib and --r8lib-no-deps specified.'
)
exit(1)
if options.r8lib_no_deps:
gradle_args.append('-Pr8lib_no_deps')
elif not options.no_r8lib:
gradle_args.append('-Pr8lib')
if options.worktree:
gradle_args.append('-g=' +
os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
gradle_args.append('--no-daemon')
if options.debug_agent:
gradle_args.append('--no-daemon')
if desugar_jdk_json_dir:
gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir)
if desugar_jdk_libs:
gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
if options.no_arttests:
gradle_args.append('-Pno_arttests=true')
if options.rerun:
testing_state.set_up_test_state(gradle_args, options.rerun,
options.testing_state_dir)
# Enable completeness testing of ART profile rewriting.
gradle_args.append('-Part_profile_rewriting_completeness_check=true')
# Build an R8 with dependencies for bootstrapping tests before adding test sources.
gradle_args.append(utils.GRADLE_TASK_R8)
gradle_args.append(utils.GRADLE_TASK_CLEAN_TEST)
gradle_args.append(utils.GRADLE_TASK_TEST)
gradle_args.append('--stacktrace')
gradle_args.append('-Pprint_full_stacktraces')
if options.debug_agent:
gradle_args.append('--debug-jvm')
if options.fail_fast:
gradle_args.append('--fail-fast')
if options.failed:
args = compute_failed_tests(args)
if args is None:
return 1
if len(args) == 0:
print("No failing tests")
return 0
# Test filtering. Must always follow the 'test' task.
testFilterProperty = []
for testFilter in args:
gradle_args.append('--tests')
gradle_args.append(testFilter)
testFilterProperty.append(testFilter)
assert not ("|" in testFilter), "| is used as separating character"
if len(testFilterProperty) > 0:
gradle_args.append("-Ptestfilter=" + "|".join(testFilterProperty))
if options.with_code_coverage:
# Create Jacoco report after tests.
gradle_args.append('jacocoTestReport')
if options.use_golden_files_in:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(options.use_golden_files_in):
utils.download_file_from_cloud_storage(
'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
'%s.tar.gz' % sha1)
utils.unpack_archive('%s.tar.gz' % sha1)
print_stacks_timeout = options.print_hanging_stacks
if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
if os.path.exists(timestamp_file):
os.remove(timestamp_file)
gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
print_stacks_timeout = (print_stacks_timeout if print_stacks_timeout
!= -1 else TIMEOUT_HANDLER_PERIOD)
if utils.is_python3():
threading.Thread(target=timeout_handler,
args=(timestamp_file, print_stacks_timeout),
daemon=True).start()
else:
thread.start_new_thread(timeout_handler, (
timestamp_file,
print_stacks_timeout,
))
rotate_test_reports()
# Now run tests on selected runtime(s).
if options.runtimes:
if options.dex_vm != 'default':
print('Unexpected runtimes and dex_vm argument: ' + options.dex_vm)
sys.exit(1)
if options.runtimes == 'empty':
# Set runtimes with no content will configure no runtimes.
gradle_args.append('-Pruntimes=')
elif options.runtimes == 'all':
# An unset runtimes will configure all runtimes
pass
else:
prefixes = [
prefix.strip() for prefix in options.runtimes.split(':')
]
runtimes = []
for prefix in prefixes:
matches = [rt for rt in VALID_RUNTIMES if rt.startswith(prefix)]
if len(matches) == 0:
print("Invalid runtime prefix '%s'." % prefix)
print("Must be just 'all', 'empty'," \
" or a prefix of %s" % ', '.join(VALID_RUNTIMES))
sys.exit(1)
runtimes.extend(matches)
gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
return archive_and_return(return_code, options)
# Legacy testing populates the runtimes based on dex_vm.
vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
if options.print_times:
gradle_args.append('-Pprint_times=true')
for art_vm in vms_to_test:
vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
runtimes = ['dex-' + art_vm]
# Append the "none" runtime and default JVM if running the "default" DEX VM.
if art_vm == "default":
runtimes.extend(['jdk11', 'none'])
return_code = gradle.RunGradle(gradle_args + [
'-Pdex_vm=%s' % art_vm + vm_suffix,
'-Pruntimes=%s' % ':'.join(runtimes),
],
throw_on_failure=False)
if options.generate_golden_files_to:
sha1 = '%s' % utils.get_HEAD_sha1()
with utils.ChangedWorkingDirectory(
options.generate_golden_files_to):
archive = utils.create_archive(sha1)
utils.upload_file_to_cloud_storage(
archive, 'gs://r8-test-results/golden-files/' + archive)
return archive_and_return(return_code, options)
return 0
def archive_and_return(return_code, options):
if return_code != 0:
if options.archive_failures:
archive_failures(options)
if options.command_cache_stats:
stats_dir = os.path.join(options.command_cache_dir, 'stats')
cache_hit = 0
cache_miss = 0
cache_put = 0
for (_, _, file_names) in os.walk(stats_dir):
for f in file_names:
if f.endswith('CACHEHIT'):
cache_hit += os.stat(os.path.join(stats_dir, f)).st_size
if f.endswith('CACHEMISS'):
cache_miss += os.stat(os.path.join(stats_dir, f)).st_size
if f.endswith('CACHEPUT'):
cache_put += os.stat(os.path.join(stats_dir, f)).st_size
print('Command cache stats')
print(' Cache hits: ' + str(cache_hit))
print(' Cache miss: ' + str(cache_miss))
print(' Cache puts: ' + str(cache_put))
return return_code
def print_jstacks():
processes = subprocess.check_output(['ps', 'aux']).decode('utf-8')
for l in processes.splitlines():
if 'art' in l or 'dalvik' in l:
print('Running art of dalvik process: \n%s' % l)
if 'java' in l and 'openjdk' in l:
print('Running jstack on process: \n%s' % l)
# Example line:
# ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
columns = l.split()
pid = columns[1]
return_value = subprocess.call(['jstack', pid])
if return_value:
print('Could not jstack %s' % l)
def get_time_from_file(timestamp_file):
if os.path.exists(timestamp_file):
timestamp = os.stat(timestamp_file).st_mtime
print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
sys.stdout.flush()
return timestamp
else:
print('TIMEOUT HANDLER no timestamp file yet')
sys.stdout.flush()
return None
def timeout_handler(timestamp_file, timeout_handler_period):
last_timestamp = None
while True:
time.sleep(timeout_handler_period)
new_timestamp = get_time_from_file(timestamp_file)
if last_timestamp and new_timestamp == last_timestamp:
print_jstacks()
last_timestamp = new_timestamp
def report_dir_path(index):
if index == 0:
return REPORTS_PATH
return '%s%d' % (REPORTS_PATH, index)
def report_index_path(index):
return os.path.join(report_dir_path(index), *REPORT_INDEX)
# Rotate test results so previous results are still accessible.
def rotate_test_reports():
if not os.path.exists(report_dir_path(0)):
return
i = 1
while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
i += 1
if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
shutil.rmtree(report_dir_path(i))
while i > 0:
shutil.move(report_dir_path(i - 1), report_dir_path(i))
i -= 1
def compute_failed_tests(args):
if len(args) > 1:
print(
"Running with --failed can take an optional path to a report index (or report number)."
)
return None
report = report_index_path(0)
# If the default report does not exist, fall back to the previous report as it may be a failed
# gradle run which has already moved the report to report1, but did not produce a new report.
if not os.path.exists(report):
report1 = report_index_path(1)
if os.path.exists(report1):
report = report1
if len(args) == 1:
try:
# try to parse the arg as a report index.
index = int(args[0])
report = report_index_path(index)
except ValueError:
# if integer parsing failed assume it is a report file path.
report = args[0]
if not os.path.exists(report):
print("Can't re-run failing, no report at:", report)
return None
print("Reading failed tests in", report)
failing = set()
inFailedSection = False
for line in open(report):
l = line.strip()
if l == "<h2>Failed tests</h2>":
inFailedSection = True
elif l.startswith("<h2>"):
inFailedSection = False
prefix = '<a href="classes/'
if inFailedSection and l.startswith(prefix):
href = l[len(prefix):l.index('">')]
# Ignore enties ending with .html which are test classes, not test methods.
if not href.endswith('.html'):
# Remove the .html and anchor separateor, also, a classMethod test is the static
# setup failing so rerun the full class of tests.
test = href.replace('.html', '').replace('#', '.').replace(
'.classMethod', '')
failing.add(test)
return list(failing)
if __name__ == '__main__':
return_code = Main()
if return_code != 0:
notify.notify("Tests failed.")
else:
notify.notify("Tests passed.")
sys.exit(return_code)