tools/test.py

#!/usr/bin/env python3
# Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# Convenience script for running tests. If no argument is given run all tests,
# if an argument is given, run only tests with that pattern. This script will
# force the tests to run, even if no input changed.

import argparse
import os
import shutil
import subprocess
import sys
import time
import uuid

import archive_desugar_jdk_libs
import download_kotlin_dev
import gradle
import notify
import testing_state
import utils

if utils.is_python3():
    import threading
else:
    import thread

ALL_ART_VMS = [
    "default", "15.0.0", "14.0.0", "13.0.0", "12.0.0", "10.0.0", "9.0.0",
    "8.1.0", "7.0.0", "6.0.1", "5.1.1", "4.4.4", "4.0.4"
]

# How often do we check for progress on the bots:
# Should be long enough that a normal run would always have med progress
# Should be short enough that we ensure that two calls are close enough
# to happen before bot times out.
# A false positiv, i.e., printing the stacks of non hanging processes
# is not a problem, no harm done except some logging in stdout.
TIMEOUT_HANDLER_PERIOD = 60 * 18

BUCKET = 'r8-test-results'

NUMBER_OF_TEST_REPORTS = 5
REPORTS_PATH = os.path.join(utils.BUILD, 'reports')
REPORT_INDEX = ['tests', 'test', 'index.html']
VALID_RUNTIMES = [
    'none',
    'jdk8',
    'jdk9',
    'jdk11',
    'jdk17',
    'jdk21',
    'jdk22',
] + ['dex-%s' % dexvm for dexvm in ALL_ART_VMS]


def ParseOptions():
    result = argparse.ArgumentParser()
    result.add_argument('--no-internal',
                        '--no_internal',
                        help='Do not run Google internal tests.',
                        default=False,
                        action='store_true')
    result.add_argument('--archive-failures',
                        '--archive_failures',
                        help='Upload test results to cloud storage on failure.',
                        default=False,
                        action='store_true')
    result.add_argument(
        '--archive-failures-file-name',
        '--archive_failures_file_name',
        help='Set file name for the archived failures file name',
        default=uuid.uuid4())
    result.add_argument('--only-internal',
                        '--only_internal',
                        help='Only run Google internal tests.',
                        default=False,
                        action='store_true')
    result.add_argument('--all-tests',
                        '--all_tests',
                        help='Run tests in all configurations.',
                        default=False,
                        action='store_true')
    result.add_argument('--slow-tests',
                        '--slow_tests',
                        help='Also run slow tests.',
                        default=False,
                        action='store_true')
    result.add_argument('-v',
                        '--verbose',
                        help='Print test stdout to, well, stdout.',
                        default=False,
                        action='store_true')
    result.add_argument(
        '--dex-vm',
        '--dex_vm',
        help='The android version of the vm to use. "all" will run the tests on '
        'all art vm versions (stopping after first failed execution)',
        default="default",
        choices=ALL_ART_VMS + ["all"])
    result.add_argument('--dex-vm-kind',
                        '--dex_vm_kind',
                        help='Whether to use host or target version of runtime',
                        default="host",
                        nargs=1,
                        choices=["host", "target"])
    result.add_argument(
        '--one-line-per-test',
        '--one_line_per_test',
        help='Print a line before a tests starts and after it ends to stdout.',
        default=False,
        action='store_true')
    result.add_argument(
        '--tool',
        help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
        ' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
        default=None,
        choices=["r8", "d8", "r8cf"])
    result.add_argument(
        '--disable-assertions',
        '--disable_assertions',
        '-da',
        help='Disable Java assertions when running the compiler '
        '(default enabled)',
        default=False,
        action='store_true')
    result.add_argument('--with-code-coverage',
                        '--with_code_coverage',
                        help='Enable code coverage with Jacoco.',
                        default=False,
                        action='store_true')
    result.add_argument(
        '--test-dir',
        '--test_dir',
        help='Use a custom directory for the test artifacts instead of a'
        ' temporary (which is automatically removed after the test).'
        ' Note that the directory will not be cleared before the test.')
    result.add_argument(
        '--command-cache-dir',
        '--command_cache_dir',
        help='Cache command invocations to this directory, speeds up test runs',
        default=os.environ.get('R8_COMMAND_CACHE_DIR'))
    result.add_argument(
        '--command-cache-stats',
        '--command_cache_stats',
        help='Collect and print statistics about the command cache.',
        default=False,
        action='store_true')
    result.add_argument('--java-home',
                        '--java_home',
                        help='Use a custom java version to run tests.')
    result.add_argument('--java-max-memory-size',
                        '--java_max_memory_size',
                        help='Set memory for running tests, default 4G',
                        default=os.environ.get('R8_JAVA_MAX_MEMORY_SIZE', '4G'))
    result.add_argument(
        '--test-namespace',
        '--test_namespace',
        help='Only run tests in  this namespace. The namespace is relative to '
        'com/android/tools/r8, e.g., desugar/desugaredlibrary',
        default=None)
    result.add_argument('--shard-count',
                        '--shard_count',
                        help='We are running this many shards.')
    result.add_argument('--shard-number',
                        '--shard_number',
                        help='We are running this shard.')
    result.add_argument(
        '--generate-golden-files-to',
        '--generate_golden_files_to',
        help='Store dex files produced by tests in the specified directory.'
        ' It is aimed to be read on platforms with no host runtime available'
        ' for comparison.')
    result.add_argument(
        '--use-golden-files-in',
        '--use_golden_files_in',
        help='Download golden files hierarchy for this commit in the specified'
        ' location and use them instead of executing on host runtime.')
    result.add_argument(
        '--no-r8lib',
        '--no_r8lib',
        default=False,
        action='store_true',
        help='Run the tests on R8 full with relocated dependencies.')
    result.add_argument('--no-arttests',
                        '--no_arttests',
                        default=False,
                        action='store_true',
                        help='Do not run the art tests.')
    result.add_argument(
        '--r8lib-no-deps',
        '--r8lib_no_deps',
        default=False,
        action='store_true',
        help='Run the tests on r8lib without relocated dependencies.')
    result.add_argument('--failed',
                        default=False,
                        action='store_true',
                        help='Run the tests that failed last execution.')
    result.add_argument(
        '--fail-fast',
        '--fail_fast',
        default=False,
        action='store_true',
        help='Stop on first failure. Passes --fail-fast to gradle test runner.')
    result.add_argument(
        '--worktree',
        default=False,
        action='store_true',
        help='Tests are run in worktree and should not use gradle user home.')
    result.add_argument(
        '--runtimes',
        default=None,
        help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
        ' Special values include: all (for all runtimes)'
        ' and empty (for no runtimes).')
    result.add_argument('--print-hanging-stacks',
                        '--print_hanging_stacks',
                        default=-1,
                        type=int,
                        help='Print hanging stacks after timeout in seconds')
    result.add_argument(
        '--print-full-stacktraces',
        '--print_full_stacktraces',
        default=False,
        action='store_true',
        help='Print the full stacktraces without any filtering applied')
    result.add_argument('--print-obfuscated-stacktraces',
                        '--print_obfuscated_stacktraces',
                        default=False,
                        action='store_true',
                        help='Print the obfuscated stacktraces')
    result.add_argument(
        '--debug-agent',
        '--debug_agent',
        help=
        'Enable Java debug agent and suspend compilation (default disabled)',
        default=False,
        action='store_true')
    result.add_argument('--desugared-library-configuration',
                        '--desugared_library-configuration',
                        help='Use alternative desugared library configuration.')
    result.add_argument('--desugared-library',
                        '--desugared_library',
                        help='Build and use desugared library from GitHub.')
    result.add_argument('--print-times',
                        '--print_times',
                        help='Print the execution time of the slowest tests..',
                        default=False,
                        action='store_true')
    result.add_argument('--testing-state-dir',
                        help='Explicitly set the testing state directory '
                        '(defaults to build/test-state/<git-branch>).')
    result.add_argument('--rerun',
                        help='Rerun tests (implicitly enables testing state).',
                        choices=testing_state.CHOICES)
    result.add_argument('--stacktrace',
                        help='Pass --stacktrace to the gradle run',
                        default=False,
                        action='store_true')
    result.add_argument('--no-daemon',
                        help='Pass --no-daemon to the gradle run',
                        default=False,
                        action='store_true')
    result.add_argument(
        '--kotlin-compiler-dev',
        help='Specify to download a kotlin dev compiler and run '
        'tests with that',
        default=False,
        action='store_true')
    result.add_argument('--kotlin-compiler-old',
                        help='Specify to run tests on older kotlin compilers',
                        default=False,
                        action='store_true')
    return result.parse_known_args()


def has_failures(classes_file):
    with open(classes_file) as f:
        contents = f.read()
        # The report has a div tag with the percentage of tests that succeeded.
        assert '<div class="percent">' in contents
        return '<div class="percent">100%</div>' not in contents


def should_upload(filename, absolute_filename):
    # filename is relative to REPO_ROOT/build/reports/tests
    if filename.startswith('test/packages'):
        # We don't upload the package overview
        return False
    if filename.startswith('test/classes'):
        return has_failures(absolute_filename)
    # Always upload index, css and js
    return True


def archive_failures(options):
    upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
    file_name = options.archive_failures_file_name
    destination_dir = 'gs://%s/%s/' % (BUCKET, file_name)
    for (dir_path, dir_names, file_names) in os.walk(upload_dir):
        for f in file_names:
            absolute_file = os.path.join(dir_path, f)
            relative_file = absolute_file[len(upload_dir) + 1:]
            if (should_upload(relative_file, absolute_file)):
                utils.upload_file_to_cloud_storage(
                    absolute_file, destination_dir + relative_file)
    url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET,
                                                                    file_name)
    print('Test results available at: %s' % url)


def Main():
    (options, args) = ParseOptions()
    if utils.is_bot():
        gradle.RunGradle(['--no-daemon', 'clean'])
        print('Running with python ' + str(sys.version_info))
        # Always print stats on bots if command cache is enabled
        options.command_cache_stats = options.command_cache_dir is not None

    desugar_jdk_json_dir = None
    if options.desugared_library_configuration:
        if options.desugared_library_configuration != 'jdk11':
            print("Only value supported for --desugared-library is 'jdk11'")
            exit(1)
        desugar_jdk_json_dir = 'src/library_desugar/jdk11'

    desugar_jdk_libs = None
    if options.desugared_library:
        if options.desugared_library != 'HEAD':
            print("Only value supported for --desugared-library is 'HEAD'")
            exit(1)
        desugar_jdk_libs_dir = 'build/desugar_jdk_libs'
        shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True)
        os.makedirs(desugar_jdk_libs_dir)
        print('Building desugared library.')
        with utils.TempDir() as checkout_dir:
            archive_desugar_jdk_libs.CloneDesugaredLibrary(
                'google', checkout_dir, 'HEAD')
            # Make sure bazel is extracted in third_party.
            utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
            utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
            utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
            (library_jar,
             maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(
                 checkout_dir, 'jdk11_legacy' if
                 options.desugared_library_configuration == 'jdk11' else 'jdk8')
            desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir,
                                            os.path.basename(library_jar))
            shutil.copyfile(library_jar, desugar_jdk_libs)
            print('Desugared library for test in ' + desugar_jdk_libs)

    gradle_args = []

    if options.stacktrace or utils.is_bot():
        gradle_args.append('--stacktrace')

    if options.no_daemon or utils.is_bot():
        # Bots don't like dangling processes.
        gradle_args.append('--no-daemon')

    # Set all necessary Gradle properties and options first.
    if options.shard_count:
        assert options.shard_number
        gradle_args.append('-Pshard_count=%s' % options.shard_count)
        gradle_args.append('-Pshard_number=%s' % options.shard_number)
    if options.verbose:
        gradle_args.append('-Pprint_test_stdout')
    if options.no_internal:
        gradle_args.append('-Pno_internal')
    if options.only_internal:
        gradle_args.append('-Ponly_internal')
    if options.all_tests:
        gradle_args.append('-Pall_tests')
    if options.slow_tests:
        gradle_args.append('-Pslow_tests=1')
    if options.tool:
        gradle_args.append('-Ptool=%s' % options.tool)
    if options.one_line_per_test:
        gradle_args.append('-Pone_line_per_test')
    if options.test_namespace:
        gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
    if options.disable_assertions:
        gradle_args.append('-Pdisable_assertions')
    if options.with_code_coverage:
        gradle_args.append('-Pwith_code_coverage')
    if options.print_full_stacktraces:
        gradle_args.append('-Pprint_full_stacktraces')
    if options.print_obfuscated_stacktraces:
        gradle_args.append('-Pprint_obfuscated_stacktraces')
    if options.kotlin_compiler_old:
        gradle_args.append('-Pkotlin_compiler_old')
    if options.kotlin_compiler_dev:
        gradle_args.append('-Pkotlin_compiler_dev')
        download_kotlin_dev.download_newest()
    if os.name == 'nt':
        gradle_args.append('-Pno_internal')
    if options.test_dir:
        gradle_args.append('-Ptest_dir=' + options.test_dir)
        if not os.path.exists(options.test_dir):
            os.makedirs(options.test_dir)
    if options.command_cache_dir:
        gradle_args.append('-Pcommand_cache_dir=' + options.command_cache_dir)
        if not os.path.exists(options.command_cache_dir):
            os.makedirs(options.command_cache_dir)
        if options.command_cache_stats:
            stats_dir = os.path.join(options.command_cache_dir, 'stats')
            gradle_args.append('-Pcommand_cache_stats_dir=' + stats_dir)
            if not os.path.exists(stats_dir):
                os.makedirs(stats_dir)
            # Clean out old stats files
            for (_, _, file_names) in os.walk(stats_dir):
                for f in file_names:
                    os.remove(os.path.join(stats_dir, f))
    if options.java_home:
        gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
    if options.java_max_memory_size:
        gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
    if options.generate_golden_files_to:
        gradle_args.append('-Pgenerate_golden_files_to=' +
                           options.generate_golden_files_to)
        if not os.path.exists(options.generate_golden_files_to):
            os.makedirs(options.generate_golden_files_to)
        gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
    if options.use_golden_files_in:
        gradle_args.append('-Puse_golden_files_in=' +
                           options.use_golden_files_in)
        if not os.path.exists(options.use_golden_files_in):
            os.makedirs(options.use_golden_files_in)
        gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
    if options.r8lib_no_deps and options.no_r8lib:
        print(
            'Inconsistent arguments: both --no-r8lib and --r8lib-no-deps specified.'
        )
        exit(1)
    if options.r8lib_no_deps:
        gradle_args.append('-Pr8lib_no_deps')
    elif not options.no_r8lib:
        gradle_args.append('-Pr8lib')
    if options.worktree:
        gradle_args.append('-g=' +
                           os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
        gradle_args.append('--no-daemon')
    if options.debug_agent:
        gradle_args.append('--no-daemon')
    if desugar_jdk_json_dir:
        gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir)
    if desugar_jdk_libs:
        gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
    if options.no_arttests:
        gradle_args.append('-Pno_arttests=true')

    if options.rerun:
        testing_state.set_up_test_state(gradle_args, options.rerun,
                                        options.testing_state_dir)

    # Enable completeness testing of ART profile rewriting.
    gradle_args.append('-Part_profile_rewriting_completeness_check=true')

    # Build an R8 with dependencies for bootstrapping tests before adding test sources.
    gradle_args.append(utils.GRADLE_TASK_R8)
    gradle_args.append(utils.GRADLE_TASK_CLEAN_TEST)
    gradle_args.append(utils.GRADLE_TASK_TEST)
    gradle_args.append('--stacktrace')
    gradle_args.append('-Pprint_full_stacktraces')

    if options.debug_agent:
        gradle_args.append('--debug-jvm')
    if options.fail_fast:
        gradle_args.append('--fail-fast')
    if options.failed:
        args = compute_failed_tests(args)
        if args is None:
            return 1
        if len(args) == 0:
            print("No failing tests")
            return 0
    # Test filtering. Must always follow the 'test' task.
    testFilterProperty = []
    for testFilter in args:
        gradle_args.append('--tests')
        gradle_args.append(testFilter)
        testFilterProperty.append(testFilter)
        assert not ("|" in testFilter), "| is used as separating character"
    if len(testFilterProperty) > 0:
        gradle_args.append("-Ptestfilter=" + "|".join(testFilterProperty))
    if options.with_code_coverage:
        # Create Jacoco report after tests.
        gradle_args.append('jacocoTestReport')

    if options.use_golden_files_in:
        sha1 = '%s' % utils.get_HEAD_sha1()
        with utils.ChangedWorkingDirectory(options.use_golden_files_in):
            utils.download_file_from_cloud_storage(
                'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
                '%s.tar.gz' % sha1)
            utils.unpack_archive('%s.tar.gz' % sha1)

    print_stacks_timeout = options.print_hanging_stacks
    if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
        timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
        if os.path.exists(timestamp_file):
            os.remove(timestamp_file)
        gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
        print_stacks_timeout = (print_stacks_timeout if print_stacks_timeout
                                != -1 else TIMEOUT_HANDLER_PERIOD)
        if utils.is_python3():
            threading.Thread(target=timeout_handler,
                             args=(timestamp_file, print_stacks_timeout),
                             daemon=True).start()
        else:
            thread.start_new_thread(timeout_handler, (
                timestamp_file,
                print_stacks_timeout,
            ))
    rotate_test_reports()

    if options.print_times:
        gradle_args.append('-Pprint_times=true')

    # Now run tests on selected runtime(s).
    if options.runtimes:
        if options.dex_vm != 'default':
            print('Unexpected runtimes and dex_vm argument: ' + options.dex_vm)
            sys.exit(1)
        if options.runtimes == 'empty':
            # Set runtimes with no content will configure no runtimes.
            gradle_args.append('-Pruntimes=')
        elif options.runtimes == 'all':
            # An unset runtimes will configure all runtimes
            pass
        else:
            prefixes = [
                prefix.strip() for prefix in options.runtimes.split(':')
            ]
            runtimes = []
            for prefix in prefixes:
                matches = [rt for rt in VALID_RUNTIMES if rt.startswith(prefix)]
                if len(matches) == 0:
                    print("Invalid runtime prefix '%s'." % prefix)
                    print("Must be just 'all', 'empty'," \
                          " or a prefix of %s" % ', '.join(VALID_RUNTIMES))
                    sys.exit(1)
                runtimes.extend(matches)
            gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))

        return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
        return archive_and_return(return_code, options)

    # Legacy testing populates the runtimes based on dex_vm.
    vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS

    for art_vm in vms_to_test:
        vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
        runtimes = ['dex-' + art_vm]
        # Append the "none" runtime and default JVM if running the "default" DEX VM.
        if art_vm == "default":
            runtimes.extend(['jdk11', 'none'])
        return_code = gradle.RunGradle(gradle_args + [
            '-Pdex_vm=%s' % art_vm + vm_suffix,
            '-Pruntimes=%s' % ':'.join(runtimes),
        ],
                                       throw_on_failure=False)
        if options.generate_golden_files_to:
            sha1 = '%s' % utils.get_HEAD_sha1()
            with utils.ChangedWorkingDirectory(
                    options.generate_golden_files_to):
                archive = utils.create_archive(sha1)
                utils.upload_file_to_cloud_storage(
                    archive, 'gs://r8-test-results/golden-files/' + archive)

        return archive_and_return(return_code, options)

    return 0


def archive_and_return(return_code, options):
    if return_code != 0:
        if options.archive_failures:
            archive_failures(options)
    if options.command_cache_stats:
        stats_dir = os.path.join(options.command_cache_dir, 'stats')
        cache_hit = 0
        cache_miss = 0
        cache_put = 0
        for (_, _, file_names) in os.walk(stats_dir):
            for f in file_names:
                if f.endswith('CACHEHIT'):
                    cache_hit += os.stat(os.path.join(stats_dir, f)).st_size
                if f.endswith('CACHEMISS'):
                    cache_miss += os.stat(os.path.join(stats_dir, f)).st_size
                if f.endswith('CACHEPUT'):
                    cache_put += os.stat(os.path.join(stats_dir, f)).st_size
        print('Command cache stats')
        print('  Cache hits: ' + str(cache_hit))
        print('  Cache miss: ' + str(cache_miss))
        print('  Cache puts: ' + str(cache_put))
    return return_code


def print_jstacks():
    processes = subprocess.check_output(['ps', 'aux']).decode('utf-8')
    for l in processes.splitlines():
        if 'art' in l or 'dalvik' in l:
            print('Running art of dalvik process: \n%s' % l)
        if 'java' in l and 'openjdk' in l:
            print('Running jstack on process: \n%s' % l)
            # Example line:
            # ricow    184313  2.6  0.0 36839068 31808 ?      Sl   09:53   0:00 /us..
            columns = l.split()
            pid = columns[1]
            return_value = subprocess.call(['jstack', pid])
            if return_value:
                print('Could not jstack %s' % l)


def get_time_from_file(timestamp_file):
    if os.path.exists(timestamp_file):
        timestamp = os.stat(timestamp_file).st_mtime
        print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
        sys.stdout.flush()
        return timestamp
    else:
        print('TIMEOUT HANDLER no timestamp file yet')
        sys.stdout.flush()
        return None


def timeout_handler(timestamp_file, timeout_handler_period):
    last_timestamp = None
    while True:
        time.sleep(timeout_handler_period)
        new_timestamp = get_time_from_file(timestamp_file)
        if last_timestamp and new_timestamp == last_timestamp:
            print_jstacks()
        last_timestamp = new_timestamp


def report_dir_path(index):
    if index == 0:
        return REPORTS_PATH
    return '%s%d' % (REPORTS_PATH, index)


def report_index_path(index):
    return os.path.join(report_dir_path(index), *REPORT_INDEX)


# Rotate test results so previous results are still accessible.
def rotate_test_reports():
    if not os.path.exists(report_dir_path(0)):
        return
    i = 1
    while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
        i += 1
    if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
        shutil.rmtree(report_dir_path(i))
    while i > 0:
        shutil.move(report_dir_path(i - 1), report_dir_path(i))
        i -= 1


def compute_failed_tests(args):
    if len(args) > 1:
        print(
            "Running with --failed can take an optional path to a report index (or report number)."
        )
        return None
    report = report_index_path(0)
    # If the default report does not exist, fall back to the previous report as it may be a failed
    # gradle run which has already moved the report to report1, but did not produce a new report.
    if not os.path.exists(report):
        report1 = report_index_path(1)
        if os.path.exists(report1):
            report = report1
    if len(args) == 1:
        try:
            # try to parse the arg as a report index.
            index = int(args[0])
            report = report_index_path(index)
        except ValueError:
            # if integer parsing failed assume it is a report file path.
            report = args[0]
    if not os.path.exists(report):
        print("Can't re-run failing, no report at:", report)
        return None
    print("Reading failed tests in", report)
    failing = set()
    inFailedSection = False
    for line in open(report):
        l = line.strip()
        if l == "<h2>Failed tests</h2>":
            inFailedSection = True
        elif l.startswith("<h2>"):
            inFailedSection = False
        prefix = '<a href="classes/'
        if inFailedSection and l.startswith(prefix):
            href = l[len(prefix):l.index('">')]
            # Ignore enties ending with .html which are test classes, not test methods.
            if not href.endswith('.html'):
                # Remove the .html and anchor separateor, also, a classMethod test is the static
                # setup failing so rerun the full class of tests.
                test = href.replace('.html', '').replace('#', '.').replace(
                    '.classMethod', '')
                failing.add(test)
    return list(failing)


if __name__ == '__main__':
    return_code = Main()
    if return_code != 0:
        notify.notify("Tests failed.")
    else:
        notify.notify("Tests passed.")
    sys.exit(return_code)