Support rerunning all previously failing tests
This also moves the "outstanding" functionality to a "run mode".
It appears to often time out if the test state is large. A mode
to rerun all tests is also added and replaces the need to delete
the test state to start over.
The testing state functionality is removed from the old build.
Bug: b/297316723
Change-Id: I5e3e239c6409d1644305599e28b92d5b3730686c
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
index 240e656..53c0192 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
@@ -16,41 +16,69 @@
import org.gradle.api.tasks.testing.TestOutputEvent
import org.gradle.api.tasks.testing.TestOutputListener
import org.gradle.api.tasks.testing.TestResult
+import org.gradle.api.tasks.testing.TestResult.ResultType
// Utility to install tracking of test results in status files.
class TestingState {
companion object {
- fun setUpTestingState(task: Test) {
- val project = task.project
- if (!project.hasProperty("testing-state")) {
- return
+ const val MODE_PROPERTY = "testing-state-mode"
+ const val PATH_PROPERTY = "testing-state-path"
+
+ const val PAST_FAILURE = "PAST_FAILURE"
+
+ // Operating mode for the test state.
+ enum class Mode { ALL, OUTSTANDING, FAILING, PAST_FAILING }
+
+ // These are the files that are allowed for tracking test status.
+ enum class StatusFile { SUCCESS, FAILURE, PAST_FAILURE }
+
+ fun getRerunMode(project: Project) : Mode? {
+ val prop = project.property(MODE_PROPERTY) ?: return null
+ return when (prop.toString().lowercase()) {
+ "all" -> Mode.ALL
+ "failing" -> Mode.FAILING
+ "past-failing" -> Mode.PAST_FAILING
+ "past_failing" -> Mode.PAST_FAILING
+ "outstanding" -> Mode.OUTSTANDING
+ else -> null
}
+ }
+
+ fun setUpTestingState(task: Test) {
+ // Both the path and the mode must be defined for the testing state to be active.
+ val testingStatePath = task.project.property(PATH_PROPERTY) ?: return
+ val testingStateMode = getRerunMode(task.project) ?: return
+
val projectName = task.project.name
- val indexDir = File(project.property("testing-state")!!.toString())
- val reportDir = indexDir.resolve(project.name)
+ val indexDir = File(testingStatePath.toString())
+ val reportDir = indexDir.resolve(projectName)
val index = indexDir.resolve("index.html")
val resuming = reportDir.exists()
if (resuming) {
- applyTestFilters(task, reportDir)
+ applyTestFilters(testingStateMode, task, reportDir)
}
addTestHandler(task, projectName, index, reportDir)
}
- private fun applyTestFilters(task: Test, reportDir: File) {
- // If there are failing tests only rerun those.
- val hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
- task.filter.includeTestsMatching("$clazz.$name")
- })
- if (hasFailingTests) {
+ private fun applyTestFilters(mode: Mode, task: Test, reportDir: File) {
+ if (mode == Mode.ALL) {
+ // Running without filters will (re)run all tests.
return
}
- // Otherwise exclude all the tests already marked as succeeding or skipped.
- forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
+ if (mode == Mode.OUTSTANDING) {
+ task.logger.lifecycle(
+ "Note: the building of an exclude list often times out."
+ + "You may need to simply rerun all tests.")
+ forEachTestReportStatusMatching(task, reportDir, StatusFile.SUCCESS, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ return
+ }
+ assert(mode == Mode.FAILING || mode == Mode.PAST_FAILING)
+ val result = if (mode == Mode.FAILING) StatusFile.FAILURE else StatusFile.PAST_FAILURE
+ forEachTestReportStatusMatching(task, reportDir, result, { clazz, name ->
+ task.filter.includeTestsMatching("$clazz.$name")
})
}
@@ -119,20 +147,13 @@
if (result.testCount != 1L) {
throw IllegalStateException("Unexpected test with more than one result: ${desc}")
}
- // Clear any previous result files.
- for (resultType in TestResult.ResultType.values()) {
- getTestResultEntryOutputFile(reportDir, desc, resultType.name).delete()
- }
- // Emit the result type status in a file of the same name: SUCCESS, FAILURE or SKIPPED.
- withTestResultEntryWriter(reportDir, desc, result.getResultType().name, false, {
- it.append(result.getResultType().name)
- })
+ updateStatusFiles(reportDir, desc, result.resultType)
// Emit the test time.
withTestResultEntryWriter(reportDir, desc, "time", false, {
it.append("${result.getEndTime() - result.getStartTime()}")
})
// For failed tests, update the index and emit stack trace information.
- if (result.resultType == TestResult.ResultType.FAILURE) {
+ if (result.resultType == ResultType.FAILURE) {
val title = testLinkContent(desc)
val link = getTestReportEntryURL(reportDir, desc)
index.appendText("<li><a href=\"${link}\">${title}</a></li>")
@@ -162,6 +183,14 @@
})
}
+ private fun getStatusFile(result: ResultType) : StatusFile {
+ return when (result) {
+ ResultType.FAILURE -> StatusFile.FAILURE
+ ResultType.SUCCESS -> StatusFile.SUCCESS
+ ResultType.SKIPPED -> StatusFile.SUCCESS
+ }
+ }
+
private fun escapeHtml(string: String): String {
return string
.replace("&", "&")
@@ -219,25 +248,31 @@
}
// Some of our test parameters have new lines :-( We really don't want test names to span lines.
- private fun sanitizedTestName(testDesc: TestDescriptor): String {
- if (testDesc.getName().contains("\n")) {
- throw RuntimeException("Unsupported use of newline in test name: '${testDesc.getName()}'")
+ private fun sanitizedTestName(testName: String): String {
+ if (testName.contains("\n")) {
+ throw RuntimeException("Unsupported use of newline in test name: '${testName}'")
}
- return testDesc.getName()
+ return testName
}
- private fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
+ private fun getTestReportClassDirPath(reportDir: File, testClass: String): Path {
+ return reportDir.toPath().resolve(testClass)
+ }
+
+ private fun getTestReportEntryDirFromString(reportDir: File, testClass: String, testName: String): File {
return ensureDir(
- reportDir.toPath()
- .resolve(testDesc.getClassName()!!)
- .resolve(sanitizedTestName(testDesc))
- .toFile()
- )
+ getTestReportClassDirPath(reportDir, testClass)
+ .resolve(sanitizedTestName(testName))
+ .toFile())
+ }
+
+ private fun getTestReportEntryDirFromTest(reportDir: File, testDesc: TestDescriptor): File {
+ return getTestReportEntryDirFromString(reportDir, testDesc.className!!, testDesc.name)
}
private fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
- val classDir = urlEncode(testDesc.getClassName()!!)
- val testDir = urlEncode(sanitizedTestName(testDesc))
+ val classDir = urlEncode(testDesc.className!!)
+ val testDir = urlEncode(sanitizedTestName(testDesc.name))
return reportDir.toPath().resolve(classDir).resolve(testDir)
}
@@ -246,10 +281,29 @@
testDesc: TestDescriptor,
fileName: String
): File {
- val dir = getTestReportEntryDir(reportDir, testDesc).toPath()
+ val dir = getTestReportEntryDirFromTest(reportDir, testDesc).toPath()
return dir.resolve(fileName).toFile()
}
+ private fun updateStatusFiles(
+ reportDir: File,
+ desc: TestDescriptor,
+ result: ResultType) {
+ val statusFile = getStatusFile(result)
+ withTestResultEntryWriter(reportDir, desc, statusFile.name, false, {
+ it.append(statusFile.name)
+ })
+ if (statusFile == StatusFile.FAILURE) {
+ getTestResultEntryOutputFile(reportDir, desc, StatusFile.SUCCESS.name).delete()
+ val pastFailure = StatusFile.PAST_FAILURE.name
+ withTestResultEntryWriter(reportDir, desc, pastFailure, false, {
+ it.append(pastFailure)
+ })
+ } else {
+ getTestResultEntryOutputFile(reportDir, desc, StatusFile.FAILURE.name).delete()
+ }
+ }
+
private fun withTestResultEntryWriter(
reportDir: File,
testDesc: TestDescriptor,
@@ -261,51 +315,13 @@
FileWriter(file, append).use(fn)
}
- private fun forEachTestReportAlreadyFailing(
+ fun forEachTestReportStatusMatching(
test: Test,
reportDir: File,
- onFailureTest: (String, String) -> Unit
- ): Boolean {
- return internalForEachTestReportState(
- test,
- reportDir,
- TestResult.ResultType.FAILURE.name,
- onFailureTest
- )
- }
-
- fun forEachTestReportAlreadyPassing(
- test: Test,
- reportDir: File,
- onSucceededTest: (String, String) -> Unit
- ): Boolean {
- return internalForEachTestReportState(
- test,
- reportDir,
- TestResult.ResultType.SUCCESS.name,
- onSucceededTest
- )
- }
-
- fun forEachTestReportAlreadySkipped(
- test: Test,
- reportDir: File,
- onSucceededTest: (String, String) -> Unit
- ): Boolean {
- return internalForEachTestReportState(
- test,
- reportDir,
- TestResult.ResultType.SKIPPED.name,
- onSucceededTest
- )
- }
-
- fun internalForEachTestReportState(
- test: Test,
- reportDir: File,
- fileName: String,
+ statusFile: StatusFile,
onTest: (String, String) -> Unit
): Boolean {
+ val fileName = statusFile.name
val logger = test.logger
val proc = ProcessBuilder("find", ".", "-name", fileName)
.directory(reportDir)
diff --git a/tools/test.py b/tools/test.py
index a6c34f0..40dc33c 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -19,7 +19,7 @@
import download_kotlin_dev
import gradle
import notify
-import test_state
+import testing_state
import utils
if utils.is_python3():
@@ -183,17 +183,13 @@
help='Print the execution time of the slowest tests..',
default=False, action='store_true')
result.add_option(
- '--testing-state-name',
- help='Set an explict name for the testing state '
- '(used in conjunction with --with/reset-testing-state).')
+ '--testing-state-dir',
+ help='Explicitly set the testing state directory '
+ '(defaults to build/test-state/<git-branch>).')
result.add_option(
- '--with-testing-state',
- help='Run/resume tests using testing state.',
- default=False, action='store_true')
- result.add_option(
- '--reset-testing-state',
- help='Clean the testing state and rerun tests (implies --with-testing-state).',
- default=False, action='store_true')
+ '--rerun',
+ help='Rerun tests (implicitly enables testing state).',
+ choices=testing_state.CHOICES)
result.add_option(
'--stacktrace',
help='Pass --stacktrace to the gradle run',
@@ -276,9 +272,6 @@
gradle_args = []
- testing_state = False
- testing_state_path = None
-
if options.stacktrace or utils.is_bot():
gradle_args.append('--stacktrace')
@@ -366,20 +359,10 @@
gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
if options.no_arttests:
gradle_args.append('-Pno_arttests=true')
- if options.reset_testing_state:
- testing_state = True
- gradle_args.append('-Preset-testing-state')
- elif options.with_testing_state:
- testing_state = True
- if options.testing_state_name:
- gradle_args.append('-Ptesting-state-name=' + options.testing_state_name)
- testing_state_path = "%s/test-state/%s" % (utils.BUILD, options.testing_state_name)
- if testing_state:
- if options.new_gradle:
- test_state.set_up_test_state(gradle_args, testing_state_path)
- else:
- gradle_args.append('-Ptesting-state')
+ # Testing state is only supported in new-gradle going forward
+ if options.new_gradle and options.rerun:
+ testing_state.set_up_test_state(gradle_args, options.rerun, options.testing_state_dir)
# Enable completeness testing of ART profile rewriting.
gradle_args.append('-Part_profile_rewriting_completeness_check=true')
diff --git a/tools/test_state.py b/tools/testing_state.py
similarity index 74%
rename from tools/test_state.py
rename to tools/testing_state.py
index 4541533..7c51790 100644
--- a/tools/test_state.py
+++ b/tools/testing_state.py
@@ -8,13 +8,17 @@
import datetime
import os
-def set_up_test_state(gradle_args, testing_state_path):
- # In the new build the test state directory must be passed explictitly.
- # TODO(b/297316723): Simplify this and just support a single flag: --testing-state <path>
+CHOICES = ["all", "failing", "past-failing", "outstanding"]
+DEFAULT_REPORTS_ROOT = os.path.join(utils.BUILD, "testing-state")
+
+def set_up_test_state(gradle_args, testing_state_mode, testing_state_path):
+ if not testing_state_mode:
+ return
if not testing_state_path:
- testing_state_path = "%s/test-state/%s" % (utils.BUILD, utils.get_HEAD_branch())
- gradle_args.append('-Ptesting-state=%s' % testing_state_path)
- prepare_testing_index(testing_state_path)
+ testing_state_path = os.path.join(DEFAULT_REPORTS_ROOT, utils.get_HEAD_branch())
+ gradle_args.append('-Ptesting-state-mode=%s' % testing_state_mode)
+ gradle_args.append('-Ptesting-state-path=%s' % testing_state_path)
+ prepare_testing_index(testing_state_mode, testing_state_path)
def fresh_testing_index(testing_state_dir):
number = 0
@@ -24,22 +28,22 @@
if not os.path.exists(freshIndex):
return freshIndex
-def prepare_testing_index(testing_state_dir):
+def prepare_testing_index(testing_state_mode, testing_state_dir):
if not os.path.exists(testing_state_dir):
os.makedirs(testing_state_dir)
index_path = os.path.join(testing_state_dir, "index.html")
parent_report = None
resuming = os.path.exists(index_path)
+ mode = testing_state_mode if resuming else f"starting (flag: {testing_state_mode})"
if (resuming):
parent_report = fresh_testing_index(testing_state_dir)
os.rename(index_path, parent_report)
index = open(index_path, "a")
- run_prefix = "Resuming" if resuming else "Starting"
relative_state_dir = os.path.relpath(testing_state_dir)
- title = f"{run_prefix} @ {relative_state_dir}"
+ title = relative_state_dir
# Print a console link to the test report for easy access.
print("=" * 70)
- print(f"{run_prefix} test, report written to:")
+ print("Test report written to:")
print(f" file://{index_path}")
print("=" * 70)
# Print the new index content.
@@ -47,6 +51,7 @@
index.write("<style> * { font-family: monospace; }</style>")
index.write("<meta http-equiv='refresh' content='10' />")
index.write(f"</head><body><h1>{title}</h1>")
+ index.write(f"<h2>Mode: {mode}</h2>")
# write index links first to avoid jumping when browsing.
if parent_report:
index.write(f"<p><a href=\"file://{parent_report}\">Previous result index</a></p>")
