Reland "Support rerunning all previously failing tests"
This reverts commit 237fc81dbf5a3e0fc101b183bf7c77eedcd63044.
Bug: b/297316723
Change-Id: I59fabab716461650fedc4f89c0909c16267689f1
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
index 240e656..2ef455f 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
@@ -16,41 +16,67 @@
import org.gradle.api.tasks.testing.TestOutputEvent
import org.gradle.api.tasks.testing.TestOutputListener
import org.gradle.api.tasks.testing.TestResult
+import org.gradle.api.tasks.testing.TestResult.ResultType
// Utility to install tracking of test results in status files.
class TestingState {
companion object {
- fun setUpTestingState(task: Test) {
- val project = task.project
- if (!project.hasProperty("testing-state")) {
- return
+ const val MODE_PROPERTY = "testing-state-mode"
+ const val PATH_PROPERTY = "testing-state-path"
+
+ // Operating mode for the test state.
+ enum class Mode { ALL, OUTSTANDING, FAILING, PAST_FAILING }
+
+ // These are the files that are allowed for tracking test status.
+ enum class StatusFile { SUCCESS, FAILURE, PAST_FAILURE }
+
+ fun getRerunMode(project: Project) : Mode? {
+ val prop = project.findProperty(MODE_PROPERTY) ?: return null
+ return when (prop.toString().lowercase()) {
+ "all" -> Mode.ALL
+ "failing" -> Mode.FAILING
+ "past-failing" -> Mode.PAST_FAILING
+ "past_failing" -> Mode.PAST_FAILING
+ "outstanding" -> Mode.OUTSTANDING
+ else -> null
}
+ }
+
+ fun setUpTestingState(task: Test) {
+ // Both the path and the mode must be defined for the testing state to be active.
+ val testingStatePath = task.project.findProperty(PATH_PROPERTY) ?: return
+ val testingStateMode = getRerunMode(task.project) ?: return
+
val projectName = task.project.name
- val indexDir = File(project.property("testing-state")!!.toString())
- val reportDir = indexDir.resolve(project.name)
+ val indexDir = File(testingStatePath.toString())
+ val reportDir = indexDir.resolve(projectName)
val index = indexDir.resolve("index.html")
val resuming = reportDir.exists()
if (resuming) {
- applyTestFilters(task, reportDir)
+ applyTestFilters(testingStateMode, task, reportDir)
}
addTestHandler(task, projectName, index, reportDir)
}
- private fun applyTestFilters(task: Test, reportDir: File) {
- // If there are failing tests only rerun those.
- val hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
- task.filter.includeTestsMatching("$clazz.$name")
- })
- if (hasFailingTests) {
+ private fun applyTestFilters(mode: Mode, task: Test, reportDir: File) {
+ if (mode == Mode.ALL) {
+ // Running without filters will (re)run all tests.
return
}
- // Otherwise exclude all the tests already marked as succeeding or skipped.
- forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
+ if (mode == Mode.OUTSTANDING) {
+ task.logger.lifecycle(
+ "Note: the building of an exclude list often times out."
+ + "You may need to simply rerun all tests.")
+ forEachTestReportStatusMatching(task, reportDir, StatusFile.SUCCESS, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ return
+ }
+ assert(mode == Mode.FAILING || mode == Mode.PAST_FAILING)
+ val result = if (mode == Mode.FAILING) StatusFile.FAILURE else StatusFile.PAST_FAILURE
+ forEachTestReportStatusMatching(task, reportDir, result, { clazz, name ->
+ task.filter.includeTestsMatching("$clazz.$name")
})
}
@@ -119,20 +145,13 @@
if (result.testCount != 1L) {
throw IllegalStateException("Unexpected test with more than one result: ${desc}")
}
- // Clear any previous result files.
- for (resultType in TestResult.ResultType.values()) {
- getTestResultEntryOutputFile(reportDir, desc, resultType.name).delete()
- }
- // Emit the result type status in a file of the same name: SUCCESS, FAILURE or SKIPPED.
- withTestResultEntryWriter(reportDir, desc, result.getResultType().name, false, {
- it.append(result.getResultType().name)
- })
+ updateStatusFiles(reportDir, desc, result.resultType)
// Emit the test time.
withTestResultEntryWriter(reportDir, desc, "time", false, {
it.append("${result.getEndTime() - result.getStartTime()}")
})
// For failed tests, update the index and emit stack trace information.
- if (result.resultType == TestResult.ResultType.FAILURE) {
+ if (result.resultType == ResultType.FAILURE) {
val title = testLinkContent(desc)
val link = getTestReportEntryURL(reportDir, desc)
index.appendText("<li><a href=\"${link}\">${title}</a></li>")
@@ -162,6 +181,14 @@
})
}
+ private fun getStatusFile(result: ResultType) : StatusFile {
+ return when (result) {
+ ResultType.FAILURE -> StatusFile.FAILURE
+ ResultType.SUCCESS -> StatusFile.SUCCESS
+ ResultType.SKIPPED -> StatusFile.SUCCESS
+ }
+ }
+
private fun escapeHtml(string: String): String {
return string
.replace("&", "&")
@@ -219,25 +246,31 @@
}
// Some of our test parameters have new lines :-( We really don't want test names to span lines.
- private fun sanitizedTestName(testDesc: TestDescriptor): String {
- if (testDesc.getName().contains("\n")) {
- throw RuntimeException("Unsupported use of newline in test name: '${testDesc.getName()}'")
+ private fun sanitizedTestName(testName: String): String {
+ if (testName.contains("\n")) {
+ throw RuntimeException("Unsupported use of newline in test name: '${testName}'")
}
- return testDesc.getName()
+ return testName
}
- private fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
+ private fun getTestReportClassDirPath(reportDir: File, testClass: String): Path {
+ return reportDir.toPath().resolve(testClass)
+ }
+
+ private fun getTestReportEntryDirFromString(reportDir: File, testClass: String, testName: String): File {
return ensureDir(
- reportDir.toPath()
- .resolve(testDesc.getClassName()!!)
- .resolve(sanitizedTestName(testDesc))
- .toFile()
- )
+ getTestReportClassDirPath(reportDir, testClass)
+ .resolve(sanitizedTestName(testName))
+ .toFile())
+ }
+
+ private fun getTestReportEntryDirFromTest(reportDir: File, testDesc: TestDescriptor): File {
+ return getTestReportEntryDirFromString(reportDir, testDesc.className!!, testDesc.name)
}
private fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
- val classDir = urlEncode(testDesc.getClassName()!!)
- val testDir = urlEncode(sanitizedTestName(testDesc))
+ val classDir = urlEncode(testDesc.className!!)
+ val testDir = urlEncode(sanitizedTestName(testDesc.name))
return reportDir.toPath().resolve(classDir).resolve(testDir)
}
@@ -246,10 +279,29 @@
testDesc: TestDescriptor,
fileName: String
): File {
- val dir = getTestReportEntryDir(reportDir, testDesc).toPath()
+ val dir = getTestReportEntryDirFromTest(reportDir, testDesc).toPath()
return dir.resolve(fileName).toFile()
}
+ private fun updateStatusFiles(
+ reportDir: File,
+ desc: TestDescriptor,
+ result: ResultType) {
+ val statusFile = getStatusFile(result)
+ withTestResultEntryWriter(reportDir, desc, statusFile.name, false, {
+ it.append(statusFile.name)
+ })
+ if (statusFile == StatusFile.FAILURE) {
+ getTestResultEntryOutputFile(reportDir, desc, StatusFile.SUCCESS.name).delete()
+ val pastFailure = StatusFile.PAST_FAILURE.name
+ withTestResultEntryWriter(reportDir, desc, pastFailure, false, {
+ it.append(pastFailure)
+ })
+ } else {
+ getTestResultEntryOutputFile(reportDir, desc, StatusFile.FAILURE.name).delete()
+ }
+ }
+
private fun withTestResultEntryWriter(
reportDir: File,
testDesc: TestDescriptor,
@@ -261,51 +313,13 @@
FileWriter(file, append).use(fn)
}
- private fun forEachTestReportAlreadyFailing(
+ fun forEachTestReportStatusMatching(
test: Test,
reportDir: File,
- onFailureTest: (String, String) -> Unit
- ): Boolean {
- return internalForEachTestReportState(
- test,
- reportDir,
- TestResult.ResultType.FAILURE.name,
- onFailureTest
- )
- }
-
- fun forEachTestReportAlreadyPassing(
- test: Test,
- reportDir: File,
- onSucceededTest: (String, String) -> Unit
- ): Boolean {
- return internalForEachTestReportState(
- test,
- reportDir,
- TestResult.ResultType.SUCCESS.name,
- onSucceededTest
- )
- }
-
- fun forEachTestReportAlreadySkipped(
- test: Test,
- reportDir: File,
- onSucceededTest: (String, String) -> Unit
- ): Boolean {
- return internalForEachTestReportState(
- test,
- reportDir,
- TestResult.ResultType.SKIPPED.name,
- onSucceededTest
- )
- }
-
- fun internalForEachTestReportState(
- test: Test,
- reportDir: File,
- fileName: String,
+ statusFile: StatusFile,
onTest: (String, String) -> Unit
): Boolean {
+ val fileName = statusFile.name
val logger = test.logger
val proc = ProcessBuilder("find", ".", "-name", fileName)
.directory(reportDir)
diff --git a/tools/test.py b/tools/test.py
index a6c34f0..40dc33c 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -19,7 +19,7 @@
import download_kotlin_dev
import gradle
import notify
-import test_state
+import testing_state
import utils
if utils.is_python3():
@@ -183,17 +183,13 @@
help='Print the execution time of the slowest tests..',
default=False, action='store_true')
result.add_option(
- '--testing-state-name',
- help='Set an explict name for the testing state '
- '(used in conjunction with --with/reset-testing-state).')
+ '--testing-state-dir',
+ help='Explicitly set the testing state directory '
+ '(defaults to build/test-state/<git-branch>).')
result.add_option(
- '--with-testing-state',
- help='Run/resume tests using testing state.',
- default=False, action='store_true')
- result.add_option(
- '--reset-testing-state',
- help='Clean the testing state and rerun tests (implies --with-testing-state).',
- default=False, action='store_true')
+ '--rerun',
+ help='Rerun tests (implicitly enables testing state).',
+ choices=testing_state.CHOICES)
result.add_option(
'--stacktrace',
help='Pass --stacktrace to the gradle run',
@@ -276,9 +272,6 @@
gradle_args = []
- testing_state = False
- testing_state_path = None
-
if options.stacktrace or utils.is_bot():
gradle_args.append('--stacktrace')
@@ -366,20 +359,10 @@
gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
if options.no_arttests:
gradle_args.append('-Pno_arttests=true')
- if options.reset_testing_state:
- testing_state = True
- gradle_args.append('-Preset-testing-state')
- elif options.with_testing_state:
- testing_state = True
- if options.testing_state_name:
- gradle_args.append('-Ptesting-state-name=' + options.testing_state_name)
- testing_state_path = "%s/test-state/%s" % (utils.BUILD, options.testing_state_name)
- if testing_state:
- if options.new_gradle:
- test_state.set_up_test_state(gradle_args, testing_state_path)
- else:
- gradle_args.append('-Ptesting-state')
+ # Testing state is only supported in new-gradle going forward
+ if options.new_gradle and options.rerun:
+ testing_state.set_up_test_state(gradle_args, options.rerun, options.testing_state_dir)
# Enable completeness testing of ART profile rewriting.
gradle_args.append('-Part_profile_rewriting_completeness_check=true')
diff --git a/tools/test_state.py b/tools/testing_state.py
similarity index 74%
rename from tools/test_state.py
rename to tools/testing_state.py
index 4541533..7c51790 100644
--- a/tools/test_state.py
+++ b/tools/testing_state.py
@@ -8,13 +8,17 @@
import datetime
import os
-def set_up_test_state(gradle_args, testing_state_path):
- # In the new build the test state directory must be passed explictitly.
- # TODO(b/297316723): Simplify this and just support a single flag: --testing-state <path>
+CHOICES = ["all", "failing", "past-failing", "outstanding"]
+DEFAULT_REPORTS_ROOT = os.path.join(utils.BUILD, "testing-state")
+
+def set_up_test_state(gradle_args, testing_state_mode, testing_state_path):
+ if not testing_state_mode:
+ return
if not testing_state_path:
- testing_state_path = "%s/test-state/%s" % (utils.BUILD, utils.get_HEAD_branch())
- gradle_args.append('-Ptesting-state=%s' % testing_state_path)
- prepare_testing_index(testing_state_path)
+ testing_state_path = os.path.join(DEFAULT_REPORTS_ROOT, utils.get_HEAD_branch())
+ gradle_args.append('-Ptesting-state-mode=%s' % testing_state_mode)
+ gradle_args.append('-Ptesting-state-path=%s' % testing_state_path)
+ prepare_testing_index(testing_state_mode, testing_state_path)
def fresh_testing_index(testing_state_dir):
number = 0
@@ -24,22 +28,22 @@
if not os.path.exists(freshIndex):
return freshIndex
-def prepare_testing_index(testing_state_dir):
+def prepare_testing_index(testing_state_mode, testing_state_dir):
if not os.path.exists(testing_state_dir):
os.makedirs(testing_state_dir)
index_path = os.path.join(testing_state_dir, "index.html")
parent_report = None
resuming = os.path.exists(index_path)
+ mode = testing_state_mode if resuming else f"starting (flag: {testing_state_mode})"
if (resuming):
parent_report = fresh_testing_index(testing_state_dir)
os.rename(index_path, parent_report)
index = open(index_path, "a")
- run_prefix = "Resuming" if resuming else "Starting"
relative_state_dir = os.path.relpath(testing_state_dir)
- title = f"{run_prefix} @ {relative_state_dir}"
+ title = relative_state_dir
# Print a console link to the test report for easy access.
print("=" * 70)
- print(f"{run_prefix} test, report written to:")
+ print("Test report written to:")
print(f" file://{index_path}")
print("=" * 70)
# Print the new index content.
@@ -47,6 +51,7 @@
index.write("<style> * { font-family: monospace; }</style>")
index.write("<meta http-equiv='refresh' content='10' />")
index.write(f"</head><body><h1>{title}</h1>")
+ index.write(f"<h2>Mode: {mode}</h2>")
# write index links first to avoid jumping when browsing.
if parent_report:
index.write(f"<p><a href=\"file://{parent_report}\">Previous result index</a></p>")
