Revert "Support rerunning all previously failing tests"
This reverts commit 1a0408281c4f0781a62ce078c32204f73ab4c150.
Reason for revert: breaks new gradle bot
Change-Id: Iac2e69dbc124cf47e0b108ef4edb06c14c80c76e
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
index 53c0192..240e656 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
@@ -16,70 +16,42 @@
import org.gradle.api.tasks.testing.TestOutputEvent
import org.gradle.api.tasks.testing.TestOutputListener
import org.gradle.api.tasks.testing.TestResult
-import org.gradle.api.tasks.testing.TestResult.ResultType
// Utility to install tracking of test results in status files.
class TestingState {
companion object {
- const val MODE_PROPERTY = "testing-state-mode"
- const val PATH_PROPERTY = "testing-state-path"
-
- const val PAST_FAILURE = "PAST_FAILURE"
-
- // Operating mode for the test state.
- enum class Mode { ALL, OUTSTANDING, FAILING, PAST_FAILING }
-
- // These are the files that are allowed for tracking test status.
- enum class StatusFile { SUCCESS, FAILURE, PAST_FAILURE }
-
- fun getRerunMode(project: Project) : Mode? {
- val prop = project.property(MODE_PROPERTY) ?: return null
- return when (prop.toString().lowercase()) {
- "all" -> Mode.ALL
- "failing" -> Mode.FAILING
- "past-failing" -> Mode.PAST_FAILING
- "past_failing" -> Mode.PAST_FAILING
- "outstanding" -> Mode.OUTSTANDING
- else -> null
- }
- }
-
fun setUpTestingState(task: Test) {
- // Both the path and the mode must be defined for the testing state to be active.
- val testingStatePath = task.project.property(PATH_PROPERTY) ?: return
- val testingStateMode = getRerunMode(task.project) ?: return
-
+ val project = task.project
+ if (!project.hasProperty("testing-state")) {
+ return
+ }
val projectName = task.project.name
- val indexDir = File(testingStatePath.toString())
- val reportDir = indexDir.resolve(projectName)
+ val indexDir = File(project.property("testing-state")!!.toString())
+ val reportDir = indexDir.resolve(project.name)
val index = indexDir.resolve("index.html")
val resuming = reportDir.exists()
if (resuming) {
- applyTestFilters(testingStateMode, task, reportDir)
+ applyTestFilters(task, reportDir)
}
addTestHandler(task, projectName, index, reportDir)
}
- private fun applyTestFilters(mode: Mode, task: Test, reportDir: File) {
- if (mode == Mode.ALL) {
- // Running without filters will (re)run all tests.
- return
- }
- if (mode == Mode.OUTSTANDING) {
- task.logger.lifecycle(
- "Note: the building of an exclude list often times out."
- + "You may need to simply rerun all tests.")
- forEachTestReportStatusMatching(task, reportDir, StatusFile.SUCCESS, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- return
- }
- assert(mode == Mode.FAILING || mode == Mode.PAST_FAILING)
- val result = if (mode == Mode.FAILING) StatusFile.FAILURE else StatusFile.PAST_FAILURE
- forEachTestReportStatusMatching(task, reportDir, result, { clazz, name ->
+ private fun applyTestFilters(task: Test, reportDir: File) {
+ // If there are failing tests only rerun those.
+ val hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
task.filter.includeTestsMatching("$clazz.$name")
})
+ if (hasFailingTests) {
+ return
+ }
+ // Otherwise exclude all the tests already marked as succeeding or skipped.
+ forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
}
private fun addTestHandler(
@@ -147,13 +119,20 @@
if (result.testCount != 1L) {
throw IllegalStateException("Unexpected test with more than one result: ${desc}")
}
- updateStatusFiles(reportDir, desc, result.resultType)
+ // Clear any previous result files.
+ for (resultType in TestResult.ResultType.values()) {
+ getTestResultEntryOutputFile(reportDir, desc, resultType.name).delete()
+ }
+ // Emit the result type status in a file of the same name: SUCCESS, FAILURE or SKIPPED.
+ withTestResultEntryWriter(reportDir, desc, result.getResultType().name, false, {
+ it.append(result.getResultType().name)
+ })
// Emit the test time.
withTestResultEntryWriter(reportDir, desc, "time", false, {
it.append("${result.getEndTime() - result.getStartTime()}")
})
// For failed tests, update the index and emit stack trace information.
- if (result.resultType == ResultType.FAILURE) {
+ if (result.resultType == TestResult.ResultType.FAILURE) {
val title = testLinkContent(desc)
val link = getTestReportEntryURL(reportDir, desc)
index.appendText("<li><a href=\"${link}\">${title}</a></li>")
@@ -183,14 +162,6 @@
})
}
- private fun getStatusFile(result: ResultType) : StatusFile {
- return when (result) {
- ResultType.FAILURE -> StatusFile.FAILURE
- ResultType.SUCCESS -> StatusFile.SUCCESS
- ResultType.SKIPPED -> StatusFile.SUCCESS
- }
- }
-
private fun escapeHtml(string: String): String {
return string
.replace("&", "&")
@@ -248,31 +219,25 @@
}
// Some of our test parameters have new lines :-( We really don't want test names to span lines.
- private fun sanitizedTestName(testName: String): String {
- if (testName.contains("\n")) {
- throw RuntimeException("Unsupported use of newline in test name: '${testName}'")
+ private fun sanitizedTestName(testDesc: TestDescriptor): String {
+ if (testDesc.getName().contains("\n")) {
+ throw RuntimeException("Unsupported use of newline in test name: '${testDesc.getName()}'")
}
- return testName
+ return testDesc.getName()
}
- private fun getTestReportClassDirPath(reportDir: File, testClass: String): Path {
- return reportDir.toPath().resolve(testClass)
- }
-
- private fun getTestReportEntryDirFromString(reportDir: File, testClass: String, testName: String): File {
+ private fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
return ensureDir(
- getTestReportClassDirPath(reportDir, testClass)
- .resolve(sanitizedTestName(testName))
- .toFile())
- }
-
- private fun getTestReportEntryDirFromTest(reportDir: File, testDesc: TestDescriptor): File {
- return getTestReportEntryDirFromString(reportDir, testDesc.className!!, testDesc.name)
+ reportDir.toPath()
+ .resolve(testDesc.getClassName()!!)
+ .resolve(sanitizedTestName(testDesc))
+ .toFile()
+ )
}
private fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
- val classDir = urlEncode(testDesc.className!!)
- val testDir = urlEncode(sanitizedTestName(testDesc.name))
+ val classDir = urlEncode(testDesc.getClassName()!!)
+ val testDir = urlEncode(sanitizedTestName(testDesc))
return reportDir.toPath().resolve(classDir).resolve(testDir)
}
@@ -281,29 +246,10 @@
testDesc: TestDescriptor,
fileName: String
): File {
- val dir = getTestReportEntryDirFromTest(reportDir, testDesc).toPath()
+ val dir = getTestReportEntryDir(reportDir, testDesc).toPath()
return dir.resolve(fileName).toFile()
}
- private fun updateStatusFiles(
- reportDir: File,
- desc: TestDescriptor,
- result: ResultType) {
- val statusFile = getStatusFile(result)
- withTestResultEntryWriter(reportDir, desc, statusFile.name, false, {
- it.append(statusFile.name)
- })
- if (statusFile == StatusFile.FAILURE) {
- getTestResultEntryOutputFile(reportDir, desc, StatusFile.SUCCESS.name).delete()
- val pastFailure = StatusFile.PAST_FAILURE.name
- withTestResultEntryWriter(reportDir, desc, pastFailure, false, {
- it.append(pastFailure)
- })
- } else {
- getTestResultEntryOutputFile(reportDir, desc, StatusFile.FAILURE.name).delete()
- }
- }
-
private fun withTestResultEntryWriter(
reportDir: File,
testDesc: TestDescriptor,
@@ -315,13 +261,51 @@
FileWriter(file, append).use(fn)
}
- fun forEachTestReportStatusMatching(
+ private fun forEachTestReportAlreadyFailing(
test: Test,
reportDir: File,
- statusFile: StatusFile,
+ onFailureTest: (String, String) -> Unit
+ ): Boolean {
+ return internalForEachTestReportState(
+ test,
+ reportDir,
+ TestResult.ResultType.FAILURE.name,
+ onFailureTest
+ )
+ }
+
+ fun forEachTestReportAlreadyPassing(
+ test: Test,
+ reportDir: File,
+ onSucceededTest: (String, String) -> Unit
+ ): Boolean {
+ return internalForEachTestReportState(
+ test,
+ reportDir,
+ TestResult.ResultType.SUCCESS.name,
+ onSucceededTest
+ )
+ }
+
+ fun forEachTestReportAlreadySkipped(
+ test: Test,
+ reportDir: File,
+ onSucceededTest: (String, String) -> Unit
+ ): Boolean {
+ return internalForEachTestReportState(
+ test,
+ reportDir,
+ TestResult.ResultType.SKIPPED.name,
+ onSucceededTest
+ )
+ }
+
+ fun internalForEachTestReportState(
+ test: Test,
+ reportDir: File,
+ fileName: String,
onTest: (String, String) -> Unit
): Boolean {
- val fileName = statusFile.name
val logger = test.logger
val proc = ProcessBuilder("find", ".", "-name", fileName)
.directory(reportDir)
diff --git a/tools/test.py b/tools/test.py
index 40dc33c..a6c34f0 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -19,7 +19,7 @@
import download_kotlin_dev
import gradle
import notify
-import testing_state
+import test_state
import utils
if utils.is_python3():
@@ -183,13 +183,17 @@
help='Print the execution time of the slowest tests..',
default=False, action='store_true')
result.add_option(
- '--testing-state-dir',
- help='Explicitly set the testing state directory '
- '(defaults to build/test-state/<git-branch>).')
+ '--testing-state-name',
+ help='Set an explict name for the testing state '
+ '(used in conjunction with --with/reset-testing-state).')
result.add_option(
- '--rerun',
- help='Rerun tests (implicitly enables testing state).',
- choices=testing_state.CHOICES)
+ '--with-testing-state',
+ help='Run/resume tests using testing state.',
+ default=False, action='store_true')
+ result.add_option(
+ '--reset-testing-state',
+ help='Clean the testing state and rerun tests (implies --with-testing-state).',
+ default=False, action='store_true')
result.add_option(
'--stacktrace',
help='Pass --stacktrace to the gradle run',
@@ -272,6 +276,9 @@
gradle_args = []
+ testing_state = False
+ testing_state_path = None
+
if options.stacktrace or utils.is_bot():
gradle_args.append('--stacktrace')
@@ -359,10 +366,20 @@
gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
if options.no_arttests:
gradle_args.append('-Pno_arttests=true')
+ if options.reset_testing_state:
+ testing_state = True
+ gradle_args.append('-Preset-testing-state')
+ elif options.with_testing_state:
+ testing_state = True
+ if options.testing_state_name:
+ gradle_args.append('-Ptesting-state-name=' + options.testing_state_name)
+ testing_state_path = "%s/test-state/%s" % (utils.BUILD, options.testing_state_name)
- # Testing state is only supported in new-gradle going forward
- if options.new_gradle and options.rerun:
- testing_state.set_up_test_state(gradle_args, options.rerun, options.testing_state_dir)
+ if testing_state:
+ if options.new_gradle:
+ test_state.set_up_test_state(gradle_args, testing_state_path)
+ else:
+ gradle_args.append('-Ptesting-state')
# Enable completeness testing of ART profile rewriting.
gradle_args.append('-Part_profile_rewriting_completeness_check=true')
diff --git a/tools/testing_state.py b/tools/test_state.py
similarity index 74%
rename from tools/testing_state.py
rename to tools/test_state.py
index 7c51790..4541533 100644
--- a/tools/testing_state.py
+++ b/tools/test_state.py
@@ -8,17 +8,13 @@
import datetime
import os
-CHOICES = ["all", "failing", "past-failing", "outstanding"]
-DEFAULT_REPORTS_ROOT = os.path.join(utils.BUILD, "testing-state")
-
-def set_up_test_state(gradle_args, testing_state_mode, testing_state_path):
- if not testing_state_mode:
- return
+def set_up_test_state(gradle_args, testing_state_path):
+ # In the new build the test state directory must be passed explictitly.
+ # TODO(b/297316723): Simplify this and just support a single flag: --testing-state <path>
if not testing_state_path:
- testing_state_path = os.path.join(DEFAULT_REPORTS_ROOT, utils.get_HEAD_branch())
- gradle_args.append('-Ptesting-state-mode=%s' % testing_state_mode)
- gradle_args.append('-Ptesting-state-path=%s' % testing_state_path)
- prepare_testing_index(testing_state_mode, testing_state_path)
+ testing_state_path = "%s/test-state/%s" % (utils.BUILD, utils.get_HEAD_branch())
+ gradle_args.append('-Ptesting-state=%s' % testing_state_path)
+ prepare_testing_index(testing_state_path)
def fresh_testing_index(testing_state_dir):
number = 0
@@ -28,22 +24,22 @@
if not os.path.exists(freshIndex):
return freshIndex
-def prepare_testing_index(testing_state_mode, testing_state_dir):
+def prepare_testing_index(testing_state_dir):
if not os.path.exists(testing_state_dir):
os.makedirs(testing_state_dir)
index_path = os.path.join(testing_state_dir, "index.html")
parent_report = None
resuming = os.path.exists(index_path)
- mode = testing_state_mode if resuming else f"starting (flag: {testing_state_mode})"
if (resuming):
parent_report = fresh_testing_index(testing_state_dir)
os.rename(index_path, parent_report)
index = open(index_path, "a")
+ run_prefix = "Resuming" if resuming else "Starting"
relative_state_dir = os.path.relpath(testing_state_dir)
- title = relative_state_dir
+ title = f"{run_prefix} @ {relative_state_dir}"
# Print a console link to the test report for easy access.
print("=" * 70)
- print("Test report written to:")
+ print(f"{run_prefix} test, report written to:")
print(f" file://{index_path}")
print("=" * 70)
# Print the new index content.
@@ -51,7 +47,6 @@
index.write("<style> * { font-family: monospace; }</style>")
index.write("<meta http-equiv='refresh' content='10' />")
index.write(f"</head><body><h1>{title}</h1>")
- index.write(f"<h2>Mode: {mode}</h2>")
# write index links first to avoid jumping when browsing.
if parent_report:
index.write(f"<p><a href=\"file://{parent_report}\">Previous result index</a></p>")
