Per module testing state with shared index
This moved the creation of the shared state index to test.py such
that it is created once and ahead of test runs. Each test module
emits to the shared index and appends a status block on completion.
Test state is moved to per-module sub-directories and filters are
computed based on the state of the tests in each modules sub-dir.
Bug: b/297316723
Change-Id: Ie20754600125c3d53194bd684f7c976ca96d4926
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
index f427bd4..240e656 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
@@ -8,8 +8,8 @@
import java.io.PrintStream
import java.net.URLEncoder
import java.nio.file.Path
-import java.util.Date
import java.util.concurrent.TimeUnit
+import org.gradle.api.Project
import org.gradle.api.tasks.testing.Test
import org.gradle.api.tasks.testing.TestDescriptor
import org.gradle.api.tasks.testing.TestListener
@@ -17,140 +17,92 @@
import org.gradle.api.tasks.testing.TestOutputListener
import org.gradle.api.tasks.testing.TestResult
+// Utility to install tracking of test results in status files.
class TestingState {
companion object {
fun setUpTestingState(task: Test) {
val project = task.project
- val reportDir = File(project.property("testing-state")!!.toString())
- val index = reportDir.resolve("index.html")
- val reportDirExists = reportDir.exists()
- val resuming = reportDirExists
-
- var hasFailingTests = false
- if (resuming) {
- // Test filtering happens before the test execution is initiated so compute it here.
- // If there are still failing tests in the report, include only those.
- hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
- task.filter.includeTestsMatching("$clazz.$name")
- })
- // Otherwise exclude all of the test already marked as succeeding.
- if (!hasFailingTests) {
- // Also allow the test to overall succeed if there are no remaining tests that match,
- // which is natural if the state already succeeded in full.
- task.filter.isFailOnNoMatchingTests = false
- forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- }
+ if (!project.hasProperty("testing-state")) {
+ return
}
+ val projectName = task.project.name
+ val indexDir = File(project.property("testing-state")!!.toString())
+ val reportDir = indexDir.resolve(project.name)
+ val index = indexDir.resolve("index.html")
+ val resuming = reportDir.exists()
+ if (resuming) {
+ applyTestFilters(task, reportDir)
+ }
+ addTestHandler(task, projectName, index, reportDir)
+ }
+ private fun applyTestFilters(task: Test, reportDir: File) {
+ // If there are failing tests only rerun those.
+ val hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
+ task.filter.includeTestsMatching("$clazz.$name")
+ })
+ if (hasFailingTests) {
+ return
+ }
+ // Otherwise exclude all the tests already marked as succeeding or skipped.
+ forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ }
+
+ private fun addTestHandler(
+ task: Test,
+ projectName: String,
+ index: File,
+ reportDir: File) {
+ task.addTestOutputListener(object : TestOutputListener {
+ override fun onOutput(desc: TestDescriptor, event: TestOutputEvent) {
+ withTestResultEntryWriter(reportDir, desc, event.getDestination().name, true, {
+ it.append(event.getMessage())
+ })
+ }
+ })
task.addTestListener(object : TestListener {
- fun isRoot(desc: TestDescriptor): Boolean {
- return desc.parent == null
- }
- fun getFreshTestReportIndex(reportDir: File): File {
- var number = 0
- while (true) {
- val freshIndex = reportDir.toPath().resolve("index.${number++}.html").toFile()
- if (!freshIndex.exists()) {
- return freshIndex
- }
- }
- }
-
- fun escapeHtml(string: String): String {
- return string.replace("&", "&").replace("<", "<").replace(">", ">")
- }
-
- fun filterStackTraces(result: TestResult) {
- for (throwable in result.getExceptions()) {
- filterStackTrace(throwable)
- }
- }
-
- // It would be nice to do this in a non-destructive way...
- fun filterStackTrace(exception: Throwable) {
- if (!project.hasProperty("print_full_stacktraces")) {
- val elements = ArrayList<StackTraceElement>()
- val skipped = ArrayList<StackTraceElement>()
- for (element in exception.getStackTrace()) {
- if (element.toString().contains("com.android.tools.r8")) {
- elements.addAll(skipped)
- elements.add(element)
- skipped.clear()
- } else {
- skipped.add(element)
- }
- }
- exception.setStackTrace(elements.toTypedArray())
- }
- }
-
- fun printAllStackTracesToFile(exceptions: List<Throwable>, out: File) {
- PrintStream(FileOutputStream(out))
- .use({ printer -> exceptions.forEach { it.printStackTrace(printer) } })
- }
-
- override fun beforeSuite(desc: TestDescriptor) {
- if (!isRoot(desc)) {
- return
- }
- var parentReport: File? = null
- if (resuming) {
- if (index.exists()) {
- parentReport = getFreshTestReportIndex(reportDir)
- index.renameTo(parentReport)
- }
- } else {
- reportDir.mkdirs()
- }
- val runPrefix = if (resuming) "Resuming" else "Starting"
- val title = "${runPrefix} @ ${reportDir}"
- // Print a console link to the test report for easy access.
- println("${runPrefix} test, report written to:")
- println(" file://${index}")
- // Print the new index content.
- index.appendText("<html><head><title>${title}</title>")
- index.appendText("<style> * { font-family: monospace; }</style>")
- index.appendText("<meta http-equiv='refresh' content='10' />")
- index.appendText("</head><body><h1>${title}</h1>")
- index.appendText("<p>Run on: ${Date()}</p>")
- if (parentReport != null) {
- index.appendText("<p><a href=\"file://${parentReport}\">Previous result index</a></p>")
- }
- index.appendText("<p><a href=\"file://${index}\">Most recent result index</a></p>")
- index.appendText("<p><a href=\"file://${reportDir}\">Test directories</a></p>")
- index.appendText("<h2>Failing tests (refreshing automatically every 10 seconds)</h2><ul>")
- }
+ override fun beforeSuite(desc: TestDescriptor) {}
override fun afterSuite(desc: TestDescriptor, result: TestResult) {
- if (!isRoot(desc)) {
+ if (desc.parent != null) {
return
}
// Update the final test results in the index.
- index.appendText("</ul>")
- if (result.resultType == TestResult.ResultType.SUCCESS) {
- if (hasFailingTests) {
- index.appendText("<h2>Rerun of failed tests now pass!</h2>")
- index.appendText("<h2>Rerun again to continue with outstanding tests!</h2>")
- } else {
- index.appendText("<h2 style=\"background-color:#62D856\">GREEN BAR == YOU ROCK!</h2>")
- }
+ val text = StringBuilder()
+ val successColor = "#a2ff99"
+ val failureColor = "#ff6454"
+ val emptyColor = "#d4d4d4"
+ val color: String;
+ if (result.testCount == 0L) {
+ color = emptyColor
+ } else if (result.resultType == TestResult.ResultType.SUCCESS) {
+ color = successColor
} else if (result.resultType == TestResult.ResultType.FAILURE) {
- index.appendText("<h2 style=\"background-color:#6D130A\">Some tests failed: ${result.resultType.name}</h2><ul>")
+ color = failureColor
} else {
- index.appendText("<h2>Tests finished: ${result.resultType.name}</h2><ul>")
+ color = failureColor
}
- index.appendText("<li>Number of tests: ${result.testCount}")
- index.appendText("<li>Failing tests: ${result.failedTestCount}")
- index.appendText("<li>Successful tests: ${result.successfulTestCount}")
- index.appendText("<li>Skipped tests: ${result.skippedTestCount}")
- index.appendText("</ul></body></html>")
+ // The failure list has an open <ul> so close it before appending the module results.
+ text.append("</ul>")
+ text.append("<div style=\"background-color:${color}\">")
+ text.append("<h2>${projectName}: ${result.resultType.name}</h2>")
+ text.append("<ul>")
+ text.append("<li>Number of tests: ${result.testCount}")
+ text.append("<li>Failing tests: ${result.failedTestCount}")
+ text.append("<li>Successful tests: ${result.successfulTestCount}")
+ text.append("<li>Skipped tests: ${result.skippedTestCount}")
+ text.append("</ul></div>")
+ // Reopen a <ul> as other modules may still append test failures.
+ text.append("<ul>")
+
+ index.appendText(text.toString())
}
override fun beforeTest(desc: TestDescriptor) {
@@ -181,7 +133,7 @@
})
// For failed tests, update the index and emit stack trace information.
if (result.resultType == TestResult.ResultType.FAILURE) {
- val title = escapeHtml("${desc.className}.${desc.name}")
+ val title = testLinkContent(desc)
val link = getTestReportEntryURL(reportDir, desc)
index.appendText("<li><a href=\"${link}\">${title}</a></li>")
if (!result.exceptions.isEmpty()) {
@@ -193,6 +145,8 @@
"exceptions-raw.txt"
)
)
+ // The raw stacktrace has lots of useless gradle test runner frames.
+ // As a convenience filter out those so the stack is just easier to read.
filterStackTraces(result)
printAllStackTracesToFile(
result.exceptions,
@@ -206,35 +160,73 @@
}
}
})
-
- task.addTestOutputListener(object : TestOutputListener {
- override fun onOutput(desc: TestDescriptor, event: TestOutputEvent) {
- withTestResultEntryWriter(reportDir, desc, event.getDestination().name, true, {
- it.append(event.getMessage())
- })
- }
- })
}
- fun urlEncode(string: String): String {
+ private fun escapeHtml(string: String): String {
+ return string
+ .replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ }
+
+ private fun urlEncode(string: String): String {
// Not sure why, but the + also needs to be converted to have working links.
return URLEncoder.encode(string, "UTF-8").replace("+", "%20")
}
- fun ensureDir(dir: File): File {
+ private fun testLinkContent(desc: TestDescriptor) : String {
+ val pkgR8 = "com.android.tools.r8."
+ val className = desc.className!!
+ val shortClassName =
+ if (className.startsWith(pkgR8)) {
+ className.substring(pkgR8.length)
+ } else {
+ className
+ }
+ return escapeHtml("${shortClassName}.${desc.name}")
+ }
+
+ private fun filterStackTraces(result: TestResult) {
+ for (throwable in result.getExceptions()) {
+ filterStackTrace(throwable)
+ }
+ }
+
+ // It would be nice to do this in a non-destructive way...
+ private fun filterStackTrace(exception: Throwable) {
+ val elements = ArrayList<StackTraceElement>()
+ val skipped = ArrayList<StackTraceElement>()
+ for (element in exception.getStackTrace()) {
+ if (element.toString().contains("com.android.tools.r8")) {
+ elements.addAll(skipped)
+ elements.add(element)
+ skipped.clear()
+ } else {
+ skipped.add(element)
+ }
+ }
+ exception.setStackTrace(elements.toTypedArray())
+ }
+
+ private fun printAllStackTracesToFile(exceptions: List<Throwable>, out: File) {
+ PrintStream(FileOutputStream(out))
+ .use({ printer -> exceptions.forEach { it.printStackTrace(printer) } })
+ }
+
+ private fun ensureDir(dir: File): File {
dir.mkdirs()
return dir
}
// Some of our test parameters have new lines :-( We really don't want test names to span lines.
- fun sanitizedTestName(testDesc: TestDescriptor): String {
+ private fun sanitizedTestName(testDesc: TestDescriptor): String {
if (testDesc.getName().contains("\n")) {
throw RuntimeException("Unsupported use of newline in test name: '${testDesc.getName()}'")
}
return testDesc.getName()
}
- fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
+ private fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
return ensureDir(
reportDir.toPath()
.resolve(testDesc.getClassName()!!)
@@ -243,13 +235,13 @@
)
}
- fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
+ private fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
val classDir = urlEncode(testDesc.getClassName()!!)
val testDir = urlEncode(sanitizedTestName(testDesc))
return reportDir.toPath().resolve(classDir).resolve(testDir)
}
- fun getTestResultEntryOutputFile(
+ private fun getTestResultEntryOutputFile(
reportDir: File,
testDesc: TestDescriptor,
fileName: String
@@ -258,7 +250,7 @@
return dir.resolve(fileName).toFile()
}
- fun withTestResultEntryWriter(
+ private fun withTestResultEntryWriter(
reportDir: File,
testDesc: TestDescriptor,
fileName: String,
@@ -269,7 +261,7 @@
FileWriter(file, append).use(fn)
}
- fun forEachTestReportAlreadyFailing(
+ private fun forEachTestReportAlreadyFailing(
test: Test,
reportDir: File,
onFailureTest: (String, String) -> Unit
