Per module testing state with shared index
This moved the creation of the shared state index to test.py such
that it is created once and ahead of test runs. Each test module
emits to the shared index and appends a status block on completion.
Test state is moved to per-module sub-directories and filters are
computed based on the state of the tests in each modules sub-dir.
Bug: b/297316723
Change-Id: Ie20754600125c3d53194bd684f7c976ca96d4926
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt
index 2ae1cf4..eb432eb 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt
@@ -74,10 +74,6 @@
test.maxHeapSize = "4G"
}
- if (project.hasProperty("testing-state")) {
- TestingState.setUpTestingState(test)
- }
-
if (project.hasProperty("one_line_per_test")
|| project.hasProperty("update_test_timestamp")) {
test.addTestListener(object : TestListener {
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
index f427bd4..240e656 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestingState.kt
@@ -8,8 +8,8 @@
import java.io.PrintStream
import java.net.URLEncoder
import java.nio.file.Path
-import java.util.Date
import java.util.concurrent.TimeUnit
+import org.gradle.api.Project
import org.gradle.api.tasks.testing.Test
import org.gradle.api.tasks.testing.TestDescriptor
import org.gradle.api.tasks.testing.TestListener
@@ -17,140 +17,92 @@
import org.gradle.api.tasks.testing.TestOutputListener
import org.gradle.api.tasks.testing.TestResult
+// Utility to install tracking of test results in status files.
class TestingState {
companion object {
fun setUpTestingState(task: Test) {
val project = task.project
- val reportDir = File(project.property("testing-state")!!.toString())
- val index = reportDir.resolve("index.html")
- val reportDirExists = reportDir.exists()
- val resuming = reportDirExists
-
- var hasFailingTests = false
- if (resuming) {
- // Test filtering happens before the test execution is initiated so compute it here.
- // If there are still failing tests in the report, include only those.
- hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
- task.filter.includeTestsMatching("$clazz.$name")
- })
- // Otherwise exclude all of the test already marked as succeeding.
- if (!hasFailingTests) {
- // Also allow the test to overall succeed if there are no remaining tests that match,
- // which is natural if the state already succeeded in full.
- task.filter.isFailOnNoMatchingTests = false
- forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
- task.filter.excludeTestsMatching("$clazz.$name")
- })
- }
+ if (!project.hasProperty("testing-state")) {
+ return
}
+ val projectName = task.project.name
+ val indexDir = File(project.property("testing-state")!!.toString())
+ val reportDir = indexDir.resolve(project.name)
+ val index = indexDir.resolve("index.html")
+ val resuming = reportDir.exists()
+ if (resuming) {
+ applyTestFilters(task, reportDir)
+ }
+ addTestHandler(task, projectName, index, reportDir)
+ }
+ private fun applyTestFilters(task: Test, reportDir: File) {
+ // If there are failing tests only rerun those.
+ val hasFailingTests = forEachTestReportAlreadyFailing(task, reportDir, { clazz, name ->
+ task.filter.includeTestsMatching("$clazz.$name")
+ })
+ if (hasFailingTests) {
+ return
+ }
+ // Otherwise exclude all the tests already marked as succeeding or skipped.
+ forEachTestReportAlreadyPassing(task, reportDir, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ forEachTestReportAlreadySkipped(task, reportDir, { clazz, name ->
+ task.filter.excludeTestsMatching("$clazz.$name")
+ })
+ }
+
+ private fun addTestHandler(
+ task: Test,
+ projectName: String,
+ index: File,
+ reportDir: File) {
+ task.addTestOutputListener(object : TestOutputListener {
+ override fun onOutput(desc: TestDescriptor, event: TestOutputEvent) {
+ withTestResultEntryWriter(reportDir, desc, event.getDestination().name, true, {
+ it.append(event.getMessage())
+ })
+ }
+ })
task.addTestListener(object : TestListener {
- fun isRoot(desc: TestDescriptor): Boolean {
- return desc.parent == null
- }
- fun getFreshTestReportIndex(reportDir: File): File {
- var number = 0
- while (true) {
- val freshIndex = reportDir.toPath().resolve("index.${number++}.html").toFile()
- if (!freshIndex.exists()) {
- return freshIndex
- }
- }
- }
-
- fun escapeHtml(string: String): String {
- return string.replace("&", "&").replace("<", "<").replace(">", ">")
- }
-
- fun filterStackTraces(result: TestResult) {
- for (throwable in result.getExceptions()) {
- filterStackTrace(throwable)
- }
- }
-
- // It would be nice to do this in a non-destructive way...
- fun filterStackTrace(exception: Throwable) {
- if (!project.hasProperty("print_full_stacktraces")) {
- val elements = ArrayList<StackTraceElement>()
- val skipped = ArrayList<StackTraceElement>()
- for (element in exception.getStackTrace()) {
- if (element.toString().contains("com.android.tools.r8")) {
- elements.addAll(skipped)
- elements.add(element)
- skipped.clear()
- } else {
- skipped.add(element)
- }
- }
- exception.setStackTrace(elements.toTypedArray())
- }
- }
-
- fun printAllStackTracesToFile(exceptions: List<Throwable>, out: File) {
- PrintStream(FileOutputStream(out))
- .use({ printer -> exceptions.forEach { it.printStackTrace(printer) } })
- }
-
- override fun beforeSuite(desc: TestDescriptor) {
- if (!isRoot(desc)) {
- return
- }
- var parentReport: File? = null
- if (resuming) {
- if (index.exists()) {
- parentReport = getFreshTestReportIndex(reportDir)
- index.renameTo(parentReport)
- }
- } else {
- reportDir.mkdirs()
- }
- val runPrefix = if (resuming) "Resuming" else "Starting"
- val title = "${runPrefix} @ ${reportDir}"
- // Print a console link to the test report for easy access.
- println("${runPrefix} test, report written to:")
- println(" file://${index}")
- // Print the new index content.
- index.appendText("<html><head><title>${title}</title>")
- index.appendText("<style> * { font-family: monospace; }</style>")
- index.appendText("<meta http-equiv='refresh' content='10' />")
- index.appendText("</head><body><h1>${title}</h1>")
- index.appendText("<p>Run on: ${Date()}</p>")
- if (parentReport != null) {
- index.appendText("<p><a href=\"file://${parentReport}\">Previous result index</a></p>")
- }
- index.appendText("<p><a href=\"file://${index}\">Most recent result index</a></p>")
- index.appendText("<p><a href=\"file://${reportDir}\">Test directories</a></p>")
- index.appendText("<h2>Failing tests (refreshing automatically every 10 seconds)</h2><ul>")
- }
+ override fun beforeSuite(desc: TestDescriptor) {}
override fun afterSuite(desc: TestDescriptor, result: TestResult) {
- if (!isRoot(desc)) {
+ if (desc.parent != null) {
return
}
// Update the final test results in the index.
- index.appendText("</ul>")
- if (result.resultType == TestResult.ResultType.SUCCESS) {
- if (hasFailingTests) {
- index.appendText("<h2>Rerun of failed tests now pass!</h2>")
- index.appendText("<h2>Rerun again to continue with outstanding tests!</h2>")
- } else {
- index.appendText("<h2 style=\"background-color:#62D856\">GREEN BAR == YOU ROCK!</h2>")
- }
+ val text = StringBuilder()
+ val successColor = "#a2ff99"
+ val failureColor = "#ff6454"
+ val emptyColor = "#d4d4d4"
+ val color: String;
+ if (result.testCount == 0L) {
+ color = emptyColor
+ } else if (result.resultType == TestResult.ResultType.SUCCESS) {
+ color = successColor
} else if (result.resultType == TestResult.ResultType.FAILURE) {
- index.appendText("<h2 style=\"background-color:#6D130A\">Some tests failed: ${result.resultType.name}</h2><ul>")
+ color = failureColor
} else {
- index.appendText("<h2>Tests finished: ${result.resultType.name}</h2><ul>")
+ color = failureColor
}
- index.appendText("<li>Number of tests: ${result.testCount}")
- index.appendText("<li>Failing tests: ${result.failedTestCount}")
- index.appendText("<li>Successful tests: ${result.successfulTestCount}")
- index.appendText("<li>Skipped tests: ${result.skippedTestCount}")
- index.appendText("</ul></body></html>")
+ // The failure list has an open <ul> so close it before appending the module results.
+ text.append("</ul>")
+ text.append("<div style=\"background-color:${color}\">")
+ text.append("<h2>${projectName}: ${result.resultType.name}</h2>")
+ text.append("<ul>")
+ text.append("<li>Number of tests: ${result.testCount}")
+ text.append("<li>Failing tests: ${result.failedTestCount}")
+ text.append("<li>Successful tests: ${result.successfulTestCount}")
+ text.append("<li>Skipped tests: ${result.skippedTestCount}")
+ text.append("</ul></div>")
+ // Reopen a <ul> as other modules may still append test failures.
+ text.append("<ul>")
+
+ index.appendText(text.toString())
}
override fun beforeTest(desc: TestDescriptor) {
@@ -181,7 +133,7 @@
})
// For failed tests, update the index and emit stack trace information.
if (result.resultType == TestResult.ResultType.FAILURE) {
- val title = escapeHtml("${desc.className}.${desc.name}")
+ val title = testLinkContent(desc)
val link = getTestReportEntryURL(reportDir, desc)
index.appendText("<li><a href=\"${link}\">${title}</a></li>")
if (!result.exceptions.isEmpty()) {
@@ -193,6 +145,8 @@
"exceptions-raw.txt"
)
)
+ // The raw stacktrace has lots of useless gradle test runner frames.
+ // As a convenience filter out those so the stack is just easier to read.
filterStackTraces(result)
printAllStackTracesToFile(
result.exceptions,
@@ -206,35 +160,73 @@
}
}
})
-
- task.addTestOutputListener(object : TestOutputListener {
- override fun onOutput(desc: TestDescriptor, event: TestOutputEvent) {
- withTestResultEntryWriter(reportDir, desc, event.getDestination().name, true, {
- it.append(event.getMessage())
- })
- }
- })
}
- fun urlEncode(string: String): String {
+ private fun escapeHtml(string: String): String {
+ return string
+ .replace("&", "&")
+ .replace("<", "<")
+ .replace(">", ">")
+ }
+
+ private fun urlEncode(string: String): String {
// Not sure why, but the + also needs to be converted to have working links.
return URLEncoder.encode(string, "UTF-8").replace("+", "%20")
}
- fun ensureDir(dir: File): File {
+ private fun testLinkContent(desc: TestDescriptor) : String {
+ val pkgR8 = "com.android.tools.r8."
+ val className = desc.className!!
+ val shortClassName =
+ if (className.startsWith(pkgR8)) {
+ className.substring(pkgR8.length)
+ } else {
+ className
+ }
+ return escapeHtml("${shortClassName}.${desc.name}")
+ }
+
+ private fun filterStackTraces(result: TestResult) {
+ for (throwable in result.getExceptions()) {
+ filterStackTrace(throwable)
+ }
+ }
+
+ // It would be nice to do this in a non-destructive way...
+ private fun filterStackTrace(exception: Throwable) {
+ val elements = ArrayList<StackTraceElement>()
+ val skipped = ArrayList<StackTraceElement>()
+ for (element in exception.getStackTrace()) {
+ if (element.toString().contains("com.android.tools.r8")) {
+ elements.addAll(skipped)
+ elements.add(element)
+ skipped.clear()
+ } else {
+ skipped.add(element)
+ }
+ }
+ exception.setStackTrace(elements.toTypedArray())
+ }
+
+ private fun printAllStackTracesToFile(exceptions: List<Throwable>, out: File) {
+ PrintStream(FileOutputStream(out))
+ .use({ printer -> exceptions.forEach { it.printStackTrace(printer) } })
+ }
+
+ private fun ensureDir(dir: File): File {
dir.mkdirs()
return dir
}
// Some of our test parameters have new lines :-( We really don't want test names to span lines.
- fun sanitizedTestName(testDesc: TestDescriptor): String {
+ private fun sanitizedTestName(testDesc: TestDescriptor): String {
if (testDesc.getName().contains("\n")) {
throw RuntimeException("Unsupported use of newline in test name: '${testDesc.getName()}'")
}
return testDesc.getName()
}
- fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
+ private fun getTestReportEntryDir(reportDir: File, testDesc: TestDescriptor): File {
return ensureDir(
reportDir.toPath()
.resolve(testDesc.getClassName()!!)
@@ -243,13 +235,13 @@
)
}
- fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
+ private fun getTestReportEntryURL(reportDir: File, testDesc: TestDescriptor): Path {
val classDir = urlEncode(testDesc.getClassName()!!)
val testDir = urlEncode(sanitizedTestName(testDesc))
return reportDir.toPath().resolve(classDir).resolve(testDir)
}
- fun getTestResultEntryOutputFile(
+ private fun getTestResultEntryOutputFile(
reportDir: File,
testDesc: TestDescriptor,
fileName: String
@@ -258,7 +250,7 @@
return dir.resolve(fileName).toFile()
}
- fun withTestResultEntryWriter(
+ private fun withTestResultEntryWriter(
reportDir: File,
testDesc: TestDescriptor,
fileName: String,
@@ -269,7 +261,7 @@
FileWriter(file, append).use(fn)
}
- fun forEachTestReportAlreadyFailing(
+ private fun forEachTestReportAlreadyFailing(
test: Test,
reportDir: File,
onFailureTest: (String, String) -> Unit
diff --git a/d8_r8/test_modules/tests_bootstrap/build.gradle.kts b/d8_r8/test_modules/tests_bootstrap/build.gradle.kts
index b196db9..903f546 100644
--- a/d8_r8/test_modules/tests_bootstrap/build.gradle.kts
+++ b/d8_r8/test_modules/tests_bootstrap/build.gradle.kts
@@ -55,6 +55,8 @@
}
withType<Test> {
+ TestingState.setUpTestingState(this)
+
environment.put("USE_NEW_GRADLE_SETUP", "true")
dependsOn(mainR8RelocatedTask)
environment.put("R8_WITH_RELOCATED_DEPS", mainR8RelocatedTask.outputs.files.getSingleFile())
diff --git a/d8_r8/test_modules/tests_java_8/build.gradle.kts b/d8_r8/test_modules/tests_java_8/build.gradle.kts
index fa45254..588063b 100644
--- a/d8_r8/test_modules/tests_java_8/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_8/build.gradle.kts
@@ -155,6 +155,8 @@
}
withType<Test> {
+ TestingState.setUpTestingState(this)
+
environment.put("USE_NEW_GRADLE_SETUP", "true")
dependsOn(mainDepsJarTask)
dependsOn(thirdPartyRuntimeDependenciesTask)
diff --git a/tools/test.py b/tools/test.py
index 5820590..a6c34f0 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -19,6 +19,7 @@
import download_kotlin_dev
import gradle
import notify
+import test_state
import utils
if utils.is_python3():
@@ -376,11 +377,7 @@
if testing_state:
if options.new_gradle:
- # In the new build the test state directory must be passed explictitly.
- # TODO(b/297316723): Simplify this and just support a single flag: --testing-state <path>
- if not testing_state_path:
- testing_state_path = "%s/test-state/%s" % (utils.BUILD, utils.get_HEAD_branch())
- gradle_args.append('-Ptesting-state=%s' % testing_state_path)
+ test_state.set_up_test_state(gradle_args, testing_state_path)
else:
gradle_args.append('-Ptesting-state')
@@ -503,7 +500,6 @@
return 0
-
def archive_and_return(return_code, options):
if return_code != 0:
if options.archive_failures:
diff --git a/tools/test_state.py b/tools/test_state.py
new file mode 100644
index 0000000..4541533
--- /dev/null
+++ b/tools/test_state.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import utils
+
+import datetime
+import os
+
+def set_up_test_state(gradle_args, testing_state_path):
+ # In the new build the test state directory must be passed explictitly.
+ # TODO(b/297316723): Simplify this and just support a single flag: --testing-state <path>
+ if not testing_state_path:
+ testing_state_path = "%s/test-state/%s" % (utils.BUILD, utils.get_HEAD_branch())
+ gradle_args.append('-Ptesting-state=%s' % testing_state_path)
+ prepare_testing_index(testing_state_path)
+
+def fresh_testing_index(testing_state_dir):
+ number = 0
+ while True:
+ freshIndex = os.path.join(testing_state_dir, "index.%d.html" % number)
+ number += 1
+ if not os.path.exists(freshIndex):
+ return freshIndex
+
+def prepare_testing_index(testing_state_dir):
+ if not os.path.exists(testing_state_dir):
+ os.makedirs(testing_state_dir)
+ index_path = os.path.join(testing_state_dir, "index.html")
+ parent_report = None
+ resuming = os.path.exists(index_path)
+ if (resuming):
+ parent_report = fresh_testing_index(testing_state_dir)
+ os.rename(index_path, parent_report)
+ index = open(index_path, "a")
+ run_prefix = "Resuming" if resuming else "Starting"
+ relative_state_dir = os.path.relpath(testing_state_dir)
+ title = f"{run_prefix} @ {relative_state_dir}"
+ # Print a console link to the test report for easy access.
+ print("=" * 70)
+ print(f"{run_prefix} test, report written to:")
+ print(f" file://{index_path}")
+ print("=" * 70)
+ # Print the new index content.
+ index.write(f"<html><head><title>{title}</title>")
+ index.write("<style> * { font-family: monospace; }</style>")
+ index.write("<meta http-equiv='refresh' content='10' />")
+ index.write(f"</head><body><h1>{title}</h1>")
+ # write index links first to avoid jumping when browsing.
+ if parent_report:
+ index.write(f"<p><a href=\"file://{parent_report}\">Previous result index</a></p>")
+ index.write(f"<p><a href=\"file://{index_path}\">Most recent result index</a></p>")
+ index.write(f"<p><a href=\"file://{testing_state_dir}\">Test directories</a></p>")
+ # git branch/hash and diff for future reference
+ index.write(f"<p>Run on: {datetime.datetime.now()}</p>")
+ index.write(f"<p>Git branch: {utils.get_HEAD_branch()}")
+ index.write(f"</br>Git SHA: {utils.get_HEAD_sha1()}")
+ index.write(f'</br>Git diff summary:\n')
+ index.write(f'<pre style="background-color: lightgray">{utils.get_HEAD_diff_stat()}</pre></p>')
+ # header for the failing tests
+ index.write("<h2>Failing tests (refreshing automatically every 10 seconds)</h2><ul>")
diff --git a/tools/utils.py b/tools/utils.py
index 279e1d0..fe8b539 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -362,6 +362,9 @@
def get_HEAD_sha1():
return get_HEAD_sha1_for_checkout(REPO_ROOT)
+def get_HEAD_diff_stat():
+ return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8')
+
def get_HEAD_sha1_for_checkout(checkout):
cmd = ['git', 'rev-parse', 'HEAD']
PrintCmd(cmd)
