Introduce "enable_r8_turbo_builds" gradle property
Use a separate sourceSet for files that have been modified when doing
incremental builds. Speeds up compile times where the list of files
isn't changed from 1-2 minutes -> 1-2 seconds.
Caveat: Unmodified sources that depend on modified ones will *not be
rebuilt* when modified sources change. This is where the speed-up comes
from, but can lead to runtime crashes if signatures change without
references to them being updated.
I've so far not actually hit this caveat, as IntelliJ is quite good at
pointing out problems without needing to do a compile.
Bug: 458494845
Change-Id: I5aea9d8dd36e8990e23032a4238f4e527c33a06d
diff --git a/d8_r8/main/build.gradle.kts b/d8_r8/main/build.gradle.kts
index cc18e39..5c0508f 100644
--- a/d8_r8/main/build.gradle.kts
+++ b/d8_r8/main/build.gradle.kts
@@ -2,19 +2,26 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import com.google.gson.Gson
+import java.io.ByteArrayOutputStream
import java.net.URI
-import java.nio.file.Paths
+import java.nio.charset.Charset
import java.nio.file.Files.readString
+import java.nio.file.Paths
+import java.util.UUID
+import javax.inject.Inject
import net.ltgt.gradle.errorprone.errorprone
import org.gradle.api.artifacts.ModuleVersionIdentifier
import org.gradle.api.artifacts.component.ModuleComponentIdentifier
+import org.gradle.api.provider.Provider
+import org.gradle.api.provider.ValueSource
+import org.gradle.api.provider.ValueSourceParameters
+import org.gradle.api.tasks.bundling.Jar
+import org.gradle.process.ExecOperations
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.spdx.sbom.gradle.SpdxSbomTask
import org.spdx.sbom.gradle.extensions.DefaultSpdxSbomTaskExtension
-import com.google.gson.Gson
-import java.util.UUID
-
plugins {
`kotlin-dsl`
id("dependencies-plugin")
@@ -30,12 +37,180 @@
// Disable Error Prone checks (can make compiles marginally faster).
var enableErrorProne = !project.hasProperty("disable_errorprone")
+// Use a separate sourceSet for files that have been modified when doing incremental builds.
+// Speeds up compile times where the list of files isn't changed from 1-2 minutes -> 1-2 seconds.
+//
+// Modified files are determined using git, and the list of modified files never shrinks (since
+// that would cause build errors). However, it is safe to fully reset the list of modified files,
+// which you can do by deleting d8_r8/main/build/turbo-paths.txt.
+//
+// What's the catch?
+// Unmodified sources that depend on modified ones will *not be rebuilt* when modified sources
+// change. This is where the speed-up comes from, but can lead to runtime crashes if signatures
+// change without references to them being updated.
+// Be sure to fix problems reported by IntelliJ when using this mode.
+var enableTurboBuilds = project.hasProperty("enable_r8_turbo_builds")
+
+val MAIN_JAVA_PATH_PREFIX = "src/main/java/"
+
+interface TurboPathsValueSourceParameters : ValueSourceParameters {
+ val pathPrefix: Property<String>
+ val turboPathsFile: Property<File>
+ val extraGlobs: ListProperty<String>
+ val mainOutputDir: Property<File>
+}
+
+enum class TurboReason {
+ FIRST_BUILD,
+ PATHS_CHANGED,
+ PATHS_UNCHANGED,
+ CORRUPT_FILE,
+ TOO_MANY_PATHS,
+}
+
+data class TurboState(val paths: List<String>, val reason: TurboReason)
+
+abstract class TurboPathsValueSource : ValueSource<TurboState, TurboPathsValueSourceParameters> {
+ @get:Inject abstract val execOperations: ExecOperations
+
+ fun isDirectoryEmpty(path: File): Boolean {
+ if (!path.exists()) {
+ return true
+ }
+
+ val files = path.listFiles()
+ return files == null || files.isEmpty()
+ }
+
+ override fun obtain(): TurboState? {
+ val prefix = parameters.pathPrefix.get()
+ val turboPathsFile = parameters.turboPathsFile.get()
+ val extraGlobs = parameters.extraGlobs.get()
+ val mainOutputDir = parameters.mainOutputDir.get()
+
+ // Check for first build (since the turbo sourceSet requires the main one
+ // to have been built already).
+ if (isDirectoryEmpty(mainOutputDir)) {
+ return TurboState(listOf(), TurboReason.FIRST_BUILD)
+ }
+
+ var mergeBase = "origin/main"
+ val pathSet: MutableSet<String> = mutableSetOf()
+
+ if (turboPathsFile.exists()) {
+ val lines = turboPathsFile.readLines()
+ if (!lines.isEmpty() && lines[0].startsWith("mergebase=")) {
+ mergeBase = lines[0].removePrefix("mergebase=")
+ pathSet.addAll(lines.drop(1))
+ } else {
+ // Corrupt file.
+ turboPathsFile.delete()
+ return TurboState(listOf(), TurboReason.CORRUPT_FILE)
+ }
+ }
+
+ val prevNumSource = pathSet.size
+ val output = ByteArrayOutputStream()
+ execOperations.exec {
+ commandLine = listOf("git", "diff", "--name-only", "--merge-base", mergeBase)
+ standardOutput = output
+ }
+ val result = String(output.toByteArray(), Charset.defaultCharset())
+ val gitPaths =
+ result
+ .lines()
+ .filter { it.startsWith(prefix) && it.endsWith(".java") }
+ .map { it.trim().removePrefix(prefix) }
+ pathSet.addAll(gitPaths)
+
+ val ret = pathSet.toMutableList()
+ ret.sort()
+ // Allow users to specify extra globs.
+ ret += extraGlobs
+
+ if (mergeBase == "origin/main") {
+ output.reset()
+ execOperations.exec {
+ commandLine = listOf("git", "rev-parse", "origin/main")
+ standardOutput = output
+ }
+ mergeBase = String(output.toByteArray(), Charset.defaultCharset()).trim()
+ }
+
+ if (pathSet.size > 200 && gitPaths.size < 40) {
+ // File has gotten too big. Start fresh.
+ turboPathsFile.delete()
+ return TurboState(listOf(), TurboReason.TOO_MANY_PATHS)
+ }
+
+ turboPathsFile.writeText("mergebase=$mergeBase\n" + ret.joinToString("\n"))
+ val changed = prevNumSource != pathSet.size
+ val reason =
+ if (pathSet.isEmpty()) TurboReason.FIRST_BUILD
+ else if (changed) TurboReason.PATHS_CHANGED else TurboReason.PATHS_UNCHANGED
+ return TurboState(ret, reason)
+ }
+}
+
+val turboPathsProvider: Provider<TurboState> =
+ providers.of(TurboPathsValueSource::class.java) {
+ parameters.pathPrefix.set(MAIN_JAVA_PATH_PREFIX)
+
+ // Wipe this file to remove files from the active set.
+ parameters.turboPathsFile.set(layout.buildDirectory.file("turbo-paths.txt").get().asFile)
+
+ parameters.extraGlobs.set(
+ project.findProperty("turbo_build_globs")?.toString()?.split(',') ?: emptyList()
+ )
+
+ parameters.mainOutputDir.set(sourceSets["main"].java.destinationDirectory.get().getAsFile())
+ }
+
+// Add all changed files to the "turbo" source set.
+val turboState = if (enableTurboBuilds) turboPathsProvider.get() else null
+
+if (turboState != null) {
+ val numFiles = turboState.paths.size
+ val msg =
+ when (turboState.reason) {
+ TurboReason.FIRST_BUILD -> "First build detected. Build will be slow."
+ TurboReason.PATHS_CHANGED -> "Paths in active set have changed. Build will be slow."
+ TurboReason.PATHS_UNCHANGED -> "Paths unchanged. Size=$numFiles. Build should be fast!"
+ TurboReason.CORRUPT_FILE -> "turbo-paths.txt was invalid. Build will be slow."
+ TurboReason.TOO_MANY_PATHS -> "Paths were compacted. Build will be slow."
+ }
+ logger.warn("Turbo: $msg")
+} else {
+ logger.warn("Turbo: enable_r8_turbo_builds=false")
+}
java {
- sourceSets.main.configure {
- java.srcDir(getRoot().resolveAll("src", "main", "java"))
- resources.srcDirs(getRoot().resolveAll("third_party", "api_database", "api_database"))
+ sourceSets {
+ val srcDir = getRoot().resolveAll("src", "main", "java")
+
+ main {
+ resources.srcDirs(getRoot().resolveAll("third_party", "api_database", "api_database"))
+ java {
+ srcDir(srcDir)
+ if (turboState != null && !turboState.paths.isEmpty()) {
+ exclude(turboState.paths)
+ }
+ }
+ }
+
+ // Must be created unconditionally so that other targets can depend on it.
+ create("turbo") {
+ java {
+ srcDir(srcDir)
+ if (turboState != null && !turboState.paths.isEmpty()) {
+ include(turboState.paths)
+ } else {
+ exclude("*")
+ }
+ }
+ }
}
+
sourceCompatibility = JvmCompatibility.sourceCompatibility
targetCompatibility = JvmCompatibility.targetCompatibility
toolchain {
@@ -45,13 +220,13 @@
}
dependencies {
+ implementation(":assistant")
implementation(":keepanno")
implementation(":resourceshrinker")
compileOnly(Deps.androidxCollection)
compileOnly(Deps.androidxTracingDriver)
compileOnly(Deps.androidxTracingDriverWire)
compileOnly(Deps.asm)
- implementation(":assistant")
compileOnly(Deps.asmCommons)
compileOnly(Deps.asmUtil)
compileOnly(Deps.fastUtil)
@@ -62,6 +237,46 @@
errorprone(Deps.errorprone)
}
+if (enableTurboBuilds) {
+ tasks.named("compileJava") {
+ // Makes compileTurboJava run first, but does not cause compileJava to re-run if
+ // compileTurboJava changes.
+ dependsOn(tasks.named("compileTurboJava"))
+ }
+
+ // Does not include main's output directory, which must also be added when compilation avoidance
+ // causes only a subset of sources to be recompiled.
+ val mainClasspath = sourceSets["main"].compileClasspath.getAsPath()
+
+ tasks.named<JavaCompile>("compileTurboJava") {
+ // Add the main's classes to the classpath without letting gradle know about this dependency
+ // (as it's a circular one).
+ options.compilerArgs.add("-classpath")
+ options.compilerArgs.add(
+ "" +
+ sourceSets["turbo"].java.destinationDirectory.get() +
+ File.pathSeparator +
+ mainClasspath +
+ File.pathSeparator +
+ sourceSets["main"].java.destinationDirectory.get()
+ )
+ }
+
+ tasks.named<JavaCompile>("compileJava") {
+ // Add the turbo's classes to the classpath without letting gradle know about this dependency
+ // (or else it will cause it to rebuild whenever files in it change).
+ options.compilerArgs.add("-classpath")
+ options.compilerArgs.add(
+ "" +
+ sourceSets["main"].java.destinationDirectory.get() +
+ File.pathSeparator +
+ mainClasspath +
+ File.pathSeparator +
+ sourceSets["turbo"].java.destinationDirectory.get()
+ )
+ }
+}
+
if (project.hasProperty("spdxVersion")) {
project.version = project.property("spdxVersion")!!
}
@@ -118,6 +333,10 @@
}
tasks {
+ jar {
+ from(sourceSets["turbo"].output)
+ }
+
withType<Exec> {
doFirst {
println("Executing command: ${commandLine.joinToString(" ")}")
@@ -261,6 +480,7 @@
}
val depsJar by registering(Zip::class) {
+ from(sourceSets["turbo"].output)
dependsOn(gradle.includedBuild("shared").task(":downloadDeps"))
dependsOn(resourceShrinkerDepsTask)
dependsOn(threadingModuleBlockingJar)
diff --git a/d8_r8/test_modules/testbase/build.gradle.kts b/d8_r8/test_modules/testbase/build.gradle.kts
index 615b6a0..8d553fa 100644
--- a/d8_r8/test_modules/testbase/build.gradle.kts
+++ b/d8_r8/test_modules/testbase/build.gradle.kts
@@ -33,6 +33,7 @@
// incompatible java class file version. By depending on the jar we circumvent that.
val keepAnnoJarTask = projectTask("keepanno", "jar")
val keepAnnoCompileTask = projectTask("keepanno", "compileJava")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
val mainCompileTask = projectTask("main", "compileJava")
val mainDepsJarTask = projectTask("main", "depsJar")
val resourceShrinkerJavaCompileTask = projectTask("resourceshrinker", "compileJava")
@@ -41,6 +42,7 @@
dependencies {
implementation(keepAnnoJarTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
implementation(resourceShrinkerJavaCompileTask.outputs.files)
diff --git a/d8_r8/test_modules/tests_java_11/build.gradle.kts b/d8_r8/test_modules/tests_java_11/build.gradle.kts
index d14c599..00c23cd 100644
--- a/d8_r8/test_modules/tests_java_11/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_11/build.gradle.kts
@@ -22,11 +22,13 @@
val testbaseJavaCompileTask = projectTask("testbase", "compileJava")
val testbaseDepsJarTask = projectTask("testbase", "depsJar")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
val mainCompileTask = projectTask("main", "compileJava")
dependencies {
implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile()))
implementation(testbaseJavaCompileTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
}
diff --git a/d8_r8/test_modules/tests_java_17/build.gradle.kts b/d8_r8/test_modules/tests_java_17/build.gradle.kts
index 473f4da..01d25ea 100644
--- a/d8_r8/test_modules/tests_java_17/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_17/build.gradle.kts
@@ -25,12 +25,14 @@
val testbaseJavaCompileTask = projectTask("testbase", "compileJava")
val testbaseDepsJarTask = projectTask("testbase", "depsJar")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
val mainCompileTask = projectTask("main", "compileJava")
dependencies {
implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile()))
implementation(testbaseJavaCompileTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
}
diff --git a/d8_r8/test_modules/tests_java_21/build.gradle.kts b/d8_r8/test_modules/tests_java_21/build.gradle.kts
index a44697d..915ca0b 100644
--- a/d8_r8/test_modules/tests_java_21/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_21/build.gradle.kts
@@ -22,12 +22,14 @@
val testbaseJavaCompileTask = projectTask("testbase", "compileJava")
val testbaseDepsJarTask = projectTask("testbase", "depsJar")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
val mainCompileTask = projectTask("main", "compileJava")
val assistantCompileTask = projectTask("assistant", "compileJava")
dependencies {
implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile()))
implementation(testbaseJavaCompileTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
implementation(assistantCompileTask.outputs.files)
diff --git a/d8_r8/test_modules/tests_java_25/build.gradle.kts b/d8_r8/test_modules/tests_java_25/build.gradle.kts
index c0f8a64..8856e7b 100644
--- a/d8_r8/test_modules/tests_java_25/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_25/build.gradle.kts
@@ -22,11 +22,13 @@
val testbaseJavaCompileTask = projectTask("testbase", "compileJava")
val testbaseDepsJarTask = projectTask("testbase", "depsJar")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
val mainCompileTask = projectTask("main", "compileJava")
dependencies {
implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile()))
implementation(testbaseJavaCompileTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
}
diff --git a/d8_r8/test_modules/tests_java_8/build.gradle.kts b/d8_r8/test_modules/tests_java_8/build.gradle.kts
index a718ef2..c5e07c1 100644
--- a/d8_r8/test_modules/tests_java_8/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_8/build.gradle.kts
@@ -40,6 +40,7 @@
val keepAnnoCompileTask = projectTask("keepanno", "compileJava")
val assistantCompileTask = projectTask("assistant", "compileJava")
val keepAnnoCompileKotlinTask = projectTask("keepanno", "compileKotlin")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
val mainCompileTask = projectTask("main", "compileJava")
val mainDepsJarTask = projectTask("main", "depsJar")
val resourceShrinkerJavaCompileTask = projectTask("resourceshrinker", "compileJava")
@@ -48,6 +49,7 @@
dependencies {
implementation(keepAnnoJarTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
implementation(assistantCompileTask.outputs.files)
diff --git a/d8_r8/test_modules/tests_java_9/build.gradle.kts b/d8_r8/test_modules/tests_java_9/build.gradle.kts
index 6ac2d1b..ab048bb 100644
--- a/d8_r8/test_modules/tests_java_9/build.gradle.kts
+++ b/d8_r8/test_modules/tests_java_9/build.gradle.kts
@@ -27,10 +27,12 @@
val testbaseJavaCompileTask = projectTask("testbase", "compileJava")
val testbaseDepsJarTask = projectTask("testbase", "depsJar")
val mainCompileTask = projectTask("main", "compileJava")
+val mainTurboCompileTask = projectTask("main", "compileTurboJava")
dependencies {
implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile()))
implementation(testbaseJavaCompileTask.outputs.files)
+ implementation(mainTurboCompileTask.outputs.files)
implementation(mainCompileTask.outputs.files)
implementation(projectTask("main", "processResources").outputs.files)
}
