Merge commit '0418ebca1a206d3202ac6f7327846de002b67191' into dev-release Change-Id: Id030e2ad9d0d37e5dae3a673401e0dee9e6e4a75
diff --git a/d8_r8/main/build.gradle.kts b/d8_r8/main/build.gradle.kts index cc18e39..5c0508f 100644 --- a/d8_r8/main/build.gradle.kts +++ b/d8_r8/main/build.gradle.kts
@@ -2,19 +2,26 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import com.google.gson.Gson +import java.io.ByteArrayOutputStream import java.net.URI -import java.nio.file.Paths +import java.nio.charset.Charset import java.nio.file.Files.readString +import java.nio.file.Paths +import java.util.UUID +import javax.inject.Inject import net.ltgt.gradle.errorprone.errorprone import org.gradle.api.artifacts.ModuleVersionIdentifier import org.gradle.api.artifacts.component.ModuleComponentIdentifier +import org.gradle.api.provider.Provider +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters +import org.gradle.api.tasks.bundling.Jar +import org.gradle.process.ExecOperations import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.spdx.sbom.gradle.SpdxSbomTask import org.spdx.sbom.gradle.extensions.DefaultSpdxSbomTaskExtension -import com.google.gson.Gson -import java.util.UUID - plugins { `kotlin-dsl` id("dependencies-plugin") @@ -30,12 +37,180 @@ // Disable Error Prone checks (can make compiles marginally faster). var enableErrorProne = !project.hasProperty("disable_errorprone") +// Use a separate sourceSet for files that have been modified when doing incremental builds. +// Speeds up compile times where the list of files isn't changed from 1-2 minutes -> 1-2 seconds. +// +// Modified files are determined using git, and the list of modified files never shrinks (since +// that would cause build errors). However, it is safe to fully reset the list of modified files, +// which you can do by deleting d8_r8/main/build/turbo-paths.txt. +// +// What's the catch? +// Unmodified sources that depend on modified ones will *not be rebuilt* when modified sources +// change. This is where the speed-up comes from, but can lead to runtime crashes if signatures +// change without references to them being updated. +// Be sure to fix problems reported by IntelliJ when using this mode. +var enableTurboBuilds = project.hasProperty("enable_r8_turbo_builds") + +val MAIN_JAVA_PATH_PREFIX = "src/main/java/" + +interface TurboPathsValueSourceParameters : ValueSourceParameters { + val pathPrefix: Property<String> + val turboPathsFile: Property<File> + val extraGlobs: ListProperty<String> + val mainOutputDir: Property<File> +} + +enum class TurboReason { + FIRST_BUILD, + PATHS_CHANGED, + PATHS_UNCHANGED, + CORRUPT_FILE, + TOO_MANY_PATHS, +} + +data class TurboState(val paths: List<String>, val reason: TurboReason) + +abstract class TurboPathsValueSource : ValueSource<TurboState, TurboPathsValueSourceParameters> { + @get:Inject abstract val execOperations: ExecOperations + + fun isDirectoryEmpty(path: File): Boolean { + if (!path.exists()) { + return true + } + + val files = path.listFiles() + return files == null || files.isEmpty() + } + + override fun obtain(): TurboState? { + val prefix = parameters.pathPrefix.get() + val turboPathsFile = parameters.turboPathsFile.get() + val extraGlobs = parameters.extraGlobs.get() + val mainOutputDir = parameters.mainOutputDir.get() + + // Check for first build (since the turbo sourceSet requires the main one + // to have been built already). + if (isDirectoryEmpty(mainOutputDir)) { + return TurboState(listOf(), TurboReason.FIRST_BUILD) + } + + var mergeBase = "origin/main" + val pathSet: MutableSet<String> = mutableSetOf() + + if (turboPathsFile.exists()) { + val lines = turboPathsFile.readLines() + if (!lines.isEmpty() && lines[0].startsWith("mergebase=")) { + mergeBase = lines[0].removePrefix("mergebase=") + pathSet.addAll(lines.drop(1)) + } else { + // Corrupt file. + turboPathsFile.delete() + return TurboState(listOf(), TurboReason.CORRUPT_FILE) + } + } + + val prevNumSource = pathSet.size + val output = ByteArrayOutputStream() + execOperations.exec { + commandLine = listOf("git", "diff", "--name-only", "--merge-base", mergeBase) + standardOutput = output + } + val result = String(output.toByteArray(), Charset.defaultCharset()) + val gitPaths = + result + .lines() + .filter { it.startsWith(prefix) && it.endsWith(".java") } + .map { it.trim().removePrefix(prefix) } + pathSet.addAll(gitPaths) + + val ret = pathSet.toMutableList() + ret.sort() + // Allow users to specify extra globs. + ret += extraGlobs + + if (mergeBase == "origin/main") { + output.reset() + execOperations.exec { + commandLine = listOf("git", "rev-parse", "origin/main") + standardOutput = output + } + mergeBase = String(output.toByteArray(), Charset.defaultCharset()).trim() + } + + if (pathSet.size > 200 && gitPaths.size < 40) { + // File has gotten too big. Start fresh. + turboPathsFile.delete() + return TurboState(listOf(), TurboReason.TOO_MANY_PATHS) + } + + turboPathsFile.writeText("mergebase=$mergeBase\n" + ret.joinToString("\n")) + val changed = prevNumSource != pathSet.size + val reason = + if (pathSet.isEmpty()) TurboReason.FIRST_BUILD + else if (changed) TurboReason.PATHS_CHANGED else TurboReason.PATHS_UNCHANGED + return TurboState(ret, reason) + } +} + +val turboPathsProvider: Provider<TurboState> = + providers.of(TurboPathsValueSource::class.java) { + parameters.pathPrefix.set(MAIN_JAVA_PATH_PREFIX) + + // Wipe this file to remove files from the active set. + parameters.turboPathsFile.set(layout.buildDirectory.file("turbo-paths.txt").get().asFile) + + parameters.extraGlobs.set( + project.findProperty("turbo_build_globs")?.toString()?.split(',') ?: emptyList() + ) + + parameters.mainOutputDir.set(sourceSets["main"].java.destinationDirectory.get().getAsFile()) + } + +// Add all changed files to the "turbo" source set. +val turboState = if (enableTurboBuilds) turboPathsProvider.get() else null + +if (turboState != null) { + val numFiles = turboState.paths.size + val msg = + when (turboState.reason) { + TurboReason.FIRST_BUILD -> "First build detected. Build will be slow." + TurboReason.PATHS_CHANGED -> "Paths in active set have changed. Build will be slow." + TurboReason.PATHS_UNCHANGED -> "Paths unchanged. Size=$numFiles. Build should be fast!" + TurboReason.CORRUPT_FILE -> "turbo-paths.txt was invalid. Build will be slow." + TurboReason.TOO_MANY_PATHS -> "Paths were compacted. Build will be slow." + } + logger.warn("Turbo: $msg") +} else { + logger.warn("Turbo: enable_r8_turbo_builds=false") +} java { - sourceSets.main.configure { - java.srcDir(getRoot().resolveAll("src", "main", "java")) - resources.srcDirs(getRoot().resolveAll("third_party", "api_database", "api_database")) + sourceSets { + val srcDir = getRoot().resolveAll("src", "main", "java") + + main { + resources.srcDirs(getRoot().resolveAll("third_party", "api_database", "api_database")) + java { + srcDir(srcDir) + if (turboState != null && !turboState.paths.isEmpty()) { + exclude(turboState.paths) + } + } + } + + // Must be created unconditionally so that other targets can depend on it. + create("turbo") { + java { + srcDir(srcDir) + if (turboState != null && !turboState.paths.isEmpty()) { + include(turboState.paths) + } else { + exclude("*") + } + } + } } + sourceCompatibility = JvmCompatibility.sourceCompatibility targetCompatibility = JvmCompatibility.targetCompatibility toolchain { @@ -45,13 +220,13 @@ } dependencies { + implementation(":assistant") implementation(":keepanno") implementation(":resourceshrinker") compileOnly(Deps.androidxCollection) compileOnly(Deps.androidxTracingDriver) compileOnly(Deps.androidxTracingDriverWire) compileOnly(Deps.asm) - implementation(":assistant") compileOnly(Deps.asmCommons) compileOnly(Deps.asmUtil) compileOnly(Deps.fastUtil) @@ -62,6 +237,46 @@ errorprone(Deps.errorprone) } +if (enableTurboBuilds) { + tasks.named("compileJava") { + // Makes compileTurboJava run first, but does not cause compileJava to re-run if + // compileTurboJava changes. + dependsOn(tasks.named("compileTurboJava")) + } + + // Does not include main's output directory, which must also be added when compilation avoidance + // causes only a subset of sources to be recompiled. + val mainClasspath = sourceSets["main"].compileClasspath.getAsPath() + + tasks.named<JavaCompile>("compileTurboJava") { + // Add the main's classes to the classpath without letting gradle know about this dependency + // (as it's a circular one). + options.compilerArgs.add("-classpath") + options.compilerArgs.add( + "" + + sourceSets["turbo"].java.destinationDirectory.get() + + File.pathSeparator + + mainClasspath + + File.pathSeparator + + sourceSets["main"].java.destinationDirectory.get() + ) + } + + tasks.named<JavaCompile>("compileJava") { + // Add the turbo's classes to the classpath without letting gradle know about this dependency + // (or else it will cause it to rebuild whenever files in it change). + options.compilerArgs.add("-classpath") + options.compilerArgs.add( + "" + + sourceSets["main"].java.destinationDirectory.get() + + File.pathSeparator + + mainClasspath + + File.pathSeparator + + sourceSets["turbo"].java.destinationDirectory.get() + ) + } +} + if (project.hasProperty("spdxVersion")) { project.version = project.property("spdxVersion")!! } @@ -118,6 +333,10 @@ } tasks { + jar { + from(sourceSets["turbo"].output) + } + withType<Exec> { doFirst { println("Executing command: ${commandLine.joinToString(" ")}") @@ -261,6 +480,7 @@ } val depsJar by registering(Zip::class) { + from(sourceSets["turbo"].output) dependsOn(gradle.includedBuild("shared").task(":downloadDeps")) dependsOn(resourceShrinkerDepsTask) dependsOn(threadingModuleBlockingJar)
diff --git a/d8_r8/shared/build.gradle.kts b/d8_r8/shared/build.gradle.kts index 7f9865d..05e1447 100644 --- a/d8_r8/shared/build.gradle.kts +++ b/d8_r8/shared/build.gradle.kts
@@ -7,21 +7,27 @@ id("dependencies-plugin") } +val enableDownloadDeps = !project.hasProperty("disable_download_deps") + tasks { val downloadDeps by registering(DownloadAllDependenciesTask::class) { this.setDependencies(getRoot(), allPublicDependencies()) + onlyIf { enableDownloadDeps } } val downloadTestDeps by registering(DownloadAllDependenciesTask::class) { this.setDependencies(getRoot(), allPublicTestDependencies()) + onlyIf { enableDownloadDeps } } val downloadDepsInternal by registering(DownloadAllDependenciesTask::class) { this.setDependencies(getRoot(), allInternalDependencies()) + onlyIf { enableDownloadDeps } } val downloadTestDepsInternal by registering(DownloadAllDependenciesTask::class) { this.setDependencies(getRoot(), allInternalTestDependencies()) + onlyIf { enableDownloadDeps } } -} \ No newline at end of file +}
diff --git a/d8_r8/test_modules/testbase/build.gradle.kts b/d8_r8/test_modules/testbase/build.gradle.kts index 615b6a0..8d553fa 100644 --- a/d8_r8/test_modules/testbase/build.gradle.kts +++ b/d8_r8/test_modules/testbase/build.gradle.kts
@@ -33,6 +33,7 @@ // incompatible java class file version. By depending on the jar we circumvent that. val keepAnnoJarTask = projectTask("keepanno", "jar") val keepAnnoCompileTask = projectTask("keepanno", "compileJava") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") val mainCompileTask = projectTask("main", "compileJava") val mainDepsJarTask = projectTask("main", "depsJar") val resourceShrinkerJavaCompileTask = projectTask("resourceshrinker", "compileJava") @@ -41,6 +42,7 @@ dependencies { implementation(keepAnnoJarTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) implementation(resourceShrinkerJavaCompileTask.outputs.files)
diff --git a/d8_r8/test_modules/tests_java_11/build.gradle.kts b/d8_r8/test_modules/tests_java_11/build.gradle.kts index d14c599..00c23cd 100644 --- a/d8_r8/test_modules/tests_java_11/build.gradle.kts +++ b/d8_r8/test_modules/tests_java_11/build.gradle.kts
@@ -22,11 +22,13 @@ val testbaseJavaCompileTask = projectTask("testbase", "compileJava") val testbaseDepsJarTask = projectTask("testbase", "depsJar") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") val mainCompileTask = projectTask("main", "compileJava") dependencies { implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile())) implementation(testbaseJavaCompileTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) }
diff --git a/d8_r8/test_modules/tests_java_17/build.gradle.kts b/d8_r8/test_modules/tests_java_17/build.gradle.kts index 473f4da..01d25ea 100644 --- a/d8_r8/test_modules/tests_java_17/build.gradle.kts +++ b/d8_r8/test_modules/tests_java_17/build.gradle.kts
@@ -25,12 +25,14 @@ val testbaseJavaCompileTask = projectTask("testbase", "compileJava") val testbaseDepsJarTask = projectTask("testbase", "depsJar") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") val mainCompileTask = projectTask("main", "compileJava") dependencies { implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile())) implementation(testbaseJavaCompileTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) }
diff --git a/d8_r8/test_modules/tests_java_21/build.gradle.kts b/d8_r8/test_modules/tests_java_21/build.gradle.kts index a44697d..915ca0b 100644 --- a/d8_r8/test_modules/tests_java_21/build.gradle.kts +++ b/d8_r8/test_modules/tests_java_21/build.gradle.kts
@@ -22,12 +22,14 @@ val testbaseJavaCompileTask = projectTask("testbase", "compileJava") val testbaseDepsJarTask = projectTask("testbase", "depsJar") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") val mainCompileTask = projectTask("main", "compileJava") val assistantCompileTask = projectTask("assistant", "compileJava") dependencies { implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile())) implementation(testbaseJavaCompileTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) implementation(assistantCompileTask.outputs.files)
diff --git a/d8_r8/test_modules/tests_java_25/build.gradle.kts b/d8_r8/test_modules/tests_java_25/build.gradle.kts index c0f8a64..8856e7b 100644 --- a/d8_r8/test_modules/tests_java_25/build.gradle.kts +++ b/d8_r8/test_modules/tests_java_25/build.gradle.kts
@@ -22,11 +22,13 @@ val testbaseJavaCompileTask = projectTask("testbase", "compileJava") val testbaseDepsJarTask = projectTask("testbase", "depsJar") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") val mainCompileTask = projectTask("main", "compileJava") dependencies { implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile())) implementation(testbaseJavaCompileTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) }
diff --git a/d8_r8/test_modules/tests_java_8/build.gradle.kts b/d8_r8/test_modules/tests_java_8/build.gradle.kts index a718ef2..c5e07c1 100644 --- a/d8_r8/test_modules/tests_java_8/build.gradle.kts +++ b/d8_r8/test_modules/tests_java_8/build.gradle.kts
@@ -40,6 +40,7 @@ val keepAnnoCompileTask = projectTask("keepanno", "compileJava") val assistantCompileTask = projectTask("assistant", "compileJava") val keepAnnoCompileKotlinTask = projectTask("keepanno", "compileKotlin") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") val mainCompileTask = projectTask("main", "compileJava") val mainDepsJarTask = projectTask("main", "depsJar") val resourceShrinkerJavaCompileTask = projectTask("resourceshrinker", "compileJava") @@ -48,6 +49,7 @@ dependencies { implementation(keepAnnoJarTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) implementation(assistantCompileTask.outputs.files)
diff --git a/d8_r8/test_modules/tests_java_9/build.gradle.kts b/d8_r8/test_modules/tests_java_9/build.gradle.kts index 6ac2d1b..ab048bb 100644 --- a/d8_r8/test_modules/tests_java_9/build.gradle.kts +++ b/d8_r8/test_modules/tests_java_9/build.gradle.kts
@@ -27,10 +27,12 @@ val testbaseJavaCompileTask = projectTask("testbase", "compileJava") val testbaseDepsJarTask = projectTask("testbase", "depsJar") val mainCompileTask = projectTask("main", "compileJava") +val mainTurboCompileTask = projectTask("main", "compileTurboJava") dependencies { implementation(files(testbaseDepsJarTask.outputs.files.getSingleFile())) implementation(testbaseJavaCompileTask.outputs.files) + implementation(mainTurboCompileTask.outputs.files) implementation(mainCompileTask.outputs.files) implementation(projectTask("main", "processResources").outputs.files) }
diff --git a/src/main/java/com/android/tools/r8/BaseCompilerCommandParser.java b/src/main/java/com/android/tools/r8/BaseCompilerCommandParser.java index 8359c03..2fca0cd 100644 --- a/src/main/java/com/android/tools/r8/BaseCompilerCommandParser.java +++ b/src/main/java/com/android/tools/r8/BaseCompilerCommandParser.java
@@ -28,6 +28,7 @@ protected static final String MAP_DIAGNOSTICS = "--map-diagnostics"; protected static final String DUMP_INPUT_TO_FILE = "--dumpinputtofile"; protected static final String DUMP_INPUT_TO_DIRECTORY = "--dumpinputtodirectory"; + protected static final String VERBOSE_SYNTHETIC_NAMES = "--verbose-synthetic-names"; public static void parsePositiveIntArgument( Consumer<Diagnostic> errorConsumer,
diff --git a/src/main/java/com/android/tools/r8/D8CommandParser.java b/src/main/java/com/android/tools/r8/D8CommandParser.java index 42315ea..21b5d83 100644 --- a/src/main/java/com/android/tools/r8/D8CommandParser.java +++ b/src/main/java/com/android/tools/r8/D8CommandParser.java
@@ -91,6 +91,7 @@ .add(ParseFlagInfoImpl.getAndroidPlatformBuild()) .add(ParseFlagInfoImpl.getArtProfile()) .add(ParseFlagInfoImpl.getStartupProfile()) + .add(ParseFlagInfoImpl.getVerboseSyntheticNames()) .add(ParseFlagInfoImpl.getVersion("d8")) .add(ParseFlagInfoImpl.getHelp()) .build(); @@ -366,6 +367,8 @@ continue; } buildMetadataOutputPath = Paths.get(nextArg); + } else if (arg.equals(VERBOSE_SYNTHETIC_NAMES)) { + builder.setEnableVerboseSyntheticNames(true); } else if (arg.startsWith("--")) { if (tryParseAssertionArgument(builder, arg, origin)) { continue;
diff --git a/src/main/java/com/android/tools/r8/ParseFlagInfoImpl.java b/src/main/java/com/android/tools/r8/ParseFlagInfoImpl.java index eb1fed3..29f66b3 100644 --- a/src/main/java/com/android/tools/r8/ParseFlagInfoImpl.java +++ b/src/main/java/com/android/tools/r8/ParseFlagInfoImpl.java
@@ -7,6 +7,7 @@ import static com.android.tools.r8.BaseCompilerCommandParser.MAP_DIAGNOSTICS; import static com.android.tools.r8.BaseCompilerCommandParser.MIN_API_FLAG; import static com.android.tools.r8.BaseCompilerCommandParser.THREAD_COUNT_FLAG; +import static com.android.tools.r8.BaseCompilerCommandParser.VERBOSE_SYNTHETIC_NAMES; import static com.android.tools.r8.D8CommandParser.STARTUP_PROFILE_FLAG; import static com.android.tools.r8.R8CommandParser.ISOLATED_SPLITS_FLAG; @@ -200,6 +201,12 @@ return flag1(STARTUP_PROFILE_FLAG, "<file>", "Startup profile <file> to use for dex layout."); } + public static ParseFlagInfoImpl getVerboseSyntheticNames() { + return flag0( + VERBOSE_SYNTHETIC_NAMES, + "Enable verbose synthetic names that use the `$$ExternalSynthetic` marker."); + } + public static ParseFlagInfoImpl getIsolatedSplits() { return flag0( ISOLATED_SPLITS_FLAG,