Add caching of kotlin compilations in tests
This extends the command cache with output caching.
Only succeeding kotlin compilations are cached.
Change-Id: I03ae1fce9ed82a3af33a407a6d1ca868aa428d49
diff --git a/src/test/java/com/android/tools/r8/KotlinCompilerTool.java b/src/test/java/com/android/tools/r8/KotlinCompilerTool.java
index 642e319..63616f3 100644
--- a/src/test/java/com/android/tools/r8/KotlinCompilerTool.java
+++ b/src/test/java/com/android/tools/r8/KotlinCompilerTool.java
@@ -12,13 +12,20 @@
import static org.junit.Assert.assertTrue;
import com.android.tools.r8.TestRuntime.CfRuntime;
+import com.android.tools.r8.ToolHelper.CacheLookupKey;
+import com.android.tools.r8.ToolHelper.CommandResultCache;
import com.android.tools.r8.ToolHelper.ProcessResult;
import com.android.tools.r8.errors.Unimplemented;
import com.android.tools.r8.utils.ArrayUtils;
import com.android.tools.r8.utils.FileUtils;
+import com.android.tools.r8.utils.Pair;
+import com.android.tools.r8.utils.StringUtils;
+import com.android.tools.r8.utils.ThrowingConsumer;
import com.android.tools.r8.utils.structural.Ordered;
+import com.google.common.hash.Hasher;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -339,7 +346,45 @@
}
private ProcessResult compileInternal(Path output) throws IOException {
- List<String> cmdline = new ArrayList<>();
+ CommandLineAndHasherConsumers commandLineAndHasherConsumers =
+ buildCommandLineAndHasherConsumers(output);
+ CacheLookupKey cacheLookupKey = null;
+ if (CommandResultCache.isEnabled()) {
+ cacheLookupKey =
+ new CacheLookupKey(
+ hasher ->
+ commandLineAndHasherConsumers.hasherConsumers.forEach(
+ hasherConsumer -> hasherConsumer.acceptWithRuntimeException(hasher)));
+ Pair<ProcessResult, Path> lookupResult =
+ CommandResultCache.getInstance().lookup(cacheLookupKey);
+ if (lookupResult != null
+ && lookupResult.getFirst().exitCode == 0
+ && lookupResult.getSecond() != null) {
+ Files.copy(lookupResult.getSecond(), output);
+ return lookupResult.getFirst();
+ }
+ }
+ ProcessBuilder builder = new ProcessBuilder(commandLineAndHasherConsumers.cmdline);
+ if (ToolHelper.isNewGradleSetup()) {
+ builder.directory(new File(ToolHelper.getProjectRoot()));
+ }
+ ProcessResult processResult = ToolHelper.runProcess(builder);
+ if (CommandResultCache.isEnabled()) {
+ CommandResultCache.getInstance().putResult(processResult, cacheLookupKey, output);
+ }
+ return processResult;
+ }
+
+ public static class CommandLineAndHasherConsumers {
+ final List<String> cmdline = new ArrayList<>();
+ final List<ThrowingConsumer<Hasher, IOException>> hasherConsumers = new ArrayList<>();
+ }
+
+ private CommandLineAndHasherConsumers buildCommandLineAndHasherConsumers(Path output)
+ throws IOException {
+ CommandLineAndHasherConsumers commandLineAndHasherConsumers =
+ new CommandLineAndHasherConsumers();
+ List<String> cmdline = commandLineAndHasherConsumers.cmdline;
cmdline.add(jdk.getJavaExecutable().toString());
if (enableAssertions) {
cmdline.add("-ea");
@@ -354,8 +399,15 @@
cmdline.add(jdk.getJavaHome().toString());
cmdline.add("-jvm-target");
cmdline.add(targetVersion.getJvmTargetString());
+ // Until now this is just command line files, no inputs, hash existing command
+ String noneFileCommandLineArguments = StringUtils.join("", cmdline);
+ commandLineAndHasherConsumers.hasherConsumers.add(
+ hasher -> hasher.putString(noneFileCommandLineArguments, StandardCharsets.UTF_8));
+
for (Path source : sources) {
cmdline.add(source.toString());
+ commandLineAndHasherConsumers.hasherConsumers.add(
+ hasher -> hasher.putBytes(Files.readAllBytes(source)));
}
cmdline.add("-d");
cmdline.add(output.toString());
@@ -365,12 +417,19 @@
.stream()
.map(Path::toString)
.collect(Collectors.joining(isWindows() ? ";" : ":")));
+ for (Path path : classpath) {
+ commandLineAndHasherConsumers.hasherConsumers.add(
+ hasher -> {
+ hasher.putString("--cp", StandardCharsets.UTF_8);
+ hasher.putBytes(Files.readAllBytes(path));
+ });
+ }
}
cmdline.addAll(additionalArguments);
- ProcessBuilder builder = new ProcessBuilder(cmdline);
- if (ToolHelper.isNewGradleSetup()) {
- builder.directory(new File(ToolHelper.getProjectRoot()));
- }
- return ToolHelper.runProcess(builder);
+ commandLineAndHasherConsumers.hasherConsumers.add(
+ hasher -> additionalArguments.forEach(s -> hasher.putString(s, StandardCharsets.UTF_8)));
+ return commandLineAndHasherConsumers;
}
+
+
}
diff --git a/src/test/java/com/android/tools/r8/ToolHelper.java b/src/test/java/com/android/tools/r8/ToolHelper.java
index 8f31142..270396d 100644
--- a/src/test/java/com/android/tools/r8/ToolHelper.java
+++ b/src/test/java/com/android/tools/r8/ToolHelper.java
@@ -39,6 +39,7 @@
import com.android.tools.r8.utils.FileUtils;
import com.android.tools.r8.utils.InternalOptions;
import com.android.tools.r8.utils.ListUtils;
+import com.android.tools.r8.utils.Pair;
import com.android.tools.r8.utils.Reporter;
import com.android.tools.r8.utils.StringUtils;
import com.android.tools.r8.utils.Timing;
@@ -591,7 +592,7 @@
private DexVm version;
private boolean withArtFrameworks;
- private ArtResultCacheLookupKey artResultCacheLookupKey;
+ private CacheLookupKey artResultCacheLookupKey;
private boolean noCaching = false;
public ArtCommandBuilder() {
@@ -646,7 +647,7 @@
}
private boolean useCache() {
- return !noCaching && CommandResultCache.getInstance() != null;
+ return !noCaching && CommandResultCache.isEnabled();
}
public void cacheResult(ProcessResult result) {
@@ -654,18 +655,20 @@
// put invalid entries into the cache.
if (useCache() && result.exitCode == 0) {
assert artResultCacheLookupKey != null;
- CommandResultCache.getInstance().putResult(result, artResultCacheLookupKey);
+ CommandResultCache.getInstance().putResult(result, artResultCacheLookupKey, null);
}
}
- public ProcessResult getCachedResults() {
+ public ProcessResult getCachedResults() throws IOException {
if (!useCache()) {
return null;
}
assert artResultCacheLookupKey == null;
// Reuse the key when storing results if this is not already cached.
- artResultCacheLookupKey = new ArtResultCacheLookupKey(this::hashParts);
- return CommandResultCache.getInstance().lookup(artResultCacheLookupKey);
+ artResultCacheLookupKey = new CacheLookupKey(this::hashParts);
+ Pair<ProcessResult, Path> lookup =
+ CommandResultCache.getInstance().lookup(artResultCacheLookupKey);
+ return lookup == null ? null : lookup.getFirst();
}
private void hashParts(Hasher hasher) {
@@ -696,11 +699,11 @@
}
}
- private static class ArtResultCacheLookupKey {
+ public static class CacheLookupKey {
private final Consumer<Hasher> hasherConsumer;
private String hash;
- public ArtResultCacheLookupKey(Consumer<Hasher> hasherConsumer) {
+ public CacheLookupKey(Consumer<Hasher> hasherConsumer) {
this.hasherConsumer = hasherConsumer;
}
@@ -714,7 +717,7 @@
}
}
- private static class CommandResultCache {
+ public static class CommandResultCache {
private static CommandResultCache INSTANCE =
System.getProperty("command_cache_dir") != null
? new CommandResultCache(Paths.get(System.getProperty("command_cache_dir")))
@@ -730,16 +733,24 @@
return INSTANCE;
}
- private Path getStdoutFile(ArtResultCacheLookupKey artResultCacheLookupKey) {
- return path.resolve(artResultCacheLookupKey.getHash() + ".stdout");
+ public static boolean isEnabled() {
+ return getInstance() != null;
}
- private Path getStderrFile(ArtResultCacheLookupKey artResultCacheLookupKey) {
- return path.resolve(artResultCacheLookupKey.getHash() + ".stderr");
+ private Path getStdoutFile(CacheLookupKey cacheLookupKey) {
+ return path.resolve(cacheLookupKey.getHash() + ".stdout");
}
- private Path getExitCodeFile(ArtResultCacheLookupKey artResultCacheLookupKey) {
- return path.resolve(artResultCacheLookupKey.getHash());
+ private Path getStderrFile(CacheLookupKey cacheLookupKey) {
+ return path.resolve(cacheLookupKey.getHash() + ".stderr");
+ }
+
+ private Path getOutputFile(CacheLookupKey cacheLookupKey) {
+ return path.resolve(cacheLookupKey.getHash() + ".output");
+ }
+
+ private Path getExitCodeFile(CacheLookupKey cacheLookupKey) {
+ return path.resolve(cacheLookupKey.getHash());
}
private Path getTempFile(Path path) {
@@ -758,38 +769,47 @@
return "";
}
- public ProcessResult lookup(ArtResultCacheLookupKey artResultCacheLookupKey) {
+ public Pair<ProcessResult, Path> lookup(CacheLookupKey cacheLookupKey) {
// TODO Add concurrency handling!
- Path exitCodeFile = getExitCodeFile(artResultCacheLookupKey);
+ Path exitCodeFile = getExitCodeFile(cacheLookupKey);
if (exitCodeFile.toFile().exists()) {
int exitCode = Integer.parseInt(getStringContent(exitCodeFile));
// Because of the temp files and order of writing we should never get here with an
// inconsistent state. It is possible, although unlikely, that the stdout/stderr
// (and even exitcode if art is non deterministic) are from different, process ids etc,
// but this should have no impact.
- return new ProcessResult(
- exitCode,
- getStringContent(getStdoutFile(artResultCacheLookupKey)),
- getStringContent(getStderrFile(artResultCacheLookupKey)));
+
+ Path outputFile = getOutputFile(cacheLookupKey);
+ return new Pair(
+ new ProcessResult(
+ exitCode,
+ getStringContent(getStdoutFile(cacheLookupKey)),
+ getStringContent(getStderrFile(cacheLookupKey))),
+ outputFile.toFile().exists() ? outputFile : null);
}
return null;
}
- public void putResult(ProcessResult result, ArtResultCacheLookupKey artResultCacheLookupKey) {
+ public void putResult(ProcessResult result, CacheLookupKey cacheLookupKey, Path output) {
try {
String exitCode = "" + result.exitCode;
// We avoid race conditions of writing vs reading by first writing all 3 files to temp
// files, then moving these to the result files, moving last the exitcode file (which is
// what we use as cache present check)
- Path exitCodeFile = getExitCodeFile(artResultCacheLookupKey);
+ Path exitCodeFile = getExitCodeFile(cacheLookupKey);
Path exitCodeTempFile = getTempFile(exitCodeFile);
- Path stdoutFile = getStdoutFile(artResultCacheLookupKey);
+ Path stdoutFile = getStdoutFile(cacheLookupKey);
Path stdoutTempFile = getTempFile(stdoutFile);
- Path stderrFile = getStderrFile(artResultCacheLookupKey);
+ Path stderrFile = getStderrFile(cacheLookupKey);
Path stderrTempFile = getTempFile(stderrFile);
+ Path outputfile = getOutputFile(cacheLookupKey);
+ Path outputTempFile = getTempFile(outputfile);
Files.write(exitCodeTempFile, exitCode.getBytes(StandardCharsets.UTF_8));
Files.write(stdoutTempFile, result.stdout.getBytes(StandardCharsets.UTF_8));
Files.write(stderrTempFile, result.stderr.getBytes(StandardCharsets.UTF_8));
+ if (output != null) {
+ Files.copy(output, outputTempFile);
+ }
// Order is important, move exitcode file last!
Files.move(
stdoutTempFile,
@@ -801,6 +821,13 @@
stderrFile,
StandardCopyOption.ATOMIC_MOVE,
StandardCopyOption.REPLACE_EXISTING);
+ if (output != null) {
+ Files.move(
+ outputTempFile,
+ outputfile,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ }
Files.move(
exitCodeTempFile,
exitCodeFile,
