Support command cache statistics collection
This will add per process files to a specified directory with one byte per cache hit/miss/put
tools/test.py will clean out the stats on each run with the flag, then sum them up on completion
Change-Id: Ib182851002a0a45371e5d89f6a6a92496d2a7f1b
diff --git a/build.gradle b/build.gradle
index 7f2f61d..f7b3a44 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1709,6 +1709,9 @@
if (project.hasProperty('command_cache_dir')) {
systemProperty 'command_cache_dir', project.property('command_cache_dir')
}
+ if (project.hasProperty('command_cache_stats_dir')) {
+ systemProperty 'command_cache_stats_dir', project.property('command_cache_stats_dir')
+ }
if (project.hasProperty('r8lib')) {
dependsOn configureTestForR8Lib
// R8lib should be used instead of the main output and all the tests in
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt b/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt
index 9a8c17f..943feea 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/TestConfigurationHelper.kt
@@ -90,7 +90,8 @@
"desugar_jdk_json_dir",
"desugar_jdk_libs",
"test_dir",
- "command_cache_dir").forEach {
+ "command_cache_dir",
+ "command_cache_stats_dir").forEach {
val propertyName = it
if (project.hasProperty(propertyName)) {
project.property(propertyName)?.let { v -> test.systemProperty(propertyName, v) }
diff --git a/src/test/java/com/android/tools/r8/ToolHelper.java b/src/test/java/com/android/tools/r8/ToolHelper.java
index d209cb8..8cacb48 100644
--- a/src/test/java/com/android/tools/r8/ToolHelper.java
+++ b/src/test/java/com/android/tools/r8/ToolHelper.java
@@ -70,6 +70,7 @@
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
@@ -753,6 +754,61 @@
}
}
+ public static class CommandCacheStatistics {
+
+ public static CommandCacheStatistics INSTANCE = new CommandCacheStatistics();
+ private final Path cachePutCounter;
+ private final Path cacheMissCounter;
+ private final Path cacheHitCounter;
+
+ private CommandCacheStatistics() {
+ String commandCacheStatsDir = System.getProperty("command_cache_stats_dir");
+ if (commandCacheStatsDir != null) {
+ String processSpecificUUID = UUID.randomUUID().toString();
+ cachePutCounter = Paths.get(commandCacheStatsDir, processSpecificUUID + "CACHEPUT");
+ cacheMissCounter = Paths.get(commandCacheStatsDir, processSpecificUUID + "CACHEFAIL");
+ cacheHitCounter = Paths.get(commandCacheStatsDir, processSpecificUUID + "CACHEHIT");
+ try {
+ Files.createFile(cachePutCounter);
+ Files.createFile(cacheMissCounter);
+ Files.createFile(cacheHitCounter);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ cachePutCounter = null;
+ cacheMissCounter = null;
+ cacheHitCounter = null;
+ }
+ }
+
+ private static void increaseCount(Path path) {
+ // Not enabled
+ if (path == null) {
+ return;
+ }
+ synchronized (path) {
+ try {
+ Files.write(path, "X".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public void addCachePut() {
+ increaseCount(cachePutCounter);
+ }
+
+ public void addCacheHit() {
+ increaseCount(cacheHitCounter);
+ }
+
+ public void addCacheMiss() {
+ increaseCount(cacheMissCounter);
+ }
+ }
+
public static class CommandResultCache {
private static CommandResultCache INSTANCE =
System.getProperty("command_cache_dir") != null
@@ -816,6 +872,7 @@
// but this should have no impact.
Path outputFile = getOutputFile(cacheLookupKey);
+ CommandCacheStatistics.INSTANCE.addCacheHit();
return new Pair(
new ProcessResult(
exitCode,
@@ -823,6 +880,7 @@
getStringContent(getStderrFile(cacheLookupKey))),
outputFile.toFile().exists() ? outputFile : null);
}
+ CommandCacheStatistics.INSTANCE.addCacheMiss();
return null;
}
@@ -870,6 +928,7 @@
exitCodeFile,
StandardCopyOption.ATOMIC_MOVE,
StandardCopyOption.REPLACE_EXISTING);
+ CommandCacheStatistics.INSTANCE.addCachePut();
} catch (IOException e) {
StringBuilder exceptionMessage = new StringBuilder();
exceptionMessage.append(
diff --git a/tools/test.py b/tools/test.py
index 0dbc827..55c25b7 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -118,6 +118,9 @@
result.add_argument('--command-cache-dir', '--command_cache_dir',
help='Cache command invocations to this directory, speeds up test runs',
default=os.environ.get('R8_COMMAND_CACHE_DIR'))
+ result.add_argument('--command-cache-stats', '--command_cache_stats',
+ help='Collect and print statistics about the command cache.',
+ default=False, action='store_true')
result.add_argument('--java-home', '--java_home',
help='Use a custom java version to run tests.')
result.add_argument('--java-max-memory-size', '--java_max_memory_size',
@@ -324,6 +327,15 @@
gradle_args.append('-Pcommand_cache_dir=' + options.command_cache_dir)
if not os.path.exists(options.command_cache_dir):
os.makedirs(options.command_cache_dir)
+ if options.command_cache_stats:
+ stats_dir = os.path.join(options.command_cache_dir, 'stats')
+ gradle_args.append('-Pcommand_cache_stats_dir=' + stats_dir)
+ if not os.path.exists(stats_dir):
+ os.makedirs(stats_dir)
+ # Clean out old stats files
+ for (_, _, file_names) in os.walk(stats_dir):
+ for f in file_names:
+ os.remove(os.path.join(stats_dir, f))
if options.java_home:
gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
if options.java_max_memory_size:
@@ -490,8 +502,7 @@
utils.upload_file_to_cloud_storage(archive,
'gs://r8-test-results/golden-files/' + archive)
- if return_code != 0:
- return archive_and_return(return_code, options)
+ return archive_and_return(return_code, options)
return 0
@@ -499,6 +510,23 @@
if return_code != 0:
if options.archive_failures:
archive_failures(options)
+ if options.command_cache_stats:
+ stats_dir = os.path.join(options.command_cache_dir, 'stats')
+ cache_hit = 0
+ cache_miss = 0
+ cache_put = 0
+ for (_, _, file_names) in os.walk(stats_dir):
+ for f in file_names:
+ if f.endswith('CACHEHIT'):
+ cache_hit += os.stat(os.path.join(stats_dir, f)).st_size
+ if f.endswith('CACHEMISS'):
+ cache_miss += os.stat(os.path.join(stats_dir, f)).st_size
+ if f.endswith('CACHEPUT'):
+ cache_put += os.stat(os.path.join(stats_dir, f)).st_size
+ print('Command cache stats')
+ print(' Cache hits: ' + str(cache_hit))
+ print(' Cache miss: ' + str(cache_miss))
+ print(' Cache puts: ' + str(cache_put))
return return_code
def print_jstacks():
