Add some gradle benchmarks
Adds tools/test_gradle_benchmarks.py which
outputs information about Gradle tasks duration
for Golem.
Adds an init-script which is used to override
Gradle-based benchmark project configuration
so that the desired versions of r8 and android
Gradle plugin is used.
Change-Id: I050671bd8f62820d8108eb53ef38407a3ac6462a
Bug: 77613196
diff --git a/build.gradle b/build.gradle
index 8237781..ff22c66 100644
--- a/build.gradle
+++ b/build.gradle
@@ -348,7 +348,6 @@
def x20Dependencies = [
"third_party": [
- "benchmarks/santa-tracker",
"gmail/gmail_android_170604.16",
"gmscore/v4",
"gmscore/v5",
@@ -358,7 +357,6 @@
"gmscore/gmscore_v9",
"gmscore/gmscore_v10",
"gmscore/latest",
- "gradle-plugin",
"photos/2017-06-06",
"youtube/youtube.android_12.10",
"youtube/youtube.android_12.17",
diff --git a/third_party/benchmarks/android-sdk.tar.gz.sha1 b/third_party/benchmarks/android-sdk.tar.gz.sha1
new file mode 100644
index 0000000..efc177e
--- /dev/null
+++ b/third_party/benchmarks/android-sdk.tar.gz.sha1
@@ -0,0 +1 @@
+d033ea956b548b5fb87a2b37c584e87cfca33717
\ No newline at end of file
diff --git a/third_party/benchmarks/antenna-pod.tar.gz.sha1 b/third_party/benchmarks/antenna-pod.tar.gz.sha1
new file mode 100644
index 0000000..c3703e0
--- /dev/null
+++ b/third_party/benchmarks/antenna-pod.tar.gz.sha1
@@ -0,0 +1 @@
+f6f15c4efb79772ce8af147e0e2639db1c585fc7
\ No newline at end of file
diff --git a/third_party/benchmarks/gradle-java-1.6.tar.gz.sha1 b/third_party/benchmarks/gradle-java-1.6.tar.gz.sha1
new file mode 100644
index 0000000..e30dbfc
--- /dev/null
+++ b/third_party/benchmarks/gradle-java-1.6.tar.gz.sha1
@@ -0,0 +1 @@
+e87ccb7a5cb95555f46a6c8adbbcec99ec14b578
\ No newline at end of file
diff --git a/third_party/benchmarks/init-script.gradle b/third_party/benchmarks/init-script.gradle
new file mode 100644
index 0000000..3434a4a
--- /dev/null
+++ b/third_party/benchmarks/init-script.gradle
@@ -0,0 +1,74 @@
+def r8RootDir = System.properties['r8.root.dir']
+
+allprojects {
+ buildscript {
+ repositories {
+ maven { url r8RootDir+ '/third_party/gradle-plugin' }
+ // We don't use 'google()' in order to support projects using gradle
+ // lower to 4.1 version.
+ maven { url 'https://maven.google.com' }
+ jcenter()
+ }
+ dependencies {
+ classpath files(r8RootDir + '/build/libs/r8.jar')
+ classpath 'com.android.tools.build:gradle:3.2.0-dev'
+ }
+ }
+ repositories {
+ maven { url r8RootDir+ '/third_party/gradle-plugin' }
+ maven { url 'https://maven.google.com' }
+ jcenter()
+ mavenCentral()
+ }
+}
+
+//
+// Dump detailed timings per subtask
+//
+import java.util.concurrent.TimeUnit;
+class TimingsListener implements TaskExecutionListener, BuildListener {
+ private long startTimeInNs;
+ private timings = []
+
+ @Override
+ void beforeExecute(Task task) {
+ startTimeInNs = System.nanoTime();
+ }
+
+ @Override
+ void afterExecute(Task task, TaskState taskState) {
+ def ms = TimeUnit.MILLISECONDS.convert(
+ System.nanoTime() - startTimeInNs, TimeUnit.NANOSECONDS);
+ timings.add([task.path,ms])
+ }
+
+ @Override
+ void buildStarted(Gradle gradle) {
+ }
+
+ @Override
+ void buildFinished(BuildResult result) {
+ def total=0
+ for (timing in timings) {
+ total += timing[1]
+ }
+
+ for (timing in timings) {
+ printf "BENCH,%s,%s\n", timing
+ }
+
+ printf "BENCH,totalGradleTasks,%s\n", total
+
+ }
+
+ @Override
+ void projectsEvaluated(Gradle gradle) {}
+
+ @Override
+ void projectsLoaded(Gradle gradle) {}
+
+ @Override
+ void settingsEvaluated(Settings settings) {}
+}
+
+gradle.addListener new TimingsListener()
\ No newline at end of file
diff --git a/third_party/benchmarks/santa-tracker.tar.gz.sha1 b/third_party/benchmarks/santa-tracker.tar.gz.sha1
index d37a000..93b21bd 100644
--- a/third_party/benchmarks/santa-tracker.tar.gz.sha1
+++ b/third_party/benchmarks/santa-tracker.tar.gz.sha1
@@ -1 +1 @@
-88150a9a2215f1f821ea9321e61b5fc8276dffd3
\ No newline at end of file
+c0985a5e801ca74dad932dcd734b297e9fabe374
\ No newline at end of file
diff --git a/third_party/benchmarks/tachiyomi.tar.gz.sha1 b/third_party/benchmarks/tachiyomi.tar.gz.sha1
new file mode 100644
index 0000000..b3cf670
--- /dev/null
+++ b/third_party/benchmarks/tachiyomi.tar.gz.sha1
@@ -0,0 +1 @@
+9101f02e39bb938d44794d22b9813628f63f87af
\ No newline at end of file
diff --git a/third_party/benchmarks/wordpress.tar.gz.sha1 b/third_party/benchmarks/wordpress.tar.gz.sha1
new file mode 100644
index 0000000..dea2249
--- /dev/null
+++ b/third_party/benchmarks/wordpress.tar.gz.sha1
@@ -0,0 +1 @@
+f06cd747d2e2880cd450e67a4a21c28cd24a8dd5
\ No newline at end of file
diff --git a/tools/gradle.py b/tools/gradle.py
index 2303bce..eff949a 100755
--- a/tools/gradle.py
+++ b/tools/gradle.py
@@ -58,30 +58,42 @@
EnsureGradle()
EnsureShadow()
-def RunGradle(args, throw_on_failure=True):
+def RunGradleIn(gradleCmd, args, cwd, throw_on_failure=True):
EnsureDeps()
- cmd = [GRADLE]
+ cmd = [gradleCmd]
cmd.extend(args)
utils.PrintCmd(cmd)
- with utils.ChangedWorkingDirectory(utils.REPO_ROOT):
+ with utils.ChangedWorkingDirectory(cwd):
return_value = subprocess.call(cmd)
if throw_on_failure and return_value != 0:
raise Exception('Failed to execute gradle')
return return_value
+def RunGradleWrapperIn(args, cwd, throw_on_failure=True):
+ return RunGradleIn('./gradlew', args, cwd, throw_on_failure)
+
+def RunGradle(args, throw_on_failure=True):
+ return RunGradleIn(GRADLE, args, utils.REPO_ROOT, throw_on_failure)
+
def RunGradleExcludeDeps(args, throw_on_failure=True):
EnsureDeps()
args.append('-Pexclude_deps')
- RunGradle(args, throw_on_failure)
+ return RunGradle(args, throw_on_failure)
-def RunGradleGetOutput(args):
+def RunGradleInGetOutput(gradleCmd, args, cwd):
EnsureDeps()
- cmd = [GRADLE]
+ cmd = [gradleCmd]
cmd.extend(args)
utils.PrintCmd(cmd)
- with utils.ChangedWorkingDirectory(utils.REPO_ROOT):
+ with utils.ChangedWorkingDirectory(cwd):
return subprocess.check_output(cmd)
+def RunGradleWrapperInGetOutput(args, cwd):
+ return RunGradleInGetOutput('./gradlew', args, cwd)
+
+def RunGradleGetOutput(args):
+ return RunGradleInGetOutput(GRADLE, args, utils.REPO_ROOT)
+
def Main():
RunGradle(sys.argv[1:])
diff --git a/tools/test_gradle_benchmarks.py b/tools/test_gradle_benchmarks.py
new file mode 100755
index 0000000..e843753
--- /dev/null
+++ b/tools/test_gradle_benchmarks.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+
+from __future__ import print_function
+import argparse
+import os
+import sys
+import utils
+import gradle
+from enum import Enum
+
+
+BENCHMARKS_ROOT_DIR = os.path.join(utils.REPO_ROOT, 'third_party', 'benchmarks')
+
+def parse_arguments():
+ parser = argparse.ArgumentParser(
+ description='Run D8 or DX on gradle apps located in'
+ ' third_party/benchmarks/.'
+ ' Report Golem-compatible RunTimeRaw values.')
+ parser.add_argument('--tool',
+ choices=['dx', 'd8'],
+ required=True,
+ help='Compiler tool to use.')
+ return parser.parse_args()
+
+
+class Benchmark:
+ class Tools(Enum):
+ D8 = 1
+ DX = 2
+
+ class DesugarMode(Enum):
+ D8_DESUGARING = 1
+ DESUGAR_TOOL = 2
+
+ displayName = ""
+ rootDirPath = ""
+ appPath = ""
+ moduleName = ""
+ buildCommand = ""
+ cleanCommand = ""
+
+ def __init__(self, displayName, benchmarkDir, moduleName, buildCommand, cleanCommand):
+ self.displayName = displayName
+ self.rootDirPath = os.path.join(BENCHMARKS_ROOT_DIR, benchmarkDir.split(os.sep)[0])
+ self.appPath = os.path.join(BENCHMARKS_ROOT_DIR, benchmarkDir)
+ self.moduleName = moduleName
+ self.buildCommand = buildCommand
+ self.cleanCommand = cleanCommand
+
+ def RunGradle(self, command, tool, desugarMode):
+
+ args = ['-Dr8.root.dir=' + utils.REPO_ROOT, '--init-script',
+ os.path.join(BENCHMARKS_ROOT_DIR, 'init-script.gradle')]
+
+ if tool == self.Tools.D8:
+ args.append('-Dandroid.enableD8=true')
+ elif tool == self.Tools.DX:
+ args.append('-Dandroid.enableD8=false')
+ else:
+ raise AssertionError("Unknown tool: " + repr(tool))
+
+ if desugarMode == self.DesugarMode.D8_DESUGARING:
+ args.append('-Dandroid.enableDesugar=false')
+ elif desugarMode == self.DesugarMode.DESUGAR_TOOL:
+ args.append('-Dandroid.enableDesugar=true')
+ else:
+ raise AssertionError("Unknown desugar mode: " + repr(desugarMode))
+
+ args.extend(command)
+
+ return gradle.RunGradleWrapperInGetOutput(args, self.appPath)
+
+ def Build(self, tool, desugarMode):
+ return self.RunGradle(self.buildCommand, tool, desugarMode)
+
+ def Clean(self):
+ # tools and desugar mode not relevant for clean
+ return self.RunGradle(self.cleanCommand, self.Tools.D8, self.DesugarMode.D8_DESUGARING)
+
+ def EnsurePresence(self):
+ EnsurePresence(self.rootDirPath, self.displayName)
+
+
+def EnsurePresence(dir, displayName):
+ if not os.path.exists(dir) or os.path.getmtime(dir + '.tar.gz')\
+ < os.path.getmtime(dir + '.tar.gz.sha1'):
+ utils.DownloadFromX20(dir + '.tar.gz.sha1')
+ # Update the mtime of the tar file to make sure we do not run again unless
+ # there is an update.
+ os.utime(dir + '.tar.gz', None)
+ else:
+ print('test_gradle_benchmarks.py: benchmark {} is present'.format(displayName))
+
+def TaskFilter(taskname):
+ acceptedGradleTasks = [
+ 'dex',
+ 'Dex',
+ 'proguard',
+ 'Proguard',
+ 'kotlin',
+ 'Kotlin',
+ ]
+
+ return any(namePattern in taskname for namePattern in acceptedGradleTasks)
+
+
+def PrintBuildTimeForGolem(benchmark, stdOut):
+ for line in stdOut.splitlines():
+ if 'BENCH' in line and benchmark.moduleName in line:
+ commaSplit = line.split(',')
+ assert len(commaSplit) == 3
+
+ # Keep only module that have been configured to use R8
+ if benchmark.moduleName + ':' not in commaSplit[1]:
+ continue
+
+ # remove <module-name> + ':'
+ taskName = commaSplit[1][(len(benchmark.moduleName) + 1):]
+
+ # Just a temporary assumption.
+ # This means we have submodules, so we'll need to check their configuration
+ # so that the right r8/d8 is taken. For now it shouldn't be the case.
+ assert taskName.find(':') == -1, taskName
+
+ # Output example:
+ # SantaTracker-transformClassesWithDexBuilderForDevelopmentDebug(RunTimeRaw): 748 ms
+
+ golemBenchmarkValue = benchmark.displayName + '-' + taskName + '(RunTimeRaw): '
+ if TaskFilter(taskName):
+ print('{}(RunTimeRaw): {} ms'
+ .format(benchmark.displayName + '-' + taskName, commaSplit[2]))
+
+
+def Main():
+ args = parse_arguments()
+
+ if args.tool == 'd8':
+ tool = Benchmark.Tools.D8
+ desugarMode = Benchmark.DesugarMode.D8_DESUGARING
+ else:
+ tool = Benchmark.Tools.DX
+ desugarMode = Benchmark.DesugarMode.DESUGAR_TOOL
+
+ buildTimeBenchmarks = [
+ Benchmark('AntennaPod',
+ os.path.join('antenna-pod', 'AntennaPod'),
+ ':app',
+ [':app:assembleDebug'],
+ ['clean']),
+ Benchmark('Maps',
+ 'gradle-java-1.6',
+ ':maps',
+ [':maps:assembleDebug', '--settings-file', 'settings.gradle.maps'],
+ ['clean']),
+ Benchmark('Music2',
+ 'gradle-java-1.6',
+ ':music2Old',
+ [':music2Old:assembleDebug', '--settings-file', 'settings.gradle.music2Old'],
+ ['clean']),
+ Benchmark('Velvet',
+ 'gradle-java-1.6',
+ ':velvet',
+ [':velvet:assembleDebug', '--settings-file', 'settings.gradle.velvet'],
+ ['clean']),
+ Benchmark('SantaTracker',
+ 'santa-tracker',
+ ':santa-tracker',
+ [':santa-tracker:assembleDebug'],
+ ['clean']),
+
+ # disabled for now, apparently because of b/74227571
+ # Benchmark('Tachiyomi',
+ # 'tachiyomi',
+ # ':app',
+ # ['assembleStandardDebug'],
+ # ['clean']),
+
+ Benchmark('WordPress',
+ 'wordpress',
+ ':WordPress',
+ ['assembleVanillaDebug'],
+ ['clean']),
+
+ ]
+
+ EnsurePresence(os.path.join('third_party', 'benchmarks', 'android-sdk'), 'android SDK')
+ EnsurePresence(os.path.join('third_party', 'gradle-plugin'), 'Android Gradle plugin')
+
+ for benchmark in buildTimeBenchmarks:
+ benchmark.EnsurePresence()
+ benchmark.Clean()
+ stdOut = benchmark.Build(tool, desugarMode)
+ PrintBuildTimeForGolem(benchmark, stdOut)
+
+
+if __name__ == '__main__':
+ sys.exit(Main())
diff --git a/tools/utils.py b/tools/utils.py
index 8716f65..a138461 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -48,6 +48,12 @@
def IsWindows():
return os.name == 'nt'
+def DownloadFromX20(sha1_file):
+ download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
+ cmd = [download_script, sha1_file]
+ PrintCmd(cmd)
+ subprocess.check_call(cmd)
+
def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps'):
suffix = '.bat' if IsWindows() else ''
download_script = 'download_from_google_storage%s' % suffix
