diff --git a/build.gradle b/build.gradle
index 8237781..ff22c66 100644
--- a/build.gradle
+++ b/build.gradle
@@ -348,7 +348,6 @@
 
 def x20Dependencies = [
     "third_party": [
-        "benchmarks/santa-tracker",
         "gmail/gmail_android_170604.16",
         "gmscore/v4",
         "gmscore/v5",
@@ -358,7 +357,6 @@
         "gmscore/gmscore_v9",
         "gmscore/gmscore_v10",
         "gmscore/latest",
-        "gradle-plugin",
         "photos/2017-06-06",
         "youtube/youtube.android_12.10",
         "youtube/youtube.android_12.17",
diff --git a/third_party/benchmarks/android-sdk.tar.gz.sha1 b/third_party/benchmarks/android-sdk.tar.gz.sha1
new file mode 100644
index 0000000..efc177e
--- /dev/null
+++ b/third_party/benchmarks/android-sdk.tar.gz.sha1
@@ -0,0 +1 @@
+d033ea956b548b5fb87a2b37c584e87cfca33717
\ No newline at end of file
diff --git a/third_party/benchmarks/antenna-pod.tar.gz.sha1 b/third_party/benchmarks/antenna-pod.tar.gz.sha1
new file mode 100644
index 0000000..c3703e0
--- /dev/null
+++ b/third_party/benchmarks/antenna-pod.tar.gz.sha1
@@ -0,0 +1 @@
+f6f15c4efb79772ce8af147e0e2639db1c585fc7
\ No newline at end of file
diff --git a/third_party/benchmarks/gradle-java-1.6.tar.gz.sha1 b/third_party/benchmarks/gradle-java-1.6.tar.gz.sha1
new file mode 100644
index 0000000..e30dbfc
--- /dev/null
+++ b/third_party/benchmarks/gradle-java-1.6.tar.gz.sha1
@@ -0,0 +1 @@
+e87ccb7a5cb95555f46a6c8adbbcec99ec14b578
\ No newline at end of file
diff --git a/third_party/benchmarks/init-script.gradle b/third_party/benchmarks/init-script.gradle
new file mode 100644
index 0000000..3434a4a
--- /dev/null
+++ b/third_party/benchmarks/init-script.gradle
@@ -0,0 +1,74 @@
+def r8RootDir = System.properties['r8.root.dir']
+
+allprojects {
+    buildscript {
+        repositories {
+            maven { url r8RootDir+ '/third_party/gradle-plugin' }
+            // We don't use 'google()' in order to support projects using gradle
+            // lower to 4.1 version.
+            maven { url 'https://maven.google.com' }
+            jcenter()
+        }
+        dependencies {
+            classpath files(r8RootDir + '/build/libs/r8.jar')
+            classpath 'com.android.tools.build:gradle:3.2.0-dev'
+        }
+    }
+    repositories {
+        maven { url r8RootDir+ '/third_party/gradle-plugin' }
+        maven { url 'https://maven.google.com' }
+        jcenter()
+        mavenCentral()
+    }
+}
+
+//
+// Dump detailed timings per subtask
+//
+import java.util.concurrent.TimeUnit;
+class TimingsListener implements TaskExecutionListener, BuildListener {
+    private long startTimeInNs;
+    private timings = []
+
+    @Override
+    void beforeExecute(Task task) {
+        startTimeInNs = System.nanoTime();
+    }
+
+    @Override
+    void afterExecute(Task task, TaskState taskState) {
+        def ms = TimeUnit.MILLISECONDS.convert(
+                System.nanoTime() - startTimeInNs, TimeUnit.NANOSECONDS);
+        timings.add([task.path,ms])
+    }
+
+    @Override
+    void buildStarted(Gradle gradle) {
+    }
+
+    @Override
+    void buildFinished(BuildResult result) {
+        def total=0
+        for (timing in timings) {
+            total += timing[1]
+        }
+
+        for (timing in timings) {
+            printf "BENCH,%s,%s\n", timing
+        }
+
+        printf "BENCH,totalGradleTasks,%s\n", total
+
+    }
+
+    @Override
+    void projectsEvaluated(Gradle gradle) {}
+
+    @Override
+    void projectsLoaded(Gradle gradle) {}
+
+    @Override
+    void settingsEvaluated(Settings settings) {}
+}
+
+gradle.addListener new TimingsListener()
\ No newline at end of file
diff --git a/third_party/benchmarks/santa-tracker.tar.gz.sha1 b/third_party/benchmarks/santa-tracker.tar.gz.sha1
index d37a000..93b21bd 100644
--- a/third_party/benchmarks/santa-tracker.tar.gz.sha1
+++ b/third_party/benchmarks/santa-tracker.tar.gz.sha1
@@ -1 +1 @@
-88150a9a2215f1f821ea9321e61b5fc8276dffd3
\ No newline at end of file
+c0985a5e801ca74dad932dcd734b297e9fabe374
\ No newline at end of file
diff --git a/third_party/benchmarks/tachiyomi.tar.gz.sha1 b/third_party/benchmarks/tachiyomi.tar.gz.sha1
new file mode 100644
index 0000000..b3cf670
--- /dev/null
+++ b/third_party/benchmarks/tachiyomi.tar.gz.sha1
@@ -0,0 +1 @@
+9101f02e39bb938d44794d22b9813628f63f87af
\ No newline at end of file
diff --git a/third_party/benchmarks/wordpress.tar.gz.sha1 b/third_party/benchmarks/wordpress.tar.gz.sha1
new file mode 100644
index 0000000..dea2249
--- /dev/null
+++ b/third_party/benchmarks/wordpress.tar.gz.sha1
@@ -0,0 +1 @@
+f06cd747d2e2880cd450e67a4a21c28cd24a8dd5
\ No newline at end of file
diff --git a/tools/gradle.py b/tools/gradle.py
index 2303bce..eff949a 100755
--- a/tools/gradle.py
+++ b/tools/gradle.py
@@ -58,30 +58,42 @@
   EnsureGradle()
   EnsureShadow()
 
-def RunGradle(args, throw_on_failure=True):
+def RunGradleIn(gradleCmd, args, cwd, throw_on_failure=True):
   EnsureDeps()
-  cmd = [GRADLE]
+  cmd = [gradleCmd]
   cmd.extend(args)
   utils.PrintCmd(cmd)
-  with utils.ChangedWorkingDirectory(utils.REPO_ROOT):
+  with utils.ChangedWorkingDirectory(cwd):
     return_value = subprocess.call(cmd)
     if throw_on_failure and return_value != 0:
       raise Exception('Failed to execute gradle')
     return return_value
 
+def RunGradleWrapperIn(args, cwd, throw_on_failure=True):
+  return RunGradleIn('./gradlew', args, cwd, throw_on_failure)
+
+def RunGradle(args, throw_on_failure=True):
+  return RunGradleIn(GRADLE, args, utils.REPO_ROOT, throw_on_failure)
+
 def RunGradleExcludeDeps(args, throw_on_failure=True):
   EnsureDeps()
   args.append('-Pexclude_deps')
-  RunGradle(args, throw_on_failure)
+  return RunGradle(args, throw_on_failure)
 
-def RunGradleGetOutput(args):
+def RunGradleInGetOutput(gradleCmd, args, cwd):
   EnsureDeps()
-  cmd = [GRADLE]
+  cmd = [gradleCmd]
   cmd.extend(args)
   utils.PrintCmd(cmd)
-  with utils.ChangedWorkingDirectory(utils.REPO_ROOT):
+  with utils.ChangedWorkingDirectory(cwd):
     return subprocess.check_output(cmd)
 
+def RunGradleWrapperInGetOutput(args, cwd):
+  return RunGradleInGetOutput('./gradlew', args, cwd)
+
+def RunGradleGetOutput(args):
+  return RunGradleInGetOutput(GRADLE, args, utils.REPO_ROOT)
+
 def Main():
   RunGradle(sys.argv[1:])
 
diff --git a/tools/test_gradle_benchmarks.py b/tools/test_gradle_benchmarks.py
new file mode 100755
index 0000000..e843753
--- /dev/null
+++ b/tools/test_gradle_benchmarks.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+
+from __future__ import print_function
+import argparse
+import os
+import sys
+import utils
+import gradle
+from enum import Enum
+
+
+BENCHMARKS_ROOT_DIR = os.path.join(utils.REPO_ROOT, 'third_party', 'benchmarks')
+
+def parse_arguments():
+  parser = argparse.ArgumentParser(
+    description='Run D8 or DX on gradle apps located in'
+                ' third_party/benchmarks/.'
+                ' Report Golem-compatible RunTimeRaw values.')
+  parser.add_argument('--tool',
+                      choices=['dx', 'd8'],
+                      required=True,
+                      help='Compiler tool to use.')
+  return parser.parse_args()
+
+
+class Benchmark:
+  class Tools(Enum):
+    D8 = 1
+    DX = 2
+
+  class DesugarMode(Enum):
+    D8_DESUGARING = 1
+    DESUGAR_TOOL = 2
+
+  displayName = ""
+  rootDirPath = ""
+  appPath = ""
+  moduleName = ""
+  buildCommand = ""
+  cleanCommand = ""
+
+  def __init__(self, displayName, benchmarkDir, moduleName, buildCommand, cleanCommand):
+    self.displayName = displayName
+    self.rootDirPath = os.path.join(BENCHMARKS_ROOT_DIR, benchmarkDir.split(os.sep)[0])
+    self.appPath = os.path.join(BENCHMARKS_ROOT_DIR, benchmarkDir)
+    self.moduleName = moduleName
+    self.buildCommand = buildCommand
+    self.cleanCommand = cleanCommand
+
+  def RunGradle(self, command, tool, desugarMode):
+
+    args = ['-Dr8.root.dir=' + utils.REPO_ROOT, '--init-script',
+            os.path.join(BENCHMARKS_ROOT_DIR, 'init-script.gradle')]
+
+    if tool == self.Tools.D8:
+      args.append('-Dandroid.enableD8=true')
+    elif tool == self.Tools.DX:
+      args.append('-Dandroid.enableD8=false')
+    else:
+      raise AssertionError("Unknown tool: " + repr(tool))
+
+    if desugarMode == self.DesugarMode.D8_DESUGARING:
+      args.append('-Dandroid.enableDesugar=false')
+    elif desugarMode == self.DesugarMode.DESUGAR_TOOL:
+      args.append('-Dandroid.enableDesugar=true')
+    else:
+      raise AssertionError("Unknown desugar mode: " + repr(desugarMode))
+
+    args.extend(command)
+
+    return gradle.RunGradleWrapperInGetOutput(args, self.appPath)
+
+  def Build(self, tool, desugarMode):
+    return self.RunGradle(self.buildCommand, tool, desugarMode)
+
+  def Clean(self):
+    # tools and desugar mode not relevant for clean
+    return self.RunGradle(self.cleanCommand, self.Tools.D8, self.DesugarMode.D8_DESUGARING)
+
+  def EnsurePresence(self):
+    EnsurePresence(self.rootDirPath, self.displayName)
+
+
+def EnsurePresence(dir, displayName):
+  if not os.path.exists(dir) or os.path.getmtime(dir + '.tar.gz')\
+          < os.path.getmtime(dir + '.tar.gz.sha1'):
+    utils.DownloadFromX20(dir + '.tar.gz.sha1')
+    # Update the mtime of the tar file to make sure we do not run again unless
+    # there is an update.
+    os.utime(dir + '.tar.gz', None)
+  else:
+    print('test_gradle_benchmarks.py: benchmark {} is present'.format(displayName))
+
+def TaskFilter(taskname):
+  acceptedGradleTasks = [
+    'dex',
+    'Dex',
+    'proguard',
+    'Proguard',
+    'kotlin',
+    'Kotlin',
+  ]
+
+  return any(namePattern in taskname for namePattern in acceptedGradleTasks)
+
+
+def PrintBuildTimeForGolem(benchmark, stdOut):
+  for line in stdOut.splitlines():
+    if 'BENCH' in line and benchmark.moduleName in line:
+      commaSplit = line.split(',')
+      assert len(commaSplit) == 3
+
+      # Keep only module that have been configured to use R8
+      if benchmark.moduleName + ':' not in commaSplit[1]:
+        continue
+
+      # remove <module-name> + ':'
+      taskName = commaSplit[1][(len(benchmark.moduleName) + 1):]
+
+      # Just a temporary assumption.
+      # This means we have submodules, so we'll need to check their configuration
+      # so that the right r8/d8 is taken. For now it shouldn't be the case.
+      assert taskName.find(':') == -1, taskName
+
+      # Output example:
+      # SantaTracker-transformClassesWithDexBuilderForDevelopmentDebug(RunTimeRaw): 748 ms
+
+      golemBenchmarkValue = benchmark.displayName + '-' + taskName + '(RunTimeRaw): '
+      if TaskFilter(taskName):
+        print('{}(RunTimeRaw): {} ms'
+              .format(benchmark.displayName + '-' + taskName, commaSplit[2]))
+
+
+def Main():
+  args = parse_arguments()
+
+  if args.tool == 'd8':
+    tool = Benchmark.Tools.D8
+    desugarMode = Benchmark.DesugarMode.D8_DESUGARING
+  else:
+    tool = Benchmark.Tools.DX
+    desugarMode = Benchmark.DesugarMode.DESUGAR_TOOL
+
+  buildTimeBenchmarks = [
+    Benchmark('AntennaPod',
+              os.path.join('antenna-pod', 'AntennaPod'),
+              ':app',
+              [':app:assembleDebug'],
+              ['clean']),
+    Benchmark('Maps',
+              'gradle-java-1.6',
+              ':maps',
+              [':maps:assembleDebug', '--settings-file', 'settings.gradle.maps'],
+              ['clean']),
+    Benchmark('Music2',
+              'gradle-java-1.6',
+              ':music2Old',
+              [':music2Old:assembleDebug', '--settings-file', 'settings.gradle.music2Old'],
+              ['clean']),
+    Benchmark('Velvet',
+              'gradle-java-1.6',
+              ':velvet',
+              [':velvet:assembleDebug', '--settings-file', 'settings.gradle.velvet'],
+              ['clean']),
+    Benchmark('SantaTracker',
+              'santa-tracker',
+              ':santa-tracker',
+              [':santa-tracker:assembleDebug'],
+              ['clean']),
+
+    # disabled for now, apparently because of b/74227571
+    # Benchmark('Tachiyomi',
+    #           'tachiyomi',
+    #           ':app',
+    #           ['assembleStandardDebug'],
+    #           ['clean']),
+
+    Benchmark('WordPress',
+              'wordpress',
+              ':WordPress',
+              ['assembleVanillaDebug'],
+              ['clean']),
+
+  ]
+
+  EnsurePresence(os.path.join('third_party', 'benchmarks', 'android-sdk'), 'android SDK')
+  EnsurePresence(os.path.join('third_party', 'gradle-plugin'), 'Android Gradle plugin')
+
+  for benchmark in buildTimeBenchmarks:
+    benchmark.EnsurePresence()
+    benchmark.Clean()
+    stdOut = benchmark.Build(tool, desugarMode)
+    PrintBuildTimeForGolem(benchmark, stdOut)
+
+
+if __name__ == '__main__':
+  sys.exit(Main())
diff --git a/tools/utils.py b/tools/utils.py
index 8716f65..a138461 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -48,6 +48,12 @@
 def IsWindows():
   return os.name == 'nt'
 
+def DownloadFromX20(sha1_file):
+  download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
+  cmd = [download_script, sha1_file]
+  PrintCmd(cmd)
+  subprocess.check_call(cmd)
+
 def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps'):
   suffix = '.bat' if IsWindows() else ''
   download_script = 'download_from_google_storage%s' % suffix