Custom download dependency task with modified time check.
Change-Id: I01c0756d6c1941af45bd6f1c75198c809cdaf62d
diff --git a/build.gradle b/build.gradle
index 7f6d5fa..a247b4d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -5,6 +5,7 @@
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import net.ltgt.gradle.errorprone.CheckSeverity
import org.gradle.internal.os.OperatingSystem
+import tasks.DownloadDependency
import tasks.GetJarsFromConfiguration
import utils.Utils
@@ -325,23 +326,15 @@
return "download_deps_${entryKey}_${entryFile.replace('/', '_').replace('\\', '_')}"
}
+def getFetchDepsTaskName(entryKey, entryFile) {
+ return "fetch_deps_${entryKey}_${entryFile.replace('/', '_').replace('\\', '_')}"
+}
+
cloudDependencies.each { entry ->
entry.value.each { entryFile ->
- task "${getDownloadDepsTaskName(entry.key, entryFile)}"(type: Exec) {
- def outputDir = "${entry.key}/${entryFile}"
- def gzFile = "${outputDir}.tar.gz"
- def sha1File = "${gzFile}.sha1"
- // Make the output file part of the input dependencies explictly.
- inputs.files files(sha1File, gzFile)
- outputs.dir outputDir
- List<String> dlFromStorageArgs = ["-n", "-b", "r8-deps", "-u", "-s", "${sha1File}"]
- if (OperatingSystem.current().isWindows()) {
- executable "download_from_google_storage.bat"
- args dlFromStorageArgs
- } else {
- executable "bash"
- args "-c", "download_from_google_storage " + String.join(" ", dlFromStorageArgs)
- }
+ task "${getDownloadDepsTaskName(entry.key, entryFile)}"(type: DownloadDependency) {
+ type DownloadDependency.Type.GOOGLE_STORAGE
+ dependency "${entry.key}/${entryFile}"
}
}
}
@@ -377,14 +370,9 @@
x20Dependencies.each { entry ->
entry.value.each { entryFile ->
- task "${getDownloadDepsTaskName(entry.key, entryFile)}"(type: Exec) {
- def outputDir = "${entry.key}/${entryFile}"
- def gzFile = "${outputDir}.tar.gz"
- def sha1File = "${gzFile}.sha1"
- inputs.files files(sha1File, gzFile)
- outputs.dir outputDir
- executable "bash"
- args "-c", "tools/download_from_x20.py ${sha1File}"
+ task "${getDownloadDepsTaskName(entry.key, entryFile)}"(type: DownloadDependency) {
+ type DownloadDependency.Type.X20
+ dependency "${entry.key}/${entryFile}"
}
}
}
diff --git a/buildSrc/src/main/java/tasks/DownloadDependency.java b/buildSrc/src/main/java/tasks/DownloadDependency.java
new file mode 100644
index 0000000..61fb95f
--- /dev/null
+++ b/buildSrc/src/main/java/tasks/DownloadDependency.java
@@ -0,0 +1,126 @@
+// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package tasks;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.internal.os.OperatingSystem;
+
+public class DownloadDependency extends DefaultTask {
+
+ public enum Type {
+ GOOGLE_STORAGE,
+ X20
+ }
+
+ private Type type;
+ private String dependency;
+ private File outputDir;
+ private File tarGzFile;
+ private File sha1File;
+
+ public void setType(Type type) {
+ this.type = type;
+ }
+
+ public void setDependency(String dependency) {
+ this.dependency = dependency;
+ outputDir = new File(dependency);
+ tarGzFile = new File(dependency + ".tar.gz");
+ sha1File = new File(dependency + ".tar.gz.sha1");
+ }
+
+ @InputFiles
+ public Collection<File> getInputFiles() {
+ return Arrays.asList(sha1File, tarGzFile);
+ }
+
+ @OutputDirectory
+ public File getOutputDir() {
+ return outputDir;
+ }
+
+ public File getSha1File() {
+ return sha1File;
+ }
+
+ public File getTarGzFile() {
+ return tarGzFile;
+ }
+
+ @TaskAction
+ public void execute() throws IOException, InterruptedException {
+ // First run will write the tar.gz file, causing the second run to still be out-of-date.
+ // Check if the modification time of the tar is newer than the sha in which case we are done.
+ // Also, check the contents of the out directory because gradle appears to create it for us...
+ if (outputDir.exists()
+ && outputDir.isDirectory()
+ && outputDir.list().length > 0
+ && tarGzFile.exists()
+ && sha1File.lastModified() <= tarGzFile.lastModified()) {
+ return;
+ }
+ if (outputDir.exists() && outputDir.isDirectory()) {
+ outputDir.delete();
+ }
+ if (type == Type.GOOGLE_STORAGE) {
+ downloadFromGoogleStorage();
+ } else if (type == Type.X20) {
+ downloadFromX20();
+ } else {
+ throw new RuntimeException("Unexpected or missing dependency type: " + type);
+ }
+ }
+
+ private void downloadFromGoogleStorage() throws IOException, InterruptedException {
+ List<String> args = Arrays.asList("-n", "-b", "r8-deps", "-s", "-u", sha1File.toString());
+ if (OperatingSystem.current().isWindows()) {
+ List<String> command = new ArrayList<>();
+ command.add("download_from_google_storage.bat");
+ command.addAll(args);
+ runProcess(new ProcessBuilder().command(command));
+ } else {
+ runProcess(
+ new ProcessBuilder()
+ .command("bash", "-c", "download_from_google_storage " + String.join(" ", args)));
+ }
+ }
+
+ private void downloadFromX20() throws IOException, InterruptedException {
+ if (OperatingSystem.current().isWindows()) {
+ throw new RuntimeException("Downloading from x20 unsupported on windows");
+ }
+ runProcess(
+ new ProcessBuilder()
+ .command("bash", "-c", "tools/download_from_x20.py " + sha1File.toString()));
+ }
+
+ private static void runProcess(ProcessBuilder builder) throws IOException, InterruptedException {
+ String command = String.join(" ", builder.command());
+ Process p = builder.start();
+ int exit = p.waitFor();
+ if (exit != 0) {
+ throw new IOException(
+ "Process failed for "
+ + command
+ + "\n"
+ + new BufferedReader(
+ new InputStreamReader(p.getErrorStream(), StandardCharsets.UTF_8))
+ .lines()
+ .collect(Collectors.joining("\n")));
+ }
+ }
+}
