Merge changes I721183e5,I1c0489d1 * changes: Rewrite signature annotations for applications which are not minified Refactor GenericSignatureRewriter to be independent of Minifier
diff --git a/infra/config/cr-buildbucket.cfg b/infra/config/global/cr-buildbucket.cfg similarity index 100% rename from infra/config/cr-buildbucket.cfg rename to infra/config/global/cr-buildbucket.cfg
diff --git a/infra/config/luci-logdog.cfg b/infra/config/global/luci-logdog.cfg similarity index 100% rename from infra/config/luci-logdog.cfg rename to infra/config/global/luci-logdog.cfg
diff --git a/infra/config/luci-milo.cfg b/infra/config/global/luci-milo.cfg similarity index 100% rename from infra/config/luci-milo.cfg rename to infra/config/global/luci-milo.cfg
diff --git a/infra/config/luci-scheduler.cfg b/infra/config/global/luci-scheduler.cfg similarity index 100% rename from infra/config/luci-scheduler.cfg rename to infra/config/global/luci-scheduler.cfg
diff --git a/infra/config/project.cfg b/infra/config/global/project.cfg similarity index 100% rename from infra/config/project.cfg rename to infra/config/global/project.cfg
diff --git a/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphEdgeInfo.java b/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphEdgeInfo.java index ca3a732..3e55242 100644 --- a/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphEdgeInfo.java +++ b/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphEdgeInfo.java
@@ -3,10 +3,14 @@ // BSD-style license that can be found in the LICENSE file. package com.android.tools.r8.experimental.graphinfo; -import com.android.tools.r8.errors.Unreachable; - public class GraphEdgeInfo { + private static GraphEdgeInfo UNKNOWN = new GraphEdgeInfo(EdgeKind.Unknown); + + public static GraphEdgeInfo unknown() { + return UNKNOWN; + } + // TODO(b/120959039): Simplify these. Most of the information is present in the source node. public enum EdgeKind { // Prioritized list of edge types. @@ -23,7 +27,8 @@ ReachableFromLiveType, ReferencedInAnnotation, IsLibraryMethod, - MethodHandleUseFrom + MethodHandleUseFrom, + Unknown } private final EdgeKind kind; @@ -66,7 +71,10 @@ case MethodHandleUseFrom: return "referenced by method handle"; default: - throw new Unreachable("Unexpected edge kind: " + edgeKind()); + assert false : "Unknown edge kind: " + edgeKind(); + // fall through + case Unknown: + return "kept for unknown reasons"; } }
diff --git a/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphNode.java b/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphNode.java index b99485a..12661c0 100644 --- a/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphNode.java +++ b/src/main/java/com/android/tools/r8/experimental/graphinfo/GraphNode.java
@@ -8,12 +8,38 @@ @Keep public abstract class GraphNode { + private static final GraphNode CYCLE = + new GraphNode(false) { + @Override + public boolean equals(Object o) { + return o == this; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public String toString() { + return "cycle"; + } + }; + private final boolean isLibraryNode; public GraphNode(boolean isLibraryNode) { this.isLibraryNode = isLibraryNode; } + public static GraphNode cycle() { + return CYCLE; + } + + public final boolean isCycle() { + return this == cycle(); + } + public boolean isLibraryNode() { return isLibraryNode; }
diff --git a/src/main/java/com/android/tools/r8/shaking/WhyAreYouKeepingConsumer.java b/src/main/java/com/android/tools/r8/shaking/WhyAreYouKeepingConsumer.java index 89a6f0e..d56e455 100644 --- a/src/main/java/com/android/tools/r8/shaking/WhyAreYouKeepingConsumer.java +++ b/src/main/java/com/android/tools/r8/shaking/WhyAreYouKeepingConsumer.java
@@ -3,7 +3,6 @@ // BSD-style license that can be found in the LICENSE file. package com.android.tools.r8.shaking; -import com.android.tools.r8.errors.Unreachable; import com.android.tools.r8.experimental.graphinfo.ClassGraphNode; import com.android.tools.r8.experimental.graphinfo.FieldGraphNode; import com.android.tools.r8.experimental.graphinfo.GraphConsumer; @@ -32,6 +31,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -125,50 +125,53 @@ out.println(DescriptorUtils.descriptorToJavaType(clazz.getDescriptor())); } - public List<Pair<GraphNode, GraphEdgeInfo>> findShortestPathTo(final GraphNode node) { + private List<Pair<GraphNode, GraphEdgeInfo>> findShortestPathTo(final GraphNode node) { if (node == null) { return null; } - Deque<GraphPath> queue; - { - Map<GraphNode, Set<GraphEdgeInfo>> sources = getSourcesTargeting(node); - if (sources == null) { - // The node is not targeted at all (it is not reachable). - return null; - } - queue = new LinkedList<>(); - for (GraphNode source : sources.keySet()) { - queue.addLast(new GraphPath(source, null)); - } - } Map<GraphNode, GraphNode> seen = new IdentityHashMap<>(); - while (!queue.isEmpty()) { - GraphPath path = queue.removeFirst(); - Map<GraphNode, Set<GraphEdgeInfo>> sources = getSourcesTargeting(path.node); + Deque<GraphPath> queue = new LinkedList<>(); + GraphPath path = null; + GraphNode current = node; + while (true) { + Map<GraphNode, Set<GraphEdgeInfo>> sources = getSourcesTargeting(current); if (sources == null) { + // We have reached a root or the current node is not targeted at all. return getCanonicalPath(path, node); } + assert !sources.isEmpty(); for (GraphNode source : sources.keySet()) { - if (seen.containsKey(source)) { - continue; + if (!seen.containsKey(source)) { + seen.put(source, source); + queue.addLast(new GraphPath(source, path)); } - seen.put(source, source); - queue.addLast(new GraphPath(source, path)); } + // The source set was not empty, thus we don't have a real root, but all sources are seen! + if (queue.isEmpty()) { + return getCanonicalPath(new GraphPath(GraphNode.cycle(), path), node); + } + path = queue.removeFirst(); + current = path.node; } - throw new Unreachable("Failed to find a root from node: " + node); } // Convert a internal path representation to the external API and compute the edge reasons. private List<Pair<GraphNode, GraphEdgeInfo>> getCanonicalPath( GraphPath path, GraphNode endTarget) { - assert path != null; + if (path == null) { + // If there is no path to endTarget, treat it as not kept by returning a null path. + return null; + } List<Pair<GraphNode, GraphEdgeInfo>> canonical = new ArrayList<>(); while (path.path != null) { GraphNode source = path.node; - GraphNode target = path.path.node; - Set<GraphEdgeInfo> infos = getSourcesTargeting(target).get(source); - canonical.add(new Pair<>(source, getCanonicalInfo(infos))); + if (source.isCycle()) { + canonical.add(new Pair<>(source, new GraphEdgeInfo(EdgeKind.Unknown))); + } else { + GraphNode target = path.path.node; + Set<GraphEdgeInfo> infos = getSourcesTargeting(target).get(source); + canonical.add(new Pair<>(source, getCanonicalInfo(infos))); + } path = path.path; } Set<GraphEdgeInfo> infos = getSourcesTargeting(endTarget).get(path.node); @@ -186,7 +189,8 @@ } } } - throw new Unreachable("Unexpected empty set of graph edge info"); + assert false : "Unexpected empty set of graph edge info"; + return GraphEdgeInfo.unknown(); } private void printEdge(GraphNode node, GraphEdgeInfo info, Formatter formatter) { @@ -225,7 +229,11 @@ ? keepRuleNode.getContent() : keepRuleNode.getOrigin() + ":" + shortPositionInfo(keepRuleNode.getPosition()); } - throw new Unreachable("Unexpected graph node type: " + node); + if (node == GraphNode.cycle()) { + return "only cyclic dependencies remain, failed to determine a path from a keep rule"; + } + assert false : "Unexpected graph node type: " + node; + return Objects.toString(node); } private void addNodeMessage(GraphNode node, Formatter formatter) {
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java index 85c4876..64a9c67 100644 --- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java +++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -288,8 +288,7 @@ public LineNumberOptimization lineNumberOptimization = LineNumberOptimization.ON; public static boolean shouldEnableKeepRuleSynthesisForRecompilation() { - return Version.isDev() - && System.getProperty("com.android.tools.r8.keepRuleSynthesisForRecompilation") != null; + return System.getProperty("com.android.tools.r8.keepRuleSynthesisForRecompilation") != null; } private static Set<String> getExtensiveLoggingFilter() {
diff --git a/src/test/java/com/android/tools/r8/shaking/ServiceLoaderTest.java b/src/test/java/com/android/tools/r8/shaking/ServiceLoaderTest.java index 29e5bcd..ac4ac5f 100644 --- a/src/test/java/com/android/tools/r8/shaking/ServiceLoaderTest.java +++ b/src/test/java/com/android/tools/r8/shaking/ServiceLoaderTest.java
@@ -4,6 +4,8 @@ package com.android.tools.r8.shaking; +import static com.android.tools.r8.references.Reference.classFromClass; +import static com.android.tools.r8.references.Reference.methodFromMethod; import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent; import static org.hamcrest.core.IsNot.not; import static org.junit.Assert.assertEquals; @@ -11,14 +13,19 @@ import static org.junit.Assert.assertTrue; import com.android.tools.r8.DataEntryResource; +import com.android.tools.r8.R8TestRunResult; import com.android.tools.r8.TestBase; import com.android.tools.r8.graph.AppServices; import com.android.tools.r8.naming.AdaptResourceFileContentsTest.DataResourceConsumerForTesting; import com.android.tools.r8.origin.Origin; +import com.android.tools.r8.references.ClassReference; +import com.android.tools.r8.references.MethodReference; import com.android.tools.r8.utils.BooleanUtils; import com.android.tools.r8.utils.StringUtils; import com.android.tools.r8.utils.codeinspector.ClassSubject; import com.android.tools.r8.utils.codeinspector.CodeInspector; +import com.android.tools.r8.utils.graphinspector.GraphInspector; +import com.android.tools.r8.utils.graphinspector.GraphInspector.QueryNode; import com.google.common.collect.Lists; import java.util.List; import java.util.ServiceLoader; @@ -54,7 +61,7 @@ serviceImplementations.add(WorldGreeter.class.getTypeName()); } - CodeInspector inspector = + R8TestRunResult result = testForR8(backend) .addInnerClasses(ServiceLoaderTest.class) .addKeepMainRule(TestClass.class) @@ -69,9 +76,11 @@ new DataResourceConsumerForTesting(options.dataResourceConsumer); options.dataResourceConsumer = dataResourceConsumer; }) + .enableGraphInspector() .run(TestClass.class) - .assertSuccessWithOutput(expectedOutput) - .inspector(); + .assertSuccessWithOutput(expectedOutput); + + CodeInspector inspector = result.inspector(); ClassSubject greeterSubject = inspector.clazz(Greeter.class); assertEquals(includeWorldGreeter, greeterSubject.isPresent()); @@ -92,7 +101,45 @@ assertEquals(worldGreeterSubject.getFinalName(), lines.get(1)); } - // TODO(b/124181030): Verify that -whyareyoukeeping works as intended. + verifyGraphInformation(result.graphInspector()); + } + + private void verifyGraphInformation(GraphInspector graphInspector) throws Exception { + assertEquals(1, graphInspector.getRoots().size()); + QueryNode keepMain = graphInspector.rule(Origin.unknown(), 1, 1).assertRoot(); + + MethodReference mainMethod = + methodFromMethod(TestClass.class.getDeclaredMethod("main", String[].class)); + graphInspector.method(mainMethod).assertKeptBy(keepMain); + + ClassReference helloGreeterClass = classFromClass(HelloGreeter.class); + MethodReference helloGreeterInitMethod = methodFromMethod(HelloGreeter.class.getConstructor()); + MethodReference helloGreeterGreetingMethod = + methodFromMethod(HelloGreeter.class.getDeclaredMethod("greeting")); + + graphInspector.clazz(helloGreeterClass).assertKeptBy(graphInspector.method(mainMethod)); + graphInspector.clazz(helloGreeterClass).assertReflectedFrom(mainMethod); + graphInspector.method(helloGreeterInitMethod).assertReflectedFrom(mainMethod); + + // TODO(b/121313747): greeting() is called from main(), so this should be strengthened to + // `assertInvokedFrom(mainMethod)`. + graphInspector.method(helloGreeterGreetingMethod).assertPresent(); + + if (includeWorldGreeter) { + ClassReference worldGreeterClass = classFromClass(WorldGreeter.class); + MethodReference worldGreeterInitMethod = + methodFromMethod(WorldGreeter.class.getConstructor()); + MethodReference worldGreeterGreetingMethod = + methodFromMethod(WorldGreeter.class.getDeclaredMethod("greeting")); + + graphInspector.clazz(worldGreeterClass).assertKeptBy(graphInspector.method(mainMethod)); + graphInspector.clazz(worldGreeterClass).assertReflectedFrom(mainMethod); + graphInspector.method(worldGreeterInitMethod).assertReflectedFrom(mainMethod); + + // TODO(b/121313747): greeting() is called from main(), so this should be strengthened to + // `assertInvokedFrom(mainMethod)`. + graphInspector.method(worldGreeterGreetingMethod).assertPresent(); + } } @Test
diff --git a/src/test/java/com/android/tools/r8/shaking/keptgraph/KeptViaClassInitializerTestRunner.java b/src/test/java/com/android/tools/r8/shaking/keptgraph/KeptViaClassInitializerTestRunner.java new file mode 100644 index 0000000..344721f --- /dev/null +++ b/src/test/java/com/android/tools/r8/shaking/keptgraph/KeptViaClassInitializerTestRunner.java
@@ -0,0 +1,126 @@ +// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +package com.android.tools.r8.shaking.keptgraph; + +import static com.android.tools.r8.references.Reference.classFromClass; +import static com.android.tools.r8.references.Reference.methodFromMethod; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; + +import com.android.tools.r8.NeverClassInline; +import com.android.tools.r8.NeverMerge; +import com.android.tools.r8.TestBase; +import com.android.tools.r8.ToolHelper; +import com.android.tools.r8.ToolHelper.DexVm.Version; +import com.android.tools.r8.VmTestRunner.IgnoreIfVmOlderThan; +import com.android.tools.r8.origin.Origin; +import com.android.tools.r8.references.MethodReference; +import com.android.tools.r8.shaking.WhyAreYouKeepingConsumer; +import com.android.tools.r8.utils.AndroidApiLevel; +import com.android.tools.r8.utils.StringUtils; +import com.android.tools.r8.utils.graphinspector.GraphInspector; +import com.android.tools.r8.utils.graphinspector.GraphInspector.QueryNode; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.function.Supplier; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class KeptViaClassInitializerTestRunner extends TestBase { + + @NeverMerge + @NeverClassInline + public static class A { + + @Override + public String toString() { + return "I'm an A"; + } + } + + @NeverMerge + @NeverClassInline + public enum T { + A(A::new); + + private final Supplier<Object> factory; + + T(Supplier<Object> factory) { + this.factory = factory; + } + + public Object create() { + return factory.get(); + } + } + + public static class Main { + + public static void main(String[] args) { + System.out.println(T.A.create()); + } + } + + private static final Class<?> CLASS = Main.class; + private static final String EXPECTED = StringUtils.lines("I'm an A"); + + private final Backend backend; + + @Parameters(name = "{0}") + public static Backend[] data() { + return Backend.values(); + } + + public KeptViaClassInitializerTestRunner(Backend backend) { + this.backend = backend; + } + + @Test + @IgnoreIfVmOlderThan(Version.V7_0_0) + public void testKeptMethod() throws Exception { + MethodReference mainMethod = + methodFromMethod(Main.class.getDeclaredMethod("main", String[].class)); + + WhyAreYouKeepingConsumer consumer = new WhyAreYouKeepingConsumer(null); + GraphInspector inspector = + testForR8(backend) + .enableGraphInspector(consumer) + .enableInliningAnnotations() + .addProgramClassesAndInnerClasses(Main.class, A.class, T.class) + .addKeepMethodRules(mainMethod) + .apply( + b -> { + if (backend == Backend.DEX) { + b.setMinApi(AndroidApiLevel.N); + b.addLibraryFiles(ToolHelper.getAndroidJar(AndroidApiLevel.N)); + } + }) + .run(Main.class) + .assertSuccessWithOutput(EXPECTED) + .graphInspector(); + + // The only root should be the keep main-method rule. + assertEquals(1, inspector.getRoots().size()); + QueryNode root = inspector.rule(Origin.unknown(), 1, 1).assertRoot(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + consumer.printWhyAreYouKeeping(classFromClass(A.class), new PrintStream(baos)); + + // TODO(b/124501298): Currently the rooted path is not found. + assertThat(baos.toString(), containsString("is kept for unknown reason")); + + // TODO(b/124499108): Currently synthetic lambda classes are referenced, + // should be their originating context. + if (backend == Backend.DEX) { + assertThat(baos.toString(), containsString("-$$Lambda$")); + } else { + assertThat(baos.toString(), not(containsString("-$$Lambda$"))); + } + } +}
diff --git a/src/test/java/com/android/tools/r8/shaking/keptgraph/WhyAreYouKeepingAllTest.java b/src/test/java/com/android/tools/r8/shaking/keptgraph/WhyAreYouKeepingAllTest.java new file mode 100644 index 0000000..da94e04 --- /dev/null +++ b/src/test/java/com/android/tools/r8/shaking/keptgraph/WhyAreYouKeepingAllTest.java
@@ -0,0 +1,44 @@ +// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +package com.android.tools.r8.shaking.keptgraph; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.android.tools.r8.TestBase; +import com.android.tools.r8.ToolHelper; +import com.android.tools.r8.utils.StringUtils; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.Test; + +/** + * Run compiling R8 with R8 using a match-all -whyareyoukeeping rule to check that it does not cause + * compilation to fail. + */ +public class WhyAreYouKeepingAllTest extends TestBase { + + private static final Path MAIN_KEEP = Paths.get("src/main/keep.txt"); + + private static final String WHY_ARE_YOU_KEEPING_ALL = StringUtils.lines( + "-whyareyoukeeping class ** { *; }", + "-whyareyoukeeping @interface ** { *; }" + ); + + @Test + public void test() throws Throwable { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + testForR8(Backend.CF) + .addProgramFiles(ToolHelper.R8_WITH_RELOCATED_DEPS_JAR) + .addKeepRuleFiles(MAIN_KEEP) + .addKeepRules(WHY_ARE_YOU_KEEPING_ALL) + .redirectStdOut(new PrintStream(baos)) + .compile(); + assertThat(baos.toString(), containsString("referenced in keep rule")); + // TODO(b/124655065): We should always know the reason for keeping. + assertThat(baos.toString(), containsString("kept for unknown reasons")); + } +}
diff --git a/third_party/opensource_apps.tar.gz.sha1 b/third_party/opensource_apps.tar.gz.sha1 index 4ee5113..1695642 100644 --- a/third_party/opensource_apps.tar.gz.sha1 +++ b/third_party/opensource_apps.tar.gz.sha1
@@ -1 +1 @@ -efed1b4bf62340b626cf2d0addb4f334664c0727 \ No newline at end of file +682d306b6e9a1c6356f1dbfe23f5acef4bc23a2f \ No newline at end of file
diff --git a/tools/run_on_as_app.py b/tools/run_on_as_app.py index f3cc2eb..7f5e79f 100755 --- a/tools/run_on_as_app.py +++ b/tools/run_on_as_app.py
@@ -27,8 +27,7 @@ WORKING_DIR = os.environ['R8_BENCHMARK_DIR'] # For running on Golem all APPS are bundled as an x20-dependency and then copied -# to WORKING_DIR. To make it easier to update the app-bundle, remove the folder -# WORKING_DIR and then run run_on_as_app.py --download-only. +# to WORKING_DIR. To update the app-bundle use 'run_on_as_app_x20_packager.py'. APPS = { # 'app-name': { # 'git_repo': ... @@ -100,7 +99,7 @@ 'app_id': 'io.rover.app.debug', 'app_module': 'debug-app', 'git_repo': 'https://github.com/mkj-gram/rover-android.git', - 'revision': 'd2e876e597b3af7eab406e38a0e08327a38bd942', + 'revision': '859af82ba56fe9035ae9949156c7a88e6012d930', }, 'Signal-Android': { 'app_id': 'org.thoughtcrime.securesms', @@ -108,7 +107,7 @@ 'flavor': 'play', 'git_repo': 'https://github.com/mkj-gram/Signal-Android.git', 'main_dex_rules': 'multidex-config.pro', - 'revision': '85e1a10993e5e9ffe923f0798b26cbc44068ba31', + 'revision': 'a45d0c1fed20fa39e8b9445fe7790326f46b3166', 'releaseTarget': 'assemblePlayRelease', 'signed-apk-name': 'Signal-play-release-4.32.7.apk', }, @@ -193,14 +192,14 @@ def IsTrackedByGit(file): return subprocess.check_output(['git', 'ls-files', file]).strip() != '' -def GitClone(git_url, revision, checkout_dir, options): +def GitClone(git_url, revision, checkout_dir, quiet): result = subprocess.check_output( ['git', 'clone', git_url, checkout_dir]).strip() head_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir) if revision == head_rev: return result warn('Target revision is not head in {}.'.format(checkout_dir)) - with utils.ChangedWorkingDirectory(checkout_dir, quiet=options.quiet): + with utils.ChangedWorkingDirectory(checkout_dir, quiet=quiet): subprocess.check_output(['git', 'reset', '--hard', revision]) return result @@ -268,7 +267,8 @@ if not os.path.exists(checkout_dir) and not options.golem: with utils.ChangedWorkingDirectory(WORKING_DIR, quiet=options.quiet): - GitClone(config['git_repo'], config['revision'], checkout_dir, options) + GitClone( + config['git_repo'], config['revision'], checkout_dir, options.quiet) checkout_rev = utils.get_HEAD_sha1_for_checkout(checkout_dir) if config['revision'] != checkout_rev: @@ -346,7 +346,12 @@ # Sanity check that keep rules have changed. with open(ext_proguard_config_file) as new: with open(proguard_config_file) as old: - assert(sum(1 for line in new) > sum(1 for line in old)) + assert( + sum(1 for line in new + if line.strip() and '-printconfiguration' not in line) + > + sum(1 for line in old + if line.strip() and '-printconfiguration' not in line)) # Extract min-sdk and target-sdk (min_sdk, compile_sdk) = as_utils.GetMinAndCompileSdk(app, config, @@ -714,13 +719,13 @@ assert shrinker in SHRINKERS return (options, args) -def download_apps(options): +def download_apps(quiet): # Download apps and place in build with utils.ChangedWorkingDirectory(WORKING_DIR): for app, config in APPS.iteritems(): app_dir = os.path.join(WORKING_DIR, app) if not os.path.exists(app_dir): - GitClone(config['git_repo'], config['revision'], app_dir, options) + GitClone(config['git_repo'], config['revision'], app_dir, quiet) def main(argv): @@ -738,7 +743,7 @@ os.makedirs(WORKING_DIR) if options.download_only: - download_apps(options) + download_apps(options.quiet) return with utils.TempDir() as temp_dir:
diff --git a/tools/run_on_as_app_x20_packager.py b/tools/run_on_as_app_x20_packager.py new file mode 100755 index 0000000..5c84626 --- /dev/null +++ b/tools/run_on_as_app_x20_packager.py
@@ -0,0 +1,50 @@ +#!/usr/bin/env python +# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import os +import run_on_as_app +import shutil +import sys +import upload_to_x20 +import utils + +def main(): + # We need prodaccess to upload to x20 + utils.check_prodacces() + + working_dir = run_on_as_app.WORKING_DIR + + print 'Removing directories that do not match checked out revision' + with utils.ChangedWorkingDirectory(working_dir): + for app, config in run_on_as_app.APPS.iteritems(): + app_dir = os.path.join(working_dir, app) + if os.path.exists(app_dir) \ + and utils.get_HEAD_sha1_for_checkout(app_dir) != config['revision']: + print 'Removing %s' % app_dir + shutil.rmtree(app_dir) + + print 'Downloading all missing apps' + run_on_as_app.download_apps(quiet=False) + + # Package all files as x20 dependency + parent_dir = os.path.dirname(working_dir) + with utils.ChangedWorkingDirectory(parent_dir): + print 'Creating archive for opensource_apps (this may take some time)' + working_dir_name = os.path.basename(working_dir) + app_dirs = [working_dir_name + '/' + name + for name in run_on_as_app.APPS.keys()] + filename = utils.create_archive("opensource_apps", app_dirs) + sha1 = utils.get_sha1(filename) + dest = os.path.join(upload_to_x20.GMSCORE_DEPS, sha1) + upload_to_x20.uploadFile(filename, dest) + sha1_file = '%s.sha1' % filename + with open(sha1_file, 'w') as output: + output.write(sha1) + shutil.move(sha1_file, + os.path.join(utils.THIRD_PARTY, 'opensource_apps.tar.gz.sha1')) + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/tools/utils.py b/tools/utils.py index 892fe7d..f6e0a70 100644 --- a/tools/utils.py +++ b/tools/utils.py
@@ -271,9 +271,13 @@ subprocess.check_call(cmd) def create_archive(name): + return create_archive(name, [name]) + +def create_archive(name, sources): tarname = '%s.tar.gz' % name with tarfile.open(tarname, 'w:gz') as tar: - tar.add(name) + for source in sources: + tar.add(source) return tarname def extract_dir(filename): @@ -288,6 +292,9 @@ with tarfile.open(filename, 'r:gz') as tar: tar.extractall(path=dirname) +def check_prodacces(): + subprocess.check_call(['prodaccess']) + # Note that gcs is eventually consistent with regards to list operations. # This is not a problem in our case, but don't ever use this method # for synchronization.