Merge commit '03f371dee7472e26f7806f358a958f194708a294' into dev-release
Change-Id: I50051a1c4fcec72968033db118f0974f80d440ce
diff --git a/.gitignore b/.gitignore
index 5b8ffd2..2fcc852 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
-
!third_party/chrome/*.sha1
!third_party/gmail/*.sha1
!third_party/gmscore/*.sha1
@@ -116,6 +115,8 @@
third_party/gmscore/*
third_party/google/google-java-format/1.14.0
third_party/google/google-java-format/1.14.0.tar.gz
+third_party/google/yapf/20231013
+third_party/google/yapf/20231013.tar.gz
third_party/google-java-format
third_party/google-java-format.tar.gz
third_party/goyt
diff --git a/d8_r8/build.gradle.kts b/d8_r8/build.gradle.kts
index 704cb23..850a0c5 100644
--- a/d8_r8/build.gradle.kts
+++ b/d8_r8/build.gradle.kts
@@ -15,4 +15,8 @@
dependsOn(gradle.includedBuild("library_desugar").task(":clean"))
dependsOn(gradle.includedBuild("test").task(":clean"))
}
+
+ val r8 by registering() {
+ dependsOn(gradle.includedBuild("main").task(":r8WithRelocatedDeps"))
+ }
}
diff --git a/d8_r8/commonBuildSrc/src/main/kotlin/DependenciesPlugin.kt b/d8_r8/commonBuildSrc/src/main/kotlin/DependenciesPlugin.kt
index 0a2f070..794e58e 100644
--- a/d8_r8/commonBuildSrc/src/main/kotlin/DependenciesPlugin.kt
+++ b/d8_r8/commonBuildSrc/src/main/kotlin/DependenciesPlugin.kt
@@ -444,6 +444,10 @@
"google-java-format-1.14",
Paths.get("third_party", "google", "google-java-format", "1.14.0").toFile(),
Paths.get("third_party", "google", "google-java-format", "1.14.0.tar.gz.sha1").toFile())
+ val googleYapf_20231013 = ThirdPartyDependency(
+ "google-yapf-20231013",
+ Paths.get("third_party", "google", "yapf", "20231013").toFile(),
+ Paths.get("third_party", "google", "yapf", "20231013.tar.gz.sha1").toFile())
val gson = ThirdPartyDependency(
"gson",
Paths.get("third_party", "gson", "gson-2.10.1").toFile(),
diff --git a/d8_r8/test/build.gradle.kts b/d8_r8/test/build.gradle.kts
index ee967fa..dfd5d24 100644
--- a/d8_r8/test/build.gradle.kts
+++ b/d8_r8/test/build.gradle.kts
@@ -264,20 +264,23 @@
fun Exec.rewriteTestsForR8Lib(
keepRulesFileProvider: TaskProvider<Task>,
+ r8JarProvider: Task,
testJarProvider: TaskProvider<*>,
artifactName: String) {
dependsOn(
keepRulesFileProvider,
packageTestDeps,
relocateTestsForR8LibWithRelocatedDeps,
+ r8JarProvider,
r8WithRelocatedDepsTask,
testJarProvider)
val keepRulesFile = keepRulesFileProvider.getSingleOutputFile()
val rtJar = resolve(ThirdPartyDeps.java8Runtime, "rt.jar").getSingleFile()
+ val r8Jar = r8JarProvider.getSingleOutputFile()
val r8WithRelocatedDepsJar = r8WithRelocatedDepsTask.getSingleOutputFile()
val testDepsJar = packageTestDeps.getSingleOutputFile()
val testJar = testJarProvider.getSingleOutputFile()
- inputs.files(keepRulesFile, rtJar, r8WithRelocatedDepsJar, testDepsJar, testJar)
+ inputs.files(keepRulesFile, rtJar, r8Jar, r8WithRelocatedDepsJar, testDepsJar, testJar)
val outputJar = getRoot().resolveAll("build", "libs", artifactName)
outputs.file(outputJar)
commandLine = baseCompilerCommandLine(
@@ -289,7 +292,7 @@
"--lib",
"$rtJar",
"--classpath",
- "$r8WithRelocatedDepsJar",
+ "$r8Jar",
"--classpath",
"$testDepsJar",
"--output",
@@ -302,6 +305,7 @@
val rewriteTestsForR8LibWithRelocatedDeps by registering(Exec::class) {
rewriteTestsForR8Lib(
generateTestKeepRulesR8LibWithRelocatedDeps,
+ r8WithRelocatedDepsTask,
relocateTestsForR8LibWithRelocatedDeps,
"r8libtestdeps-cf.jar")
}
@@ -309,6 +313,7 @@
val rewriteTestsForR8LibNoDeps by registering(Exec::class) {
rewriteTestsForR8Lib(
generateTestKeepRulesR8LibNoDeps,
+ swissArmyKnifeTask,
packageTests,
"r8lib-exclude-deps-testdeps-cf.jar")
}
diff --git a/gradle.properties b/gradle.properties
index 4bd94ac..5804b94 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
+# Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
@@ -12,6 +12,6 @@
org.gradle.parallel=true
org.gradle.caching=false
-# Do not download any jdks or detect them. We provide them
+# Do not download any jdks or detect them. We provide them.
org.gradle.java.installations.auto-detect=false
org.gradle.java.installations.auto-download=false
diff --git a/old.gradle.deprecated b/old.gradle.deprecated
deleted file mode 100644
index f7b3a44..0000000
--- a/old.gradle.deprecated
+++ /dev/null
@@ -1,1844 +0,0 @@
-// Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-
-
-import net.ltgt.gradle.errorprone.CheckSeverity
-import org.gradle.internal.os.OperatingSystem
-import tasks.DownloadDependency
-
-plugins {
- id "net.ltgt.errorprone" version "2.0.2"
-}
-
-apply plugin: 'java'
-apply plugin: 'idea'
-
-ext {
- // When updating dependencies also update and run
- // tools/create_local_maven_with_dependencies.py
- androidSupportVersion = '25.4.0'
- asmVersion = '9.5' // When updating update tools/asmifier.py, build.src and Toolhelper as well.
- javassistVersion = '3.29.2-GA'
- espressoVersion = '3.0.0'
- fastutilVersion = '7.2.1'
- guavaVersion = '31.1-jre'
- gsonVersion = '2.10.1'
- junitVersion = '4.13-beta-2'
- mockitoVersion = '2.10.0'
- // The kotlin version is only here to specify the kotlin language level,
- // all kotlin compilations are done in tests.
- kotlinVersion = '1.8.0'
- kotlinExtMetadataJVMVersion = '0.7.0'
- smaliVersion = '3.0.3'
- errorproneVersion = '2.18.0'
- testngVersion = '6.10'
-}
-
-repositories {
- maven {
- url uri('file:third_party/dependencies')
- }
-}
-
-if (project.hasProperty('with_code_coverage')) {
- apply plugin: 'jacoco'
-}
-
-// Custom source set for example tests and generated tests.
-sourceSets {
- main {
- java {
- srcDirs = ['src/main/java', 'src/keepanno/java']
- // Exclude in old setup, resource shrinker is only compiled into R8 with 8.0
- exclude 'com/android/tools/r8/utils/resourceshrinker/*.java'
- }
- resources {
- srcDirs "third_party/api_database/api_database"
- }
- }
- main17 {
- java {
- srcDirs = ['src/main/java', 'src/keepanno/java']
- // Exclude in old setup, resource shrinker is only compiled into R8 with 8.0
- exclude 'com/android/tools/r8/utils/resourceshrinker/*.java'
- }
- resources {
- srcDirs "third_party/api_database/api_database"
- }
- }
- test {
- java {
- srcDirs = [
- 'src/test/java',
- 'src/test/bootstrap',
- 'build/generated/test/java',
- ]
- }
- }
- apiUsageSample {
- java {
- srcDirs = ['src/test/apiUsageSample', 'src/main/java']
- include 'com/android/tools/apiusagesample/*.java'
- include 'com/android/tools/r8/BaseCompilerCommandParser.java'
- include 'com/android/tools/r8/D8CommandParser.java'
- include 'com/android/tools/r8/R8CommandParser.java'
- include 'com/android/tools/r8/utils/FlagFile.java'
- }
- }
- cfSegments {
- java {
- srcDirs = ['third_party/classlib/java', 'src/cf_segments/java']
- }
- output.resourcesDir = 'build/classes/cfSegments'
- }
- debugTestResources {
- java {
- srcDirs = ['src/test/debugTestResources']
- }
- output.resourcesDir = 'build/classes/debugTestResources'
- }
- debugTestResourcesJava8 {
- java {
- srcDirs = ['src/test/debugTestResourcesJava8']
- }
- output.resourcesDir = 'build/classes/debugTestResourcesJava8'
- }
- examplesJava9 {
- java {
- srcDirs = ['src/test/examplesJava9']
- }
- }
- examplesJava10 {
- java {
- srcDirs = ['src/test/examplesJava10']
- }
- }
- examplesJava11 {
- java {
- srcDirs = ['src/test/examplesJava11']
- }
- }
- examplesJava17 {
- java {
- srcDirs = ['src/test/examplesJava17']
- }
- }
- examplesJava20 {
- java {
- srcDirs = ['src/test/examplesJava20']
- }
- }
- keepanno {
- java {
- srcDirs = ['src/keepanno/java']
- include 'com/android/tools/r8/keepanno/annotations/*.java'
- }
- }
-}
-
-// Ensure importing into IntelliJ IDEA use the same output directories as Gradle. In tests we
-// use the output path for tests (ultimately through ToolHelper.getClassPathForTests()) and
-// therefore these paths need to be the same. See https://youtrack.jetbrains.com/issue/IDEA-175172
-// for context.
-idea {
- sourceSets.all { SourceSet sources ->
- module {
- if (sources.name == "main") {
- sourceDirs += sources.java.srcDirs
- outputDir sources.output.classesDirs[0]
- } else {
- testSourceDirs += sources.java.srcDirs
- testOutputDir sources.output.classesDirs[0]
- }
- }
- }
-}
-
-dependencies {
- implementation "com.google.code.gson:gson:$gsonVersion"
- // Include all of guava when compiling the code, but exclude annotations that we don't
- // need from the packaging.
- compileOnly("com.google.guava:guava:$guavaVersion")
- implementation("com.google.guava:guava:$guavaVersion", {
- exclude group: 'com.google.errorprone'
- exclude group: 'com.google.code.findbugs'
- exclude group: 'com.google.j2objc'
- exclude group: 'org.codehaus.mojo'
- exclude group: 'com.google.guava', module: 'listenablefuture'
- exclude group: 'com.google.guava', module: 'failureaccess'
- })
- implementation group: 'it.unimi.dsi', name: 'fastutil', version: fastutilVersion
- implementation "org.jetbrains.kotlinx:kotlinx-metadata-jvm:$kotlinExtMetadataJVMVersion"
- implementation group: 'org.ow2.asm', name: 'asm', version: asmVersion
- implementation group: 'org.ow2.asm', name: 'asm-commons', version: asmVersion
- implementation group: 'org.ow2.asm', name: 'asm-tree', version: asmVersion
- implementation group: 'org.ow2.asm', name: 'asm-analysis', version: asmVersion
- implementation group: 'org.ow2.asm', name: 'asm-util', version: asmVersion
-
- main17Implementation "com.google.code.gson:gson:$gsonVersion"
- // Include all of guava when compiling the code, but exclude annotations that we don't
- // need from the packaging.
- main17CompileOnly("com.google.guava:guava:$guavaVersion")
- main17Implementation("com.google.guava:guava:$guavaVersion", {
- exclude group: 'com.google.errorprone'
- exclude group: 'com.google.code.findbugs'
- exclude group: 'com.google.j2objc'
- exclude group: 'org.codehaus.mojo'
- })
- main17Implementation group: 'it.unimi.dsi', name: 'fastutil', version: fastutilVersion
- main17Implementation "org.jetbrains.kotlinx:kotlinx-metadata-jvm:$kotlinExtMetadataJVMVersion"
- main17Implementation group: 'org.ow2.asm', name: 'asm', version: asmVersion
- main17Implementation group: 'org.ow2.asm', name: 'asm-commons', version: asmVersion
- main17Implementation group: 'org.ow2.asm', name: 'asm-tree', version: asmVersion
- main17Implementation group: 'org.ow2.asm', name: 'asm-analysis', version: asmVersion
- main17Implementation group: 'org.ow2.asm', name: 'asm-util', version: asmVersion
-
- testCompile "junit:junit:$junitVersion"
- testCompile "com.google.guava:guava:$guavaVersion"
- testCompile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
- testCompile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
- testCompile group: 'com.android.tools.smali', name: 'smali', version: smaliVersion
- testCompile files('third_party/jasmin/jasmin-2.4.jar')
- testCompile files('third_party/jdwp-tests/apache-harmony-jdwp-tests-host.jar')
- testCompile files('third_party/ddmlib/ddmlib.jar')
- testCompile group: 'org.ow2.asm', name: 'asm', version: asmVersion
- testCompile group: 'org.ow2.asm', name: 'asm-commons', version: asmVersion
- testCompile group: 'org.ow2.asm', name: 'asm-tree', version: asmVersion
- testCompile group: 'org.ow2.asm', name: 'asm-analysis', version: asmVersion
- testCompile group: 'org.ow2.asm', name: 'asm-util', version: asmVersion
- testCompile group: 'it.unimi.dsi', name: 'fastutil', version: fastutilVersion
- testCompile group: 'org.javassist', name: 'javassist', version: javassistVersion
-
- apiUsageSampleCompile sourceSets.main.output
- apiUsageSampleCompile "com.google.guava:guava:$guavaVersion"
- errorprone("com.google.errorprone:error_prone_core:$errorproneVersion")
-
- keepannoCompile group: 'org.ow2.asm', name: 'asm', version: asmVersion
- keepannoCompile "com.google.guava:guava:$guavaVersion"
-}
-
-def r8LibPath = "$buildDir/libs/r8lib.jar"
-def r8LibExludeDepsPath = "$buildDir/libs/r8lib-exclude-deps.jar"
-def r8DesugaredPath = "$buildDir/libs/r8desugared.jar"
-def r8LibGeneratedKeepRulesPath = "$buildDir/generated/keep.txt"
-def r8LibTestPath = "$buildDir/classes/r8libtest"
-def java11ClassFiles = "$buildDir/classes/java/mainJava11"
-def r8RetracePath = "$buildDir/libs/r8retrace.jar"
-def r8RetraceExludeDepsPath = "$buildDir/libs/r8retrace-exclude-deps.jar"
-
-def osString = OperatingSystem.current().isLinux() ? "linux" :
- OperatingSystem.current().isMacOsX() ? "mac" : "windows"
-
-def cloudDependencies = [
- "tests" : [
- "2017-10-04/art",
- "2016-12-19/art"
- ],
- "third_party": [
- "aapt2",
- "android_cts_baseline",
- "android_jar/libcore_latest",
- "android_jar/lib-v14",
- "android_jar/lib-v15",
- "android_jar/lib-v19",
- "android_jar/lib-v21",
- "android_jar/lib-v22",
- "android_jar/lib-v23",
- "android_jar/lib-v24",
- "android_jar/lib-v25",
- "android_jar/lib-v26",
- "android_jar/lib-v27",
- "android_jar/lib-v28",
- "android_jar/lib-v29",
- "android_jar/lib-v30",
- "android_jar/lib-v31",
- "android_jar/lib-v32",
- "android_jar/lib-v33",
- "android_jar/lib-v34",
- "android_jar/lib-master",
- "api_database/api_database",
- "api-outlining/simple-app-dump",
- "binary_compatibility_tests/compiler_api_tests",
- "bundletool/bundletool-1.11.0",
- "core-lambda-stubs",
- "dagger/2.41",
- "dart-sdk",
- "ddmlib",
- "examples",
- "examplesAndroidN",
- "examplesAndroidO",
- "examplesAndroidOLegacy",
- "examplesAndroidOGenerated",
- "examplesAndroidP",
- "examplesAndroidPGenerated",
- "gradle/gradle",
- "google/google-java-format/1.14.0",
- "google-java-format",
- "gson/gson-2.10.1",
- "guava/guava-32.1.2-android",
- "guava/guava-32.1.2-jre",
- "jacoco/0.8.2",
- "jacoco/0.8.6",
- "jasmin",
- "junit",
- "jdwp-tests",
- "jsr223-api-1.0",
- "rhino-1.7.10",
- "rhino-android-1.1.1",
- "kotlin/kotlin-compiler-1.3.72",
- "kotlin/kotlin-compiler-1.4.20",
- "kotlin/kotlin-compiler-1.5.0",
- "kotlin/kotlin-compiler-1.6.0",
- "kotlin/kotlin-compiler-1.7.0",
- "kotlin/kotlin-compiler-1.8.0",
- "kotlinx-coroutines-1.3.6",
- "kotlinR8TestResources",
- "multidex",
- "openjdk/custom_conversion",
- "openjdk/openjdk-rt-1.8",
- "openjdk/desugar_jdk_libs",
- "openjdk/desugar_jdk_libs_11",
- "openjdk/desugar_jdk_libs_releases/1.0.9",
- "openjdk/desugar_jdk_libs_releases/1.0.10",
- "openjdk/desugar_jdk_libs_releases/1.1.0",
- "openjdk/desugar_jdk_libs_releases/1.1.1",
- "openjdk/desugar_jdk_libs_releases/1.1.5",
- "openjdk/desugar_jdk_libs_releases/2.0.3",
- "openjdk/jdk-11-test",
- "opensource-apps/tivi",
- "proguard/proguard5.2.1",
- "proguard/proguard6.0.1",
- "proguard/proguard-7.0.0",
- "proguard/proguard-7.3.2",
- "retrace_benchmark",
- "retrace/binary_compatibility",
- "r8",
- "r8-releases/2.0.74",
- "r8-releases/3.2.54",
- "r8mappings",
- "smali"
- ],
- // All dex-vms have a fixed OS of Linux, as they are only supported on Linux, and will be run in a Docker
- // container on other platforms where supported.
- "tools" : [
- "linux/art",
- "linux/art-5.1.1",
- "linux/art-6.0.1",
- "linux/art-7.0.0",
- "linux/art-8.1.0",
- "linux/art-9.0.0",
- "linux/art-10.0.0",
- "linux/host/art-12.0.0-beta4",
- "linux/host/art-13.0.0",
- "linux/host/art-14.0.0-beta3",
- "linux/host/art-master",
- "linux/dalvik",
- "linux/dalvik-4.0.4",
- ]
-]
-
-def cloudSystemDependencies = [
- linux: [
- "third_party": ["openjdk/openjdk-9.0.4/linux",
- "openjdk/jdk8/linux-x86",
- "openjdk/jdk-11/linux",
- "openjdk/jdk-17/linux",
- "openjdk/jdk-20/linux"],
- ],
- osx: [
- "third_party": ["openjdk/openjdk-9.0.4/osx",
- "openjdk/jdk8/darwin-x86",
- "openjdk/jdk-11/osx",
- "openjdk/jdk-17/osx",
- "openjdk/jdk-20/osx"],
- ],
- windows: [
- "third_party": ["openjdk/openjdk-9.0.4/windows",
- "openjdk/jdk-11/windows",
- "openjdk/jdk-17/windows",
- "openjdk/jdk-20/windows"],
- ],
-]
-
-if (OperatingSystem.current().isWindows()) {
- cloudSystemDependencies.windows.each { entry ->
- cloudDependencies.get(entry.key).addAll(entry.value)
- }
-} else if (OperatingSystem.current().isLinux()) {
- cloudSystemDependencies.linux.each { entry ->
- cloudDependencies.get(entry.key).addAll(entry.value)
- }
-} else if (OperatingSystem.current().isMacOsX()) {
- cloudSystemDependencies.osx.each { entry ->
- cloudDependencies.get(entry.key).addAll(entry.value)
- }
-} else {
- println "WARNING: Unsupported system: " + OperatingSystem.current()
-}
-
-def getDownloadDepsTaskName(entryKey, entryFile) {
- return "download_deps_${entryKey}_${entryFile.replace('/', '_').replace('\\', '_')}"
-}
-
-cloudDependencies.each { entry ->
- entry.value.each { entryFile ->
- task (getDownloadDepsTaskName(entry.key, entryFile), type: DownloadDependency) {
- type DownloadDependency.Type.GOOGLE_STORAGE
- dependency "${entry.key}/${entryFile}"
- }
- }
-}
-
-def x20Dependencies = [
- "third_party": [
- "benchmarks/kotlin-benches",
- "chrome/chrome_200430",
- "chrome/monochrome_public_minimal_apks/chrome_200520",
- "chrome/clank_google3_prebuilt",
- "classlib",
- "cf_segments",
- "desugar/desugar_20180308",
- "internal/issue-127524985",
- "framework",
- "gmail/gmail_android_180826.15",
- "gmscore/gmscore_v10",
- "gmscore/latest",
- "nest/nest_20180926_7c6cfb",
- "proguard/proguard_internal_159423826",
- "proguardsettings",
- "proto",
- "protobuf-lite",
- "retrace_internal",
- "youtube/youtube.android_17.19"
- ],
-]
-
-x20Dependencies.each { entry ->
- entry.value.each { entryFile ->
- task "${getDownloadDepsTaskName(entry.key, entryFile)}"(type: DownloadDependency) {
- type DownloadDependency.Type.X20
- dependency "${entry.key}/${entryFile}"
- }
- }
-}
-
-task downloadOpenJDKrt {
- cloudDependencies.each { entry ->
- entry.value.each { entryFile ->
- if (entryFile.contains("openjdk-rt")) {
- dependsOn "${getDownloadDepsTaskName(entry.key, entryFile)}"
- }
- }
- }
-}
-
-task downloadAndroidCts {
- cloudDependencies.each { entry ->
- entry.value.each { entryFile ->
- if (entryFile.contains("android_cts_baseline")) {
- dependsOn "${getDownloadDepsTaskName(entry.key, entryFile)}"
- }
- }
- }
-}
-
-task downloadCloudDeps() {
- cloudDependencies.each { entry ->
- entry.value.each { entryFile ->
- dependsOn "${getDownloadDepsTaskName(entry.key, entryFile)}"
- }
- }
-}
-
-task downloadX20Deps() {
- x20Dependencies.each { entry ->
- entry.value.each { entryFile ->
- dependsOn "${getDownloadDepsTaskName(entry.key, entryFile)}"
- }
- }
-}
-
-task downloadDeps {
- dependsOn downloadCloudDeps
- if (!project.hasProperty('no_internal')) {
- dependsOn downloadX20Deps
- }
-}
-
-allprojects {
- sourceCompatibility = JavaVersion.VERSION_1_8
- targetCompatibility = JavaVersion.VERSION_1_8
-}
-
-// TODO(ricow): Remove debug prints
-println("NOTE: Current operating system: " + OperatingSystem.current())
-println("NOTE: Current operating system isWindows: " + OperatingSystem.current().isWindows())
-
-// Check if running with the JDK location from tools/jdk.py.
-if (OperatingSystem.current().isWindows()) {
- println "NOTE: Running with JDK: " + org.gradle.internal.jvm.Jvm.current().javaHome
- compileJava.options.encoding = "UTF-8"
- compileTestJava.options.encoding = "UTF-8"
-} else {
- def javaHomeOut = new StringBuilder()
- def javaHomeErr = new StringBuilder()
- def javaHomeProc = './tools/jdk.py'.execute([], projectDir)
- javaHomeProc.waitForProcessOutput(javaHomeOut, javaHomeErr)
- def jdkHome = new File(javaHomeOut.toString().trim())
- if (!jdkHome.exists()) {
- println "WARNING: Failed to find the ./tools/jdk.py specified JDK: " + jdkHome
- } else if (jdkHome != org.gradle.internal.jvm.Jvm.current().javaHome) {
- println("WARNING: Gradle is running in a non-pinned Java"
- + ". Gradle Java Home: " + org.gradle.internal.jvm.Jvm.current().javaHome
- + ". Expected: " + jdkHome)
- } else {
- println("NOTE: Running with jdk from tools/jdk.py: " + jdkHome)
- }
-}
-
-compileJava.dependsOn downloadCloudDeps
-
-sourceSets.configureEach { sourceSet ->
- tasks.named(sourceSet.compileJavaTaskName).configure {
- // Default disable errorprone (enabled and setup below).
- options.errorprone.enabled = false
- options.compilerArgs << '-Xlint:unchecked'
- // Run all compilation tasks in a forked subprocess.
- options.fork = true
- // Javac often runs out of stack space when compiling the tests.
- // Increase the stack size for the javac process.
- options.forkOptions.jvmArgs << "-Xss256m"
- // Test compilation is sometimes hitting the default limit at 1g, increase it.
- options.forkOptions.jvmArgs << "-Xmx3g"
- }
-}
-
-def setJdkCompilationWithCompatibility(String sourceSet, String javaHome, JavaVersion compatibility, boolean enablePreview) {
- tasks.named(sourceSet).get().configure {
- def jdkDir = "third_party/openjdk/${javaHome}/"
- options.fork = true
- options.forkOptions.jvmArgs = []
- if (enablePreview) {
- options.compilerArgs.add('--enable-preview')
- }
- if (OperatingSystem.current().isLinux()) {
- dependsOn getDownloadDepsTaskName("third_party", "openjdk/" + javaHome + "/linux")
- options.forkOptions.javaHome = file(jdkDir + 'linux')
- } else if (OperatingSystem.current().isMacOsX()) {
- dependsOn getDownloadDepsTaskName("third_party", "openjdk/" + javaHome + "/osx")
- options.forkOptions.javaHome = compatibility > JavaVersion.VERSION_1_9
- ? file(jdkDir + 'osx/Contents/Home')
- : file(jdkDir + 'osx')
- } else {
- dependsOn getDownloadDepsTaskName("third_party", "openjdk/" + javaHome + "/windows")
- options.forkOptions.javaHome = file(jdkDir + 'windows')
- }
- sourceCompatibility = compatibility
- targetCompatibility = compatibility
- }
-}
-
-setJdkCompilationWithCompatibility(
- sourceSets.main.compileJavaTaskName,
- 'jdk-11',
- JavaVersion.VERSION_11,
- false);
-
-setJdkCompilationWithCompatibility(
- sourceSets.examplesJava9.compileJavaTaskName,
- 'openjdk-9.0.4',
- JavaVersion.VERSION_1_9,
- false)
-setJdkCompilationWithCompatibility(
- sourceSets.examplesJava10.compileJavaTaskName,
- 'jdk-11',
- JavaVersion.VERSION_1_10,
- false)
-setJdkCompilationWithCompatibility(
- sourceSets.main17.compileJavaTaskName,
- 'jdk-17',
- JavaVersion.VERSION_17,
- false)
-setJdkCompilationWithCompatibility(
- sourceSets.examplesJava11.compileJavaTaskName,
- 'jdk-11',
- JavaVersion.VERSION_11,
- false)
-setJdkCompilationWithCompatibility(
- sourceSets.examplesJava17.compileJavaTaskName,
- 'jdk-17',
- JavaVersion.VERSION_17,
- false)
-setJdkCompilationWithCompatibility(
- sourceSets.examplesJava20.compileJavaTaskName,
- 'jdk-20',
- // TODO(b/218293990): Update Gradle to get JavaVersion.VERSION_18.
- JavaVersion.VERSION_17,
- false)
-
-if (!project.hasProperty('without_error_prone') &&
- // Don't enable error prone on Java 8 as the plugin setup does not support it.
- !org.gradle.internal.jvm.Jvm.current().javaVersion.java8) {
- compileJava {
-
- // Enable error prone for D8/R8 main sources.
- options.errorprone.enabled = true
-
- // Make all warnings errors. Warnings that we have chosen not to fix (or suppress) are
- // disabled outright below.
- options.compilerArgs << "-Werror"
-
- // Increase number of reported errors to 1000 (default is 100).
- options.compilerArgs << "-Xmaxerrs"
- options.compilerArgs << "1000"
-
- // Non-default / Experimental checks - explicitly enforced.
- options.errorprone.check('RemoveUnusedImports', CheckSeverity.ERROR)
- options.errorprone.check('InconsistentOverloads', CheckSeverity.ERROR)
- options.errorprone.check('MissingDefault', CheckSeverity.ERROR)
- options.errorprone.check('MultipleTopLevelClasses', CheckSeverity.ERROR)
- options.errorprone.check('NarrowingCompoundAssignment', CheckSeverity.ERROR)
-
- // Warnings that cause unwanted edits (e.g., inability to write informative asserts).
- options.errorprone.check('AlreadyChecked', CheckSeverity.OFF)
-
- // JavaDoc related warnings. Would be nice to resolve but of no real consequence.
- options.errorprone.check('InvalidLink', CheckSeverity.OFF)
- options.errorprone.check('InvalidBlockTag', CheckSeverity.OFF)
- options.errorprone.check('InvalidInlineTag', CheckSeverity.OFF)
- options.errorprone.check('EmptyBlockTag', CheckSeverity.OFF)
- options.errorprone.check('MissingSummary', CheckSeverity.OFF)
- options.errorprone.check('UnrecognisedJavadocTag', CheckSeverity.OFF)
- options.errorprone.check('AlmostJavadoc', CheckSeverity.OFF)
-
- // Moving away from identity and canonical items is not planned.
- options.errorprone.check('IdentityHashMapUsage', CheckSeverity.OFF)
- }
-}
-
-task consolidatedLicense {
- def license = new File(new File(buildDir, 'generatedLicense'), 'LICENSE')
-
- inputs.files files('LICENSE', 'LIBRARY-LICENSE') + fileTree(dir: 'library-licensing')
- def runtimeClasspath = configurations.findByName("runtimeClasspath")
- inputs.files { runtimeClasspath.getResolvedConfiguration().files }
-
- outputs.files license
-
- doLast {
- def dependencies = []
- runtimeClasspath.resolvedConfiguration.resolvedArtifacts.each {
- def identifier = (ModuleComponentIdentifier) it.id.componentIdentifier
- dependencies.add("${identifier.group}:${identifier.module}")
- }
- def libraryLicenses = file('LIBRARY-LICENSE').text
- dependencies.each {
- if (!libraryLicenses.contains("- artifact: $it")) {
- throw new GradleException("No license for $it in LIBRARY_LICENSE")
- }
- }
-
- license.getParentFile().mkdirs()
- license.createNewFile()
- license.text = "This file lists all licenses for code distributed.\n"
- license.text += "All non-library code has the following 3-Clause BSD license.\n"
- license.text += "\n"
- license.text += "\n"
- license.text += file('LICENSE').text
- license.text += "\n"
- license.text += "\n"
- license.text += "Summary of distributed libraries:\n"
- license.text += "\n"
- license.text += libraryLicenses
- license.text += "\n"
- license.text += "\n"
- license.text += "Licenses details:\n"
- fileTree(dir: 'library-licensing').getFiles().stream().sorted().forEach { file ->
- license.text += "\n"
- license.text += "\n"
- license.text += file.text
- }
- }
-}
-
-def repackageDepFile(file) {
- if (file.getName().endsWith('.jar')) {
- return zipTree(file).matching {
- exclude '**/module-info.class'
- exclude 'META-INF/maven/**'
- exclude 'META-INF/LICENSE.txt'
- exclude 'META-INF/MANIFEST.MF'
- }
- } else {
- return fileTree(file)
- }
-}
-
-task repackageDeps(type: Jar) {
- dependsOn downloadCloudDeps
- dependsOn project.configurations.runtimeClasspath
- project.configurations.runtimeClasspath.forEach {
- from repackageDepFile(it)
- }
- archiveFileName = 'deps_all.jar'
-}
-
-task repackageTestDeps(type: Jar) {
- dependsOn downloadCloudDeps
- dependsOn project.configurations.testCompile
- project.configurations.testCompile.forEach {
- from repackageDepFile(it)
- }
- archiveFileName = 'test_deps_all.jar'
-}
-
-task repackageSources(type: Jar) {
- // If this fails then remove all generated folders from
- // build/classes/java/test that is not {com,dalvik}
- from sourceSets.main.output
- archiveFileName = 'sources_main.jar'
-}
-
-task repackageSources17(type: Jar) {
- from sourceSets.main17.output
- archiveFileName = 'sources_main_17.jar'
-}
-
-def r8CreateTask(name, baseName, sources, includeLibraryLicenses, includeSwissArmyKnife) {
- return tasks.create("r8Create${name}", Jar) {
- entryCompression ZipEntryCompression.STORED
- dependsOn sources
- dependsOn files('LICENSE')
- if (includeLibraryLicenses) {
- from consolidatedLicense.outputs.files
- } else {
- from files('LICENSE')
- }
- from sources.collect { zipTree(it) }
- exclude "$buildDir/classes/**"
- archiveFileName = baseName
- if (includeSwissArmyKnife) {
- manifest {
- attributes 'Main-Class': 'com.android.tools.r8.SwissArmyKnife'
- }
- }
- exclude "META-INF/*.kotlin_module"
- exclude "**/*.kotlin_metadata"
- }
-}
-
-def r8RelocateTask(r8Task, output) {
- return tasks.create("r8Relocate_${r8Task.name}", Exec) {
- dependsOn r8WithDeps
- dependsOn r8Task
- outputs.file output
- workingDir = projectDir
- inputs.files r8Task.outputs.files + r8WithDeps.outputs.files
- commandLine baseCompilerCommandLine([
- "relocator",
- "--input",
- r8Task.outputs.files[0],
- "--output",
- output,
- "--map",
- "com.google.common.**->com.android.tools.r8.com.google.common",
- "--map",
- "com.google.gson.**->com.android.tools.r8.com.google.gson",
- "--map",
- "com.google.thirdparty.**->com.android.tools.r8.com.google.thirdparty",
- "--map",
- "org.objectweb.asm.**->com.android.tools.r8.org.objectweb.asm",
- "--map",
- "it.unimi.dsi.fastutil.**->com.android.tools.r8.it.unimi.dsi.fastutil",
- "--map",
- "kotlin.**->com.android.tools.r8.jetbrains.kotlin",
- "--map",
- "kotlinx.**->com.android.tools.r8.jetbrains.kotlinx",
- "--map",
- "org.jetbrains.**->com.android.tools.r8.org.jetbrains",
- "--map",
- "org.intellij.**->com.android.tools.r8.org.intellij",
- "--map",
- "org.checkerframework.**->com.android.tools.r8.org.checkerframework"
- ])
- }
-}
-
-task r8WithDeps {
- dependsOn repackageSources
- dependsOn repackageDeps
- inputs.files ([repackageSources.outputs, repackageDeps.outputs])
- def r8Task = r8CreateTask(
- 'WithDeps',
- 'r8_with_deps.jar',
- repackageSources.outputs.files + repackageDeps.outputs.files,
- true,
- true)
- dependsOn r8Task
- outputs.files r8Task.outputs.files
-}
-
-task r8WithDeps17 {
- dependsOn repackageSources17
- dependsOn repackageDeps
- inputs.files ([repackageSources17.outputs, repackageDeps.outputs])
- def r8Task = r8CreateTask(
- 'WithDeps17',
- 'r8_with_deps_17.jar',
- repackageSources17.outputs.files + repackageDeps.outputs.files,
- true,
- true)
- dependsOn r8Task
- outputs.files r8Task.outputs.files
-}
-
-task r8WithRelocatedDeps {
- def output = "${buildDir}/libs/r8_with_relocated_deps.jar"
- dependsOn r8RelocateTask(r8WithDeps, output)
- inputs.files r8WithDeps.outputs.files
- outputs.file output
-}
-
-task r8WithRelocatedDeps17 {
- def output = "${buildDir}/libs/r8_with_relocated_deps_17.jar"
- dependsOn r8RelocateTask(r8WithDeps17, output)
- inputs.files r8WithDeps17.outputs.files
- outputs.file output
-}
-
-task r8WithoutDeps {
- dependsOn repackageSources
- inputs.files repackageSources.outputs
- def r8Task = r8CreateTask(
- 'WithoutDeps',
- 'r8_without_deps.jar',
- repackageSources.outputs.files,
- false,
- true)
- dependsOn r8Task
- outputs.files r8Task.outputs.files
-}
-
-task r8(type: Copy) {
- def r8Task = project.hasProperty("exclude_deps")
- ? r8WithoutDeps : r8WithRelocatedDeps
- dependsOn r8Task
- from r8Task.outputs.files[0]
- into file("${buildDir}/libs")
- rename { String fileName -> "r8.jar" }
- outputs.file "${buildDir}/libs/r8.jar"
-}
-
-task r8NoManifestWithoutDeps {
- dependsOn repackageSources
- inputs.files repackageSources.outputs
- def r8Task = r8CreateTask(
- 'NoManifestWithoutDeps',
- 'r8_no_manifest_without_deps.jar',
- repackageSources.outputs.files,
- false,
- false)
- dependsOn r8Task
- outputs.files r8Task.outputs.files
-}
-
-task r8NoManifestWithDeps {
- dependsOn repackageSources
- inputs.files ([repackageSources.outputs, repackageDeps.outputs])
- def r8Task = r8CreateTask(
- 'NoManifestWithDeps',
- 'r8_no_manifest_with_deps.jar',
- repackageSources.outputs.files + repackageDeps.outputs.files,
- true,
- false)
- dependsOn r8Task
- outputs.files r8Task.outputs.files
-}
-
-task r8NoManifestWithRelocatedDeps {
- def output = "${buildDir}/libs/r8_no_manifest_with_relocated_deps.jar"
- dependsOn r8RelocateTask(r8NoManifestWithDeps, output)
- inputs.files r8NoManifestWithDeps.outputs.files
- outputs.file output
-}
-
-task r8NoManifest(type: Copy) {
- def r8Task = project.hasProperty("exclude_deps")
- ? r8NoManifestWithoutDeps : r8NoManifestWithRelocatedDeps
- dependsOn r8Task
- from r8Task.outputs.files[0]
- into file("${buildDir}/libs")
- rename { String fileName -> "r8_no_manifest.jar" }
- outputs.file "${buildDir}/libs/r8_no_manifest.jar"
-}
-
-def baseCompilerCommandLine(compiler, args = []) {
- // Execute r8 commands against a stable r8 with dependencies.
- // TODO(b/139725780): See if we can remove or lower the heap size (-Xmx8g).
- return [org.gradle.internal.jvm.Jvm.current().getJavaExecutable(),
- "-Xmx8g", "-ea", "-jar", r8WithDeps.outputs.files[0]] + compiler + args
-}
-
-def baseR8CommandLine(args = []) {
- // Execute r8 commands against a stable r8 with dependencies.
- return baseCompilerCommandLine("r8", args)
-}
-
-def baseD8CommandLine(args = []) {
- // Execute r8 commands against a stable r8 with dependencies.
- return baseCompilerCommandLine("d8", args)
-}
-
-def r8CfCommandLine(input, output, pgConfs = [], args = ["--release"], libs = []) {
- def allArgs = [
- "--classfile",
- input,
- "--output", output,
- "--pg-map-output", output + ".map",
- "--partition-map-output", output + "_map.zip",
- "--lib", org.gradle.internal.jvm.Jvm.current().javaHome,
- ] + args + libs.collectMany { ["--lib", it] } + pgConfs.collectMany { ["--pg-conf", it] }
- return baseR8CommandLine(allArgs)
-}
-
-def d8CfCommandLine(input, output, args = ["--release"], libs = []) {
- def allArgs = [
- "--classfile",
- input,
- "--output", output,
- "--lib", "third_party/openjdk/openjdk-rt-1.8/rt.jar"
- ] + args + libs.collectMany { ["--lib", it] }
- return baseD8CommandLine(allArgs)
-}
-
-def r8LibCreateTask(
- name, pgConfs = [], r8Task, output, libs = [], classpath = [], excldeps=false, debug=false) {
- return tasks.create("r8Lib${name}", Exec) {
- inputs.files ([pgConfs, r8WithRelocatedDeps.outputs, r8Task.outputs])
- outputs.file output
- dependsOn downloadOpenJDKrt
- dependsOn r8WithRelocatedDeps
- dependsOn r8Task
- commandLine ([
- "python3", "tools/create_r8lib.py",
- "--r8jar", r8Task.outputs.files[0],
- "--output", output]
- + (excldeps ? ['--excldeps-variant'] : [])
- + (debug ? ['--debug-variant'] : [])
- + (pgConfs.collectMany { ["--pg-conf", it] })
- + (libs.collectMany { ["--lib", it] })
- + (classpath.collectMany { ["--classpath", it] }))
- workingDir = projectDir
- }
-}
-
-task testJarSources(type: Jar, dependsOn: [testClasses]) {
- archiveFileName = "r8testsbase.jar"
- from sourceSets.test.output
- // We only want to include tests that use R8 when generating keep rules for applymapping.
- include "com/android/tools/r8/**"
- include "android/**"
- include "dalvik/**"
-}
-
-task testJar(type: Exec) {
- dependsOn r8WithDeps
- dependsOn testJarSources
- def output = "$buildDir/libs/r8tests.jar"
- outputs.file output
- workingDir = projectDir
- inputs.files (testJarSources.outputs.files + r8WithDeps.outputs.files)
- commandLine baseCompilerCommandLine([
- "relocator",
- "--input",
- testJarSources.outputs.files[0],
- "--output",
- output,
- "--map",
- "kotlinx.metadata.**->com.android.tools.r8.jetbrains.kotlinx.metadata"
- ])
-}
-
-task generateR8LibKeepRules(type: Exec) {
- // Depend on r8WithRelocatedDeps to ensure that we do not have external
- // dependencies crossing the boundary.
- dependsOn r8WithDeps
- dependsOn r8NoManifestWithRelocatedDeps
- dependsOn testJar
- dependsOn repackageTestDeps
- dependsOn downloadOpenJDKrt
- inputs.files ([
- r8WithDeps.outputs,
- r8NoManifestWithRelocatedDeps.outputs,
- repackageDeps.outputs,
- repackageTestDeps.outputs,
- testJar.outputs])
- outputs.file r8LibGeneratedKeepRulesPath
- commandLine baseCompilerCommandLine([
- "tracereferences",
- "--keep-rules",
- "--allowobfuscation",
- "--lib",
- "third_party/openjdk/openjdk-rt-1.8/rt.jar",
- "--lib",
- repackageDeps.outputs.files[0],
- "--lib",
- repackageTestDeps.outputs.files[0],
- "--target",
- r8NoManifestWithRelocatedDeps.outputs.files[0],
- "--source",
- testJar.outputs.files[0],
- "--output",
- r8LibGeneratedKeepRulesPath])
- workingDir = projectDir
-}
-
-task R8Lib {
- dependsOn r8LibCreateTask(
- "Main",
- ["src/main/keep.txt", generateR8LibKeepRules.outputs.files[0]],
- r8NoManifestWithRelocatedDeps,
- r8LibPath
- ).dependsOn(generateR8LibKeepRules)
- inputs.files r8NoManifestWithRelocatedDeps.outputs.files
- outputs.file r8LibPath
-}
-
-task R8LibNoDeps {
- dependsOn r8LibCreateTask(
- "MainNoDeps",
- ["src/main/keep.txt"],
- r8NoManifestWithoutDeps,
- r8LibExludeDepsPath,
- [],
- repackageDeps.outputs.files,
- true,
- ).dependsOn(repackageDeps)
- inputs.files ([r8NoManifestWithoutDeps.outputs, repackageDeps.outputs])
- outputs.file r8LibExludeDepsPath
-}
-
-task R8Desugared(type: Exec) {
- dependsOn downloadOpenJDKrt
- dependsOn r8NoManifestWithRelocatedDeps
- inputs.files r8NoManifestWithRelocatedDeps.outputs.files
- commandLine d8CfCommandLine(
- r8NoManifestWithRelocatedDeps.outputs.files[0],
- r8DesugaredPath,
- ["--release"])
- workingDir = projectDir
- outputs.file r8DesugaredPath
-}
-
-task R8Retrace {
- dependsOn R8Lib
- dependsOn r8LibCreateTask(
- "Retrace",
- ["src/main/keep_retrace.txt"],
- R8Lib,
- r8RetracePath,
- [],
- [],
- false,
- true
- ).dependsOn(R8Lib)
- outputs.file r8RetracePath
-}
-
-task R8RetraceNoDeps {
- dependsOn R8LibNoDeps
- dependsOn r8LibCreateTask(
- "RetraceNoDeps",
- ["src/main/keep_retrace.txt"],
- R8LibNoDeps,
- r8RetraceExludeDepsPath,
- [],
- repackageDeps.outputs.files,
- true,
- true,
- ).dependsOn(R8LibNoDeps)
- outputs.file r8RetraceExludeDepsPath
-}
-
-task sourceJar(type: Jar, dependsOn: classes) {
- classifier = 'src'
- from sourceSets.main.allSource
-}
-
-task keepAnnoJar(type: Jar) {
- archiveFileName = "keepanno-annotations.jar"
- from sourceSets.keepanno.output
-}
-
-artifacts {
- archives sourceJar
-}
-
-task createArtTests(type: Exec) {
- def outputDir = "build/generated/test/java/com/android/tools/r8/art"
- def createArtTestsScript = "tools/create_art_tests.py"
- inputs.files files("tests/2017-10-04/art.tar.gz", createArtTestsScript)
- outputs.dir outputDir
- dependsOn downloadDeps
- commandLine "python3", createArtTestsScript
- workingDir = projectDir
-}
-
-compileTestJava {
- dependsOn createArtTests
-}
-
-task buildCfSegments(type: Jar, dependsOn: downloadDeps) {
- from sourceSets.cfSegments.output
- archiveFileName = 'cf_segments.jar'
- destinationDir file('build/libs')
-}
-
-task buildR8ApiUsageSample(type: Jar) {
- from sourceSets.apiUsageSample.output
- archiveFileName = 'r8_api_usage_sample.jar'
- destinationDir file('tests')
-}
-
-task buildApiSampleJars {
- dependsOn buildR8ApiUsageSample
-}
-
-def buildExampleJarsCreateTask(javaVersion, sourceSet) {
- return tasks.create("buildExample${javaVersion}Jars") {
- def examplesDir = file("src/test/examples${javaVersion}")
- examplesDir.eachDir { dir ->
- def name = dir.getName();
- def exampleOutputDir = file("build/test/examples${javaVersion}");
- def jarName = "${name}.jar"
- dependsOn "jar_examples${javaVersion}_${name}"
- task "jar_examples${javaVersion}_${name}"(type: Jar) {
- archiveName = jarName
- destinationDir = exampleOutputDir
- from sourceSet.output
- include "**/" + name + "/**/*.class"
- }
- }
- }
-}
-
-buildExampleJarsCreateTask("Java9", sourceSets.examplesJava9)
-buildExampleJarsCreateTask("Java10", sourceSets.examplesJava10)
-buildExampleJarsCreateTask("Java11", sourceSets.examplesJava11)
-buildExampleJarsCreateTask("Java17", sourceSets.examplesJava17)
-buildExampleJarsCreateTask("Java20", sourceSets.examplesJava20)
-
-task buildExamples {
- if (OperatingSystem.current().isMacOsX() || OperatingSystem.current().isWindows()) {
- logger.lifecycle("WARNING: Testing (including building examples) is only partially supported on your " +
- "platform (" + OperatingSystem.current().getName() + ").")
- } else if (!OperatingSystem.current().isLinux()) {
- logger.lifecycle("WARNING: Testing (including building examples) is not supported on your platform. " +
- "It is fully supported on Linux and partially supported on Mac OS and Windows")
- return;
- }
- dependsOn buildExampleJava9Jars
- dependsOn buildExampleJava10Jars
- dependsOn buildExampleJava11Jars
- dependsOn buildExampleJava17Jars
- dependsOn buildExampleJava20Jars
-}
-
-tasks.withType(Test) {
- println("NOTE: Number of processors " + Runtime.runtime.availableProcessors())
- def userDefinedCoresPerFork = System.getenv('R8_GRADLE_CORES_PER_FORK')
- def processors = Runtime.runtime.availableProcessors()
- // See https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html.
- if (userDefinedCoresPerFork) {
- maxParallelForks = processors.intdiv(userDefinedCoresPerFork.toInteger()) ?: 1
- } else {
- // On normal work machines this seems to give the best test execution time (without freezing)
- maxParallelForks = processors.intdiv(3) ?: 1
- // On low cpu count machines (bots) we under subscribe, so increase the count.
- if (processors == 32) {
- maxParallelForks = 15
- }
- }
- println("NOTE: Max parallel forks " + maxParallelForks)
-
- forkEvery = 0
- if (project.hasProperty('disable_assertions')) {
- enableAssertions = false
- }
- // TODO(b/124091860): Increase the max heap size to avoid OOM when running tests.
- if (project.hasProperty('test_xmx')) {
- maxHeapSize = project.property('test_xmx')
- } else {
- maxHeapSize = "4G"
- }
-}
-
-task generateR8TestKeepRules {
- def path = "build/generated/r8tests-keep.txt"
- outputs.file path
- dependsOn R8Lib
- doLast {
- file(path).write """-keep class ** { *; }
--dontshrink
--dontoptimize
--keepattributes *
--applymapping ${R8Lib.outputs.files[0]}.map
-"""
- }
-}
-
-task buildR8LibCfTestDeps(type: Exec) {
- def outputPath = "build/libs/r8libtestdeps-cf.jar"
- dependsOn downloadDeps
- dependsOn r8NoManifest
- dependsOn R8Lib
- dependsOn generateR8TestKeepRules
- dependsOn testJar
- // Take all .jar files as libraries and append the generated test classes in classes/java/test.
- def addedLibraries = sourceSets.test.runtimeClasspath.findAll { pkg ->
- return pkg.toString().endsWith(".jar")
- } + ["${buildDir}/classes/java/test"]
- inputs.files testJar.outputs.files +
- generateR8TestKeepRules.outputs.files +
- R8Lib.outputs
- commandLine = r8CfCommandLine(
- testJar.outputs.files[0],
- outputPath,
- [generateR8TestKeepRules.outputs.files[0]],
- ["--debug", "--classpath", r8NoManifest.outputs.files[0]],
- r8NoManifest.outputs.files + addedLibraries)
- workingDir = projectDir
- outputs.file outputPath
-}
-
-task configureTestForR8Lib(type: Copy) {
- dependsOn testJar
- inputs.files buildR8LibCfTestDeps.outputs
- dependsOn R8Lib
- delete r8LibTestPath
- from zipTree(buildR8LibCfTestDeps.outputs.files[0])
- def examplesDir = file("build/test")
- if (examplesDir.exists()) {
- examplesDir.eachDir { dir ->
- from ("${buildDir}/test/${dir.getName()}/classes")
- }
- }
- from ("${buildDir}/runtime/examples")
- into r8LibTestPath
- outputs.dir r8LibTestPath
-}
-
-def shouldRetrace() {
- return project.hasProperty('r8lib') || project.hasProperty('r8lib_no_deps')
-}
-
-def retrace(Throwable exception) {
- def out = new StringBuffer()
- def err = new StringBuffer()
- def command = "python3 tools/retrace.py --quiet"
- def header = "RETRACED STACKTRACE";
- out.append("\n--------------------------------------\n")
- out.append("${header}\n")
- out.append("--------------------------------------\n")
- Process process = command.execute()
- def processIn = new PrintStream(process.getOut())
- process.consumeProcessOutput(out, err)
- exception.printStackTrace(processIn)
- processIn.flush()
- processIn.close()
- def errorDuringRetracing = process.waitFor() != 0
- if (errorDuringRetracing) {
- out.append("ERROR DURING RETRACING\n")
- out.append(err.toString())
- }
- if (project.hasProperty('print_obfuscated_stacktraces') || errorDuringRetracing) {
- out.append("\n\n--------------------------------------\n")
- out.append("OBFUSCATED STACKTRACE\n")
- out.append("--------------------------------------\n")
- }
- return out.toString()
-}
-
-def printStackTrace(TestResult result) {
- filterStackTraces(result)
- if (shouldRetrace()) {
- def exception = new Exception(retrace(result.exception))
- exception.setStackTrace([] as StackTraceElement[])
- result.exceptions.add(0, exception)
- }
-}
-
-def filterStackTraces(TestResult result) {
- for (Throwable throwable : result.getExceptions()) {
- filterStackTrace(throwable)
- }
-}
-
-// It would be nice to do this in a non-destructive way...
-def filterStackTrace(Throwable exception) {
- if (!project.hasProperty('print_full_stacktraces')) {
- def elements = []
- def skipped = []
- for (StackTraceElement element : exception.getStackTrace()) {
- if (element.toString().contains("com.android.tools.r8")) {
- elements.addAll(skipped)
- elements.add(element)
- skipped.clear()
- } else {
- skipped.add(element)
- }
- }
- exception.setStackTrace(elements as StackTraceElement[])
- }
-}
-
-def printAllStackTracesToFile(List<Throwable> exceptions, File out) {
- new PrintStream(new FileOutputStream(out)).withCloseable {printer ->
- exceptions.forEach { it.printStackTrace(printer) }
- }
-}
-
-static def escapeHtml(String string) {
- return string.replace("&", "&").replace("<", "<").replace(">", ">")
-}
-
-static def urlEncode(String string) {
- // Not sure why, but the + also needs to be converted to have working links.
- return URLEncoder.encode(string, "UTF-8").replace("+","%20")
-}
-
-def ensureDir(File dir) {
- dir.mkdirs()
- return dir
-}
-
-// Some of our test parameters have new lines :-( We really don't want test names to span lines.
-static def sanitizedTestName(testDesc) {
- if (testDesc.getName().contains("\n")) {
- throw new RuntimeException("Unsupported use of newline in test name: '${testDesc.getName()}'")
- }
- return testDesc.getName()
-}
-
-static def desanitizedTestName(testName) {
- return testName
-}
-
-def getTestReportEntryDir(reportDir, testDesc) {
- return ensureDir(reportDir.toPath()
- .resolve(testDesc.getClassName())
- .resolve(sanitizedTestName(testDesc))
- .toFile())
-}
-
-def getTestReportEntryURL(reportDir, testDesc) {
- def classDir = urlEncode(testDesc.getClassName())
- def testDir = urlEncode(sanitizedTestName(testDesc))
- return "file://${reportDir}/${classDir}/${testDir}"
-}
-
-def getTestResultEntryOutputFile(reportDir, testDesc, fileName) {
- def dir = getTestReportEntryDir(reportDir, testDesc).toPath()
- return dir.resolve(fileName).toFile()
-}
-
-def withTestResultEntryWriter(reportDir, testDesc, fileName, append, fn) {
- def file = getTestResultEntryOutputFile(reportDir, testDesc, fileName)
- new FileWriter(file, append).withCloseable fn
-}
-
-static def getGitBranchName() {
- def out = new StringBuilder()
- def err = new StringBuilder()
- def proc = "git rev-parse --abbrev-ref HEAD".execute()
- proc.waitForProcessOutput(out, err)
- return out.toString().trim()
-}
-
-static def getFreshTestReportIndex(File reportDir) {
- def number = 0
- while (true) {
- def freshIndex = reportDir.toPath().resolve("index.${number++}.html").toFile()
- if (!freshIndex.exists()) {
- return freshIndex
- }
- }
-}
-
-def forEachTestReportAlreadyX(File reportDir, fileName, onTest) {
- def out = new StringBuilder()
- def err = new StringBuilder()
- def proc = "find . -name ${fileName}".execute([], reportDir)
- proc.waitForProcessOutput(out, err)
- def outString = out.toString()
- outString.eachLine {
- // Lines are of the form: ./<class>/<name>/FAILURE
- def clazz = null
- def name = null
- try {
- def trimmed = it.trim()
- def line = trimmed.substring(2)
- def sep = line.indexOf("/")
- clazz = line.substring(0, sep)
- name = line.substring(sep + 1, line.length() - fileName.length() - 1)
- } catch (Exception e) {
- logger.lifecycle("WARNING: failed attempt to read test description from: '${it}'")
- return
- }
- onTest(clazz, desanitizedTestName(name))
- }
- return !outString.trim().isEmpty()
-}
-
-def forEachTestReportAlreadyFailing(File reportDir, onFailureTest) {
- return forEachTestReportAlreadyX(reportDir, TestResult.ResultType.FAILURE.name(), onFailureTest)
-}
-
-def forEachTestReportAlreadyPassing(File reportDir, onSucceededTest) {
- return forEachTestReportAlreadyX(reportDir, TestResult.ResultType.SUCCESS.name(), onSucceededTest)
-}
-
-def forEachTestReportAlreadySkipped(File reportDir, onSucceededTest) {
- return forEachTestReportAlreadyX(reportDir, TestResult.ResultType.SKIPPED.name(), onSucceededTest)
-}
-
-def setUpTestingState(Test task) {
- // Hide all test events from the console, they are written to the report.
- task.testLogging { events = [] }
-
- def branch = project.hasProperty('testing-state-name')
- ? project.getProperty('testing-state-name')
- : getGitBranchName()
- def reportDir = file("${buildDir}/test-state/${branch}")
- def index = reportDir.toPath().resolve("index.html").toFile()
- def resetState = project.hasProperty('reset-testing-state')
- def reportDirExists = reportDir.exists()
- def resuming = !resetState && reportDirExists
-
- def hasFailingTests = false;
- if (resuming) {
- // Test filtering happens before the test execution is initiated so compute it here.
- // If there are still failing tests in the report, include only those.
- hasFailingTests = forEachTestReportAlreadyFailing(reportDir, {
- clazz, name -> task.filter.includeTestsMatching("$clazz.$name")
- })
- // Otherwise exclude all of the test already marked as succeeding.
- if (!hasFailingTests) {
- // Also allow the test to overall succeed if there are no remaining tests that match,
- // which is natural if the state already succeeded in full.
- task.filter.failOnNoMatchingTests = false
- forEachTestReportAlreadyPassing(reportDir, {
- clazz, name -> task.filter.excludeTestsMatching("$clazz.$name")
- })
- forEachTestReportAlreadySkipped(reportDir, {
- clazz, name -> task.filter.excludeTestsMatching("$clazz.$name")
- })
- }
- }
-
- task.beforeSuite { desc ->
- if (!desc.parent) {
- def parentReport = null
- if (resetState && reportDirExists) {
- delete reportDir
- }
- if (resuming) {
- if (index.exists()) {
- parentReport = getFreshTestReportIndex(reportDir)
- index.renameTo(parentReport)
- }
- } else {
- reportDir.mkdirs()
- }
- def runPrefix = resuming ? "Resuming" : "Starting"
- def title = "${runPrefix} @ ${branch}"
- // Print a console link to the test report for easy access.
- println "${runPrefix} test, report written to:"
- println " file://${index}"
- // Print the new index content.
- index << "<html><head><title>${title}</title>"
- index << "<style> * { font-family: monospace; }</style>"
- index << "<meta http-equiv='refresh' content='10' />"
- index << "</head><body><h1>${title}</h1>"
- index << "<p>Run on: ${new Date()}</p>"
- index << "<p>Git branch: ${branch}</p>"
- if (parentReport != null) {
- index << "<p><a href=\"file://${parentReport}\">Previous result index</a></p>"
- }
- index << "<p><a href=\"file://${index}\">Most recent result index</a></p>"
- index << "<p><a href=\"file://${reportDir}\">Test directories</a></p>"
- index << "<h2>Failing tests (refreshing automatically every 10 seconds)</h2><ul>"
- }
- }
-
- task.afterSuite { desc, result ->
- if (!desc.parent) {
- // Update the final test results in the index.
- index << "</ul>"
- if (result.resultType == TestResult.ResultType.SUCCESS) {
- if (hasFailingTests) {
- index << "<h2>Rerun of failed tests now pass!</h2>"
- index << "<h2>Rerun again to continue with outstanding tests!</h2>"
- } else {
- index << "<h2 style=\"background-color:#62D856\">GREEN BAR == YOU ROCK!</h2>"
- }
- } else if (result.resultType == TestResult.ResultType.FAILURE) {
- index << "<h2 style=\"background-color:#6D130A\">Some tests failed: ${result.resultType.name()}</h2><ul>"
- } else {
- index << "<h2>Tests finished: ${result.resultType.name()}</h2><ul>"
- }
- index << "<li>Number of tests: ${result.testCount}"
- index << "<li>Failing tests: ${result.failedTestCount}"
- index << "<li>Successful tests: ${result.successfulTestCount}"
- index << "<li>Skipped tests: ${result.skippedTestCount}"
- index << "</ul></body></html>"
- }
- }
-
- // Events to stdout/err are appended to the files in the test directories.
- task.onOutput { desc, event ->
- withTestResultEntryWriter(reportDir, desc, event.getDestination().name(), true, {
- it.append(event.getMessage())
- })
- }
-
- task.beforeTest { desc ->
- // Remove any stale output files before running the test.
- for (def destType : TestOutputEvent.Destination.values()) {
- def destFile = getTestResultEntryOutputFile(reportDir, desc, destType.name())
- if (destFile.exists()) {
- delete destFile
- }
- }
- }
-
- task.afterTest { desc, result ->
- if (result.getTestCount() != 1) {
- throw new IllegalStateException("Unexpected test with more than one result: ${desc}")
- }
- // Clear any previous result files.
- for (def resultType : TestResult.ResultType.values()) {
- delete getTestResultEntryOutputFile(reportDir, desc, resultType.name())
- }
- // Emit the result type status in a file of the same name: SUCCESS, FAILURE or SKIPPED.
- withTestResultEntryWriter(reportDir, desc, result.getResultType().name(), false, {
- it.append(result.getResultType().name())
- })
- // Emit the test time.
- withTestResultEntryWriter(reportDir, desc, "time", false, {
- it.append("${result.getEndTime() - result.getStartTime()}")
- })
- // For failed tests, update the index and emit stack trace information.
- if (result.resultType == TestResult.ResultType.FAILURE) {
- def title = escapeHtml("${desc.className}.${desc.name}")
- def link = getTestReportEntryURL(reportDir, desc)
- index << "<li><a href=\"${link}\">${title}</a></li>"
- if (!result.exceptions.isEmpty()) {
- printAllStackTracesToFile(
- result.exceptions,
- getTestResultEntryOutputFile(
- reportDir,
- desc,
- "exceptions-raw.txt"))
- filterStackTraces(result)
- printAllStackTracesToFile(
- result.exceptions,
- getTestResultEntryOutputFile(
- reportDir,
- desc,
- "exceptions-filtered.txt"))
- if (shouldRetrace()) {
- withTestResultEntryWriter(reportDir, desc, "exceptions-retraced.txt", false, { writer ->
- result.exceptions.forEach { writer.append(retrace(it)) }
- })
- }
- }
- }
- }
-}
-
-def testTimes = [:]
-def numberOfTestTimesToPrint = 100
-
-test { task ->
-
- dependsOn sourceSets.keepanno.output
- // R8.jar is required for running bootstrap tests.
- dependsOn r8
-
- def useTestingState = project.hasProperty('testing-state')
- if (useTestingState) {
- setUpTestingState(task)
- }
-
- if (project.hasProperty('generate_golden_files_to')) {
- systemProperty 'generate_golden_files_to', project.property('generate_golden_files_to')
- assert project.hasProperty('HEAD_sha1')
- systemProperty 'test_git_HEAD_sha1', project.property('HEAD_sha1')
- }
-
- if (project.hasProperty('use_golden_files_in')) {
- systemProperty 'use_golden_files_in', project.property('use_golden_files_in')
- assert project.hasProperty('HEAD_sha1')
- systemProperty 'test_git_HEAD_sha1', project.property('HEAD_sha1')
- }
-
- if (project.hasProperty('kotlin_compiler_dev')) {
- systemProperty 'com.android.tools.r8.kotlincompilerdev', '1';
- }
-
- if (project.hasProperty('kotlin_compiler_old')) {
- systemProperty 'com.android.tools.r8.kotlincompilerold', '1';
- }
-
- if (!useTestingState) {
- testLogging.exceptionFormat = 'full'
- if (project.hasProperty('print_test_stdout')) {
- testLogging.showStandardStreams = true
- }
- }
- if (project.hasProperty('dex_vm') && project.property('dex_vm') != 'default') {
- println "NOTE: Running with non default vm: " + project.property('dex_vm')
- systemProperty 'dex_vm', project.property('dex_vm')
- }
-
- // Forward runtime configurations for test parameters.
- if (project.hasProperty('runtimes')) {
- println "NOTE: Running with runtimes: " + project.property('runtimes')
- systemProperty 'runtimes', project.property('runtimes')
- }
-
- if (project.hasProperty('art_profile_rewriting_completeness_check')) {
- String key = 'com.android.tools.r8.artprofilerewritingcompletenesscheck'
- String value = project.property('art_profile_rewriting_completeness_check')
- systemProperty key, value
- }
-
- if (project.hasProperty('slow_tests')) {
- systemProperty 'slow_tests', project.property('slow_tests')
- }
-
-
- if (project.hasProperty('desugar_jdk_json_dir')) {
- systemProperty 'desugar_jdk_json_dir', project.property('desugar_jdk_json_dir')
- }
- if (project.hasProperty('desugar_jdk_libs')) {
- systemProperty 'desugar_jdk_libs', project.property('desugar_jdk_libs')
- }
-
- if (!useTestingState) {
- if (project.hasProperty('print_times') || project.hasProperty('one_line_per_test')) {
- afterTest { desc, result ->
- def executionTime = (result.endTime - result.startTime) / 1000
- testTimes["${desc.name} [${desc.className}]"] = executionTime
- }
- afterSuite { desc, result ->
- // parent is null if all tests are done.
- if (desc.parent == null) {
- def sortedTimes = testTimes.sort({ e1, e2 -> e2.value <=> e1.value })
- sortedTimes.eachWithIndex { key, value, i ->
- println "$key: $value"
- }
- }
- }
- }
-
- if (project.hasProperty('one_line_per_test')) {
- beforeTest { desc ->
- println "Start executing test ${desc.name} [${desc.className}]"
- }
-
- afterTest { desc, result ->
- if (result.resultType == TestResult.ResultType.FAILURE) {
- printStackTrace(result)
- }
- if (project.hasProperty('update_test_timestamp')) {
- file(project.getProperty('update_test_timestamp')).text = new Date().getTime()
- }
- println "Done executing test ${desc.name} [${desc.className}] with result: ${result.resultType}"
- }
- } else {
- afterTest { desc, result ->
- if (result.resultType == TestResult.ResultType.FAILURE) {
- printStackTrace(result)
- }
- }
- }
- }
- if (project.hasProperty('no_internal')) {
- exclude "com/android/tools/r8/internal/**"
- }
- if (project.hasProperty('only_internal')) {
- include "com/android/tools/r8/internal/**"
- }
-
- if (project.hasProperty('test_namespace')) {
- include "com/android/tools/r8/" + project.getProperty('test_namespace') + "/**"
- }
-
- if (project.hasProperty('tool') && project.property('tool') == 'd8') {
- // Don't run anything, deprecated
- println "Running with deprecated tool d8, not running any tests"
- include ""
- }
- if (project.hasProperty('no_arttests')) {
- exclude "com/android/tools/r8/art/**"
- }
- if (project.hasProperty('shard_count') ) {
- assert project.hasProperty('shard_number')
- int shard_count = project.getProperty('shard_count') as Integer
- int shard_number = project.getProperty('shard_number') as Integer
- assert shard_count < 65536
- assert shard_number < shard_count
- exclude {
- entry ->
- // Don't leave out directories. Leaving out a directory means all entries below.
- if (entry.file.isDirectory()) {
- return false
- }
- def first4 = entry.getRelativePath().toString().md5().substring(0, 4)
- int hash = Integer.parseInt(first4, 16)
- return hash % shard_count != shard_number
- }
- }
- if (project.hasProperty('test_dir')) {
- systemProperty 'test_dir', project.property('test_dir')
- }
- if (project.hasProperty('command_cache_dir')) {
- systemProperty 'command_cache_dir', project.property('command_cache_dir')
- }
- if (project.hasProperty('command_cache_stats_dir')) {
- systemProperty 'command_cache_stats_dir', project.property('command_cache_stats_dir')
- }
- if (project.hasProperty('r8lib')) {
- dependsOn configureTestForR8Lib
- // R8lib should be used instead of the main output and all the tests in
- // r8 should be mapped and exists in r8LibTestPath.
- classpath = sourceSets.test.runtimeClasspath.filter {
- !it.getAbsolutePath().contains("/build/")
- }
- classpath += files([r8LibPath, r8LibTestPath])
- testClassesDirs = files(r8LibTestPath)
- }
- if (OperatingSystem.current().isLinux()
- || OperatingSystem.current().isMacOsX()
- || OperatingSystem.current().isWindows()) {
- if (OperatingSystem.current().isMacOsX()) {
- logger.lifecycle("WARNING: Testing in only partially supported on Mac OS. " +
- "Art only runs on Linux and tests requiring Art runs in a Docker container, which must be present. " +
- "See tools/docker/README.md for details.")
- }
- if (OperatingSystem.current().isWindows()) {
- logger.lifecycle("WARNING: Testing in only partially supported on Windows. " +
- "Art only runs on Linux and tests requiring Art will be skipped")
- }
- dependsOn downloadDeps
- dependsOn buildExamples
- } else {
- logger.lifecycle("WARNING: Testing in not supported on your platform. Testing is only fully supported on " +
- "Linux and partially supported on Mac OS and Windows. Art does not run on other platforms.")
- }
-}
-
-
-task javadocD8(type: Javadoc) {
- title "D8 API"
- classpath = sourceSets.main.compileClasspath
- source = sourceSets.main.allJava
- include '**/com/android/tools/r8/ArchiveClassFileProvider.java'
- include '**/com/android/tools/r8/ArchiveProgramResourceProvider.java'
- include '**/com/android/tools/r8/BaseCommand.java'
- include '**/com/android/tools/r8/BaseCompilerCommand.java'
- include '**/com/android/tools/r8/ClassFileResourceProvider.java'
- include '**/com/android/tools/r8/CompilationFailedException.java'
- include '**/com/android/tools/r8/CompilationMode.java'
- include '**/com/android/tools/r8/D8.java'
- include '**/com/android/tools/r8/D8Command.java'
- include '**/com/android/tools/r8/DexIndexedConsumer.java'
- include '**/com/android/tools/r8/DexFilePerClassFileConsumer.java'
- include '**/com/android/tools/r8/Diagnostic.java'
- include '**/com/android/tools/r8/DiagnosticsHandler.java'
- include '**/com/android/tools/r8/DirectoryClassFileProvider.java'
- include '**/com/android/tools/r8/OutputMode.java'
- include '**/com/android/tools/r8/ProgramConsumer.java'
- include '**/com/android/tools/r8/ProgramResource.java'
- include '**/com/android/tools/r8/ProgramResourceProvider.java'
- include '**/com/android/tools/r8/Resource.java'
- include '**/com/android/tools/r8/ResourceException.java'
- include '**/com/android/tools/r8/StringConsumer.java'
- include '**/com/android/tools/r8/StringResource.java'
- include '**/com/android/tools/r8/Version.java'
- include '**/com/android/tools/r8/origin/*.java'
-}
-
-task javadocR8(type: Javadoc) {
- title "R8 API"
- classpath = sourceSets.main.compileClasspath
- source = sourceSets.main.allJava
- include '**/com/android/tools/r8/ArchiveClassFileProvider.java'
- include '**/com/android/tools/r8/ArchiveProgramResourceProvider.java'
- include '**/com/android/tools/r8/BaseCommand.java'
- include '**/com/android/tools/r8/BaseCompilerCommand.java'
- include '**/com/android/tools/r8/ClassFileConsumer.java'
- include '**/com/android/tools/r8/ClassFileResourceProvider.java'
- include '**/com/android/tools/r8/CompilationFailedException.java'
- include '**/com/android/tools/r8/CompilationMode.java'
- include '**/com/android/tools/r8/R8.java'
- include '**/com/android/tools/r8/R8Command.java'
- include '**/com/android/tools/r8/DexIndexedConsumer.java'
- include '**/com/android/tools/r8/Diagnostic.java'
- include '**/com/android/tools/r8/DiagnosticsHandler.java'
- include '**/com/android/tools/r8/DirectoryClassFileProvider.java'
- include '**/com/android/tools/r8/OutputMode.java'
- include '**/com/android/tools/r8/ProgramConsumer.java'
- include '**/com/android/tools/r8/ProgramResource.java'
- include '**/com/android/tools/r8/ProgramResourceProvider.java'
- include '**/com/android/tools/r8/Resource.java'
- include '**/com/android/tools/r8/ResourceException.java'
- include '**/com/android/tools/r8/StringConsumer.java'
- include '**/com/android/tools/r8/StringResource.java'
- include '**/com/android/tools/r8/Version.java'
- include '**/com/android/tools/r8/origin/*.java'
-}
-
-task copyMavenDeps(type: Copy) {
- from configurations.compile into "$buildDir/deps"
- from configurations.compileClasspath into "$buildDir/deps"
- from configurations.testCompile into "$buildDir/deps"
-}
-
-task printMavenDeps {
- // Only actually print to stdout when we are updating.
- if (project.hasProperty('updatemavendeps')) {
- for (Configuration config : configurations) {
- if (!config.isCanBeResolved()) {
- continue
- }
- def componentIds = config.incoming.resolutionResult.allDependencies.collect {
- it.selected.id
- }
- def result = dependencies.createArtifactResolutionQuery()
- .forComponents(componentIds)
- .withArtifacts(MavenModule, MavenPomArtifact)
- .execute()
- for (component in result.resolvedComponents) {
- component.getArtifacts(MavenPomArtifact).each {
- println "POM: ${it.file} ${component.id}"
- }
- }
- config.each {
- println "JAR: ${it}"
- }
- }
- }
-}
-
-allprojects {
- tasks.withType(Exec) {
- doFirst {
- println commandLine.join(' ')
- }
- }
-}
diff --git a/settings.gradle b/settings.gradle
deleted file mode 100644
index 969cb9c..0000000
--- a/settings.gradle
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-import org.gradle.internal.os.OperatingSystem
-
-rootProject.name = 'r8'
-
-// Bootstrap building by downloading dependencies.
-def dependencies_bucket = "r8-deps"
-def dependencies_sha1_file = "third_party/dependencies.tar.gz.sha1"
-def extension = OperatingSystem.current().isWindows() ? ".bat" : ".py"
-
-def cmd =
- ("download_from_google_storage${extension} --extract"
- + " --bucket ${dependencies_bucket}"
- + " --sha1_file ${dependencies_sha1_file}")
-def process = cmd.execute()
-process.waitFor()
-if (process.exitValue() != 0) {
- throw new GradleException(
- "Bootstrapping dependencies download failed:\n"
- + "STDOUT:\n${process.in.text}\n"
- + "STDERR:\n${process.err.text}")
-}
\ No newline at end of file
diff --git a/src/main/java/com/android/tools/r8/GlobalSyntheticsGenerator.java b/src/main/java/com/android/tools/r8/GlobalSyntheticsGenerator.java
index fc2188f..41f4db1 100644
--- a/src/main/java/com/android/tools/r8/GlobalSyntheticsGenerator.java
+++ b/src/main/java/com/android/tools/r8/GlobalSyntheticsGenerator.java
@@ -127,6 +127,8 @@
createGlobalSynthetics(appView, timing, executorService);
timing.end();
+ assert GlobalSyntheticsGeneratorVerifier.verifyExpectedClassesArePresent(appView);
+
ApplicationWriter.create(appView, options.getMarker()).write(executorService, app);
} catch (ExecutionException e) {
throw unwrapExecutionException(e);
diff --git a/src/main/java/com/android/tools/r8/GlobalSyntheticsGeneratorVerifier.java b/src/main/java/com/android/tools/r8/GlobalSyntheticsGeneratorVerifier.java
new file mode 100644
index 0000000..c0197ae
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/GlobalSyntheticsGeneratorVerifier.java
@@ -0,0 +1,30 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8;
+
+import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.DexItemFactory;
+import com.android.tools.r8.graph.DexType;
+import java.util.function.Consumer;
+
+public class GlobalSyntheticsGeneratorVerifier {
+
+ public static void forEachExpectedClass(
+ DexItemFactory dexItemFactory, int minApi, Consumer<DexType> consumer) {
+ consumer.accept(dexItemFactory.methodHandlesType);
+ consumer.accept(dexItemFactory.methodHandlesLookupType);
+ consumer.accept(dexItemFactory.recordType);
+ consumer.accept(dexItemFactory.varHandleType);
+ }
+
+ public static boolean verifyExpectedClassesArePresent(AppView<?> appView) {
+ forEachExpectedClass(
+ appView.dexItemFactory(),
+ appView.options().getMinApiLevel().getLevel(),
+ type -> {
+ assert appView.hasDefinitionFor(type);
+ });
+ return true;
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/ResourceShrinker.java b/src/main/java/com/android/tools/r8/ResourceShrinker.java
index ee1dd76..f23dddc 100644
--- a/src/main/java/com/android/tools/r8/ResourceShrinker.java
+++ b/src/main/java/com/android/tools/r8/ResourceShrinker.java
@@ -476,10 +476,14 @@
@SuppressWarnings("RedundantThrows")
public static void run(Command command, ReferenceChecker callback)
throws IOException, ExecutionException {
- AndroidApp inputApp = command.getInputApp();
+ runForTesting(command.getInputApp(), command.getInternalOptions(), callback);
+ }
+
+ public static void runForTesting(
+ AndroidApp inputApp, InternalOptions options, ReferenceChecker callback)
+ throws IOException, ExecutionException {
Timing timing = new Timing("resource shrinker analyzer");
- DexApplication dexApplication =
- new ApplicationReader(inputApp, command.getInternalOptions(), timing).read();
+ DexApplication dexApplication = new ApplicationReader(inputApp, options, timing).read();
for (DexProgramClass programClass : dexApplication.classes()) {
new DexClassUsageVisitor(programClass, callback).visit();
}
diff --git a/src/main/java/com/android/tools/r8/dex/ApplicationReader.java b/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
index d18a979..6ccf2e6 100644
--- a/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
+++ b/src/main/java/com/android/tools/r8/dex/ApplicationReader.java
@@ -345,6 +345,13 @@
}
}
if (!options.testing.dexContainerExperiment) {
+ if (dexReader.getDexVersion().isContainerDex()) {
+ throw new ResourceException(
+ input.getOrigin(),
+ "Experimental container DEX version "
+ + dexReader.getDexVersion()
+ + " is not supported");
+ }
dexParsers.add(new DexParser<>(dexReader, PROGRAM, options));
} else {
addDexParsersForContainer(dexParsers, dexReader);
diff --git a/src/main/java/com/android/tools/r8/graph/AppInfo.java b/src/main/java/com/android/tools/r8/graph/AppInfo.java
index b02bb19..07512e5 100644
--- a/src/main/java/com/android/tools/r8/graph/AppInfo.java
+++ b/src/main/java/com/android/tools/r8/graph/AppInfo.java
@@ -226,14 +226,13 @@
* @param context the method the invoke is contained in, i.e., the caller.
* @return The actual target for {@code method} if on the holder, or {@code null}.
*/
- @SuppressWarnings("ReferenceEquality")
- public final DexEncodedMethod lookupStaticTargetOnItself(
+ public final DexClassAndMethod lookupStaticTargetOnItself(
DexMethod method, ProgramMethod context) {
- if (method.holder != context.getHolderType()) {
+ if (!method.getHolderType().isIdenticalTo(context.getHolderType())) {
return null;
}
- DexEncodedMethod singleTarget = context.getHolder().lookupDirectMethod(method);
- if (singleTarget != null && singleTarget.isStatic()) {
+ DexClassAndMethod singleTarget = context.getHolder().lookupDirectClassMethod(method);
+ if (singleTarget != null && singleTarget.getAccessFlags().isStatic()) {
return singleTarget;
}
return null;
@@ -246,14 +245,13 @@
* @param context the method the invoke is contained in, i.e., the caller.
* @return The actual target for {@code method} if on the holder, or {@code null}.
*/
- @SuppressWarnings("ReferenceEquality")
- public final DexEncodedMethod lookupDirectTargetOnItself(
+ public final DexClassAndMethod lookupDirectTargetOnItself(
DexMethod method, ProgramMethod context) {
- if (method.holder != context.getHolderType()) {
+ if (!method.getHolderType().isIdenticalTo(context.getHolderType())) {
return null;
}
- DexEncodedMethod singleTarget = context.getHolder().lookupDirectMethod(method);
- if (singleTarget != null && !singleTarget.isStatic()) {
+ DexClassAndMethod singleTarget = context.getHolder().lookupDirectClassMethod(method);
+ if (singleTarget != null && !singleTarget.getAccessFlags().isStatic()) {
return singleTarget;
}
return null;
diff --git a/src/main/java/com/android/tools/r8/graph/AppInfoWithClassHierarchy.java b/src/main/java/com/android/tools/r8/graph/AppInfoWithClassHierarchy.java
index fcfb3cf..d121559 100644
--- a/src/main/java/com/android/tools/r8/graph/AppInfoWithClassHierarchy.java
+++ b/src/main/java/com/android/tools/r8/graph/AppInfoWithClassHierarchy.java
@@ -627,14 +627,14 @@
* @return The actual target for {@code method} or {@code null} if none found.
*/
// TODO(b/155968472): This should take a parameter `boolean isInterface` and use resolveMethod().
- public DexEncodedMethod lookupStaticTarget(
+ public DexClassAndMethod lookupStaticTarget(
DexMethod method,
DexProgramClass context,
AppView<? extends AppInfoWithClassHierarchy> appView) {
return lookupStaticTarget(method, context, appView, appView.appInfo());
}
- public DexEncodedMethod lookupStaticTarget(
+ public DexClassAndMethod lookupStaticTarget(
DexMethod method,
DexProgramClass context,
AppView<?> appView,
@@ -645,14 +645,14 @@
}
// TODO(b/155968472): This should take a parameter `boolean isInterface` and use resolveMethod().
- public DexEncodedMethod lookupStaticTarget(
+ public DexClassAndMethod lookupStaticTarget(
DexMethod method,
ProgramMethod context,
AppView<? extends AppInfoWithClassHierarchy> appView) {
return lookupStaticTarget(method, context.getHolder(), appView);
}
- public DexEncodedMethod lookupStaticTarget(
+ public DexClassAndMethod lookupStaticTarget(
DexMethod method,
ProgramMethod context,
AppView<?> appView,
@@ -713,14 +713,14 @@
* @return The actual target for {@code method} or {@code null} if none found.
*/
// TODO(b/155968472): This should take a parameter `boolean isInterface` and use resolveMethod().
- public DexEncodedMethod lookupDirectTarget(
+ public DexClassAndMethod lookupDirectTarget(
DexMethod method,
DexProgramClass context,
AppView<? extends AppInfoWithClassHierarchy> appView) {
return lookupDirectTarget(method, context, appView, appView.appInfo());
}
- public DexEncodedMethod lookupDirectTarget(
+ public DexClassAndMethod lookupDirectTarget(
DexMethod method,
DexProgramClass context,
AppView<?> appView,
@@ -731,14 +731,14 @@
}
// TODO(b/155968472): This should take a parameter `boolean isInterface` and use resolveMethod().
- public DexEncodedMethod lookupDirectTarget(
+ public DexClassAndMethod lookupDirectTarget(
DexMethod method,
ProgramMethod context,
AppView<? extends AppInfoWithClassHierarchy> appView) {
return lookupDirectTarget(method, context, appView, appView.appInfo());
}
- public DexEncodedMethod lookupDirectTarget(
+ public DexClassAndMethod lookupDirectTarget(
DexMethod method,
ProgramMethod context,
AppView<?> appView,
diff --git a/src/main/java/com/android/tools/r8/graph/AppView.java b/src/main/java/com/android/tools/r8/graph/AppView.java
index a5fe760..af882b2 100644
--- a/src/main/java/com/android/tools/r8/graph/AppView.java
+++ b/src/main/java/com/android/tools/r8/graph/AppView.java
@@ -38,6 +38,7 @@
import com.android.tools.r8.naming.NamingLens;
import com.android.tools.r8.naming.SeedMapper;
import com.android.tools.r8.optimize.argumentpropagation.ArgumentPropagator;
+import com.android.tools.r8.optimize.compose.ComposeReferences;
import com.android.tools.r8.optimize.interfaces.collection.OpenClosedInterfacesCollection;
import com.android.tools.r8.profile.art.ArtProfileCollection;
import com.android.tools.r8.profile.startup.profile.StartupProfile;
@@ -105,7 +106,9 @@
// Currently however the liveness may be downgraded thus loosing the computed keep info.
private KeepInfoCollection keepInfo = null;
- private final AbstractValueFactory abstractValueFactory = new AbstractValueFactory();
+ private ComposeReferences composeReferences = null;
+
+ private final AbstractValueFactory abstractValueFactory;
private final AbstractValueConstantPropagationJoiner abstractValueConstantPropagationJoiner;
private final AbstractValueFieldJoiner abstractValueFieldJoiner;
private final AbstractValueParameterJoiner abstractValueParameterJoiner;
@@ -180,6 +183,7 @@
timing.time(
"Compilation context", () -> CompilationContext.createInitialContext(options()));
this.wholeProgramOptimizations = wholeProgramOptimizations;
+ abstractValueFactory = new AbstractValueFactory(options());
abstractValueConstantPropagationJoiner = new AbstractValueConstantPropagationJoiner(this);
if (enableWholeProgramOptimizations()) {
abstractValueFieldJoiner = new AbstractValueFieldJoiner(withClassHierarchy());
@@ -510,6 +514,14 @@
return appInfo.dexItemFactory();
}
+ public ComposeReferences getComposeReferences() {
+ assert testing().modelUnknownChangedAndDefaultArgumentsToComposableFunctions;
+ if (composeReferences == null) {
+ composeReferences = new ComposeReferences(dexItemFactory());
+ }
+ return composeReferences;
+ }
+
public boolean enableWholeProgramOptimizations() {
return wholeProgramOptimizations == WholeProgramOptimizations.ON;
}
diff --git a/src/main/java/com/android/tools/r8/graph/DexAnnotation.java b/src/main/java/com/android/tools/r8/graph/DexAnnotation.java
index 78beee1..efe3ead 100644
--- a/src/main/java/com/android/tools/r8/graph/DexAnnotation.java
+++ b/src/main/java/com/android/tools/r8/graph/DexAnnotation.java
@@ -51,6 +51,10 @@
return reference.apply(type -> TYPE, field -> FIELD, method -> METHOD);
}
+ public boolean isMethod() {
+ return this == METHOD;
+ }
+
public boolean isParameter() {
return this == PARAMETER;
}
diff --git a/src/main/java/com/android/tools/r8/graph/DexClass.java b/src/main/java/com/android/tools/r8/graph/DexClass.java
index 263335f..e8c908a 100644
--- a/src/main/java/com/android/tools/r8/graph/DexClass.java
+++ b/src/main/java/com/android/tools/r8/graph/DexClass.java
@@ -499,6 +499,10 @@
return field != null ? DexClassAndField.create(this, field) : null;
}
+ public DexClassAndMethod lookupDirectClassMethod(DexMethod method) {
+ return toClassMethodOrNull(lookupDirectMethod(method));
+ }
+
/** Find direct method in this class matching {@param method}. */
public DexEncodedMethod lookupDirectMethod(DexMethod method) {
return methodCollection.getDirectMethod(method);
@@ -509,6 +513,10 @@
return methodCollection.getDirectMethod(predicate);
}
+ public DexClassAndMethod lookupVirtualClassMethod(DexMethod method) {
+ return toClassMethodOrNull(lookupVirtualMethod(method));
+ }
+
/** Find virtual method in this class matching {@param method}. */
public DexEncodedMethod lookupVirtualMethod(DexMethod method) {
return methodCollection.getVirtualMethod(method);
diff --git a/src/main/java/com/android/tools/r8/graph/DexClassAndMethod.java b/src/main/java/com/android/tools/r8/graph/DexClassAndMethod.java
index 4af04cb..f46fc2b 100644
--- a/src/main/java/com/android/tools/r8/graph/DexClassAndMethod.java
+++ b/src/main/java/com/android/tools/r8/graph/DexClassAndMethod.java
@@ -38,7 +38,9 @@
@SuppressWarnings("ReferenceEquality")
public boolean isStructurallyEqualTo(DexClassAndMethod other) {
- return getDefinition() == other.getDefinition() && getHolder() == other.getHolder();
+ return other != null
+ && getDefinition() == other.getDefinition()
+ && getHolder() == other.getHolder();
}
@Override
diff --git a/src/main/java/com/android/tools/r8/graph/DexItemFactory.java b/src/main/java/com/android/tools/r8/graph/DexItemFactory.java
index 6e0111c..75f9968 100644
--- a/src/main/java/com/android/tools/r8/graph/DexItemFactory.java
+++ b/src/main/java/com/android/tools/r8/graph/DexItemFactory.java
@@ -990,6 +990,7 @@
objectsMethods.requireNonNullWithMessage,
objectsMethods.requireNonNullWithMessageSupplier,
stringMembers.valueOf)
+ .addAll(boxedValueOfMethods())
.addAll(stringBufferMethods.appendMethods)
.addAll(stringBuilderMethods.appendMethods)
.build();
diff --git a/src/main/java/com/android/tools/r8/graph/DispatchTargetLookupResult.java b/src/main/java/com/android/tools/r8/graph/DispatchTargetLookupResult.java
new file mode 100644
index 0000000..0190e2c
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/graph/DispatchTargetLookupResult.java
@@ -0,0 +1,41 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.graph;
+
+import com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult;
+
+public abstract class DispatchTargetLookupResult {
+
+ private final SingleResolutionResult<?> singleResolutionResult;
+
+ DispatchTargetLookupResult(SingleResolutionResult<?> singleResolutionResult) {
+ assert singleResolutionResult != null;
+ this.singleResolutionResult = singleResolutionResult;
+ }
+
+ public static DispatchTargetLookupResult create(
+ DexClassAndMethod singleDispatchTarget, SingleResolutionResult<?> singleResolutionResult) {
+ if (singleDispatchTarget != null) {
+ return new SingleDispatchTargetLookupResult(singleDispatchTarget, singleResolutionResult);
+ } else {
+ return new UnknownDispatchTargetLookupResult(singleResolutionResult);
+ }
+ }
+
+ public SingleResolutionResult<?> getResolutionResult() {
+ return singleResolutionResult;
+ }
+
+ public DexClassAndMethod getSingleDispatchTarget() {
+ return null;
+ }
+
+ public boolean isSingleResult() {
+ return false;
+ }
+
+ public SingleDispatchTargetLookupResult asSingleResult() {
+ return null;
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/graph/MethodResolutionResult.java b/src/main/java/com/android/tools/r8/graph/MethodResolutionResult.java
index 428c57d..6dbdc35 100644
--- a/src/main/java/com/android/tools/r8/graph/MethodResolutionResult.java
+++ b/src/main/java/com/android/tools/r8/graph/MethodResolutionResult.java
@@ -8,7 +8,12 @@
import com.android.tools.r8.errors.Unreachable;
import com.android.tools.r8.graph.LookupResult.LookupResultSuccess;
import com.android.tools.r8.graph.LookupResult.LookupResultSuccess.LookupResultCollectionState;
+import com.android.tools.r8.ir.analysis.type.DynamicType;
+import com.android.tools.r8.ir.code.InvokeDirect;
import com.android.tools.r8.ir.code.InvokeMethod;
+import com.android.tools.r8.ir.code.InvokeMethodWithReceiver;
+import com.android.tools.r8.ir.code.InvokeStatic;
+import com.android.tools.r8.ir.code.InvokeSuper;
import com.android.tools.r8.ir.desugar.LambdaDescriptor;
import com.android.tools.r8.ir.optimize.info.DefaultMethodOptimizationInfo;
import com.android.tools.r8.ir.optimize.info.MethodOptimizationInfo;
@@ -34,6 +39,10 @@
public abstract class MethodResolutionResult
extends MemberResolutionResult<DexEncodedMethod, DexMethod> {
+ public static UnknownMethodResolutionResult unknown() {
+ return UnknownMethodResolutionResult.get();
+ }
+
@Override
public boolean isMethodResolutionResult() {
return true;
@@ -136,7 +145,13 @@
return null;
}
- /** Short-hand to get the single resolution method if resolution finds it, null otherwise. */
+ /**
+ * Short-hand to get the single resolution method if resolution finds it, null otherwise.
+ *
+ * @deprecated Use {@link #getResolvedMethod()}.
+ */
+ @Deprecated
+ @SuppressWarnings("InlineMeSuggester")
public final DexEncodedMethod getSingleTarget() {
return isSingleResolution() ? asSingleResolution().getResolvedMethod() : null;
}
@@ -173,21 +188,21 @@
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo);
/** Lookup the single target of an invoke-direct on this resolution result if possible. */
- public final DexEncodedMethod lookupInvokeDirectTarget(
+ public final DexClassAndMethod lookupInvokeDirectTarget(
DexProgramClass context, AppView<? extends AppInfoWithClassHierarchy> appView) {
return lookupInvokeDirectTarget(context, appView, appView.appInfo());
}
- public abstract DexEncodedMethod lookupInvokeDirectTarget(
+ public abstract DexClassAndMethod lookupInvokeDirectTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo);
/** Lookup the single target of an invoke-static on this resolution result if possible. */
- public final DexEncodedMethod lookupInvokeStaticTarget(
+ public final DexClassAndMethod lookupInvokeStaticTarget(
DexProgramClass context, AppView<? extends AppInfoWithClassHierarchy> appView) {
return lookupInvokeStaticTarget(context, appView, appView.appInfo());
}
- public abstract DexEncodedMethod lookupInvokeStaticTarget(
+ public abstract DexClassAndMethod lookupInvokeStaticTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo);
public abstract LookupResult lookupVirtualDispatchTargets(
@@ -300,6 +315,127 @@
return DefaultMethodOptimizationInfo.getInstance();
}
+ public DispatchTargetLookupResult lookupDispatchTarget(
+ AppView<?> appView, InvokeMethod invoke, ProgramMethod context) {
+ switch (invoke.getType()) {
+ case DIRECT:
+ return lookupDirectDispatchTarget(appView, invoke.asInvokeDirect(), context);
+ case POLYMORPHIC:
+ return new UnknownDispatchTargetLookupResult(this);
+ case STATIC:
+ return lookupStaticDispatchTarget(appView, invoke.asInvokeStatic(), context);
+ case SUPER:
+ return lookupSuperDispatchTarget(appView, invoke.asInvokeSuper(), context);
+ default:
+ break;
+ }
+ assert invoke.isInvokeInterface() || invoke.isInvokeVirtual();
+ InvokeMethodWithReceiver invokeMethodWithReceiver = invoke.asInvokeMethodWithReceiver();
+ DynamicType dynamicReceiverType =
+ appView.hasLiveness()
+ ? invokeMethodWithReceiver.getReceiver().getDynamicType(appView.withLiveness())
+ : DynamicType.unknown();
+ return lookupVirtualDispatchTarget(
+ appView, invokeMethodWithReceiver, dynamicReceiverType, context);
+ }
+
+ private DispatchTargetLookupResult lookupDirectDispatchTarget(
+ AppView<?> appView, InvokeDirect invoke, ProgramMethod context) {
+ DexMethod invokedMethod = invoke.getInvokedMethod();
+ DexClassAndMethod result;
+ if (appView.hasLiveness()) {
+ AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
+ AppInfoWithLiveness appInfo = appViewWithLiveness.appInfo();
+ result = appInfo.lookupDirectTarget(invokedMethod, context, appViewWithLiveness);
+ assert invoke.verifyD8LookupResult(
+ result, appView.appInfo().lookupDirectTargetOnItself(invokedMethod, context));
+ } else {
+ // In D8, we can treat invoke-direct instructions as having a single target if the invoke is
+ // targeting a method in the enclosing class.
+ result = appView.appInfo().lookupDirectTargetOnItself(invokedMethod, context);
+ }
+ return DispatchTargetLookupResult.create(result, this);
+ }
+
+ private DispatchTargetLookupResult lookupStaticDispatchTarget(
+ AppView<?> appView, InvokeStatic invoke, ProgramMethod context) {
+ DexMethod invokedMethod = invoke.getInvokedMethod();
+ DexClassAndMethod result;
+ if (appView.appInfo().hasLiveness()) {
+ AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
+ AppInfoWithLiveness appInfo = appViewWithLiveness.appInfo();
+ result = appInfo.lookupStaticTarget(invokedMethod, context, appViewWithLiveness);
+ assert invoke.verifyD8LookupResult(
+ result, appInfo.lookupStaticTargetOnItself(invokedMethod, context));
+ } else {
+ // Allow optimizing static library invokes in D8.
+ DexClass clazz = appView.definitionForHolder(invokedMethod);
+ if (clazz != null
+ && (clazz.isLibraryClass() || appView.libraryMethodOptimizer().isModeled(clazz.type))) {
+ result = clazz.lookupClassMethod(invokedMethod);
+ } else {
+ // In D8, we can treat invoke-static instructions as having a single target if the invoke
+ // is targeting a method in the enclosing class.
+ result = appView.appInfo().lookupStaticTargetOnItself(invokedMethod, context);
+ }
+ }
+ return DispatchTargetLookupResult.create(result, this);
+ }
+
+ private DispatchTargetLookupResult lookupSuperDispatchTarget(
+ AppView<?> appView, InvokeSuper invoke, ProgramMethod context) {
+ DexClassAndMethod result = null;
+ if (appView.appInfo().hasLiveness() && context != null) {
+ AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
+ AppInfoWithLiveness appInfo = appViewWithLiveness.appInfo();
+ DexMethod invokedMethod = invoke.getInvokedMethod();
+ if (appInfo.isSubtype(context.getHolderType(), invokedMethod.getHolderType())) {
+ result = appInfo.lookupSuperTarget(invokedMethod, context, appViewWithLiveness);
+ }
+ }
+ return DispatchTargetLookupResult.create(result, this);
+ }
+
+ public DispatchTargetLookupResult lookupVirtualDispatchTarget(
+ AppView<?> appView,
+ InvokeMethodWithReceiver invoke,
+ DynamicType dynamicReceiverType,
+ ProgramMethod context) {
+ DexMethod invokedMethod = invoke.getInvokedMethod();
+ DexClassAndMethod result = null;
+ if (appView.appInfo().hasLiveness()) {
+ AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
+ result =
+ appViewWithLiveness
+ .appInfo()
+ .lookupSingleVirtualTarget(
+ appViewWithLiveness,
+ invokedMethod,
+ this,
+ context,
+ invoke.getInterfaceBit(),
+ appView,
+ dynamicReceiverType);
+ } else {
+ // In D8, allow lookupSingleTarget() to be used for finding final library methods. This is
+ // used for library modeling.
+ DexType holder = invokedMethod.getHolderType();
+ if (holder.isClassType()) {
+ DexClass clazz = appView.definitionFor(holder);
+ if (clazz != null
+ && (clazz.isLibraryClass()
+ || appView.libraryMethodOptimizer().isModeled(clazz.getType()))) {
+ DexClassAndMethod singleTargetCandidate = clazz.lookupClassMethod(invokedMethod);
+ if (singleTargetCandidate != null
+ && (clazz.isFinal() || singleTargetCandidate.getAccessFlags().isFinal())) {
+ result = singleTargetCandidate;
+ }
+ }
+ }
+ }
+ return DispatchTargetLookupResult.create(result, this);
+ }
+
@Override
public DexClass getInitialResolutionHolder() {
return initialResolutionHolder;
@@ -441,13 +577,13 @@
* @return The actual target or {@code null} if none found.
*/
@Override
- public DexEncodedMethod lookupInvokeStaticTarget(
+ public DexClassAndMethod lookupInvokeStaticTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
if (isAccessibleFrom(context, appView, appInfo).isFalse()) {
return null;
}
if (resolvedMethod.isStatic()) {
- return resolvedMethod;
+ return getResolutionPair();
}
return null;
}
@@ -462,13 +598,13 @@
* @return The actual target or {@code null} if none found.
*/
@Override
- public DexEncodedMethod lookupInvokeDirectTarget(
+ public DexClassAndMethod lookupInvokeDirectTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
if (isAccessibleFrom(context, appView, appInfo).isFalse()) {
return null;
}
if (resolvedMethod.isDirectMethod()) {
- return resolvedMethod;
+ return getResolutionPair();
}
return null;
}
@@ -882,23 +1018,30 @@
private static DexClassAndMethod findWideningOverride(
DexClassAndMethod resolvedMethod, DexClass clazz, AppInfoWithClassHierarchy appInfo) {
// Otherwise, lookup to first override that is distinct from resolvedMethod.
- assert resolvedMethod.getDefinition().accessFlags.isPackagePrivate();
- while (clazz.superType != null) {
- clazz = definitionForHelper(appInfo, clazz.superType);
+ assert resolvedMethod.getDefinition().getAccessFlags().isPackagePrivate();
+ while (clazz.hasSuperType()) {
+ clazz = definitionForHelper(appInfo, clazz.getSuperType());
if (clazz == null) {
return resolvedMethod;
}
- DexEncodedMethod otherOverride = clazz.lookupVirtualMethod(resolvedMethod.getReference());
+ DexClassAndMethod otherOverride =
+ clazz.lookupVirtualClassMethod(resolvedMethod.getReference());
if (otherOverride != null
- && isOverriding(resolvedMethod.getDefinition(), otherOverride)
- && (otherOverride.accessFlags.isPublic() || otherOverride.accessFlags.isProtected())) {
- assert resolvedMethod.getDefinition() != otherOverride;
- return DexClassAndMethod.create(clazz, otherOverride);
+ && isOverriding(resolvedMethod, otherOverride)
+ && (otherOverride.getAccessFlags().isPublic()
+ || otherOverride.getAccessFlags().isProtected())) {
+ assert resolvedMethod.getDefinition() != otherOverride.getDefinition();
+ return otherOverride;
}
}
return resolvedMethod;
}
+ public static boolean isOverriding(
+ DexClassAndMethod resolvedMethod, DexClassAndMethod candidate) {
+ return isOverriding(resolvedMethod.getDefinition(), candidate.getDefinition());
+ }
+
/**
* Implementation of method overriding according to the jvm specification
* https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-5.html#jvms-5.4.5
@@ -1040,13 +1183,13 @@
}
@Override
- public DexEncodedMethod lookupInvokeStaticTarget(
+ public DexClassAndMethod lookupInvokeStaticTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
return null;
}
@Override
- public DexEncodedMethod lookupInvokeDirectTarget(
+ public DexClassAndMethod lookupInvokeDirectTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
return null;
}
@@ -1429,13 +1572,13 @@
}
@Override
- public DexEncodedMethod lookupInvokeDirectTarget(
+ public DexClassAndMethod lookupInvokeDirectTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
throw new Unreachable("Should not be called on MultipleFieldResolutionResult");
}
@Override
- public DexEncodedMethod lookupInvokeStaticTarget(
+ public DexClassAndMethod lookupInvokeStaticTarget(
DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
throw new Unreachable("Should not be called on MultipleFieldResolutionResult");
}
@@ -1687,4 +1830,106 @@
: new NoSuchMethodResultDueToMultipleClassDefinitions(typesCausingError);
}
}
+
+ public static class UnknownMethodResolutionResult extends MethodResolutionResult {
+
+ private static final UnknownMethodResolutionResult INSTANCE =
+ new UnknownMethodResolutionResult();
+
+ private UnknownMethodResolutionResult() {}
+
+ static UnknownMethodResolutionResult get() {
+ return INSTANCE;
+ }
+
+ @Override
+ public OptionalBool isAccessibleFrom(
+ ProgramDefinition context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
+ return OptionalBool.unknown();
+ }
+
+ @Override
+ public OptionalBool isAccessibleForVirtualDispatchFrom(
+ ProgramDefinition context, AppView<? extends AppInfoWithClassHierarchy> appView) {
+ return OptionalBool.unknown();
+ }
+
+ @Override
+ public boolean isVirtualTarget() {
+ throw new Unreachable();
+ }
+
+ @Override
+ public DexClassAndMethod lookupInvokeSpecialTarget(
+ DexProgramClass context, AppView<? extends AppInfoWithClassHierarchy> appView) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public DexClassAndMethod lookupInvokeSuperTarget(
+ DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public DexClassAndMethod lookupInvokeDirectTarget(
+ DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public DexClassAndMethod lookupInvokeStaticTarget(
+ DexProgramClass context, AppView<?> appView, AppInfoWithClassHierarchy appInfo) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public LookupResult lookupVirtualDispatchTargets(
+ DexProgramClass context,
+ AppView<? extends AppInfoWithClassHierarchy> appView,
+ InstantiatedSubTypeInfo instantiatedInfo,
+ PinnedPredicate pinnedPredicate) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public LookupResult lookupVirtualDispatchTargets(
+ DexProgramClass context,
+ AppView<AppInfoWithLiveness> appView,
+ DexProgramClass refinedReceiverUpperBound,
+ DexProgramClass refinedReceiverLowerBound) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public LookupTarget lookupVirtualDispatchTarget(
+ InstantiatedObject instance, AppInfoWithClassHierarchy appInfo) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public LookupMethodTarget lookupVirtualDispatchTarget(
+ DexClass dynamicInstance, AppInfoWithClassHierarchy appInfo) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public LookupTarget lookupVirtualDispatchTarget(
+ LambdaDescriptor lambdaInstance,
+ AppInfoWithClassHierarchy appInfo,
+ Consumer<DexType> typeCausingFailureConsumer,
+ Consumer<? super DexEncodedMethod> methodCausingFailureConsumer) {
+ throw new Unreachable();
+ }
+
+ @Override
+ public void visitMethodResolutionResults(
+ Consumer<? super SingleResolutionResult<? extends ProgramOrClasspathClass>>
+ programOrClasspathConsumer,
+ Consumer<? super SingleLibraryResolutionResult> libraryResultConsumer,
+ Consumer<? super ArrayCloneMethodResult> cloneResultConsumer,
+ Consumer<? super FailedResolutionResult> failedResolutionConsumer) {
+ throw new Unreachable();
+ }
+ }
}
diff --git a/src/main/java/com/android/tools/r8/graph/SingleDispatchTargetLookupResult.java b/src/main/java/com/android/tools/r8/graph/SingleDispatchTargetLookupResult.java
new file mode 100644
index 0000000..5cd3952
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/graph/SingleDispatchTargetLookupResult.java
@@ -0,0 +1,33 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.graph;
+
+import com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult;
+
+public class SingleDispatchTargetLookupResult extends DispatchTargetLookupResult {
+
+ private final DexClassAndMethod singleDispatchTarget;
+
+ public SingleDispatchTargetLookupResult(
+ DexClassAndMethod singleDispatchTarget, SingleResolutionResult<?> singleResolutionResult) {
+ super(singleResolutionResult);
+ assert singleDispatchTarget != null;
+ this.singleDispatchTarget = singleDispatchTarget;
+ }
+
+ @Override
+ public DexClassAndMethod getSingleDispatchTarget() {
+ return singleDispatchTarget;
+ }
+
+ @Override
+ public boolean isSingleResult() {
+ return true;
+ }
+
+ @Override
+ public SingleDispatchTargetLookupResult asSingleResult() {
+ return this;
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/graph/UnknownDispatchTargetLookupResult.java b/src/main/java/com/android/tools/r8/graph/UnknownDispatchTargetLookupResult.java
new file mode 100644
index 0000000..0d51cff
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/graph/UnknownDispatchTargetLookupResult.java
@@ -0,0 +1,13 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.graph;
+
+import com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult;
+
+public class UnknownDispatchTargetLookupResult extends DispatchTargetLookupResult {
+
+ public UnknownDispatchTargetLookupResult(SingleResolutionResult<?> singleResolutionResult) {
+ super(singleResolutionResult);
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/ir/analysis/value/AbstractValueFactory.java b/src/main/java/com/android/tools/r8/ir/analysis/value/AbstractValueFactory.java
index 2dbb047..79eca14 100644
--- a/src/main/java/com/android/tools/r8/ir/analysis/value/AbstractValueFactory.java
+++ b/src/main/java/com/android/tools/r8/ir/analysis/value/AbstractValueFactory.java
@@ -14,17 +14,26 @@
import com.android.tools.r8.ir.analysis.value.objectstate.KnownLengthArrayState;
import com.android.tools.r8.ir.analysis.value.objectstate.ObjectState;
import com.android.tools.r8.naming.dexitembasedstring.NameComputationInfo;
+import com.android.tools.r8.utils.InternalOptions;
+import com.android.tools.r8.utils.InternalOptions.TestingOptions;
import java.util.concurrent.ConcurrentHashMap;
public class AbstractValueFactory {
- private ConcurrentHashMap<DexType, SingleConstClassValue> singleConstClassValues =
+ private final TestingOptions testingOptions;
+
+ private final ConcurrentHashMap<DexType, SingleConstClassValue> singleConstClassValues =
new ConcurrentHashMap<>();
- private ConcurrentHashMap<Long, SingleNumberValue> singleNumberValues = new ConcurrentHashMap<>();
- private ConcurrentHashMap<DexString, SingleStringValue> singleStringValues =
+ private final ConcurrentHashMap<Long, SingleNumberValue> singleNumberValues =
new ConcurrentHashMap<>();
- private ConcurrentHashMap<Integer, KnownLengthArrayState> knownArrayLengthStates =
+ private final ConcurrentHashMap<DexString, SingleStringValue> singleStringValues =
new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<Integer, KnownLengthArrayState> knownArrayLengthStates =
+ new ConcurrentHashMap<>();
+
+ public AbstractValueFactory(InternalOptions options) {
+ testingOptions = options.testing;
+ }
public SingleBoxedBooleanValue createBoxedBooleanFalse() {
return SingleBoxedBooleanValue.getFalseInstance();
@@ -71,10 +80,20 @@
int definitelySetBits, int definitelyUnsetBits) {
if (definitelySetBits != 0 || definitelyUnsetBits != 0) {
// If all bits are known, then create a single number value.
- if ((definitelySetBits | definitelyUnsetBits) == ALL_BITS_SET_MASK) {
+ boolean allBitsSet = (definitelySetBits | definitelyUnsetBits) == ALL_BITS_SET_MASK;
+ // Account for the temporary hack in the Compose modeling where we create a
+ // DefiniteBitsNumberValue with set bits=0b1^32 and unset bits = 0b1^(31)0. This value is used
+ // to simulate the effect of `x | 1` in joins.
+ if (testingOptions.modelUnknownChangedAndDefaultArgumentsToComposableFunctions) {
+ boolean overlappingSetAndUnsetBits = (definitelySetBits & definitelyUnsetBits) != 0;
+ if (overlappingSetAndUnsetBits) {
+ allBitsSet = false;
+ }
+ }
+ if (allBitsSet) {
return createUncheckedSingleNumberValue(definitelySetBits);
}
- return new DefiniteBitsNumberValue(definitelySetBits, definitelyUnsetBits);
+ return new DefiniteBitsNumberValue(definitelySetBits, definitelyUnsetBits, testingOptions);
}
return AbstractValue.unknown();
}
diff --git a/src/main/java/com/android/tools/r8/ir/analysis/value/DefiniteBitsNumberValue.java b/src/main/java/com/android/tools/r8/ir/analysis/value/DefiniteBitsNumberValue.java
index 778e61b..a18281e 100644
--- a/src/main/java/com/android/tools/r8/ir/analysis/value/DefiniteBitsNumberValue.java
+++ b/src/main/java/com/android/tools/r8/ir/analysis/value/DefiniteBitsNumberValue.java
@@ -8,6 +8,7 @@
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.lens.GraphLens;
import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.utils.InternalOptions.TestingOptions;
import com.android.tools.r8.utils.OptionalBool;
import java.util.Objects;
@@ -16,8 +17,10 @@
private final int definitelySetBits;
private final int definitelyUnsetBits;
- public DefiniteBitsNumberValue(int definitelySetBits, int definitelyUnsetBits) {
- assert (definitelySetBits & definitelyUnsetBits) == 0;
+ public DefiniteBitsNumberValue(
+ int definitelySetBits, int definitelyUnsetBits, TestingOptions testingOptions) {
+ assert (definitelySetBits & definitelyUnsetBits) == 0
+ || testingOptions.modelUnknownChangedAndDefaultArgumentsToComposableFunctions;
this.definitelySetBits = definitelySetBits;
this.definitelyUnsetBits = definitelyUnsetBits;
}
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeDirect.java b/src/main/java/com/android/tools/r8/ir/code/InvokeDirect.java
index 5bf3bd6..22cc39f 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeDirect.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeDirect.java
@@ -3,8 +3,6 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.ir.code;
-import static com.android.tools.r8.graph.DexEncodedMethod.asDexClassAndMethodOrNull;
-
import com.android.tools.r8.cf.code.CfInvoke;
import com.android.tools.r8.dex.Constants;
import com.android.tools.r8.dex.code.DexInstruction;
@@ -12,7 +10,6 @@
import com.android.tools.r8.dex.code.DexInvokeDirectRange;
import com.android.tools.r8.graph.AppView;
import com.android.tools.r8.graph.DexClassAndMethod;
-import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexItemFactory;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
@@ -23,7 +20,6 @@
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.Query;
import com.android.tools.r8.ir.analysis.fieldvalueanalysis.AbstractFieldSet;
import com.android.tools.r8.ir.analysis.modeling.LibraryMethodReadSetModeling;
-import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.ir.conversion.CfBuilder;
import com.android.tools.r8.ir.conversion.DexBuilder;
import com.android.tools.r8.ir.optimize.DeadCodeRemover.DeadInstructionResult;
@@ -137,25 +133,6 @@
}
@Override
- public DexClassAndMethod lookupSingleTarget(
- AppView<?> appView, ProgramMethod context, DynamicType dynamicReceiverType) {
- DexMethod invokedMethod = getInvokedMethod();
- DexEncodedMethod result;
- if (appView.hasLiveness()) {
- AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
- AppInfoWithLiveness appInfo = appViewWithLiveness.appInfo();
- result = appInfo.lookupDirectTarget(invokedMethod, context, appViewWithLiveness);
- assert verifyD8LookupResult(
- result, appView.appInfo().lookupDirectTargetOnItself(invokedMethod, context));
- } else {
- // In D8, we can treat invoke-direct instructions as having a single target if the invoke is
- // targeting a method in the enclosing class.
- result = appView.appInfo().lookupDirectTargetOnItself(invokedMethod, context);
- }
- return asDexClassAndMethodOrNull(result, appView);
- }
-
- @Override
public ConstraintWithTarget inliningConstraint(
InliningConstraints inliningConstraints, ProgramMethod context) {
return inliningConstraints.forInvokeDirect(getInvokedMethod(), context);
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeInterface.java b/src/main/java/com/android/tools/r8/ir/code/InvokeInterface.java
index 2936292..13f5ade 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeInterface.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeInterface.java
@@ -3,8 +3,6 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.ir.code;
-import static com.android.tools.r8.graph.DexEncodedMethod.asDexClassAndMethodOrNull;
-
import com.android.tools.r8.cf.code.CfInvoke;
import com.android.tools.r8.dex.code.DexInstruction;
import com.android.tools.r8.dex.code.DexInvokeDirect;
@@ -13,7 +11,6 @@
import com.android.tools.r8.dex.code.DexInvokeInterfaceRange;
import com.android.tools.r8.graph.AppView;
import com.android.tools.r8.graph.DexClassAndMethod;
-import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.ProgramMethod;
@@ -21,7 +18,6 @@
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis;
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.AnalysisAssumption;
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.Query;
-import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.ir.conversion.CfBuilder;
import com.android.tools.r8.ir.conversion.DexBuilder;
import com.android.tools.r8.ir.optimize.Inliner.ConstraintWithTarget;
@@ -120,26 +116,6 @@
}
@Override
- public DexClassAndMethod lookupSingleTarget(
- AppView<?> appView, ProgramMethod context, DynamicType dynamicReceiverType) {
- if (!appView.appInfo().hasLiveness()) {
- return null;
- }
- AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
- DexEncodedMethod result =
- appViewWithLiveness
- .appInfo()
- .lookupSingleVirtualTarget(
- appViewWithLiveness,
- getInvokedMethod(),
- context,
- true,
- appView,
- dynamicReceiverType);
- return asDexClassAndMethodOrNull(result, appView);
- }
-
- @Override
public ConstraintWithTarget inliningConstraint(
InliningConstraints inliningConstraints, ProgramMethod context) {
return inliningConstraints.forInvokeInterface(getInvokedMethod(), context);
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeMethod.java b/src/main/java/com/android/tools/r8/ir/code/InvokeMethod.java
index 9799455..434272d 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeMethod.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeMethod.java
@@ -11,11 +11,13 @@
import com.android.tools.r8.errors.Unreachable;
import com.android.tools.r8.graph.AppInfoWithClassHierarchy;
import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.DexClass;
import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProgramClass;
import com.android.tools.r8.graph.DexType;
+import com.android.tools.r8.graph.DispatchTargetLookupResult;
import com.android.tools.r8.graph.LookupResult;
import com.android.tools.r8.graph.MethodResolutionResult;
import com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult;
@@ -121,11 +123,53 @@
return appView.appInfo().resolveMethod(method, getInterfaceBit());
}
- // In subclasses, e.g., invoke-virtual or invoke-super, use a narrower receiver type by using
- // receiver type and calling context---the holder of the method where the current invocation is.
- // TODO(b/140204899): Refactor lookup methods to be defined in a single place.
- public abstract DexClassAndMethod lookupSingleTarget(AppView<?> appView, ProgramMethod context);
+ public MethodResolutionResult resolveMethod(AppView<?> appView, ProgramMethod context) {
+ if (appView.hasClassHierarchy()) {
+ return resolveMethod(appView.withClassHierarchy());
+ }
+ if (method.getHolderType().isIdenticalTo(context.getHolderType())) {
+ DexClass resolutionHolder = context.getHolder();
+ DexEncodedMethod lookupResult = resolutionHolder.lookupMethod(method);
+ if (lookupResult != null) {
+ return MethodResolutionResult.createSingleResolutionResult(
+ resolutionHolder, resolutionHolder, lookupResult);
+ }
+ }
+ if (appView.libraryMethodOptimizer().isModeled(method.getHolderType())) {
+ DexClassAndMethod lookupResult = appView.definitionFor(method);
+ if (lookupResult != null) {
+ DexClass resolutionHolder = lookupResult.getHolder();
+ return MethodResolutionResult.createSingleResolutionResult(
+ resolutionHolder, resolutionHolder, lookupResult.getDefinition());
+ }
+ }
+ return MethodResolutionResult.unknown();
+ }
+ /**
+ * In subclasses, e.g., invoke-virtual or invoke-super, use a narrower receiver type by using
+ * receiver type and calling context---the resolutionHolder of the method where the current
+ * invocation is.
+ *
+ * @deprecated Use {@link SingleResolutionResult#lookupDispatchTarget}.
+ */
+ @Deprecated
+ public final DexClassAndMethod lookupSingleTarget(AppView<?> appView, ProgramMethod context) {
+ MethodResolutionResult resolutionResult = resolveMethod(appView, context);
+ if (resolutionResult.isSingleResolution()) {
+ DispatchTargetLookupResult lookupResult =
+ resolutionResult.asSingleResolution().lookupDispatchTarget(appView, this, context);
+ if (lookupResult.isSingleResult()) {
+ return lookupResult.asSingleResult().getSingleDispatchTarget();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * @deprecated Use {@link SingleResolutionResult#lookupDispatchTarget}.
+ */
+ @Deprecated
public final ProgramMethod lookupSingleProgramTarget(AppView<?> appView, ProgramMethod context) {
return DexClassAndMethod.asProgramMethodOrNull(lookupSingleTarget(appView, context));
}
@@ -269,12 +313,12 @@
}
@SuppressWarnings("ReferenceEquality")
- boolean verifyD8LookupResult(
- DexEncodedMethod hierarchyResult, DexEncodedMethod lookupDirectTargetOnItself) {
+ public boolean verifyD8LookupResult(
+ DexClassAndMethod hierarchyResult, DexClassAndMethod lookupDirectTargetOnItself) {
if (lookupDirectTargetOnItself == null) {
return true;
}
- assert lookupDirectTargetOnItself == hierarchyResult;
+ assert lookupDirectTargetOnItself.isStructurallyEqualTo(hierarchyResult);
return true;
}
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java b/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java
index 2521043..edc96f6 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java
@@ -16,7 +16,6 @@
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis;
import com.android.tools.r8.ir.analysis.VerifyTypesHelper;
import com.android.tools.r8.ir.analysis.type.ClassTypeElement;
-import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.ir.analysis.type.TypeAnalysis;
import com.android.tools.r8.ir.analysis.type.TypeElement;
import com.android.tools.r8.ir.conversion.DexBuilder;
@@ -66,24 +65,6 @@
this, singleTarget, reason, whyAreYouNotInliningReporter);
}
- @Override
- public final DexClassAndMethod lookupSingleTarget(AppView<?> appView, ProgramMethod context) {
- DynamicType dynamicReceiverType =
- appView.hasLiveness()
- ? getReceiver().getDynamicType(appView.withLiveness())
- : DynamicType.unknown();
- return lookupSingleTarget(appView, context, dynamicReceiverType);
- }
-
- public abstract DexClassAndMethod lookupSingleTarget(
- AppView<?> appView, ProgramMethod context, DynamicType dynamicReceiverType);
-
- public final ProgramMethod lookupSingleProgramTarget(
- AppView<?> appView, ProgramMethod context, DynamicType dynamicReceiverType) {
- return DexClassAndMethod.asProgramMethodOrNull(
- lookupSingleTarget(appView, context, dynamicReceiverType));
- }
-
/**
* If an invoke-virtual targets a private method in the current class overriding will not apply
* (see JVM 11 spec on method selection 5.4.6. In previous jvm specs this was not explicitly
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokePolymorphic.java b/src/main/java/com/android/tools/r8/ir/code/InvokePolymorphic.java
index 5e80e07ef..0061bf3 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokePolymorphic.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokePolymorphic.java
@@ -9,7 +9,6 @@
import com.android.tools.r8.dex.code.DexInvokePolymorphicRange;
import com.android.tools.r8.errors.Unreachable;
import com.android.tools.r8.graph.AppView;
-import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexItemFactory;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProto;
@@ -135,12 +134,6 @@
}
@Override
- public DexClassAndMethod lookupSingleTarget(AppView<?> appView, ProgramMethod context) {
- // TODO(herhut): Implement lookup target for invokePolymorphic.
- return null;
- }
-
- @Override
public ProgramMethodSet lookupProgramDispatchTargets(
AppView<AppInfoWithLiveness> appView, ProgramMethod context) {
// TODO(herhut): Implement lookup target for invokePolymorphic.
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeStatic.java b/src/main/java/com/android/tools/r8/ir/code/InvokeStatic.java
index e7043b0..f1d4aea 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeStatic.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeStatic.java
@@ -3,16 +3,13 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.ir.code;
-import static com.android.tools.r8.graph.DexEncodedMethod.asDexClassAndMethodOrNull;
import com.android.tools.r8.cf.code.CfInvoke;
import com.android.tools.r8.dex.code.DexInstruction;
import com.android.tools.r8.dex.code.DexInvokeStatic;
import com.android.tools.r8.dex.code.DexInvokeStaticRange;
import com.android.tools.r8.graph.AppView;
-import com.android.tools.r8.graph.DexClass;
import com.android.tools.r8.graph.DexClassAndMethod;
-import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult;
@@ -117,31 +114,6 @@
}
@Override
- public DexClassAndMethod lookupSingleTarget(AppView<?> appView, ProgramMethod context) {
- DexMethod invokedMethod = getInvokedMethod();
- DexEncodedMethod result;
- if (appView.appInfo().hasLiveness()) {
- AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
- AppInfoWithLiveness appInfo = appViewWithLiveness.appInfo();
- result = appInfo.lookupStaticTarget(invokedMethod, context, appViewWithLiveness);
- assert verifyD8LookupResult(
- result, appInfo.lookupStaticTargetOnItself(invokedMethod, context));
- } else {
- // Allow optimizing static library invokes in D8.
- DexClass clazz = appView.definitionForHolder(getInvokedMethod());
- if (clazz != null
- && (clazz.isLibraryClass() || appView.libraryMethodOptimizer().isModeled(clazz.type))) {
- result = clazz.lookupMethod(getInvokedMethod());
- } else {
- // In D8, we can treat invoke-static instructions as having a single target if the invoke is
- // targeting a method in the enclosing class.
- result = appView.appInfo().lookupStaticTargetOnItself(invokedMethod, context);
- }
- }
- return asDexClassAndMethodOrNull(result, appView);
- }
-
- @Override
public ConstraintWithTarget inliningConstraint(
InliningConstraints inliningConstraints, ProgramMethod context) {
return inliningConstraints.forInvokeStatic(getInvokedMethod(), context);
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeSuper.java b/src/main/java/com/android/tools/r8/ir/code/InvokeSuper.java
index b63fc98..6db911d 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeSuper.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeSuper.java
@@ -3,7 +3,6 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.ir.code;
-
import com.android.tools.r8.cf.code.CfInvoke;
import com.android.tools.r8.dex.code.DexInstruction;
import com.android.tools.r8.dex.code.DexInvokeSuper;
@@ -17,7 +16,6 @@
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis;
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.AnalysisAssumption;
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.Query;
-import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.ir.conversion.CfBuilder;
import com.android.tools.r8.ir.conversion.DexBuilder;
import com.android.tools.r8.ir.optimize.Inliner.ConstraintWithTarget;
@@ -109,19 +107,6 @@
}
@Override
- public DexClassAndMethod lookupSingleTarget(
- AppView<?> appView, ProgramMethod context, DynamicType dynamicReceiverType) {
- if (appView.appInfo().hasLiveness() && context != null) {
- AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
- AppInfoWithLiveness appInfo = appViewWithLiveness.appInfo();
- if (appInfo.isSubtype(context.getHolderType(), getInvokedMethod().holder)) {
- return appInfo.lookupSuperTarget(getInvokedMethod(), context, appViewWithLiveness);
- }
- }
- return null;
- }
-
- @Override
public ConstraintWithTarget inliningConstraint(
InliningConstraints inliningConstraints, ProgramMethod context) {
return inliningConstraints.forInvokeSuper(getInvokedMethod(), context);
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeVirtual.java b/src/main/java/com/android/tools/r8/ir/code/InvokeVirtual.java
index cf04e98..cde845d 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeVirtual.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeVirtual.java
@@ -3,8 +3,6 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.ir.code;
-import static com.android.tools.r8.graph.DexEncodedMethod.asDexClassAndMethodOrNull;
-
import com.android.tools.r8.cf.code.CfInvoke;
import com.android.tools.r8.dex.code.DexInstruction;
import com.android.tools.r8.dex.code.DexInvokeDirect;
@@ -12,9 +10,7 @@
import com.android.tools.r8.dex.code.DexInvokeVirtual;
import com.android.tools.r8.dex.code.DexInvokeVirtualRange;
import com.android.tools.r8.graph.AppView;
-import com.android.tools.r8.graph.DexClass;
import com.android.tools.r8.graph.DexClassAndMethod;
-import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.ProgramMethod;
@@ -22,7 +18,6 @@
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis;
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.AnalysisAssumption;
import com.android.tools.r8.ir.analysis.ClassInitializationAnalysis.Query;
-import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.ir.conversion.CfBuilder;
import com.android.tools.r8.ir.conversion.DexBuilder;
import com.android.tools.r8.ir.optimize.Inliner.ConstraintWithTarget;
@@ -125,44 +120,6 @@
}
@Override
- public DexClassAndMethod lookupSingleTarget(
- AppView<?> appView, ProgramMethod context, DynamicType dynamicReceiverType) {
- return lookupSingleTarget(appView, context, dynamicReceiverType, getInvokedMethod());
- }
-
- public static DexClassAndMethod lookupSingleTarget(
- AppView<?> appView,
- ProgramMethod context,
- DynamicType dynamicReceiverType,
- DexMethod method) {
- DexEncodedMethod result = null;
- if (appView.appInfo().hasLiveness()) {
- AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
- result =
- appViewWithLiveness
- .appInfo()
- .lookupSingleVirtualTarget(
- appViewWithLiveness, method, context, false, appView, dynamicReceiverType);
- } else {
- // In D8, allow lookupSingleTarget() to be used for finding final library methods. This is
- // used for library modeling.
- DexType holder = method.holder;
- if (holder.isClassType()) {
- DexClass clazz = appView.definitionFor(holder);
- if (clazz != null
- && (clazz.isLibraryClass() || appView.libraryMethodOptimizer().isModeled(clazz.type))) {
- DexEncodedMethod singleTargetCandidate = clazz.lookupMethod(method);
- if (singleTargetCandidate != null
- && (clazz.isFinal() || singleTargetCandidate.isFinal())) {
- result = singleTargetCandidate;
- }
- }
- }
- }
- return asDexClassAndMethodOrNull(result, appView);
- }
-
- @Override
public ConstraintWithTarget inliningConstraint(
InliningConstraints inliningConstraints, ProgramMethod context) {
return inliningConstraints.forInvokeVirtual(getInvokedMethod(), context);
diff --git a/src/main/java/com/android/tools/r8/ir/code/Value.java b/src/main/java/com/android/tools/r8/ir/code/Value.java
index 003b119..6011eb0 100644
--- a/src/main/java/com/android/tools/r8/ir/code/Value.java
+++ b/src/main/java/com/android/tools/r8/ir/code/Value.java
@@ -788,10 +788,26 @@
boolean isConstant = definition != null && definition.isConstNumber();
if (isConstant || hasLocalInfo()) {
builder.append("(");
- if (isConstant && definition.asConstNumber().outValue != null) {
+ if (isConstant && definition.asConstNumber().hasOutValue()) {
ConstNumber constNumber = definition.asConstNumber();
if (constNumber.getOutType().isSinglePrimitive()) {
- builder.append((int) constNumber.getRawValue());
+ Value constNumberValue = constNumber.outValue();
+ int intValue = (int) constNumber.getRawValue();
+ boolean useBinaryRepresentation =
+ !constNumberValue.hasPhiUsers()
+ && constNumberValue.uniqueUsers().stream()
+ .allMatch(
+ user ->
+ user.isAnd()
+ || user.isOr()
+ || ((user.isShl() || user.isShr() || user.isUshr())
+ && constNumberValue == user.asShl().getFirstOperand())
+ || user.isXor());
+ if (useBinaryRepresentation) {
+ builder.append("0b").append(Integer.toBinaryString(intValue));
+ } else {
+ builder.append(intValue);
+ }
} else {
builder.append(constNumber.getRawValue());
}
@@ -823,6 +839,12 @@
return isConstant() && getConstInstruction().isConstNumber();
}
+ public boolean isConstNumber(long rawValue) {
+ return isConstant()
+ && getConstInstruction().isConstNumber()
+ && getConstInstruction().asConstNumber().getRawValue() == rawValue;
+ }
+
public boolean isConstBoolean(boolean value) {
return isConstNumber()
&& definition.asConstNumber().getRawValue() == BooleanUtils.longValue(value);
diff --git a/src/main/java/com/android/tools/r8/ir/conversion/callgraph/InvokeExtractor.java b/src/main/java/com/android/tools/r8/ir/conversion/callgraph/InvokeExtractor.java
index 1f0e197..eb09ac7 100644
--- a/src/main/java/com/android/tools/r8/ir/conversion/callgraph/InvokeExtractor.java
+++ b/src/main/java/com/android/tools/r8/ir/conversion/callgraph/InvokeExtractor.java
@@ -4,6 +4,8 @@
package com.android.tools.r8.ir.conversion.callgraph;
+import static com.android.tools.r8.graph.DexClassAndMethod.asProgramMethodOrNull;
+
import com.android.tools.r8.graph.AppView;
import com.android.tools.r8.graph.DexCallSite;
import com.android.tools.r8.graph.DexClass;
@@ -77,20 +79,28 @@
.lookupMethod(originalMethod, context.getReference(), originalType, getCodeLens());
DexMethod method = result.getReference();
InvokeType type = result.getType();
- if (type == InvokeType.INTERFACE || type == InvokeType.VIRTUAL) {
+ MethodResolutionResult resolutionResult =
+ type.isInterface() || type.isVirtual()
+ ? appViewWithLiveness.appInfo().resolveMethodLegacy(method, type.isInterface())
+ : appViewWithLiveness.appInfo().unsafeResolveMethodDueToDexFormatLegacy(method);
+ if (!resolutionResult.isSingleResolution()) {
+ return;
+ }
+ if (type.isInterface() || type.isVirtual()) {
// For virtual and interface calls add all potential targets that could be called.
- MethodResolutionResult resolutionResult =
- appViewWithLiveness.appInfo().resolveMethodLegacy(method, type == InvokeType.INTERFACE);
- DexClassAndMethod target = resolutionResult.getResolutionPair();
- if (target != null) {
- processInvokeWithDynamicDispatch(type, target, context);
- }
+ processInvokeWithDynamicDispatch(type, resolutionResult.getResolutionPair(), context);
} else {
ProgramMethod singleTarget =
- appViewWithLiveness
- .appInfo()
- .lookupSingleProgramTarget(
- appViewWithLiveness, type, method, context, appViewWithLiveness);
+ asProgramMethodOrNull(
+ appViewWithLiveness
+ .appInfo()
+ .lookupSingleTarget(
+ appViewWithLiveness,
+ type,
+ method,
+ resolutionResult.asSingleResolution(),
+ context,
+ appViewWithLiveness));
if (singleTarget != null) {
processSingleTarget(singleTarget, context);
}
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java b/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
index 9f4c5d0..7e9cc29 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/LambdaDescriptor.java
@@ -8,8 +8,8 @@
import com.android.tools.r8.graph.AppInfoWithClassHierarchy;
import com.android.tools.r8.graph.AppView;
import com.android.tools.r8.graph.DexCallSite;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexDefinitionSupplier;
-import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexItemFactory;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexMethodHandle;
@@ -96,10 +96,10 @@
this.captures = captures;
this.interfaces.add(mainInterface);
- DexEncodedMethod targetMethod =
+ DexClassAndMethod targetMethod =
context == null ? null : lookupTargetMethod(appView, appInfo, context);
if (targetMethod != null) {
- targetAccessFlags = targetMethod.accessFlags.copy();
+ targetAccessFlags = targetMethod.getAccessFlags().copy();
targetHolder = targetMethod.getHolderType();
} else {
targetAccessFlags = null;
@@ -116,7 +116,7 @@
return captures.length > 0 ? captures[0] : params[0];
}
- private DexEncodedMethod lookupTargetMethod(
+ private DexClassAndMethod lookupTargetMethod(
AppView<?> appView, AppInfoWithClassHierarchy appInfo, ProgramMethod context) {
assert context != null;
// Find the lambda's impl-method target.
@@ -124,10 +124,10 @@
switch (implHandle.type) {
case INVOKE_DIRECT:
case INVOKE_INSTANCE: {
- DexEncodedMethod target =
+ DexClassAndMethod target =
appInfo
.resolveMethodOnLegacy(getImplReceiverType(), method, implHandle.isInterface)
- .getSingleTarget();
+ .getResolutionPair();
if (target == null) {
target = appInfo.lookupDirectTarget(method, context, appView, appInfo);
}
@@ -139,22 +139,22 @@
}
case INVOKE_STATIC: {
- DexEncodedMethod target = appInfo.lookupStaticTarget(method, context, appView, appInfo);
- assert target == null || target.accessFlags.isStatic();
+ DexClassAndMethod target = appInfo.lookupStaticTarget(method, context, appView, appInfo);
+ assert target == null || target.getAccessFlags().isStatic();
return target;
}
case INVOKE_CONSTRUCTOR: {
- DexEncodedMethod target = appInfo.lookupDirectTarget(method, context, appView, appInfo);
- assert target == null || target.accessFlags.isConstructor();
+ DexClassAndMethod target = appInfo.lookupDirectTarget(method, context, appView, appInfo);
+ assert target == null || target.getAccessFlags().isConstructor();
return target;
}
case INVOKE_INTERFACE: {
- DexEncodedMethod target =
+ DexClassAndMethod target =
appInfo
.resolveMethodOnInterfaceLegacy(getImplReceiverType(), method)
- .getSingleTarget();
+ .getResolutionPair();
assert target == null || isInstanceMethod(target);
return target;
}
@@ -164,24 +164,23 @@
}
}
- private boolean isInstanceMethod(DexEncodedMethod encodedMethod) {
- assert encodedMethod != null;
- return !encodedMethod.accessFlags.isConstructor() && !encodedMethod.isStatic();
+ private boolean isInstanceMethod(DexClassAndMethod method) {
+ assert method != null;
+ return !method.getAccessFlags().isConstructor() && !method.getAccessFlags().isStatic();
}
- private boolean isPrivateInstanceMethod(DexEncodedMethod encodedMethod) {
- assert encodedMethod != null;
- return encodedMethod.isPrivateMethod() && isInstanceMethod(encodedMethod);
+ private boolean isPrivateInstanceMethod(DexClassAndMethod method) {
+ assert method != null;
+ return method.getAccessFlags().isPrivate() && isInstanceMethod(method);
}
- private boolean isPublicizedInstanceMethod(DexEncodedMethod encodedMethod) {
- assert encodedMethod != null;
- return encodedMethod.isPublicized() && isInstanceMethod(encodedMethod);
+ private boolean isPublicizedInstanceMethod(DexClassAndMethod method) {
+ assert method != null;
+ return method.getDefinition().isPublicized() && isInstanceMethod(method);
}
- @SuppressWarnings("ReferenceEquality")
public final boolean verifyTargetFoundInClass(DexType type) {
- return targetHolder == type;
+ return targetHolder.isIdenticalTo(type);
}
/** If the lambda delegates to lambda$ method. */
@@ -196,10 +195,6 @@
}
}
- public Iterable<DexType> getReferencedBaseTypes(DexItemFactory dexItemFactory) {
- return enforcedProto.getBaseTypes(dexItemFactory);
- }
-
/** Is a stateless lambda, i.e. lambda does not capture any values */
final boolean isStateless() {
return captures.isEmpty();
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/Devirtualizer.java b/src/main/java/com/android/tools/r8/ir/optimize/Devirtualizer.java
index f399865..a0715f7 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/Devirtualizer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/Devirtualizer.java
@@ -11,6 +11,7 @@
import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
+import com.android.tools.r8.graph.DispatchTargetLookupResult;
import com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult;
import com.android.tools.r8.graph.ProgramMethod;
import com.android.tools.r8.ir.analysis.type.TypeElement;
@@ -127,20 +128,24 @@
// Check if the instruction can be rewritten to invoke-virtual. This allows inlining of
// the enclosing method into contexts outside the current class.
if (options.testing.enableInvokeSuperToInvokeVirtualRewriting) {
- DexClassAndMethod singleTarget = invoke.lookupSingleTarget(appView, context);
- if (singleTarget != null) {
- DexMethod invokedMethod = invoke.getInvokedMethod();
- DexClassAndMethod newSingleTarget =
- InvokeVirtual.lookupSingleTarget(
- appView,
- context,
- invoke.getReceiver().getDynamicType(appView),
- invokedMethod);
- if (newSingleTarget != null
- && newSingleTarget.getReference() == singleTarget.getReference()) {
- it.replaceCurrentInstruction(
- new InvokeVirtual(invokedMethod, invoke.outValue(), invoke.arguments()));
- continue;
+ SingleResolutionResult<?> resolutionResult =
+ invoke.resolveMethod(appView).asSingleResolution();
+ if (resolutionResult != null) {
+ DispatchTargetLookupResult lookupResult =
+ resolutionResult.lookupDispatchTarget(appView, invoke, context);
+ if (lookupResult.isSingleResult()
+ && !lookupResult.getSingleDispatchTarget().getHolder().isInterface()) {
+ DexMethod invokedMethod = invoke.getInvokedMethod();
+ DispatchTargetLookupResult newLookupResult =
+ resolutionResult.lookupVirtualDispatchTarget(
+ appView, invoke, invoke.getReceiver().getDynamicType(appView), context);
+ if (lookupResult
+ .getSingleDispatchTarget()
+ .isStructurallyEqualTo(newLookupResult.getSingleDispatchTarget())) {
+ it.replaceCurrentInstruction(
+ new InvokeVirtual(invokedMethod, invoke.outValue(), invoke.arguments()));
+ continue;
+ }
}
}
}
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/InliningConstraints.java b/src/main/java/com/android/tools/r8/ir/optimize/InliningConstraints.java
index 054a256..46dd4a0 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/InliningConstraints.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/InliningConstraints.java
@@ -9,8 +9,8 @@
import com.android.tools.r8.graph.AppInfoWithClassHierarchy;
import com.android.tools.r8.graph.AppView;
import com.android.tools.r8.graph.DexClass;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexEncodedMember;
-import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexField;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProgramClass;
@@ -177,10 +177,14 @@
}
MethodResolutionResult resolutionResult =
appView.appInfo().unsafeResolveMethodDueToDexFormatLegacy(lookup);
- DexEncodedMethod target =
+ DexClassAndMethod target =
singleTargetWhileVerticalClassMerging(
resolutionResult, context, MethodResolutionResult::lookupInvokeDirectTarget);
- return forResolvedMember(resolutionResult.getInitialResolutionHolder(), context, target);
+ if (target != null) {
+ return forResolvedMember(
+ resolutionResult.getInitialResolutionHolder(), context, target.getDefinition());
+ }
+ return ConstraintWithTarget.NEVER;
}
public ConstraintWithTarget forInvokeInterface(DexMethod method, ProgramMethod context) {
@@ -209,36 +213,39 @@
}
MethodResolutionResult resolutionResult =
appView.appInfo().unsafeResolveMethodDueToDexFormatLegacy(lookup);
- DexEncodedMethod target =
+ DexClassAndMethod target =
singleTargetWhileVerticalClassMerging(
resolutionResult, context, MethodResolutionResult::lookupInvokeStaticTarget);
if (!allowStaticInterfaceMethodCalls && target != null) {
// See b/120121170.
DexClass methodClass = appView.definitionFor(graphLens.lookupType(target.getHolderType()));
- if (methodClass != null && methodClass.isInterface() && target.hasCode()) {
+ if (methodClass != null && methodClass.isInterface() && target.getDefinition().hasCode()) {
return ConstraintWithTarget.NEVER;
}
}
- return forResolvedMember(resolutionResult.getInitialResolutionHolder(), context, target);
+ if (target != null) {
+ return forResolvedMember(
+ resolutionResult.getInitialResolutionHolder(), context, target.getDefinition());
+ }
+ return ConstraintWithTarget.NEVER;
}
@SuppressWarnings({"ConstantConditions", "ReferenceEquality"})
- private DexEncodedMethod singleTargetWhileVerticalClassMerging(
+ private DexClassAndMethod singleTargetWhileVerticalClassMerging(
MethodResolutionResult resolutionResult,
ProgramMethod context,
TriFunction<
MethodResolutionResult,
DexProgramClass,
AppView<? extends AppInfoWithClassHierarchy>,
- DexEncodedMethod>
+ DexClassAndMethod>
lookup) {
if (!resolutionResult.isSingleResolution()) {
return null;
}
- DexEncodedMethod dexEncodedMethod =
- lookup.apply(resolutionResult, context.getHolder(), appView);
- if (!isVerticalClassMerging() || dexEncodedMethod != null) {
- return dexEncodedMethod;
+ DexClassAndMethod lookupResult = lookup.apply(resolutionResult, context.getHolder(), appView);
+ if (!isVerticalClassMerging() || lookupResult != null) {
+ return lookupResult;
}
assert isVerticalClassMerging();
assert graphLens.lookupType(context.getHolder().superType) == context.getHolderType();
@@ -247,7 +254,7 @@
if (superContext == null) {
return null;
}
- DexEncodedMethod alternativeDexEncodedMethod =
+ DexClassAndMethod alternativeDexEncodedMethod =
lookup.apply(resolutionResult, superContext, appView);
if (alternativeDexEncodedMethod != null
&& alternativeDexEncodedMethod.getHolderType() == superContext.type) {
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java b/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java
index c0a989b..52b3b8a 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java
@@ -19,6 +19,7 @@
import com.android.tools.r8.graph.DexItemFactory;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProgramClass;
+import com.android.tools.r8.graph.DispatchTargetLookupResult;
import com.android.tools.r8.graph.FieldResolutionResult;
import com.android.tools.r8.graph.FieldResolutionResult.SingleProgramFieldResolutionResult;
import com.android.tools.r8.graph.LibraryMethod;
@@ -491,12 +492,34 @@
continue;
}
- DynamicType exactReceiverType =
- DynamicType.createExact(
- ClassTypeElement.create(
- eligibleClass.getType(), Nullability.definitelyNotNull(), appView));
+ SingleResolutionResult<?> resolutionResult =
+ invoke.resolveMethod(appView).asSingleResolution();
+ if (resolutionResult == null) {
+ throw new IllegalClassInlinerStateException();
+ }
+
+ DispatchTargetLookupResult dispatchTargetLookupResult;
+ if (invoke.isInvokeDirect() || invoke.isInvokeSuper()) {
+ dispatchTargetLookupResult =
+ resolutionResult.lookupDispatchTarget(appView, invoke, method);
+ } else {
+ DynamicType exactReceiverType =
+ DynamicType.createExact(
+ ClassTypeElement.create(
+ eligibleClass.getType(), Nullability.definitelyNotNull(), appView));
+ dispatchTargetLookupResult =
+ resolutionResult.lookupVirtualDispatchTarget(
+ appView, invoke, exactReceiverType, method);
+ }
+ if (!dispatchTargetLookupResult.isSingleResult()) {
+ throw new IllegalClassInlinerStateException();
+ }
+
ProgramMethod singleTarget =
- invoke.lookupSingleProgramTarget(appView, method, exactReceiverType);
+ dispatchTargetLookupResult
+ .asSingleResult()
+ .getSingleDispatchTarget()
+ .asProgramMethod();
if (singleTarget == null || !indirectMethodCallsOnInstance.contains(singleTarget)) {
throw new IllegalClassInlinerStateException();
}
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/library/LibraryOptimizationInfoInitializer.java b/src/main/java/com/android/tools/r8/ir/optimize/library/LibraryOptimizationInfoInitializer.java
index 900d426..730d22e 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/library/LibraryOptimizationInfoInitializer.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/library/LibraryOptimizationInfoInitializer.java
@@ -90,15 +90,21 @@
private void modelLibraryFields() {
AbstractValueFactory abstractValueFactory = appView.abstractValueFactory();
modelLibraryField(
- dexItemFactory.booleanMembers.FALSE, abstractValueFactory.createBoxedBooleanFalse());
+ dexItemFactory.booleanMembers.FALSE,
+ abstractValueFactory.createBoxedBooleanFalse(),
+ DynamicType.definitelyNotNull());
modelLibraryField(
- dexItemFactory.booleanMembers.TRUE, abstractValueFactory.createBoxedBooleanTrue());
+ dexItemFactory.booleanMembers.TRUE,
+ abstractValueFactory.createBoxedBooleanTrue(),
+ DynamicType.definitelyNotNull());
}
- private void modelLibraryField(DexField field, AbstractValue abstractValue) {
+ private void modelLibraryField(
+ DexField field, AbstractValue abstractValue, DynamicType dynamicType) {
DexEncodedField definition = lookupField(field);
if (definition != null) {
feedback.setAbstractFieldValue(abstractValue, definition);
+ feedback.markFieldHasDynamicType(definition, dynamicType);
}
}
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/library/info/ComputedBooleanMethodOptimizationInfoCollection.java b/src/main/java/com/android/tools/r8/ir/optimize/library/info/ComputedBooleanMethodOptimizationInfoCollection.java
index c98d0cd..0748f2c 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/library/info/ComputedBooleanMethodOptimizationInfoCollection.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/library/info/ComputedBooleanMethodOptimizationInfoCollection.java
@@ -23,12 +23,13 @@
DexClassAndMethod singleTarget,
ProgramMethod context,
AbstractValueSupplier abstractValueSupplier) {
- if (singleTarget.getReference().isIdenticalTo(dexItemFactory.integerMembers.valueOf)) {
+ if (singleTarget.getReference().isIdenticalTo(dexItemFactory.booleanMembers.valueOf)) {
AbstractValue operandValue =
invoke.getFirstOperand().getAbstractValue(appView, context, abstractValueSupplier);
if (operandValue.isSingleNumberValue()) {
- return abstractValueFactory.createBoxedInteger(
- operandValue.asSingleNumberValue().getIntValue());
+ return operandValue.asSingleNumberValue().getBooleanValue()
+ ? abstractValueFactory.createBoxedBooleanTrue()
+ : abstractValueFactory.createBoxedBooleanFalse();
}
}
return AbstractValue.unknown();
diff --git a/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScanner.java b/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScanner.java
index 3a6e882..49d6400 100644
--- a/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScanner.java
+++ b/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScanner.java
@@ -67,6 +67,8 @@
private final AppView<AppInfoWithLiveness> appView;
+ private final ArgumentPropagatorCodeScannerModeling modeling;
+
private final MethodParameterFactory methodParameterFactory = new MethodParameterFactory();
private final Set<DexMethod> monomorphicVirtualMethods = Sets.newIdentityHashSet();
@@ -91,6 +93,7 @@
AppView<AppInfoWithLiveness> appView,
ArgumentPropagatorReprocessingCriteriaCollection reprocessingCriteriaCollection) {
this.appView = appView;
+ this.modeling = new ArgumentPropagatorCodeScannerModeling(appView);
this.reprocessingCriteriaCollection = reprocessingCriteriaCollection;
}
@@ -404,11 +407,11 @@
parameterStates.add(
computeParameterStateForNonReceiver(
invoke,
+ singleTarget,
argumentIndex,
invoke.getArgument(argumentIndex),
context,
- existingMethodState,
- methodReprocessingCriteria.getParameterReprocessingCriteria(argumentIndex)));
+ existingMethodState));
}
// We simulate that the return value is used for methods with void return type. This ensures
@@ -448,11 +451,18 @@
@SuppressWarnings("UnusedVariable")
private ParameterState computeParameterStateForNonReceiver(
InvokeMethod invoke,
+ ProgramMethod singleTarget,
int argumentIndex,
Value argument,
ProgramMethod context,
- ConcreteMonomorphicMethodStateOrBottom existingMethodState,
- ParameterReprocessingCriteria parameterReprocessingCriteria) {
+ ConcreteMonomorphicMethodStateOrBottom existingMethodState) {
+ ParameterState modeledState =
+ modeling.modelParameterStateForArgumentToFunction(
+ invoke, singleTarget, argumentIndex, argument);
+ if (modeledState != null) {
+ return modeledState;
+ }
+
// Don't compute a state for this parameter if the stored state is already unknown.
if (existingMethodState.isMonomorphic()
&& existingMethodState.asMonomorphic().getParameterState(argumentIndex).isUnknown()) {
diff --git a/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScannerModeling.java b/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScannerModeling.java
new file mode 100644
index 0000000..7ab506e
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/optimize/argumentpropagation/ArgumentPropagatorCodeScannerModeling.java
@@ -0,0 +1,33 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.optimize.argumentpropagation;
+
+import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.ProgramMethod;
+import com.android.tools.r8.ir.code.InvokeMethod;
+import com.android.tools.r8.ir.code.Value;
+import com.android.tools.r8.optimize.argumentpropagation.codescanner.ParameterState;
+import com.android.tools.r8.optimize.compose.ArgumentPropagatorComposeModeling;
+import com.android.tools.r8.shaking.AppInfoWithLiveness;
+
+public class ArgumentPropagatorCodeScannerModeling {
+
+ private final ArgumentPropagatorComposeModeling composeModeling;
+
+ ArgumentPropagatorCodeScannerModeling(AppView<AppInfoWithLiveness> appView) {
+ this.composeModeling =
+ appView.testing().modelUnknownChangedAndDefaultArgumentsToComposableFunctions
+ ? new ArgumentPropagatorComposeModeling(appView)
+ : null;
+ }
+
+ ParameterState modelParameterStateForArgumentToFunction(
+ InvokeMethod invoke, ProgramMethod singleTarget, int argumentIndex, Value argument) {
+ if (composeModeling != null) {
+ return composeModeling.modelParameterStateForChangedOrDefaultArgumentToComposableFunction(
+ invoke, singleTarget, argumentIndex, argument);
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/optimize/compose/ArgumentPropagatorComposeModeling.java b/src/main/java/com/android/tools/r8/optimize/compose/ArgumentPropagatorComposeModeling.java
new file mode 100644
index 0000000..4bd4242
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/optimize/compose/ArgumentPropagatorComposeModeling.java
@@ -0,0 +1,180 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.optimize.compose;
+
+import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.DexMethod;
+import com.android.tools.r8.graph.DexString;
+import com.android.tools.r8.graph.ProgramMethod;
+import com.android.tools.r8.graph.lens.GraphLens;
+import com.android.tools.r8.ir.code.InstanceGet;
+import com.android.tools.r8.ir.code.Instruction;
+import com.android.tools.r8.ir.code.InvokeMethod;
+import com.android.tools.r8.ir.code.InvokeStatic;
+import com.android.tools.r8.ir.code.Or;
+import com.android.tools.r8.ir.code.Value;
+import com.android.tools.r8.optimize.argumentpropagation.codescanner.ConcretePrimitiveTypeParameterState;
+import com.android.tools.r8.optimize.argumentpropagation.codescanner.ParameterState;
+import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.utils.BitUtils;
+import com.android.tools.r8.utils.BooleanUtils;
+
+public class ArgumentPropagatorComposeModeling {
+
+ private final AppView<AppInfoWithLiveness> appView;
+ private final ComposeReferences composeReferences;
+
+ public ArgumentPropagatorComposeModeling(AppView<AppInfoWithLiveness> appView) {
+ assert appView.testing().modelUnknownChangedAndDefaultArgumentsToComposableFunctions;
+ this.appView = appView;
+ this.composeReferences = appView.getComposeReferences();
+ }
+
+ /**
+ * Models calls to @Composable functions from Compose restart lambdas.
+ *
+ * <p>The @Composable functions are static and should have one of the following two parameter
+ * lists:
+ *
+ * <ol>
+ * <li>(..., Composable, int)
+ * <li>(..., Composable, int, int)
+ * </ol>
+ *
+ * <p>The int argument after the Composable parameter is the $$changed parameter. The second int
+ * argument after the Composable parameter (if present) is the $$default parameter.
+ *
+ * <p>The call to a @Composable function from its restart lambda follows the following code
+ * pattern:
+ *
+ * <pre>
+ * MyComposableFunction(
+ * ..., this.composer, updateChangedFlags(this.$$changed) || 1, this.$$default)
+ * </pre>
+ *
+ * <p>The modeling performed by this method assumes that updateChangedFlags() does not have any
+ * impact on $$changed (see the current implementation below). The modeling also assumes that
+ * this.$$changed and this.$$default are captures of the $$changed and $$default parameters of
+ * the @Composable function.
+ *
+ * <pre>
+ * internal fun updateChangedFlags(flags: Int): Int {
+ * val lowBits = flags and changedLowBitMask
+ * val highBits = flags and changedHighBitMask
+ * return ((flags and changedMask) or
+ * (lowBits or (highBits shr 1)) or ((lowBits shl 1) and highBits))
+ * }
+ * </pre>
+ */
+ public ParameterState modelParameterStateForChangedOrDefaultArgumentToComposableFunction(
+ InvokeMethod invoke, ProgramMethod singleTarget, int argumentIndex, Value argument) {
+ // First check if this is an invoke to a @Composable function.
+ if (singleTarget == null
+ || singleTarget
+ .getDefinition()
+ .annotations()
+ .getFirstMatching(composeReferences.composableType)
+ == null) {
+ return null;
+ }
+
+ // The @Composable function is expected to be static and have >= 2 parameters.
+ if (!invoke.isInvokeStatic()) {
+ return null;
+ }
+
+ DexMethod invokedMethod = invoke.getInvokedMethod();
+ if (invokedMethod.getArity() < 2) {
+ return null;
+ }
+
+ // Check if the parameters list is one of (..., Composer, int) or (..., Composer, int, int).
+ if (!invokedMethod.getParameter(invokedMethod.getArity() - 1).isIntType()) {
+ return null;
+ }
+
+ boolean hasDefaultParameter =
+ invokedMethod.getParameter(invokedMethod.getArity() - 2).isIntType();
+ if (hasDefaultParameter && invokedMethod.getArity() < 3) {
+ return null;
+ }
+
+ int composerParameterIndex =
+ invokedMethod.getArity() - 2 - BooleanUtils.intValue(hasDefaultParameter);
+ if (!invokedMethod
+ .getParameter(composerParameterIndex)
+ .isIdenticalTo(composeReferences.composerType)) {
+ return null;
+ }
+
+ // We only model the two last arguments ($$changed and $$default) to the @Composable function.
+ // If this argument is not on of these two int arguments, then don't apply any modeling.
+ if (argumentIndex <= composerParameterIndex) {
+ return null;
+ }
+
+ assert argument.getType().isInt();
+
+ DexString expectedFieldName;
+ ParameterState state = ParameterState.bottomPrimitiveTypeParameter();
+ if (!hasDefaultParameter || argumentIndex == invokedMethod.getArity() - 2) {
+ // We are looking at an argument to the $$changed parameter of the @Composable function.
+ // We generally expect this argument to be defined by a call to updateChangedFlags().
+ if (argument.isDefinedByInstructionSatisfying(Instruction::isInvokeStatic)) {
+ InvokeStatic invokeStatic = argument.getDefinition().asInvokeStatic();
+ DexMethod maybeUpdateChangedFlagsMethod =
+ appView
+ .graphLens()
+ .getOriginalMethodSignature(
+ invokeStatic.getInvokedMethod(), GraphLens.getIdentityLens());
+ if (!maybeUpdateChangedFlagsMethod.isIdenticalTo(
+ composeReferences.updatedChangedFlagsMethod)) {
+ return null;
+ }
+ // Assume the call does not impact the $$changed capture and strip the call.
+ argument = invokeStatic.getFirstArgument();
+ }
+ // Allow the argument to be defined by `this.$$changed | 1`.
+ if (argument.isDefinedByInstructionSatisfying(Instruction::isOr)) {
+ Or or = argument.getDefinition().asOr();
+ Value maybeNumberOperand =
+ or.leftValue().isConstNumber() ? or.leftValue() : or.rightValue();
+ Value otherOperand = or.getOperand(1 - or.inValues().indexOf(maybeNumberOperand));
+ if (!maybeNumberOperand.isConstNumber(1)) {
+ return null;
+ }
+ // Strip the OR instruction.
+ argument = otherOperand;
+ // Update the model from bottom to a special value that effectively throws away any known
+ // information about the lowermost bit of $$changed.
+ state =
+ new ConcretePrimitiveTypeParameterState(
+ appView
+ .abstractValueFactory()
+ .createDefiniteBitsNumberValue(
+ BitUtils.ALL_BITS_SET_MASK, BitUtils.ALL_BITS_SET_MASK << 1));
+ }
+ expectedFieldName = composeReferences.changedFieldName;
+ } else {
+ // We are looking at an argument to the $$default parameter of the @Composable function.
+ expectedFieldName = composeReferences.defaultFieldName;
+ }
+
+ // At this point we expect that the restart lambda is reading either this.$$changed or
+ // this.$$default using an instance-get.
+ if (!argument.isDefinedByInstructionSatisfying(Instruction::isInstanceGet)) {
+ return null;
+ }
+
+ // Check that the instance-get is reading the capture field that we expect it to.
+ InstanceGet instanceGet = argument.getDefinition().asInstanceGet();
+ if (!instanceGet.getField().getName().isIdenticalTo(expectedFieldName)) {
+ return null;
+ }
+
+ // Return the argument model. Note that, for the $$default field, this is always bottom, which
+ // is equivalent to modeling that this call does not contribute any new argument information.
+ return state;
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/optimize/compose/ComposeReferences.java b/src/main/java/com/android/tools/r8/optimize/compose/ComposeReferences.java
new file mode 100644
index 0000000..7a489a7
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/optimize/compose/ComposeReferences.java
@@ -0,0 +1,34 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.optimize.compose;
+
+import com.android.tools.r8.graph.DexItemFactory;
+import com.android.tools.r8.graph.DexMethod;
+import com.android.tools.r8.graph.DexString;
+import com.android.tools.r8.graph.DexType;
+
+public class ComposeReferences {
+
+ public final DexString changedFieldName;
+ public final DexString defaultFieldName;
+
+ public final DexType composableType;
+ public final DexType composerType;
+
+ public final DexMethod updatedChangedFlagsMethod;
+
+ public ComposeReferences(DexItemFactory factory) {
+ changedFieldName = factory.createString("$$changed");
+ defaultFieldName = factory.createString("$$default");
+
+ composableType = factory.createType("Landroidx/compose/runtime/Composable;");
+ composerType = factory.createType("Landroidx/compose/runtime/Composer;");
+
+ updatedChangedFlagsMethod =
+ factory.createMethod(
+ factory.createType("Landroidx/compose/runtime/RecomposeScopeImplKt;"),
+ factory.createProto(factory.intType, factory.intType),
+ "updateChangedFlags");
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/optimize/interfaces/analysis/CfOpenClosedInterfacesAnalysisHelper.java b/src/main/java/com/android/tools/r8/optimize/interfaces/analysis/CfOpenClosedInterfacesAnalysisHelper.java
index e06916b..5416619 100644
--- a/src/main/java/com/android/tools/r8/optimize/interfaces/analysis/CfOpenClosedInterfacesAnalysisHelper.java
+++ b/src/main/java/com/android/tools/r8/optimize/interfaces/analysis/CfOpenClosedInterfacesAnalysisHelper.java
@@ -92,7 +92,7 @@
assert false;
}
} else {
- assert false;
+ assert array.isNullType();
}
},
options);
diff --git a/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java b/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java
index 6389e26..a02420c 100644
--- a/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java
+++ b/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java
@@ -62,7 +62,7 @@
ProgramDefinition holder, DexAnnotation annotation, AnnotatedKind kind) {
return annotationsToRetain.contains(annotation)
|| shouldKeepAnnotation(
- appView, holder, annotation, isAnnotationTypeLive(annotation), kind);
+ appView, holder, annotation, isAnnotationTypeLive(annotation), kind, mode);
}
public static boolean shouldKeepAnnotation(
@@ -70,13 +70,14 @@
ProgramDefinition holder,
DexAnnotation annotation,
boolean isAnnotationTypeLive,
- AnnotatedKind kind) {
+ AnnotatedKind kind,
+ Mode mode) {
// If we cannot run the AnnotationRemover we are keeping the annotation.
- if (!appView.options().isShrinking()) {
+ InternalOptions options = appView.options();
+ if (!options.isShrinking()) {
return true;
}
- InternalOptions options = appView.options();
ProguardKeepAttributes config =
options.getProguardConfiguration() != null
? options.getProguardConfiguration().getKeepAttributes()
@@ -145,6 +146,9 @@
.startsWith(options.itemFactory.dalvikAnnotationOptimizationPrefix)) {
return true;
}
+ if (isComposableAnnotationToRetain(appView, annotation, kind, mode, options)) {
+ return true;
+ }
if (kind.isParameter()) {
if (!options.isKeepRuntimeInvisibleParameterAnnotationsEnabled()) {
return false;
@@ -291,14 +295,10 @@
boolean isAnnotation =
definition.isProgramClass() && definition.asProgramClass().isAnnotation();
if (keepInfo.isAnnotationRemovalAllowed(options)) {
- if (isAnnotation) {
+ if (isAnnotation || mode.isInitialTreeShaking()) {
definition.rewriteAllAnnotations(
- (annotation, isParameterAnnotation) ->
- shouldRetainAnnotationOnAnnotationClass(annotation) ? annotation : null);
- } else if (mode.isInitialTreeShaking()) {
- definition.rewriteAllAnnotations(
- (annotation, isParameterAnnotation) ->
- annotationsToRetain.contains(annotation) ? annotation : null);
+ (annotation, kind) ->
+ shouldRetainAnnotation(definition, annotation, kind) ? annotation : null);
} else {
definition.clearAllAnnotations();
}
@@ -308,14 +308,34 @@
}
}
- private boolean shouldRetainAnnotationOnAnnotationClass(DexAnnotation annotation) {
- if (DexAnnotation.isAnnotationDefaultAnnotation(annotation, appView.dexItemFactory())) {
- return shouldRetainAnnotationDefaultAnnotationOnAnnotationClass(annotation);
+ private boolean shouldRetainAnnotation(
+ ProgramDefinition definition, DexAnnotation annotation, AnnotatedKind kind) {
+ boolean isAnnotationOnAnnotationClass =
+ definition.isProgramClass() && definition.asProgramClass().isAnnotation();
+ if (isAnnotationOnAnnotationClass) {
+ if (DexAnnotation.isAnnotationDefaultAnnotation(annotation, appView.dexItemFactory())) {
+ return shouldRetainAnnotationDefaultAnnotationOnAnnotationClass(annotation);
+ }
+ if (DexAnnotation.isJavaLangRetentionAnnotation(annotation, appView.dexItemFactory())) {
+ return shouldRetainRetentionAnnotationOnAnnotationClass(annotation);
+ }
}
- if (DexAnnotation.isJavaLangRetentionAnnotation(annotation, appView.dexItemFactory())) {
- return shouldRetainRetentionAnnotationOnAnnotationClass(annotation);
- }
- return annotationsToRetain.contains(annotation);
+ return annotationsToRetain.contains(annotation)
+ || isComposableAnnotationToRetain(appView, annotation, kind, mode, options);
+ }
+
+ private static boolean isComposableAnnotationToRetain(
+ AppView<?> appView,
+ DexAnnotation annotation,
+ AnnotatedKind kind,
+ Mode mode,
+ InternalOptions options) {
+ return options.testing.modelUnknownChangedAndDefaultArgumentsToComposableFunctions
+ && mode.isInitialTreeShaking()
+ && kind.isMethod()
+ && annotation
+ .getAnnotationType()
+ .isIdenticalTo(appView.getComposeReferences().composableType);
}
@SuppressWarnings("UnusedVariable")
diff --git a/src/main/java/com/android/tools/r8/shaking/AppInfoWithLiveness.java b/src/main/java/com/android/tools/r8/shaking/AppInfoWithLiveness.java
index 927f14b..56c1ea4 100644
--- a/src/main/java/com/android/tools/r8/shaking/AppInfoWithLiveness.java
+++ b/src/main/java/com/android/tools/r8/shaking/AppInfoWithLiveness.java
@@ -3,8 +3,6 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.shaking;
-import static com.android.tools.r8.graph.DexEncodedMethod.asProgramMethodOrNull;
-import static com.android.tools.r8.graph.DexEncodedMethod.toMethodDefinitionOrNull;
import static com.android.tools.r8.graph.DexProgramClass.asProgramClassOrNull;
import static com.android.tools.r8.graph.MethodResolutionResult.SingleResolutionResult.isOverriding;
import static com.android.tools.r8.utils.collections.ThrowingSet.isThrowingSet;
@@ -18,6 +16,7 @@
import com.android.tools.r8.graph.DexClass;
import com.android.tools.r8.graph.DexClassAndField;
import com.android.tools.r8.graph.DexClassAndMember;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexClasspathClass;
import com.android.tools.r8.graph.DexDefinition;
import com.android.tools.r8.graph.DexDefinitionSupplier;
@@ -30,6 +29,7 @@
import com.android.tools.r8.graph.DexReference;
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.DirectMappedDexApplication;
+import com.android.tools.r8.graph.DispatchTargetLookupResult;
import com.android.tools.r8.graph.FieldAccessInfo;
import com.android.tools.r8.graph.FieldAccessInfoCollection;
import com.android.tools.r8.graph.FieldAccessInfoCollectionImpl;
@@ -46,7 +46,9 @@
import com.android.tools.r8.graph.ProgramField;
import com.android.tools.r8.graph.ProgramMethod;
import com.android.tools.r8.graph.PrunedItems;
+import com.android.tools.r8.graph.SingleDispatchTargetLookupResult;
import com.android.tools.r8.graph.SubtypingInfo;
+import com.android.tools.r8.graph.UnknownDispatchTargetLookupResult;
import com.android.tools.r8.graph.lens.GraphLens;
import com.android.tools.r8.graph.lens.NonIdentityGraphLens;
import com.android.tools.r8.horizontalclassmerging.HorizontalClassMerger;
@@ -130,12 +132,14 @@
* kept.
*/
private Set<DexMethod> liveMethods;
+
/**
* Information about all fields that are accessed by the program. The information includes whether
* a given field is read/written by the program, and it also includes all indirect accesses to
* each field. The latter is used, for example, during member rebinding.
*/
- private FieldAccessInfoCollectionImpl fieldAccessInfoCollection;
+ private final FieldAccessInfoCollectionImpl fieldAccessInfoCollection;
+
/** Set of all methods referenced in invokes along with their calling contexts. */
private final MethodAccessInfoCollection methodAccessInfoCollection;
/** Information about instantiated classes and their allocation sites. */
@@ -1281,69 +1285,67 @@
return prunedTypes;
}
- public DexEncodedMethod lookupSingleTarget(
+ public DexClassAndMethod lookupSingleTarget(
AppView<AppInfoWithLiveness> appView,
InvokeType type,
DexMethod target,
+ SingleResolutionResult<?> resolutionResult,
ProgramMethod context,
LibraryModeledPredicate modeledPredicate) {
assert checkIfObsolete();
- DexType holder = target.holder;
- if (!holder.isClassType()) {
+ if (!target.getHolderType().isClassType()) {
return null;
}
switch (type) {
- case VIRTUAL:
- return lookupSingleVirtualTarget(appView, target, context, false, modeledPredicate);
case INTERFACE:
- return lookupSingleVirtualTarget(appView, target, context, true, modeledPredicate);
+ case VIRTUAL:
+ return lookupSingleVirtualTarget(
+ appView,
+ target,
+ resolutionResult,
+ context,
+ type.isInterface(),
+ modeledPredicate,
+ DynamicType.unknown());
case DIRECT:
return lookupDirectTarget(target, context, appView);
case STATIC:
return lookupStaticTarget(target, context, appView);
case SUPER:
- return toMethodDefinitionOrNull(lookupSuperTarget(target, context, appView));
+ return lookupSuperTarget(target, context, appView);
default:
return null;
}
}
- public ProgramMethod lookupSingleProgramTarget(
- AppView<AppInfoWithLiveness> appView,
- InvokeType type,
- DexMethod target,
- ProgramMethod context,
- LibraryModeledPredicate modeledPredicate) {
- return asProgramMethodOrNull(
- lookupSingleTarget(appView, type, target, context, modeledPredicate), this);
- }
-
/** For mapping invoke virtual instruction to single target method. */
- public DexEncodedMethod lookupSingleVirtualTarget(
- AppView<AppInfoWithLiveness> appView,
- DexMethod method,
- ProgramMethod context,
- boolean isInterface) {
- assert checkIfObsolete();
- return lookupSingleVirtualTarget(appView, method, context, isInterface, type -> false);
- }
-
- /** For mapping invoke virtual instruction to single target method. */
- public DexEncodedMethod lookupSingleVirtualTarget(
+ public DexClassAndMethod lookupSingleVirtualTargetForTesting(
AppView<AppInfoWithLiveness> appView,
DexMethod method,
ProgramMethod context,
boolean isInterface,
- LibraryModeledPredicate modeledPredicate) {
+ LibraryModeledPredicate modeledPredicate,
+ DynamicType dynamicReceiverType) {
assert checkIfObsolete();
- return lookupSingleVirtualTarget(
- appView, method, context, isInterface, modeledPredicate, DynamicType.unknown());
+ SingleResolutionResult<?> resolutionResult =
+ appView.appInfo().resolveMethodLegacy(method, isInterface).asSingleResolution();
+ if (resolutionResult != null) {
+ return lookupSingleVirtualTarget(
+ appView,
+ method,
+ resolutionResult,
+ context,
+ isInterface,
+ modeledPredicate,
+ dynamicReceiverType);
+ }
+ return null;
}
- @SuppressWarnings("ReferenceEquality")
- public DexEncodedMethod lookupSingleVirtualTarget(
+ public DexClassAndMethod lookupSingleVirtualTarget(
AppView<AppInfoWithLiveness> appView,
DexMethod method,
+ SingleResolutionResult<?> resolutionResult,
ProgramMethod context,
boolean isInterface,
LibraryModeledPredicate modeledPredicate,
@@ -1370,53 +1372,57 @@
// The refined receiver is not defined in the program and we cannot determine the target.
return null;
}
- if (!dynamicReceiverType.hasDynamicLowerBoundType()
- && singleTargetLookupCache.hasCachedItem(refinedReceiverType, method)) {
- DexEncodedMethod cachedItem =
- singleTargetLookupCache.getCachedItem(refinedReceiverType, method);
- return cachedItem;
+ if (!dynamicReceiverType.hasDynamicLowerBoundType()) {
+ if (singleTargetLookupCache.hasPositiveCacheHit(refinedReceiverType, method)) {
+ return singleTargetLookupCache.getPositiveCacheHit(refinedReceiverType, method);
+ }
+ if (singleTargetLookupCache.hasNegativeCacheHit(refinedReceiverType, method)) {
+ return null;
+ }
}
- SingleResolutionResult<?> resolution =
- resolveMethodOnLegacy(initialResolutionHolder, method).asSingleResolution();
- if (resolution == null
- || resolution.isAccessibleForVirtualDispatchFrom(context.getHolder(), appView).isFalse()) {
+ if (resolutionResult
+ .isAccessibleForVirtualDispatchFrom(context.getHolder(), appView)
+ .isFalse()) {
return null;
}
// If the method is modeled, return the resolution.
- DexEncodedMethod resolvedMethod = resolution.getResolvedMethod();
- if (modeledPredicate.isModeled(resolution.getResolvedHolder().type)) {
- if (resolution.getResolvedHolder().isFinal()
- || (resolvedMethod.isFinal() && resolvedMethod.accessFlags.isPublic())) {
+ DexClassAndMethod resolvedMethod = resolutionResult.getResolutionPair();
+ if (modeledPredicate.isModeled(resolutionResult.getResolvedHolder().getType())) {
+ if (resolutionResult.getResolvedHolder().isFinal()
+ || (resolvedMethod.getAccessFlags().isFinal()
+ && resolvedMethod.getAccessFlags().isPublic())) {
singleTargetLookupCache.addToCache(refinedReceiverType, method, resolvedMethod);
return resolvedMethod;
}
}
- DexEncodedMethod exactTarget =
+ DispatchTargetLookupResult exactTarget =
getMethodTargetFromExactRuntimeInformation(
refinedReceiverType,
dynamicReceiverType.getDynamicLowerBoundType(),
- resolution,
+ resolutionResult,
refinedReceiverClass);
if (exactTarget != null) {
// We are not caching single targets here because the cache does not include the
// lower bound dimension.
- return exactTarget == DexEncodedMethod.SENTINEL ? null : exactTarget;
+ return exactTarget.isSingleResult()
+ ? exactTarget.asSingleResult().getSingleDispatchTarget()
+ : null;
}
if (refinedReceiverClass.isNotProgramClass()) {
// The refined receiver is not defined in the program and we cannot determine the target.
- singleTargetLookupCache.addToCache(refinedReceiverType, method, null);
+ singleTargetLookupCache.addNoSingleTargetToCache(refinedReceiverType, method);
return null;
}
- DexClass resolvedHolder = resolution.getResolvedHolder();
+ DexClass resolvedHolder = resolutionResult.getResolvedHolder();
// TODO(b/148769279): Disable lookup single target on lambda's for now.
if (resolvedHolder.isInterface()
&& resolvedHolder.isProgramClass()
&& objectAllocationInfoCollection.isImmediateInterfaceOfInstantiatedLambda(
resolvedHolder.asProgramClass())) {
- singleTargetLookupCache.addToCache(refinedReceiverType, method, null);
+ singleTargetLookupCache.addNoSingleTargetToCache(refinedReceiverType, method);
return null;
}
- DexEncodedMethod singleMethodTarget = null;
+ DexClassAndMethod singleMethodTarget = null;
DexProgramClass refinedLowerBound = null;
if (dynamicReceiverType.hasDynamicLowerBoundType()) {
DexClass refinedLowerBoundClass =
@@ -1430,7 +1436,7 @@
}
}
LookupResultSuccess lookupResult =
- resolution
+ resolutionResult
.lookupVirtualDispatchTargets(
context.getHolder(),
appView,
@@ -1440,7 +1446,7 @@
if (lookupResult != null && !lookupResult.isIncomplete()) {
LookupTarget singleTarget = lookupResult.getSingleLookupTarget();
if (singleTarget != null && singleTarget.isMethodTarget()) {
- singleMethodTarget = singleTarget.asMethodTarget().getDefinition();
+ singleMethodTarget = singleTarget.asMethodTarget().getTarget();
}
}
if (!dynamicReceiverType.hasDynamicLowerBoundType()) {
@@ -1450,7 +1456,7 @@
}
@SuppressWarnings("ReferenceEquality")
- private DexEncodedMethod getMethodTargetFromExactRuntimeInformation(
+ private DispatchTargetLookupResult getMethodTargetFromExactRuntimeInformation(
DexType refinedReceiverType,
ClassTypeElement receiverLowerBoundType,
SingleResolutionResult<?> resolution,
@@ -1469,21 +1475,21 @@
.getMethodInfo(methodTarget.getTarget().asProgramMethod())
.isOptimizationAllowed(options()))) {
// TODO(b/150640456): We should maybe only consider program methods.
- return DexEncodedMethod.SENTINEL;
+ return new UnknownDispatchTargetLookupResult(resolution);
}
- return methodTarget.getDefinition();
+ return new SingleDispatchTargetLookupResult(methodTarget.getTarget(), resolution);
} else {
// TODO(b/150640456): We should maybe only consider program methods.
// If we resolved to a method on the refined receiver in the library, then we report the
// method as a single target as well. This is a bit iffy since the library could change
// implementation, but we use this for library modelling.
- DexEncodedMethod resolvedMethod = resolution.getResolvedMethod();
- DexEncodedMethod targetOnReceiver =
- refinedReceiverClass.lookupVirtualMethod(resolvedMethod.getReference());
+ DexClassAndMethod resolvedMethod = resolution.getResolutionPair();
+ DexClassAndMethod targetOnReceiver =
+ refinedReceiverClass.lookupVirtualClassMethod(resolvedMethod.getReference());
if (targetOnReceiver != null && isOverriding(resolvedMethod, targetOnReceiver)) {
- return targetOnReceiver;
+ return new SingleDispatchTargetLookupResult(targetOnReceiver, resolution);
}
- return DexEncodedMethod.SENTINEL;
+ return new UnknownDispatchTargetLookupResult(resolution);
}
}
return null;
diff --git a/src/main/java/com/android/tools/r8/shaking/Enqueuer.java b/src/main/java/com/android/tools/r8/shaking/Enqueuer.java
index d07235f..fda3ddf 100644
--- a/src/main/java/com/android/tools/r8/shaking/Enqueuer.java
+++ b/src/main/java/com/android/tools/r8/shaking/Enqueuer.java
@@ -2347,7 +2347,7 @@
return true;
}
return AnnotationRemover.shouldKeepAnnotation(
- appView, annotatedItem, annotation, isLive, annotatedKind);
+ appView, annotatedItem, annotation, isLive, annotatedKind, mode);
}
private DexProgramClass resolveBaseType(DexType type, ProgramDefinition context) {
diff --git a/src/main/java/com/android/tools/r8/shaking/LibraryModeledPredicate.java b/src/main/java/com/android/tools/r8/shaking/LibraryModeledPredicate.java
index 39ee67c..00be009 100644
--- a/src/main/java/com/android/tools/r8/shaking/LibraryModeledPredicate.java
+++ b/src/main/java/com/android/tools/r8/shaking/LibraryModeledPredicate.java
@@ -10,4 +10,8 @@
public interface LibraryModeledPredicate {
boolean isModeled(DexType type);
+
+ static LibraryModeledPredicate alwaysFalse() {
+ return type -> false;
+ }
}
diff --git a/src/main/java/com/android/tools/r8/shaking/SingleTargetLookupCache.java b/src/main/java/com/android/tools/r8/shaking/SingleTargetLookupCache.java
index 392ae35..0c576e0 100644
--- a/src/main/java/com/android/tools/r8/shaking/SingleTargetLookupCache.java
+++ b/src/main/java/com/android/tools/r8/shaking/SingleTargetLookupCache.java
@@ -4,32 +4,51 @@
package com.android.tools.r8.shaking;
+import static com.android.tools.r8.utils.MapUtils.ignoreKey;
+
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
+import com.android.tools.r8.utils.ObjectUtils;
import com.android.tools.r8.utils.TraversalContinuation;
import com.google.common.collect.Sets;
+import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
public class SingleTargetLookupCache {
- private Map<DexType, Map<DexMethod, DexEncodedMethod>> cache = new ConcurrentHashMap<>();
+ private final Map<DexType, Map<DexMethod, DexClassAndMethod>> positiveCache =
+ new ConcurrentHashMap<>();
+ private final Map<DexType, Set<DexMethod>> negativeCache = new ConcurrentHashMap<>();
- @SuppressWarnings("ReferenceEquality")
- public void addToCache(DexType refinedReceiverType, DexMethod method, DexEncodedMethod target) {
- assert target != DexEncodedMethod.SENTINEL;
- Map<DexMethod, DexEncodedMethod> methodCache =
- cache.computeIfAbsent(refinedReceiverType, ignored -> new ConcurrentHashMap<>());
- target = target == null ? DexEncodedMethod.SENTINEL : target;
- assert methodCache.getOrDefault(method, target) == target;
- methodCache.putIfAbsent(method, target);
+ public void addNoSingleTargetToCache(DexType refinedReceiverType, DexMethod method) {
+ assert !hasPositiveCacheHit(refinedReceiverType, method);
+ negativeCache
+ .computeIfAbsent(refinedReceiverType, ignoreKey(ConcurrentHashMap::newKeySet))
+ .add(method);
+ }
+
+ public void addToCache(DexType refinedReceiverType, DexMethod method, DexClassAndMethod target) {
+ if (target == null) {
+ addNoSingleTargetToCache(refinedReceiverType, method);
+ return;
+ }
+ assert !ObjectUtils.identical(target.getDefinition(), DexEncodedMethod.SENTINEL);
+ assert !hasNegativeCacheHit(refinedReceiverType, method);
+ assert !hasPositiveCacheHit(refinedReceiverType, method)
+ || getPositiveCacheHit(refinedReceiverType, method).isStructurallyEqualTo(target);
+ positiveCache
+ .computeIfAbsent(refinedReceiverType, ignoreKey(ConcurrentHashMap::new))
+ .put(method, target);
}
public void removeInstantiatedType(DexType instantiatedType, AppInfoWithLiveness appInfo) {
// Remove all types in the instantiated hierarchy related to this type.
- cache.remove(instantiatedType);
+ positiveCache.remove(instantiatedType);
+ negativeCache.remove(instantiatedType);
Set<DexType> seen = Sets.newIdentityHashSet();
appInfo.forEachInstantiatedSubType(
instantiatedType,
@@ -38,7 +57,8 @@
instance,
(superType, subclass, ignore) -> {
if (seen.add(superType)) {
- cache.remove(superType);
+ positiveCache.remove(superType);
+ negativeCache.remove(superType);
return TraversalContinuation.doContinue();
} else {
return TraversalContinuation.doBreak();
@@ -49,25 +69,15 @@
});
}
- @SuppressWarnings("ReferenceEquality")
- public DexEncodedMethod getCachedItem(DexType receiverType, DexMethod method) {
- Map<DexMethod, DexEncodedMethod> cachedMethods = cache.get(receiverType);
- if (cachedMethods == null) {
- return null;
- }
- DexEncodedMethod target = cachedMethods.get(method);
- return target == DexEncodedMethod.SENTINEL ? null : target;
+ public boolean hasPositiveCacheHit(DexType receiverType, DexMethod method) {
+ return positiveCache.getOrDefault(receiverType, Collections.emptyMap()).containsKey(method);
}
- public boolean hasCachedItem(DexType receiverType, DexMethod method) {
- Map<DexMethod, DexEncodedMethod> cachedMethods = cache.get(receiverType);
- if (cachedMethods == null) {
- return false;
- }
- return cachedMethods.containsKey(method);
+ public DexClassAndMethod getPositiveCacheHit(DexType receiverType, DexMethod method) {
+ return positiveCache.getOrDefault(receiverType, Collections.emptyMap()).get(method);
}
- public void clear() {
- cache = new ConcurrentHashMap<>();
+ public boolean hasNegativeCacheHit(DexType receiverType, DexMethod method) {
+ return negativeCache.getOrDefault(receiverType, Collections.emptySet()).contains(method);
}
}
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 1367e23..51593eb 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -2398,6 +2398,10 @@
System.getProperty("com.android.tools.r8.disableMarkingClassesFinal") != null;
public boolean testEnableTestAssertions = false;
public boolean keepMetadataInR8IfNotRewritten = true;
+ public boolean modelUnknownChangedAndDefaultArgumentsToComposableFunctions =
+ SystemPropertyUtils.parseSystemPropertyForDevelopmentOrDefault(
+ "com.android.tools.r8.modelUnknownChangedAndDefaultArgumentsToComposableFunctions",
+ false);
// Flag to allow processing of resources in D8. A data resource consumer still needs to be
// specified.
diff --git a/src/main/java/com/android/tools/r8/utils/ObjectUtils.java b/src/main/java/com/android/tools/r8/utils/ObjectUtils.java
index 216f19b..9dfd1e0 100644
--- a/src/main/java/com/android/tools/r8/utils/ObjectUtils.java
+++ b/src/main/java/com/android/tools/r8/utils/ObjectUtils.java
@@ -16,6 +16,11 @@
return orElse;
}
+ @SuppressWarnings("ReferenceEquality")
+ public static boolean identical(Object a, Object b) {
+ return a == b;
+ }
+
public static <S, T> T mapNotNull(S object, Function<? super S, ? extends T> fn) {
if (object != null) {
return fn.apply(object);
diff --git a/src/main/java/com/android/tools/r8/utils/ResourceTracing.java b/src/main/java/com/android/tools/r8/utils/ResourceTracing.java
index f0bc26e..6e030ac 100644
--- a/src/main/java/com/android/tools/r8/utils/ResourceTracing.java
+++ b/src/main/java/com/android/tools/r8/utils/ResourceTracing.java
@@ -13,10 +13,9 @@
import com.android.tools.r8.ResourceException;
import com.android.tools.r8.ResourcePath;
import com.android.tools.r8.origin.Origin;
+import com.android.tools.r8.utils.resourceshrinker.ResourceTracingImpl;
import com.google.common.io.ByteStreams;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Collections;
import java.util.List;
public interface ResourceTracing {
@@ -54,19 +53,7 @@
}
static ResourceTracing getImpl() {
- try {
- Class<? extends ResourceTracing> implClass =
- Class.forName("com.android.tools.r8.utils.resourceshrinker.ResourceTracingImpl")
- .asSubclass(ResourceTracing.class);
- return implClass.getConstructor().newInstance();
- } catch (ClassNotFoundException
- | NoSuchMethodException
- | InstantiationException
- | IllegalAccessException
- | InvocationTargetException e) {
- // Old gradle setup.
- return new NoOpResourceTracingImpl();
- }
+ return new ResourceTracingImpl();
}
static void copyProviderToConsumer(
@@ -106,30 +93,4 @@
provider.finished(diagnosticsHandler);
}
}
-
- class NoOpResourceTracingImpl implements ResourceTracing {
-
- private AndroidResourceProvider provider;
- private AndroidResourceConsumer consumer;
-
- @Override
- public List<LiveMethod> traceResourceId(int id) {
- return Collections.emptyList();
- }
-
- @Override
- public void done(DiagnosticsHandler diagnosticsHandler) {
- ResourceTracing.copyProviderToConsumer(diagnosticsHandler, provider, consumer);
- }
-
- @Override
- public void setProvider(AndroidResourceProvider provider) {
- this.provider = provider;
- }
-
- @Override
- public void setConsumer(AndroidResourceConsumer consumer) {
- this.consumer = consumer;
- }
- }
}
diff --git a/src/test/java/com/android/tools/r8/TestCompilerBuilder.java b/src/test/java/com/android/tools/r8/TestCompilerBuilder.java
index 3e17faf..7db5069 100644
--- a/src/test/java/com/android/tools/r8/TestCompilerBuilder.java
+++ b/src/test/java/com/android/tools/r8/TestCompilerBuilder.java
@@ -27,6 +27,7 @@
import com.android.tools.r8.utils.codeinspector.HorizontallyMergedClassesInspector;
import com.android.tools.r8.utils.codeinspector.VerticallyMergedClassesInspector;
import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableSet;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
@@ -93,7 +94,9 @@
private Optional<Integer> isAndroidBuildVersionAdded = null;
- private static final Map<Integer, Set<String>> allowedGlobalSynthetics =
+ private static final Map<Integer, Set<String>> allGlobalSynthetics = new ConcurrentHashMap<>();
+
+ private static final Map<Integer, Set<String>> definiteGlobalSynthetics =
new ConcurrentHashMap<>();
LibraryDesugaringTestConfiguration libraryDesugaringTestConfiguration =
@@ -270,17 +273,22 @@
&& (isD8TestBuilder() || isR8TestBuilder())
&& !isBenchmarkRunner) {
int minApiLevel = builder.getMinApiLevel();
- allowedGlobalSynthetics.computeIfAbsent(
- minApiLevel, TestCompilerBuilder::computeAllGlobalSynthetics);
Consumer<InternalOptions> previousConsumer = optionsConsumer;
optionsConsumer =
options -> {
options.testing.globalSyntheticCreatedCallback =
programClass -> {
- assertTrue(
- allowedGlobalSynthetics
- .get(minApiLevel)
- .contains(programClass.getType().toDescriptorString()));
+ String descriptor = programClass.getType().toDescriptorString();
+ boolean isGlobalSynthetic =
+ definiteGlobalSynthetics
+ .computeIfAbsent(
+ minApiLevel, computeDefiniteGlobalSynthetics(options))
+ .contains(descriptor)
+ || allGlobalSynthetics
+ .computeIfAbsent(
+ minApiLevel, TestCompilerBuilder::computeAllGlobalSynthetics)
+ .contains(descriptor);
+ assertTrue(isGlobalSynthetic);
};
if (previousConsumer != null) {
previousConsumer.accept(options);
@@ -604,6 +612,16 @@
}
}
+ private static Function<Integer, Set<String>> computeDefiniteGlobalSynthetics(
+ InternalOptions options) {
+ return minApiLevel -> {
+ ImmutableSet.Builder<String> builder = ImmutableSet.builder();
+ GlobalSyntheticsGeneratorVerifier.forEachExpectedClass(
+ options.dexItemFactory(), minApiLevel, type -> builder.add(type.toDescriptorString()));
+ return builder.build();
+ };
+ }
+
private static class ChainedStringConsumer implements StringConsumer {
private final List<StringConsumer> consumers;
diff --git a/src/test/java/com/android/tools/r8/ToolHelper.java b/src/test/java/com/android/tools/r8/ToolHelper.java
index 8563271..eceaf77 100644
--- a/src/test/java/com/android/tools/r8/ToolHelper.java
+++ b/src/test/java/com/android/tools/r8/ToolHelper.java
@@ -12,6 +12,7 @@
import static org.junit.Assert.fail;
import com.android.tools.r8.DeviceRunner.DeviceRunnerConfigurationException;
+import com.android.tools.r8.ResourceShrinker.ReferenceChecker;
import com.android.tools.r8.TestBase.Backend;
import com.android.tools.r8.TestRuntime.CfRuntime;
import com.android.tools.r8.ToolHelper.DexVm.Kind;
@@ -1847,6 +1848,20 @@
return compatSink.build();
}
+ public static void runLegacyResourceShrinker(
+ ResourceShrinker.Builder builder,
+ Consumer<InternalOptions> optionsConsumer,
+ ReferenceChecker callback)
+ throws IOException, CompilationFailedException, ExecutionException {
+ ResourceShrinker.Command command = builder.build();
+ InternalOptions options = command.getInternalOptions();
+ if (optionsConsumer != null) {
+ ExceptionUtils.withD8CompilationHandler(
+ options.reporter, () -> optionsConsumer.accept(options));
+ }
+ ResourceShrinker.runForTesting(command.getInputApp(), options, callback);
+ }
+
@Deprecated
public static ProcessResult runJava(Class clazz) throws Exception {
String main = clazz.getTypeName();
diff --git a/src/test/java/com/android/tools/r8/androidresources/ResourceShrinkerIntegrationTest.java b/src/test/java/com/android/tools/r8/androidresources/ResourceShrinkerIntegrationTest.java
deleted file mode 100644
index e234c69..0000000
--- a/src/test/java/com/android/tools/r8/androidresources/ResourceShrinkerIntegrationTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-package com.android.tools.r8.androidresources;
-
-import static org.junit.Assert.assertTrue;
-
-import com.android.tools.r8.TestBase;
-import com.android.tools.r8.TestParameters;
-import com.android.tools.r8.TestParametersCollection;
-import com.android.tools.r8.ToolHelper;
-import com.android.tools.r8.utils.ResourceTracing;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class ResourceShrinkerIntegrationTest extends TestBase {
-
- @Parameter(0)
- public TestParameters parameters;
-
- @Parameters(name = "{0}")
- public static TestParametersCollection parameters() {
- return getTestParameters().withNoneRuntime().build();
- }
-
- @Test
- public void testResourceShrinkerClassAvailable() {
- if (ToolHelper.isNewGradleSetup()) {
- assertTrue(
- ResourceTracing.getImpl().getClass() != ResourceTracing.NoOpResourceTracingImpl.class);
- } else {
- assertTrue(
- ResourceTracing.getImpl().getClass() == ResourceTracing.NoOpResourceTracingImpl.class);
- }
- }
-}
diff --git a/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/ObjectsTest.java b/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/ObjectsTest.java
index 7a5fb66..add6c84 100644
--- a/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/ObjectsTest.java
+++ b/src/test/java/com/android/tools/r8/desugar/desugaredlibrary/ObjectsTest.java
@@ -15,6 +15,7 @@
import static com.android.tools.r8.utils.codeinspector.Matchers.onlyIf;
import static org.hamcrest.MatcherAssert.assertThat;
+import com.android.tools.r8.KeepConstantArguments;
import com.android.tools.r8.NeverInline;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestRuntime.CfVm;
@@ -305,6 +306,7 @@
.addProgramClassFileData(ImmutableList.of(dumpAndroidRUtilsObjectsMethods()))
.addKeepMainRule(TestClass.class)
.enableInliningAnnotations()
+ .enableConstantArgumentAnnotations()
.noMinification()
.addKeepRules("-keep class AndroidRUtilsObjectsMethods { *; }")
.addKeepRules("-neverinline class AndroidRUtilsObjectsMethods { *; }")
@@ -342,6 +344,9 @@
System.out.println(Objects.hash(o1, o2));
}
+ // We keep constant arguments to avoid the argument to be proven non-null leading to
+ // Objects#hashCode(Object) being rewritten to Object#hashCode().
+ @KeepConstantArguments
@NeverInline
private static void objectsHashCode(Object o) {
System.out.println(Objects.hashCode(o));
diff --git a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java
index 81d709c9..efc954e 100644
--- a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java
+++ b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatBasicTest.java
@@ -3,61 +3,21 @@
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.dex.container;
-import static com.android.tools.r8.dex.Constants.CHECKSUM_OFFSET;
-import static com.android.tools.r8.dex.Constants.CONTAINER_OFF_OFFSET;
-import static com.android.tools.r8.dex.Constants.CONTAINER_SIZE_OFFSET;
-import static com.android.tools.r8.dex.Constants.DATA_OFF_OFFSET;
-import static com.android.tools.r8.dex.Constants.DATA_SIZE_OFFSET;
-import static com.android.tools.r8.dex.Constants.DEX_MAGIC_SIZE;
-import static com.android.tools.r8.dex.Constants.FILE_SIZE_OFFSET;
-import static com.android.tools.r8.dex.Constants.HEADER_SIZE_OFFSET;
-import static com.android.tools.r8.dex.Constants.MAP_OFF_OFFSET;
-import static com.android.tools.r8.dex.Constants.SIGNATURE_OFFSET;
-import static com.android.tools.r8.dex.Constants.STRING_IDS_OFF_OFFSET;
-import static com.android.tools.r8.dex.Constants.STRING_IDS_SIZE_OFFSET;
-import static com.android.tools.r8.dex.Constants.TYPE_STRING_ID_ITEM;
import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import com.android.tools.r8.ByteDataView;
-import com.android.tools.r8.ClassFileConsumer.ArchiveConsumer;
-import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
-import com.android.tools.r8.dex.CompatByteBuffer;
-import com.android.tools.r8.dex.Constants;
-import com.android.tools.r8.errors.CompilationError;
-import com.android.tools.r8.maindexlist.MainDexListTests;
-import com.android.tools.r8.transformers.ClassTransformer;
import com.android.tools.r8.utils.AndroidApiLevel;
-import com.android.tools.r8.utils.BitUtils;
-import com.android.tools.r8.utils.DescriptorUtils;
-import com.android.tools.r8.utils.DexVersion;
-import com.android.tools.r8.utils.ListUtils;
-import com.android.tools.r8.utils.ZipUtils;
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.ByteStreams;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-import it.unimi.dsi.fastutil.ints.IntList;
-import java.io.IOException;
-import java.nio.ByteOrder;
import java.nio.file.Path;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.zip.Adler32;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
-import org.objectweb.asm.MethodVisitor;
-import org.objectweb.asm.Opcodes;
@RunWith(Parameterized.class)
-public class DexContainerFormatBasicTest extends TestBase {
+public class DexContainerFormatBasicTest extends DexContainerFormatTestBase {
@Parameter() public TestParameters parameters;
@@ -176,239 +136,4 @@
assertArrayEquals(unzipContent(outputBoth).get(0), unzipContent(outputMerged).get(0));
}
-
- private void validateDex(Path output, int expectedDexes, DexVersion expectedVersion)
- throws Exception {
- List<byte[]> dexes = unzipContent(output);
- assertEquals(expectedDexes, dexes.size());
- for (byte[] dex : dexes) {
- validate(dex, expectedVersion);
- }
- }
-
- private void validateSingleContainerDex(Path output) throws Exception {
- List<byte[]> dexes = unzipContent(output);
- assertEquals(1, dexes.size());
- validate(dexes.get(0), DexVersion.V41);
- }
-
- private void validate(byte[] dex, DexVersion expectedVersion) throws Exception {
- CompatByteBuffer buffer = CompatByteBuffer.wrap(dex);
- setByteOrder(buffer);
-
- IntList sections = new IntArrayList();
- int offset = 0;
- while (offset < buffer.capacity()) {
- assertTrue(BitUtils.isAligned(4, offset));
- sections.add(offset);
- int dataSize = buffer.getInt(offset + DATA_SIZE_OFFSET);
- int dataOffset = buffer.getInt(offset + DATA_OFF_OFFSET);
- int file_size = buffer.getInt(offset + FILE_SIZE_OFFSET);
- if (expectedVersion.isContainerDex()) {
- assertEquals(0, dataSize);
- assertEquals(0, dataOffset);
- } else {
- assertEquals(file_size, dataOffset + dataSize);
- }
- offset += expectedVersion.isContainerDex() ? file_size : dataOffset + dataSize;
- assertEquals(file_size, offset - ListUtils.last(sections));
- }
- assertEquals(buffer.capacity(), offset);
-
- for (Integer sectionOffset : sections) {
- validateHeader(sections, buffer, sectionOffset, expectedVersion);
- validateMap(buffer, sectionOffset);
- validateSignature(buffer, sectionOffset);
- validateChecksum(buffer, sectionOffset);
- }
- }
-
- private byte[] magicBytes(DexVersion version) {
- byte[] magic = new byte[DEX_MAGIC_SIZE];
- System.arraycopy(
- Constants.DEX_FILE_MAGIC_PREFIX, 0, magic, 0, Constants.DEX_FILE_MAGIC_PREFIX.length);
- System.arraycopy(
- version.getBytes(),
- 0,
- magic,
- Constants.DEX_FILE_MAGIC_PREFIX.length,
- version.getBytes().length);
- magic[Constants.DEX_FILE_MAGIC_PREFIX.length + version.getBytes().length] =
- Constants.DEX_FILE_MAGIC_SUFFIX;
- assertEquals(
- DEX_MAGIC_SIZE, Constants.DEX_FILE_MAGIC_PREFIX.length + version.getBytes().length + 1);
- return magic;
- }
-
- private void validateHeader(
- IntList sections, CompatByteBuffer buffer, int offset, DexVersion expectedVersion) {
- int lastOffset = sections.getInt(sections.size() - 1);
- int stringIdsSize = buffer.getInt(lastOffset + STRING_IDS_SIZE_OFFSET);
- int stringIdsOffset = buffer.getInt(lastOffset + STRING_IDS_OFF_OFFSET);
-
- byte[] magic = new byte[DEX_MAGIC_SIZE];
- buffer.get(magic);
- assertArrayEquals(magicBytes(expectedVersion), magic);
-
- assertEquals(
- expectedVersion.isContainerDex()
- ? Constants.TYPE_HEADER_ITEM_SIZE_V41
- : Constants.TYPE_HEADER_ITEM_SIZE,
- buffer.getInt(offset + HEADER_SIZE_OFFSET));
- assertEquals(stringIdsSize, buffer.getInt(offset + STRING_IDS_SIZE_OFFSET));
- assertEquals(stringIdsOffset, buffer.getInt(offset + STRING_IDS_OFF_OFFSET));
- if (expectedVersion.isContainerDex()) {
- assertEquals(0, buffer.getInt(offset + DATA_SIZE_OFFSET));
- assertEquals(0, buffer.getInt(offset + DATA_OFF_OFFSET));
- // Additional header field from V41.
- assertEquals(buffer.capacity(), buffer.getInt(offset + CONTAINER_SIZE_OFFSET));
- assertEquals(offset, buffer.getInt(offset + CONTAINER_OFF_OFFSET));
- }
- assertEquals(stringIdsSize, getSizeFromMap(TYPE_STRING_ID_ITEM, buffer, offset));
- assertEquals(stringIdsOffset, getOffsetFromMap(TYPE_STRING_ID_ITEM, buffer, offset));
- }
-
- private void validateMap(CompatByteBuffer buffer, int offset) {
- int mapOffset = buffer.getInt(offset + MAP_OFF_OFFSET);
- buffer.position(mapOffset);
- int mapSize = buffer.getInt();
- int previousOffset = Integer.MAX_VALUE;
- for (int i = 0; i < mapSize; i++) {
- buffer.getShort(); // Skip section type.
- buffer.getShort(); // Skip unused.
- buffer.getInt(); // Skip section size.
- int o = buffer.getInt();
- if (i > 0) {
- assertTrue("" + i + ": " + o + " " + previousOffset, o > previousOffset);
- }
- previousOffset = o;
- }
- }
-
- private void validateSignature(CompatByteBuffer buffer, int offset) throws Exception {
- int sectionSize = buffer.getInt(offset + FILE_SIZE_OFFSET);
- MessageDigest md = MessageDigest.getInstance("SHA-1");
- md.update(
- buffer.asByteBuffer().array(), offset + FILE_SIZE_OFFSET, sectionSize - FILE_SIZE_OFFSET);
- byte[] expectedSignature = new byte[20];
- md.digest(expectedSignature, 0, 20);
- for (int i = 0; i < expectedSignature.length; i++) {
- assertEquals(expectedSignature[i], buffer.get(offset + SIGNATURE_OFFSET + i));
- }
- }
-
- private void validateChecksum(CompatByteBuffer buffer, int offset) {
- int sectionSize = buffer.getInt(offset + FILE_SIZE_OFFSET);
- Adler32 adler = new Adler32();
- adler.update(
- buffer.asByteBuffer().array(), offset + SIGNATURE_OFFSET, sectionSize - SIGNATURE_OFFSET);
- assertEquals((int) adler.getValue(), buffer.getInt(offset + CHECKSUM_OFFSET));
- }
-
- private int getSizeFromMap(int type, CompatByteBuffer buffer, int offset) {
- int mapOffset = buffer.getInt(offset + MAP_OFF_OFFSET);
- buffer.position(mapOffset);
- int mapSize = buffer.getInt();
- for (int i = 0; i < mapSize; i++) {
- int sectionType = buffer.getShort();
- buffer.getShort(); // Skip unused.
- int sectionSize = buffer.getInt();
- buffer.getInt(); // Skip offset.
- if (type == sectionType) {
- return sectionSize;
- }
- }
- throw new RuntimeException("Not found");
- }
-
- private int getOffsetFromMap(int type, CompatByteBuffer buffer, int offset) {
- int mapOffset = buffer.getInt(offset + MAP_OFF_OFFSET);
- buffer.position(mapOffset);
- int mapSize = buffer.getInt();
- for (int i = 0; i < mapSize; i++) {
- int sectionType = buffer.getShort();
- buffer.getShort(); // Skip unused.
- buffer.getInt(); // SKip size.
- int sectionOffset = buffer.getInt();
- if (type == sectionType) {
- return sectionOffset;
- }
- }
- throw new RuntimeException("Not found");
- }
-
- private void setByteOrder(CompatByteBuffer buffer) {
- // Make sure we set the right endian for reading.
- buffer.order(ByteOrder.LITTLE_ENDIAN);
- int endian = buffer.getInt(Constants.ENDIAN_TAG_OFFSET);
- if (endian == Constants.REVERSE_ENDIAN_CONSTANT) {
- buffer.order(ByteOrder.BIG_ENDIAN);
- } else {
- if (endian != Constants.ENDIAN_CONSTANT) {
- throw new CompilationError("Unable to determine endianess for reading dex file.");
- }
- }
- }
-
- private List<byte[]> unzipContent(Path zip) throws IOException {
- List<byte[]> result = new ArrayList<>();
- ZipUtils.iter(zip, (entry, inputStream) -> result.add(ByteStreams.toByteArray(inputStream)));
- return result;
- }
-
- private static void generateApplication(Path applicationJar, String rootPackage, int methodCount)
- throws Throwable {
- ImmutableList.Builder<String> builder = ImmutableList.builder();
- for (int i = 0; i < 10000; ++i) {
- String pkg = rootPackage + "." + (i % 2 == 0 ? "a" : "b");
- String className = "Class" + i;
- builder.add(pkg + "." + className);
- }
- List<String> classes = builder.build();
-
- generateApplication(applicationJar, classes, methodCount);
- }
-
- private static void generateApplication(Path output, List<String> classes, int methodCount)
- throws IOException {
- ArchiveConsumer consumer = new ArchiveConsumer(output);
- for (String typename : classes) {
- String descriptor = DescriptorUtils.javaTypeToDescriptor(typename);
- byte[] bytes =
- transformer(MainDexListTests.ClassStub.class)
- .setClassDescriptor(descriptor)
- .addClassTransformer(
- new ClassTransformer() {
- @Override
- public MethodVisitor visitMethod(
- int access,
- String name,
- String descriptor,
- String signature,
- String[] exceptions) {
- // This strips <init>() too.
- if (name.equals("methodStub")) {
- for (int i = 0; i < methodCount; i++) {
- MethodVisitor mv =
- super.visitMethod(
- access, "method" + i, descriptor, signature, exceptions);
- mv.visitCode();
- mv.visitInsn(Opcodes.RETURN);
- mv.visitMaxs(0, 0);
- mv.visitEnd();
- }
- }
- return null;
- }
- })
- .transform();
- consumer.accept(ByteDataView.of(bytes), descriptor, null);
- }
- consumer.finished(null);
- }
-
- // Simple stub/template for generating the input classes.
- public static class ClassStub {
- public static void methodStub() {}
- }
}
diff --git a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatEmptyTest.java b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatEmptyTest.java
index b2e6aec..ec8e01a 100644
--- a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatEmptyTest.java
+++ b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatEmptyTest.java
@@ -5,16 +5,10 @@
import static org.junit.Assert.assertEquals;
-import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
import com.android.tools.r8.utils.AndroidApiLevel;
-import com.android.tools.r8.utils.ZipUtils;
-import com.google.common.io.ByteStreams;
-import java.io.IOException;
import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -22,7 +16,7 @@
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
-public class DexContainerFormatEmptyTest extends TestBase {
+public class DexContainerFormatEmptyTest extends DexContainerFormatTestBase {
@Parameter() public TestParameters parameters;
@@ -59,10 +53,4 @@
.writeToZip();
assertEquals(0, unzipContent(outputFromDexing).size());
}
-
- private List<byte[]> unzipContent(Path zip) throws IOException {
- List<byte[]> result = new ArrayList<>();
- ZipUtils.iter(zip, (entry, inputStream) -> result.add(ByteStreams.toByteArray(inputStream)));
- return result;
- }
}
diff --git a/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatTestBase.java b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatTestBase.java
new file mode 100644
index 0000000..7938844
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/dex/container/DexContainerFormatTestBase.java
@@ -0,0 +1,286 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.dex.container;
+
+import static com.android.tools.r8.dex.Constants.CHECKSUM_OFFSET;
+import static com.android.tools.r8.dex.Constants.CONTAINER_OFF_OFFSET;
+import static com.android.tools.r8.dex.Constants.CONTAINER_SIZE_OFFSET;
+import static com.android.tools.r8.dex.Constants.DATA_OFF_OFFSET;
+import static com.android.tools.r8.dex.Constants.DATA_SIZE_OFFSET;
+import static com.android.tools.r8.dex.Constants.DEX_MAGIC_SIZE;
+import static com.android.tools.r8.dex.Constants.FILE_SIZE_OFFSET;
+import static com.android.tools.r8.dex.Constants.HEADER_SIZE_OFFSET;
+import static com.android.tools.r8.dex.Constants.MAP_OFF_OFFSET;
+import static com.android.tools.r8.dex.Constants.SIGNATURE_OFFSET;
+import static com.android.tools.r8.dex.Constants.STRING_IDS_OFF_OFFSET;
+import static com.android.tools.r8.dex.Constants.STRING_IDS_SIZE_OFFSET;
+import static com.android.tools.r8.dex.Constants.TYPE_STRING_ID_ITEM;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.android.tools.r8.ByteDataView;
+import com.android.tools.r8.ClassFileConsumer.ArchiveConsumer;
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.dex.CompatByteBuffer;
+import com.android.tools.r8.dex.Constants;
+import com.android.tools.r8.errors.CompilationError;
+import com.android.tools.r8.maindexlist.MainDexListTests;
+import com.android.tools.r8.transformers.ClassTransformer;
+import com.android.tools.r8.utils.BitUtils;
+import com.android.tools.r8.utils.DescriptorUtils;
+import com.android.tools.r8.utils.DexVersion;
+import com.android.tools.r8.utils.ListUtils;
+import com.android.tools.r8.utils.ZipUtils;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.Adler32;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+public class DexContainerFormatTestBase extends TestBase {
+
+ static void validateDex(Path output, int expectedDexes, DexVersion expectedVersion)
+ throws Exception {
+ List<byte[]> dexes = unzipContent(output);
+ assertEquals(expectedDexes, dexes.size());
+ for (byte[] dex : dexes) {
+ validate(dex, expectedVersion);
+ }
+ }
+
+ static void validateSingleContainerDex(Path output) throws Exception {
+ List<byte[]> dexes = unzipContent(output);
+ assertEquals(1, dexes.size());
+ validate(dexes.get(0), DexVersion.V41);
+ }
+
+ static void validate(byte[] dex, DexVersion expectedVersion) throws Exception {
+ CompatByteBuffer buffer = CompatByteBuffer.wrap(dex);
+ setByteOrder(buffer);
+
+ IntList sections = new IntArrayList();
+ int offset = 0;
+ while (offset < buffer.capacity()) {
+ assertTrue(BitUtils.isAligned(4, offset));
+ sections.add(offset);
+ int dataSize = buffer.getInt(offset + DATA_SIZE_OFFSET);
+ int dataOffset = buffer.getInt(offset + DATA_OFF_OFFSET);
+ int file_size = buffer.getInt(offset + FILE_SIZE_OFFSET);
+ if (expectedVersion.isContainerDex()) {
+ assertEquals(0, dataSize);
+ assertEquals(0, dataOffset);
+ } else {
+ assertEquals(file_size, dataOffset + dataSize);
+ }
+ offset += expectedVersion.isContainerDex() ? file_size : dataOffset + dataSize;
+ assertEquals(file_size, offset - ListUtils.last(sections));
+ }
+ assertEquals(buffer.capacity(), offset);
+
+ for (Integer sectionOffset : sections) {
+ validateHeader(sections, buffer, sectionOffset, expectedVersion);
+ validateMap(buffer, sectionOffset);
+ validateSignature(buffer, sectionOffset);
+ validateChecksum(buffer, sectionOffset);
+ }
+ }
+
+ static byte[] magicBytes(DexVersion version) {
+ byte[] magic = new byte[DEX_MAGIC_SIZE];
+ System.arraycopy(
+ Constants.DEX_FILE_MAGIC_PREFIX, 0, magic, 0, Constants.DEX_FILE_MAGIC_PREFIX.length);
+ System.arraycopy(
+ version.getBytes(),
+ 0,
+ magic,
+ Constants.DEX_FILE_MAGIC_PREFIX.length,
+ version.getBytes().length);
+ magic[Constants.DEX_FILE_MAGIC_PREFIX.length + version.getBytes().length] =
+ Constants.DEX_FILE_MAGIC_SUFFIX;
+ assertEquals(
+ DEX_MAGIC_SIZE, Constants.DEX_FILE_MAGIC_PREFIX.length + version.getBytes().length + 1);
+ return magic;
+ }
+
+ static void validateHeader(
+ IntList sections, CompatByteBuffer buffer, int offset, DexVersion expectedVersion) {
+ int lastOffset = sections.getInt(sections.size() - 1);
+ int stringIdsSize = buffer.getInt(lastOffset + STRING_IDS_SIZE_OFFSET);
+ int stringIdsOffset = buffer.getInt(lastOffset + STRING_IDS_OFF_OFFSET);
+
+ byte[] magic = new byte[DEX_MAGIC_SIZE];
+ buffer.get(magic);
+ assertArrayEquals(magicBytes(expectedVersion), magic);
+
+ assertEquals(
+ expectedVersion.isContainerDex()
+ ? Constants.TYPE_HEADER_ITEM_SIZE_V41
+ : Constants.TYPE_HEADER_ITEM_SIZE,
+ buffer.getInt(offset + HEADER_SIZE_OFFSET));
+ assertEquals(stringIdsSize, buffer.getInt(offset + STRING_IDS_SIZE_OFFSET));
+ assertEquals(stringIdsOffset, buffer.getInt(offset + STRING_IDS_OFF_OFFSET));
+ if (expectedVersion.isContainerDex()) {
+ assertEquals(0, buffer.getInt(offset + DATA_SIZE_OFFSET));
+ assertEquals(0, buffer.getInt(offset + DATA_OFF_OFFSET));
+ // Additional header field from V41.
+ assertEquals(buffer.capacity(), buffer.getInt(offset + CONTAINER_SIZE_OFFSET));
+ assertEquals(offset, buffer.getInt(offset + CONTAINER_OFF_OFFSET));
+ }
+ assertEquals(stringIdsSize, getSizeFromMap(TYPE_STRING_ID_ITEM, buffer, offset));
+ assertEquals(stringIdsOffset, getOffsetFromMap(TYPE_STRING_ID_ITEM, buffer, offset));
+ }
+
+ static void validateMap(CompatByteBuffer buffer, int offset) {
+ int mapOffset = buffer.getInt(offset + MAP_OFF_OFFSET);
+ buffer.position(mapOffset);
+ int mapSize = buffer.getInt();
+ int previousOffset = Integer.MAX_VALUE;
+ for (int i = 0; i < mapSize; i++) {
+ buffer.getShort(); // Skip section type.
+ buffer.getShort(); // Skip unused.
+ buffer.getInt(); // Skip section size.
+ int o = buffer.getInt();
+ if (i > 0) {
+ assertTrue("" + i + ": " + o + " " + previousOffset, o > previousOffset);
+ }
+ previousOffset = o;
+ }
+ }
+
+ static void validateSignature(CompatByteBuffer buffer, int offset) throws Exception {
+ int sectionSize = buffer.getInt(offset + FILE_SIZE_OFFSET);
+ MessageDigest md = MessageDigest.getInstance("SHA-1");
+ md.update(
+ buffer.asByteBuffer().array(), offset + FILE_SIZE_OFFSET, sectionSize - FILE_SIZE_OFFSET);
+ byte[] expectedSignature = new byte[20];
+ md.digest(expectedSignature, 0, 20);
+ for (int i = 0; i < expectedSignature.length; i++) {
+ assertEquals(expectedSignature[i], buffer.get(offset + SIGNATURE_OFFSET + i));
+ }
+ }
+
+ static void validateChecksum(CompatByteBuffer buffer, int offset) {
+ int sectionSize = buffer.getInt(offset + FILE_SIZE_OFFSET);
+ Adler32 adler = new Adler32();
+ adler.update(
+ buffer.asByteBuffer().array(), offset + SIGNATURE_OFFSET, sectionSize - SIGNATURE_OFFSET);
+ assertEquals((int) adler.getValue(), buffer.getInt(offset + CHECKSUM_OFFSET));
+ }
+
+ static int getSizeFromMap(int type, CompatByteBuffer buffer, int offset) {
+ int mapOffset = buffer.getInt(offset + MAP_OFF_OFFSET);
+ buffer.position(mapOffset);
+ int mapSize = buffer.getInt();
+ for (int i = 0; i < mapSize; i++) {
+ int sectionType = buffer.getShort();
+ buffer.getShort(); // Skip unused.
+ int sectionSize = buffer.getInt();
+ buffer.getInt(); // Skip offset.
+ if (type == sectionType) {
+ return sectionSize;
+ }
+ }
+ throw new RuntimeException("Not found");
+ }
+
+ static int getOffsetFromMap(int type, CompatByteBuffer buffer, int offset) {
+ int mapOffset = buffer.getInt(offset + MAP_OFF_OFFSET);
+ buffer.position(mapOffset);
+ int mapSize = buffer.getInt();
+ for (int i = 0; i < mapSize; i++) {
+ int sectionType = buffer.getShort();
+ buffer.getShort(); // Skip unused.
+ buffer.getInt(); // SKip size.
+ int sectionOffset = buffer.getInt();
+ if (type == sectionType) {
+ return sectionOffset;
+ }
+ }
+ throw new RuntimeException("Not found");
+ }
+
+ static void setByteOrder(CompatByteBuffer buffer) {
+ // Make sure we set the right endian for reading.
+ buffer.order(ByteOrder.LITTLE_ENDIAN);
+ int endian = buffer.getInt(Constants.ENDIAN_TAG_OFFSET);
+ if (endian == Constants.REVERSE_ENDIAN_CONSTANT) {
+ buffer.order(ByteOrder.BIG_ENDIAN);
+ } else {
+ if (endian != Constants.ENDIAN_CONSTANT) {
+ throw new CompilationError("Unable to determine endianess for reading dex file.");
+ }
+ }
+ }
+
+ static List<byte[]> unzipContent(Path zip) throws IOException {
+ List<byte[]> result = new ArrayList<>();
+ ZipUtils.iter(zip, (entry, inputStream) -> result.add(ByteStreams.toByteArray(inputStream)));
+ return result;
+ }
+
+ static void generateApplication(Path applicationJar, String rootPackage, int methodCount)
+ throws Throwable {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (int i = 0; i < 10000; ++i) {
+ String pkg = rootPackage + "." + (i % 2 == 0 ? "a" : "b");
+ String className = "Class" + i;
+ builder.add(pkg + "." + className);
+ }
+ List<String> classes = builder.build();
+
+ generateApplication(applicationJar, classes, methodCount);
+ }
+
+ private static void generateApplication(Path output, List<String> classes, int methodCount)
+ throws IOException {
+ ArchiveConsumer consumer = new ArchiveConsumer(output);
+ for (String typename : classes) {
+ String descriptor = DescriptorUtils.javaTypeToDescriptor(typename);
+ byte[] bytes =
+ transformer(MainDexListTests.ClassStub.class)
+ .setClassDescriptor(descriptor)
+ .addClassTransformer(
+ new ClassTransformer() {
+ @Override
+ public MethodVisitor visitMethod(
+ int access,
+ String name,
+ String descriptor,
+ String signature,
+ String[] exceptions) {
+ // This strips <init>() too.
+ if (name.equals("methodStub")) {
+ for (int i = 0; i < methodCount; i++) {
+ MethodVisitor mv =
+ super.visitMethod(
+ access, "method" + i, descriptor, signature, exceptions);
+ mv.visitCode();
+ mv.visitInsn(Opcodes.RETURN);
+ mv.visitMaxs(0, 0);
+ mv.visitEnd();
+ }
+ }
+ return null;
+ }
+ })
+ .transform();
+ consumer.accept(ByteDataView.of(bytes), descriptor, null);
+ }
+ consumer.finished(null);
+ }
+
+ // Simple stub/template for generating the input classes.
+ public static class ClassStub {
+ public static void methodStub() {}
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/dex/container/LegacyResourceShrinkerTest.java b/src/test/java/com/android/tools/r8/dex/container/LegacyResourceShrinkerTest.java
new file mode 100644
index 0000000..70a3f03
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/dex/container/LegacyResourceShrinkerTest.java
@@ -0,0 +1,104 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.dex.container;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.android.tools.r8.ResourceShrinker;
+import com.android.tools.r8.ResourceShrinker.ReferenceChecker;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.ToolHelper;
+import com.android.tools.r8.utils.AndroidApiLevel;
+import com.android.tools.r8.utils.BooleanUtils;
+import com.android.tools.r8.utils.IntBox;
+import java.nio.file.Path;
+import java.util.List;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class LegacyResourceShrinkerTest extends DexContainerFormatTestBase {
+
+ @Parameter(0)
+ public TestParameters parameters;
+
+ @Parameter(1)
+ public boolean enableDexContainerInResourceShrinker;
+
+ @Parameters(name = "{0}, enableDexContainerInResourceShrinker: {1}")
+ public static List<Object[]> data() {
+ return buildParameters(getTestParameters().withNoneRuntime().build(), BooleanUtils.values());
+ }
+
+ private static Path inputA;
+ private static Path inputB;
+
+ @BeforeClass
+ public static void generateTestApplications() throws Throwable {
+ // Build two applications in different packages both with required multidex due to number
+ // of methods.
+ inputA = getStaticTemp().getRoot().toPath().resolve("application_a.jar");
+ inputB = getStaticTemp().getRoot().toPath().resolve("application_b.jar");
+
+ generateApplication(inputA, "a", 10);
+ generateApplication(inputB, "b", 10);
+ }
+
+ @Test
+ public void test() throws Exception {
+ Path outputBoth =
+ testForD8(Backend.DEX)
+ .addProgramFiles(inputA, inputB)
+ .setMinApi(AndroidApiLevel.L)
+ .addOptionsModification(
+ options -> options.getTestingOptions().dexContainerExperiment = true)
+ .compile()
+ .writeToZip();
+ validateSingleContainerDex(outputBoth);
+ ResourceShrinker.Command command =
+ new ResourceShrinker.Builder().addProgramFiles(outputBoth).build();
+
+ try {
+ IntBox classCount = new IntBox();
+ ToolHelper.runLegacyResourceShrinker(
+ new ResourceShrinker.Builder().addProgramFiles(outputBoth),
+ options -> options.testing.dexContainerExperiment = enableDexContainerInResourceShrinker,
+ new ReferenceChecker() {
+ @Override
+ public boolean shouldProcess(String internalName) {
+ classCount.increment();
+ return true;
+ }
+
+ @Override
+ public void referencedInt(int value) {}
+
+ @Override
+ public void referencedString(String value) {}
+
+ @Override
+ public void referencedStaticField(String internalName, String fieldName) {}
+
+ @Override
+ public void referencedMethod(
+ String internalName, String methodName, String methodDescriptor) {}
+ });
+ assertTrue(enableDexContainerInResourceShrinker);
+ assertEquals(20000, classCount.get());
+ } catch (RuntimeException e) {
+ assertThat(
+ e.getMessage(),
+ containsString("Experimental container DEX version V41 is not supported"));
+ assertFalse(enableDexContainerInResourceShrinker);
+ }
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/enumunboxing/ArgumentInfoCollectionTest.java b/src/test/java/com/android/tools/r8/enumunboxing/ArgumentInfoCollectionTest.java
index 7cd720a..556a914 100644
--- a/src/test/java/com/android/tools/r8/enumunboxing/ArgumentInfoCollectionTest.java
+++ b/src/test/java/com/android/tools/r8/enumunboxing/ArgumentInfoCollectionTest.java
@@ -16,6 +16,7 @@
import com.android.tools.r8.graph.proto.RemovedArgumentInfo;
import com.android.tools.r8.graph.proto.RewrittenTypeInfo;
import com.android.tools.r8.ir.analysis.value.AbstractValueFactory;
+import com.android.tools.r8.utils.InternalOptions;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -127,8 +128,9 @@
@Test
public void testCombineRemoveRewritten() {
- DexItemFactory factory = new DexItemFactory();
- AbstractValueFactory abstractValueFactory = new AbstractValueFactory();
+ InternalOptions options = new InternalOptions();
+ DexItemFactory factory = options.dexItemFactory();
+ AbstractValueFactory abstractValueFactory = new AbstractValueFactory(options);
ArgumentInfoCollection.Builder builder1 = ArgumentInfoCollection.builder();
builder1.addArgumentInfo(
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedBooleanReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedBooleanReturnPropagationTest.java
index fce2a69..0d1cb4e 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedBooleanReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedBooleanReturnPropagationTest.java
@@ -11,6 +11,7 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,9 +50,14 @@
mainMethodSubject
.streamInstructions()
.anyMatch(instruction -> instruction.isConstNumber(1)));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("false", "true");
+ .assertSuccessWithOutputLines("false", "true", "notnull", "notnull");
}
static class Main {
@@ -59,6 +65,21 @@
public static void main(String[] args) {
System.out.println(getFalse().booleanValue());
System.out.println(getTrue().booleanValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getFalse() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getTrue() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedByteReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedByteReturnPropagationTest.java
index ab37cf8..3043c57 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedByteReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedByteReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedBooleanReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,9 +51,14 @@
mainMethodSubject
.streamInstructions()
.anyMatch(instruction -> instruction.isConstNumber(1)));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("0", "1");
+ .assertSuccessWithOutputLines("0", "1", "notnull", "notnull");
}
static class Main {
@@ -59,6 +66,21 @@
public static void main(String[] args) {
System.out.println(getZero().byteValue());
System.out.println(getOne().byteValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getZero() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getOne() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedCharacterReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedCharacterReturnPropagationTest.java
index 7d56596..6542212 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedCharacterReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedCharacterReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedByteReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,9 +51,14 @@
mainMethodSubject
.streamInstructions()
.anyMatch(instruction -> instruction.isConstNumber('B')));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("A", "B");
+ .assertSuccessWithOutputLines("A", "B", "notnull", "notnull");
}
static class Main {
@@ -59,6 +66,21 @@
public static void main(String[] args) {
System.out.println(getA().charValue());
System.out.println(getB().charValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getA() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getB() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedDoubleReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedDoubleReturnPropagationTest.java
index 864471e..a383a2d 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedDoubleReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedDoubleReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedCharacterReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -50,9 +52,14 @@
.streamInstructions()
.anyMatch(
instruction -> instruction.isConstNumber(Double.doubleToLongBits(1))));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("0.0", "1.0");
+ .assertSuccessWithOutputLines("0.0", "1.0", "notnull", "notnull");
}
static class Main {
@@ -60,6 +67,21 @@
public static void main(String[] args) {
System.out.println(getZero().doubleValue());
System.out.println(getOne().doubleValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getZero() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getOne() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedFloatReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedFloatReturnPropagationTest.java
index daef6c7..ca68794 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedFloatReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedFloatReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedCharacterReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -50,9 +52,14 @@
.streamInstructions()
.anyMatch(
instruction -> instruction.isConstNumber(Float.floatToIntBits(1f))));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("0.0", "1.0");
+ .assertSuccessWithOutputLines("0.0", "1.0", "notnull", "notnull");
}
static class Main {
@@ -64,6 +71,21 @@
} catch (Exception e) {
System.out.println();
}
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getZero() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getOne() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedIntegerReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedIntegerReturnPropagationTest.java
index 8f9c9b9..0f205ea 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedIntegerReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedIntegerReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedCharacterReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,9 +51,14 @@
mainMethodSubject
.streamInstructions()
.anyMatch(instruction -> instruction.isConstNumber(1)));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("0", "1");
+ .assertSuccessWithOutputLines("0", "1", "notnull", "notnull");
}
static class Main {
@@ -59,6 +66,21 @@
public static void main(String[] args) {
System.out.println(getZero().intValue());
System.out.println(getOne().intValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getZero() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getOne() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedLongReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedLongReturnPropagationTest.java
index 5b08473..e588489 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedLongReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedLongReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedCharacterReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,9 +51,14 @@
mainMethodSubject
.streamInstructions()
.anyMatch(instruction -> instruction.isConstNumber(1)));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("0", "1");
+ .assertSuccessWithOutputLines("0", "1", "notnull", "notnull");
}
static class Main {
@@ -59,6 +66,21 @@
public static void main(String[] args) {
System.out.println(getZero().longValue());
System.out.println(getOne().longValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getZero() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getOne() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedPrimitiveNonNullTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedPrimitiveNonNullTest.java
new file mode 100644
index 0000000..8e2e53c
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedPrimitiveNonNullTest.java
@@ -0,0 +1,147 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.ir.optimize.boxedprimitives;
+
+import static org.junit.Assert.assertTrue;
+
+import com.android.tools.r8.NeverInline;
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.utils.codeinspector.CodeInspector;
+import com.android.tools.r8.utils.codeinspector.FoundMethodSubject;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class BoxedPrimitiveNonNullTest extends TestBase {
+
+ private final TestParameters parameters;
+
+ @Parameters(name = "{0}")
+ public static TestParametersCollection data() {
+ return getTestParameters().withAllRuntimesAndApiLevels().build();
+ }
+
+ public BoxedPrimitiveNonNullTest(TestParameters parameters) {
+ this.parameters = parameters;
+ }
+
+ @Test
+ public void testNonNull() throws Exception {
+ testForR8(parameters.getBackend())
+ .addInnerClasses(getClass())
+ .enableInliningAnnotations()
+ .addKeepMainRule(TestClass.class)
+ .setMinApi(parameters)
+ .compile()
+ .inspect(this::verifyNoBranch)
+ .run(parameters.getRuntime(), TestClass.class)
+ .assertSuccessWithOutputLines("true", "1", "1", "c", "1", "1", "1.0", "1.0");
+ }
+
+ private void verifyNoBranch(CodeInspector codeInspector) {
+ for (FoundMethodSubject nonMain :
+ codeInspector.clazz(TestClass.class).allMethods(m -> !m.getOriginalName().equals("main"))) {
+ assertTrue(nonMain.streamInstructions().noneMatch(InstructionSubject::isIf));
+ }
+ }
+
+ public static class TestClass {
+
+ public static void main(String[] args) {
+ booleanTest();
+ byteTest();
+ shortTest();
+ characterTest();
+ integerTest();
+ longTest();
+ floatTest();
+ doubleTest();
+ }
+
+ @NeverInline
+ private static void booleanTest() {
+ Boolean boxed = Boolean.valueOf(true);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.booleanValue());
+ }
+ }
+
+ @NeverInline
+ private static void byteTest() {
+ Byte boxed = Byte.valueOf((byte) 1);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.byteValue());
+ }
+ }
+
+ @NeverInline
+ private static void shortTest() {
+ Short boxed = Short.valueOf((short) 1);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.shortValue());
+ }
+ }
+
+ @NeverInline
+ private static void characterTest() {
+ Character boxed = Character.valueOf('c');
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.charValue());
+ }
+ }
+
+ @NeverInline
+ private static void integerTest() {
+ Integer boxed = Integer.valueOf(1);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.byteValue());
+ }
+ }
+
+ @NeverInline
+ private static void longTest() {
+ Long boxed = Long.valueOf(1L);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.longValue());
+ }
+ }
+
+ @NeverInline
+ private static void floatTest() {
+ Float boxed = Float.valueOf(1.0f);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.floatValue());
+ }
+ }
+
+ @NeverInline
+ private static void doubleTest() {
+ Double boxed = Double.valueOf(1.0d);
+ if (boxed == null) {
+ System.out.println("null int");
+ } else {
+ System.out.println(boxed.doubleValue());
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedShortReturnPropagationTest.java b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedShortReturnPropagationTest.java
index ccb4b56..3c8b0b8 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedShortReturnPropagationTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/boxedprimitives/BoxedShortReturnPropagationTest.java
@@ -11,6 +11,8 @@
import com.android.tools.r8.TestBase;
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.ir.optimize.boxedprimitives.BoxedCharacterReturnPropagationTest.Main;
+import com.android.tools.r8.utils.codeinspector.InstructionSubject;
import com.android.tools.r8.utils.codeinspector.MethodSubject;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,9 +51,14 @@
mainMethodSubject
.streamInstructions()
.anyMatch(instruction -> instruction.isConstNumber(1)));
+
+ MethodSubject nullTest =
+ inspector.clazz(Main.class).uniqueMethodWithOriginalName("nullTest");
+ assertThat(nullTest, isPresent());
+ assertTrue(nullTest.streamInstructions().noneMatch(InstructionSubject::isIf));
})
.run(parameters.getRuntime(), Main.class)
- .assertSuccessWithOutputLines("0", "1");
+ .assertSuccessWithOutputLines("0", "1", "notnull", "notnull");
}
static class Main {
@@ -59,6 +66,21 @@
public static void main(String[] args) {
System.out.println(getZero().shortValue());
System.out.println(getOne().shortValue());
+ nullTest();
+ }
+
+ @NeverInline
+ static void nullTest() {
+ if (getZero() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
+ if (getOne() == null) {
+ System.out.println("null");
+ } else {
+ System.out.println("notnull");
+ }
}
@NeverInline
diff --git a/src/test/java/com/android/tools/r8/resolution/SingleTargetLookupTest.java b/src/test/java/com/android/tools/r8/resolution/SingleTargetLookupTest.java
index e010bfe..afe93c0 100644
--- a/src/test/java/com/android/tools/r8/resolution/SingleTargetLookupTest.java
+++ b/src/test/java/com/android/tools/r8/resolution/SingleTargetLookupTest.java
@@ -11,12 +11,13 @@
import com.android.tools.r8.TestAppViewBuilder;
import com.android.tools.r8.graph.AppInfo;
import com.android.tools.r8.graph.AppView;
-import com.android.tools.r8.graph.DexEncodedMethod;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.LookupResult;
import com.android.tools.r8.graph.MethodResolutionResult;
import com.android.tools.r8.graph.ProgramMethod;
+import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.resolution.singletarget.Main;
import com.android.tools.r8.resolution.singletarget.one.AbstractSubClass;
import com.android.tools.r8.resolution.singletarget.one.AbstractTopClass;
@@ -34,6 +35,7 @@
import com.android.tools.r8.resolution.singletarget.two.OtherSubSubClassOne;
import com.android.tools.r8.resolution.singletarget.two.OtherSubSubClassTwo;
import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.shaking.LibraryModeledPredicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.Collections;
@@ -195,8 +197,14 @@
ProgramMethod context =
appInfo.definitionForProgramType(reference.holder).getProgramDefaultInitializer();
Assert.assertNotNull(appInfo.resolveMethodOnClassHolderLegacy(reference).getSingleTarget());
- DexEncodedMethod singleVirtualTarget =
- appInfo.lookupSingleVirtualTarget(appView, reference, context, false);
+ DexClassAndMethod singleVirtualTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView,
+ reference,
+ context,
+ false,
+ LibraryModeledPredicate.alwaysFalse(),
+ DynamicType.unknown());
if (singleTargetHolderOrNull == null) {
Assert.assertNull(singleVirtualTarget);
} else {
diff --git a/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentInterfaceTest.java b/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentInterfaceTest.java
index 244e9ea..3cbd047 100644
--- a/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentInterfaceTest.java
+++ b/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentInterfaceTest.java
@@ -11,12 +11,15 @@
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProgramClass;
import com.android.tools.r8.graph.MethodResolutionResult;
import com.android.tools.r8.graph.ProgramMethod;
+import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.shaking.LibraryModeledPredicate;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Assert;
@@ -132,8 +135,14 @@
DexEncodedMethod resolved = resolutionResult.getSingleTarget();
assertEquals(methodOnBReference, resolved.getReference());
assertFalse(resolutionResult.isVirtualTarget());
- DexEncodedMethod singleVirtualTarget =
- appInfo.lookupSingleVirtualTarget(appView, methodOnBReference, methodOnB, false);
+ DexClassAndMethod singleVirtualTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView,
+ methodOnBReference,
+ methodOnB,
+ false,
+ LibraryModeledPredicate.alwaysFalse(),
+ DynamicType.unknown());
Assert.assertNull(singleVirtualTarget);
}
diff --git a/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentTest.java b/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentTest.java
index b456774..15aa8e9 100644
--- a/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentTest.java
+++ b/src/test/java/com/android/tools/r8/resolution/VirtualOverrideOfStaticMethodWithVirtualParentTest.java
@@ -14,11 +14,14 @@
import com.android.tools.r8.TestRunResult;
import com.android.tools.r8.ToolHelper.DexVm.Version;
import com.android.tools.r8.graph.AppView;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexEncodedMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProgramClass;
import com.android.tools.r8.graph.MethodResolutionResult;
+import com.android.tools.r8.ir.analysis.type.DynamicType;
import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.shaking.LibraryModeledPredicate;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Assert;
@@ -179,9 +182,14 @@
DexEncodedMethod resolved = resolutionResult.getSingleTarget();
assertEquals(methodOnA, resolved.getReference());
assertFalse(resolutionResult.isVirtualTarget());
- DexEncodedMethod singleVirtualTarget =
- appInfo.lookupSingleVirtualTarget(
- appView, methodOnB, bClass.getProgramDefaultInitializer(), false);
+ DexClassAndMethod singleVirtualTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView,
+ methodOnB,
+ bClass.getProgramDefaultInitializer(),
+ false,
+ LibraryModeledPredicate.alwaysFalse(),
+ DynamicType.unknown());
Assert.assertNull(singleVirtualTarget);
}
diff --git a/src/test/java/com/android/tools/r8/resolution/singletarget/InstantiatedLowerBoundTest.java b/src/test/java/com/android/tools/r8/resolution/singletarget/InstantiatedLowerBoundTest.java
index 788e55b..4c11c91 100644
--- a/src/test/java/com/android/tools/r8/resolution/singletarget/InstantiatedLowerBoundTest.java
+++ b/src/test/java/com/android/tools/r8/resolution/singletarget/InstantiatedLowerBoundTest.java
@@ -14,7 +14,7 @@
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
import com.android.tools.r8.graph.AppView;
-import com.android.tools.r8.graph.DexEncodedMethod;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexProgramClass;
import com.android.tools.r8.graph.DexType;
@@ -25,6 +25,7 @@
import com.android.tools.r8.ir.analysis.type.DynamicTypeWithLowerBound;
import com.android.tools.r8.ir.analysis.type.TypeElement;
import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.shaking.LibraryModeledPredicate;
import com.google.common.collect.Sets;
import java.util.Set;
import org.junit.Test;
@@ -64,13 +65,13 @@
DexMethod fooA = buildNullaryVoidMethod(A.class, "foo", appInfo.dexItemFactory());
TypeElement latticeA = typeA.toTypeElement(appView);
ClassTypeElement latticeB = typeB.toTypeElement(appView).asClassType();
- DexEncodedMethod singleTarget =
- appInfo.lookupSingleVirtualTarget(
+ DexClassAndMethod singleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
appView,
fooA,
mainMethod,
false,
- t -> false,
+ LibraryModeledPredicate.alwaysFalse(),
DynamicTypeWithLowerBound.create(appView, latticeA, latticeB));
assertNotNull(singleTarget);
DexMethod fooB = buildNullaryVoidMethod(B.class, "foo", appInfo.dexItemFactory());
@@ -97,13 +98,13 @@
DexMethod fooA = buildNullaryVoidMethod(A.class, "foo", appInfo.dexItemFactory());
TypeElement latticeA = typeA.toTypeElement(appView);
ClassTypeElement latticeB = typeB.toTypeElement(appView).asClassType();
- DexEncodedMethod singleTarget =
- appInfo.lookupSingleVirtualTarget(
+ DexClassAndMethod singleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
appView,
fooA,
mainMethod,
false,
- t -> false,
+ LibraryModeledPredicate.alwaysFalse(),
DynamicTypeWithLowerBound.create(appView, latticeA, latticeB));
assertNotNull(singleTarget);
DexMethod fooB = buildNullaryVoidMethod(B.class, "foo", appInfo.dexItemFactory());
@@ -153,13 +154,13 @@
assertEquals(expected, actual);
TypeElement latticeA = typeA.toTypeElement(appView);
ClassTypeElement latticeC = typeC.toTypeElement(appView).asClassType();
- DexEncodedMethod singleTarget =
- appInfo.lookupSingleVirtualTarget(
+ DexClassAndMethod singleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
appView,
fooA,
mainMethod,
false,
- t -> false,
+ LibraryModeledPredicate.alwaysFalse(),
DynamicTypeWithLowerBound.create(appView, latticeA, latticeC));
assertNull(singleTarget);
}
diff --git a/src/test/java/com/android/tools/r8/resolution/singletarget/SuccessAndInvalidLookupTest.java b/src/test/java/com/android/tools/r8/resolution/singletarget/SuccessAndInvalidLookupTest.java
index ca2dee8..0a94601 100644
--- a/src/test/java/com/android/tools/r8/resolution/singletarget/SuccessAndInvalidLookupTest.java
+++ b/src/test/java/com/android/tools/r8/resolution/singletarget/SuccessAndInvalidLookupTest.java
@@ -13,7 +13,7 @@
import com.android.tools.r8.TestParameters;
import com.android.tools.r8.TestParametersCollection;
import com.android.tools.r8.graph.AppView;
-import com.android.tools.r8.graph.DexEncodedMethod;
+import com.android.tools.r8.graph.DexClassAndMethod;
import com.android.tools.r8.graph.DexMethod;
import com.android.tools.r8.graph.DexType;
import com.android.tools.r8.graph.ProgramMethod;
@@ -21,6 +21,7 @@
import com.android.tools.r8.ir.analysis.type.DynamicTypeWithUpperBound;
import com.android.tools.r8.optimize.interfaces.collection.NonEmptyOpenClosedInterfacesCollection;
import com.android.tools.r8.shaking.AppInfoWithLiveness;
+import com.android.tools.r8.shaking.LibraryModeledPredicate;
import java.util.Collections;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -58,14 +59,14 @@
DynamicType dynamicTypeA =
DynamicTypeWithUpperBound.create(appView, typeA.toTypeElement(appView));
DexMethod fooA = buildNullaryVoidMethod(A.class, "foo", appInfo.dexItemFactory());
- DexEncodedMethod singleTarget =
- appInfo.lookupSingleVirtualTarget(
- appView, fooA, mainMethod, false, t -> false, dynamicTypeA);
+ DexClassAndMethod singleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView, fooA, mainMethod, false, LibraryModeledPredicate.alwaysFalse(), dynamicTypeA);
assertNotNull(singleTarget);
assertEquals(fooA, singleTarget.getReference());
- DexEncodedMethod invalidSingleTarget =
- appInfo.lookupSingleVirtualTarget(
- appView, fooA, mainMethod, true, t -> false, dynamicTypeA);
+ DexClassAndMethod invalidSingleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView, fooA, mainMethod, true, LibraryModeledPredicate.alwaysFalse(), dynamicTypeA);
assertNull(invalidSingleTarget);
}
@@ -91,14 +92,14 @@
DynamicTypeWithUpperBound.create(appView, typeA.toTypeElement(appView));
DexMethod fooI = buildNullaryVoidMethod(I.class, "foo", appInfo.dexItemFactory());
DexMethod fooA = buildNullaryVoidMethod(A.class, "foo", appInfo.dexItemFactory());
- DexEncodedMethod singleTarget =
- appInfo.lookupSingleVirtualTarget(
- appView, fooI, mainMethod, true, t -> false, dynamicTypeA);
+ DexClassAndMethod singleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView, fooI, mainMethod, true, LibraryModeledPredicate.alwaysFalse(), dynamicTypeA);
assertNotNull(singleTarget);
assertEquals(fooA, singleTarget.getReference());
- DexEncodedMethod invalidSingleTarget =
- appInfo.lookupSingleVirtualTarget(
- appView, fooI, mainMethod, false, t -> false, dynamicTypeA);
+ DexClassAndMethod invalidSingleTarget =
+ appInfo.lookupSingleVirtualTargetForTesting(
+ appView, fooI, mainMethod, false, LibraryModeledPredicate.alwaysFalse(), dynamicTypeA);
assertNull(invalidSingleTarget);
}
diff --git a/third_party/google/yapf/20231013.tar.gz.sha1 b/third_party/google/yapf/20231013.tar.gz.sha1
new file mode 100644
index 0000000..dbcbb50
--- /dev/null
+++ b/third_party/google/yapf/20231013.tar.gz.sha1
@@ -0,0 +1 @@
+5469c71a8b6a49fed240977d0a8053c31d599ae7
\ No newline at end of file
diff --git a/tools/adb.py b/tools/adb.py
index 2e015a0..988c684 100644
--- a/tools/adb.py
+++ b/tools/adb.py
@@ -7,105 +7,117 @@
import time
import utils
+
def install_apk_on_emulator(apk, emulator_id, quiet=False):
- cmd = ['adb', '-s', emulator_id, 'install', '-r', '-d', apk]
- if quiet:
- subprocess.check_output(cmd)
- else:
- subprocess.check_call(cmd)
+ cmd = ['adb', '-s', emulator_id, 'install', '-r', '-d', apk]
+ if quiet:
+ subprocess.check_output(cmd)
+ else:
+ subprocess.check_call(cmd)
def uninstall_apk_on_emulator(app_id, emulator_id):
- process = subprocess.Popen(
- ['adb', '-s', emulator_id, 'uninstall', app_id],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- stdout = stdout.decode('UTF-8')
- stderr = stderr.decode('UTF-8')
+ process = subprocess.Popen(['adb', '-s', emulator_id, 'uninstall', app_id],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ stdout = stdout.decode('UTF-8')
+ stderr = stderr.decode('UTF-8')
- if stdout.strip() == 'Success':
- # Successfully uninstalled
- return
+ if stdout.strip() == 'Success':
+ # Successfully uninstalled
+ return
- if 'Unknown package: {}'.format(app_id) in stderr:
- # Application not installed
- return
+ if 'Unknown package: {}'.format(app_id) in stderr:
+ # Application not installed
+ return
- # Check if the app is listed in packages
- packages = subprocess.check_output(['adb', 'shell', 'pm', 'list', 'packages'])
- if not 'package:' + app_id in packages:
- return
+ # Check if the app is listed in packages
+ packages = subprocess.check_output(
+ ['adb', 'shell', 'pm', 'list', 'packages'])
+ if not 'package:' + app_id in packages:
+ return
- raise Exception(
- 'Unexpected result from `adb uninstall {}\nStdout: {}\nStderr: {}'.format(
- app_id, stdout, stderr))
+ raise Exception(
+ 'Unexpected result from `adb uninstall {}\nStdout: {}\nStderr: {}'.
+ format(app_id, stdout, stderr))
def wait_for_emulator(emulator_id):
- stdout = subprocess.check_output(['adb', 'devices']).decode('UTF-8')
- if '{}\tdevice'.format(emulator_id) in stdout:
- return True
-
- print('Emulator \'{}\' not connected; waiting for connection'.format(
- emulator_id))
-
- time_waited = 0
- while True:
- time.sleep(10)
- time_waited += 10
stdout = subprocess.check_output(['adb', 'devices']).decode('UTF-8')
- if '{}\tdevice'.format(emulator_id) not in stdout:
- print('... still waiting for connection')
- if time_waited >= 5 * 60:
- return False
- else:
- return True
+ if '{}\tdevice'.format(emulator_id) in stdout:
+ return True
+
+ print('Emulator \'{}\' not connected; waiting for connection'.format(
+ emulator_id))
+
+ time_waited = 0
+ while True:
+ time.sleep(10)
+ time_waited += 10
+ stdout = subprocess.check_output(['adb', 'devices']).decode('UTF-8')
+ if '{}\tdevice'.format(emulator_id) not in stdout:
+ print('... still waiting for connection')
+ if time_waited >= 5 * 60:
+ return False
+ else:
+ return True
def run_monkey(app_id, emulator_id, apk, monkey_events, quiet, enable_logging):
- if not wait_for_emulator(emulator_id):
- return False
+ if not wait_for_emulator(emulator_id):
+ return False
- install_apk_on_emulator(apk, emulator_id, quiet)
+ install_apk_on_emulator(apk, emulator_id, quiet)
- # Intentionally using a constant seed such that the monkey generates the same
- # event sequence for each shrinker.
- random_seed = 42
+ # Intentionally using a constant seed such that the monkey generates the same
+ # event sequence for each shrinker.
+ random_seed = 42
- cmd = ['adb', '-s', emulator_id, 'shell', 'monkey', '-p', app_id,
- '-s', str(random_seed), str(monkey_events)]
+ cmd = [
+ 'adb', '-s', emulator_id, 'shell', 'monkey', '-p', app_id, '-s',
+ str(random_seed),
+ str(monkey_events)
+ ]
- try:
- stdout = utils.RunCmd(cmd, quiet=quiet, logging=enable_logging)
- succeeded = ('Events injected: {}'.format(monkey_events) in stdout)
- except subprocess.CalledProcessError as e:
- succeeded = False
+ try:
+ stdout = utils.RunCmd(cmd, quiet=quiet, logging=enable_logging)
+ succeeded = ('Events injected: {}'.format(monkey_events) in stdout)
+ except subprocess.CalledProcessError as e:
+ succeeded = False
- uninstall_apk_on_emulator(app_id, emulator_id)
+ uninstall_apk_on_emulator(app_id, emulator_id)
- return succeeded
+ return succeeded
-def run_instrumented(app_id, test_id, emulator_id, apk, test_apk, quiet,
+def run_instrumented(app_id,
+ test_id,
+ emulator_id,
+ apk,
+ test_apk,
+ quiet,
enable_logging,
test_runner='androidx.test.runner.AndroidJUnitRunner'):
- if not wait_for_emulator(emulator_id):
- return None
+ if not wait_for_emulator(emulator_id):
+ return None
- install_apk_on_emulator(apk, emulator_id, quiet)
- install_apk_on_emulator(test_apk, emulator_id, quiet)
+ install_apk_on_emulator(apk, emulator_id, quiet)
+ install_apk_on_emulator(test_apk, emulator_id, quiet)
- cmd = ['adb', '-s', emulator_id, 'shell', 'am', 'instrument', '-w',
- '{}/{}'.format(test_id, test_runner)]
+ cmd = [
+ 'adb', '-s', emulator_id, 'shell', 'am', 'instrument', '-w',
+ '{}/{}'.format(test_id, test_runner)
+ ]
- try:
- stdout = utils.RunCmd(cmd, quiet=quiet, logging=enable_logging)
- # The runner will print OK (X tests) if completed succesfully
- succeeded = any("OK (" in s for s in stdout)
- except subprocess.CalledProcessError as e:
- succeeded = False
+ try:
+ stdout = utils.RunCmd(cmd, quiet=quiet, logging=enable_logging)
+ # The runner will print OK (X tests) if completed succesfully
+ succeeded = any("OK (" in s for s in stdout)
+ except subprocess.CalledProcessError as e:
+ succeeded = False
- uninstall_apk_on_emulator(test_id, emulator_id)
- uninstall_apk_on_emulator(app_id, emulator_id)
+ uninstall_apk_on_emulator(test_id, emulator_id)
+ uninstall_apk_on_emulator(app_id, emulator_id)
- return succeeded
+ return succeeded
diff --git a/tools/api_sample_coverage.py b/tools/api_sample_coverage.py
index 2d9b0d2..448a0be 100755
--- a/tools/api_sample_coverage.py
+++ b/tools/api_sample_coverage.py
@@ -2,7 +2,6 @@
# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
-
'''
Compare the R8 API used by the API usage sample to the API kept by @Keep.
'''
@@ -21,57 +20,67 @@
def main(output_dir=None):
- if output_dir is None:
- output_dir = ''
+ if output_dir is None:
+ output_dir = ''
- javaExecutable = jdk.GetJavaExecutable()
- printseeds_path = os.path.join(output_dir, 'keep-seeds.txt')
- printseeds_args = [
- javaExecutable, '-jar', utils.R8_JAR, 'printseeds',
- utils.RT_JAR, utils.R8_JAR, utils.R8LIB_KEEP_RULES,
- ]
- write_sorted_lines(printseeds_args, printseeds_path)
+ javaExecutable = jdk.GetJavaExecutable()
+ printseeds_path = os.path.join(output_dir, 'keep-seeds.txt')
+ printseeds_args = [
+ javaExecutable,
+ '-jar',
+ utils.R8_JAR,
+ 'printseeds',
+ utils.RT_JAR,
+ utils.R8_JAR,
+ utils.R8LIB_KEEP_RULES,
+ ]
+ write_sorted_lines(printseeds_args, printseeds_path)
- printuses_path = os.path.join(output_dir, 'sample-uses.txt')
- printuses_args = [
- javaExecutable, '-jar', utils.R8_JAR, 'printuses',
- utils.RT_JAR, utils.R8_JAR, API_SAMPLE_JAR,
- ]
- write_sorted_lines(printuses_args, printuses_path)
+ printuses_path = os.path.join(output_dir, 'sample-uses.txt')
+ printuses_args = [
+ javaExecutable,
+ '-jar',
+ utils.R8_JAR,
+ 'printuses',
+ utils.RT_JAR,
+ utils.R8_JAR,
+ API_SAMPLE_JAR,
+ ]
+ write_sorted_lines(printuses_args, printuses_path)
- print_diff(printseeds_path, printuses_path)
+ print_diff(printseeds_path, printuses_path)
def write_sorted_lines(cmd_args, output_path):
- utils.PrintCmd(cmd_args)
- output_lines = subprocess.check_output(cmd_args).splitlines(True)
- print("Write output to %s" % output_path)
- output_lines.sort()
- with open(output_path, 'w') as fp:
- for line in output_lines:
- fp.write(line)
+ utils.PrintCmd(cmd_args)
+ output_lines = subprocess.check_output(cmd_args).splitlines(True)
+ print("Write output to %s" % output_path)
+ output_lines.sort()
+ with open(output_path, 'w') as fp:
+ for line in output_lines:
+ fp.write(line)
def print_diff(printseeds_path, printuses_path):
- with open(printseeds_path) as fp:
- seeds = set(fp.read().splitlines())
- with open(printuses_path) as fp:
- uses = set(fp.read().splitlines())
- only_in_seeds = seeds - uses
- only_in_uses = uses - seeds
- if only_in_seeds:
- print("%s lines with '-' are marked @Keep " % len(only_in_seeds) +
- "but not used by sample.")
- if only_in_uses:
- print("%s lines with '+' are used by sample " % len(only_in_uses) +
- "but are missing @Keep annotations.")
- for line in sorted(only_in_seeds):
- print('-' + line)
- for line in sorted(only_in_uses):
- print('+' + line)
- if not only_in_seeds and not only_in_uses:
- print('Sample uses the entire set of members marked @Keep. Well done!')
+ with open(printseeds_path) as fp:
+ seeds = set(fp.read().splitlines())
+ with open(printuses_path) as fp:
+ uses = set(fp.read().splitlines())
+ only_in_seeds = seeds - uses
+ only_in_uses = uses - seeds
+ if only_in_seeds:
+ print("%s lines with '-' are marked @Keep " % len(only_in_seeds) +
+ "but not used by sample.")
+ if only_in_uses:
+ print("%s lines with '+' are used by sample " % len(only_in_uses) +
+ "but are missing @Keep annotations.")
+ for line in sorted(only_in_seeds):
+ print('-' + line)
+ for line in sorted(only_in_uses):
+ print('+' + line)
+ if not only_in_seeds and not only_in_uses:
+ print('Sample uses the entire set of members marked @Keep. Well done!')
if __name__ == '__main__':
- main(**vars(parser.parse_args()))
+ main(**vars(parser.parse_args()))
diff --git a/tools/apk_masseur.py b/tools/apk_masseur.py
index 1b35d23..51e9366 100755
--- a/tools/apk_masseur.py
+++ b/tools/apk_masseur.py
@@ -15,164 +15,193 @@
USAGE = 'usage: %prog [options] <apk>'
+
def parse_options():
- parser = optparse.OptionParser(usage=USAGE)
- parser.add_option('--clear-profile',
- help='To remove baseline.prof and baseline.profm from '
- 'assets/dexopt/',
- default=False,
- action='store_true')
- parser.add_option('--dex',
- help='Directory or archive with dex files to use instead '
- 'of those in the apk',
- default=None)
- parser.add_option('--desugared-library-dex',
- help='Path to desugared library dex file to use or archive '
- 'containing a single classes.dex file',
- default=None)
- parser.add_option('--resources',
- help=('pattern that matches resources to use instead of '
- + 'those in the apk'),
- default=None)
- parser.add_option('--out',
- help='output file (default ./$(basename <apk>))',
- default=None)
- parser.add_option('--keystore',
- help='keystore file (default ~/.android/app.keystore)',
- default=None)
- parser.add_option('--install',
- help='install the generated apk with adb options -t -r -d',
- default=False,
- action='store_true')
- parser.add_option('--adb-options',
- help='additional adb options when running adb',
- default=None)
- parser.add_option('--quiet',
- help='disable verbose logging',
- default=False)
- parser.add_option('--sign-before-align',
- help='Sign the apk before aligning',
- default=False,
- action='store_true')
- (options, args) = parser.parse_args()
- if len(args) != 1:
- parser.error('Expected <apk> argument, got: ' + ' '.join(args))
- apk = args[0]
- return (options, apk)
+ parser = optparse.OptionParser(usage=USAGE)
+ parser.add_option('--clear-profile',
+ help='To remove baseline.prof and baseline.profm from '
+ 'assets/dexopt/',
+ default=False,
+ action='store_true')
+ parser.add_option('--dex',
+ help='Directory or archive with dex files to use instead '
+ 'of those in the apk',
+ default=None)
+ parser.add_option(
+ '--desugared-library-dex',
+ help='Path to desugared library dex file to use or archive '
+ 'containing a single classes.dex file',
+ default=None)
+ parser.add_option(
+ '--resources',
+ help=('pattern that matches resources to use instead of ' +
+ 'those in the apk'),
+ default=None)
+ parser.add_option('--out',
+ help='output file (default ./$(basename <apk>))',
+ default=None)
+ parser.add_option('--keystore',
+ help='keystore file (default ~/.android/app.keystore)',
+ default=None)
+ parser.add_option(
+ '--install',
+ help='install the generated apk with adb options -t -r -d',
+ default=False,
+ action='store_true')
+ parser.add_option('--adb-options',
+ help='additional adb options when running adb',
+ default=None)
+ parser.add_option('--quiet', help='disable verbose logging', default=False)
+ parser.add_option('--sign-before-align',
+ help='Sign the apk before aligning',
+ default=False,
+ action='store_true')
+ (options, args) = parser.parse_args()
+ if len(args) != 1:
+ parser.error('Expected <apk> argument, got: ' + ' '.join(args))
+ apk = args[0]
+ return (options, apk)
+
def is_archive(file):
- return file.endswith('.zip') or file.endswith('.jar')
+ return file.endswith('.zip') or file.endswith('.jar')
-def repack(
- apk, clear_profile, processed_out, desugared_library_dex, resources, temp,
- quiet, logging):
- processed_apk = os.path.join(temp, 'processed.apk')
- shutil.copyfile(apk, processed_apk)
- if clear_profile:
- zip_utils.remove_files_from_zip(
- ['assets/dexopt/baseline.prof', 'assets/dexopt/baseline.profm'],
- processed_apk)
+def repack(apk, clear_profile, processed_out, desugared_library_dex, resources,
+ temp, quiet, logging):
+ processed_apk = os.path.join(temp, 'processed.apk')
+ shutil.copyfile(apk, processed_apk)
- if not processed_out:
- utils.Print('Using original dex as is', quiet=quiet)
+ if clear_profile:
+ zip_utils.remove_files_from_zip(
+ ['assets/dexopt/baseline.prof', 'assets/dexopt/baseline.profm'],
+ processed_apk)
+
+ if not processed_out:
+ utils.Print('Using original dex as is', quiet=quiet)
+ return processed_apk
+
+ utils.Print('Repacking APK with dex files from {}'.format(processed_out),
+ quiet=quiet)
+
+ # Delete original dex files in APK.
+ with utils.ChangedWorkingDirectory(temp, quiet=quiet):
+ cmd = ['zip', '-d', 'processed.apk', '*.dex']
+ utils.RunCmd(cmd, quiet=quiet, logging=logging)
+
+ # Unzip the jar or zip file into `temp`.
+ if is_archive(processed_out):
+ cmd = ['unzip', processed_out, '-d', temp]
+ if quiet:
+ cmd.insert(1, '-q')
+ utils.RunCmd(cmd, quiet=quiet, logging=logging)
+ processed_out = temp
+ elif desugared_library_dex:
+ for dex_name in glob.glob('*.dex', root_dir=processed_out):
+ src = os.path.join(processed_out, dex_name)
+ dst = os.path.join(temp, dex_name)
+ shutil.copyfile(src, dst)
+ processed_out = temp
+
+ if desugared_library_dex:
+ desugared_library_dex_index = len(glob.glob('*.dex', root_dir=temp)) + 1
+ desugared_library_dex_name = 'classes%s.dex' % desugared_library_dex_index
+ desugared_library_dex_dst = os.path.join(temp,
+ desugared_library_dex_name)
+ if is_archive(desugared_library_dex):
+ zip_utils.extract_member(desugared_library_dex, 'classes.dex',
+ desugared_library_dex_dst)
+ else:
+ shutil.copyfile(desugared_library_dex, desugared_library_dex_dst)
+
+ # Insert the new dex and resource files from `processed_out` into the APK.
+ with utils.ChangedWorkingDirectory(processed_out, quiet=quiet):
+ dex_files = glob.glob('*.dex')
+ dex_files.sort()
+ resource_files = glob.glob(resources) if resources else []
+ cmd = ['zip', '-u', '-0', processed_apk] + dex_files + resource_files
+ utils.RunCmd(cmd, quiet=quiet, logging=logging)
return processed_apk
- utils.Print(
- 'Repacking APK with dex files from {}'.format(processed_out), quiet=quiet)
-
- # Delete original dex files in APK.
- with utils.ChangedWorkingDirectory(temp, quiet=quiet):
- cmd = ['zip', '-d', 'processed.apk', '*.dex']
- utils.RunCmd(cmd, quiet=quiet, logging=logging)
-
- # Unzip the jar or zip file into `temp`.
- if is_archive(processed_out):
- cmd = ['unzip', processed_out, '-d', temp]
- if quiet:
- cmd.insert(1, '-q')
- utils.RunCmd(cmd, quiet=quiet, logging=logging)
- processed_out = temp
- elif desugared_library_dex:
- for dex_name in glob.glob('*.dex', root_dir=processed_out):
- src = os.path.join(processed_out, dex_name)
- dst = os.path.join(temp, dex_name)
- shutil.copyfile(src, dst)
- processed_out = temp
-
- if desugared_library_dex:
- desugared_library_dex_index = len(glob.glob('*.dex', root_dir=temp)) + 1
- desugared_library_dex_name = 'classes%s.dex' % desugared_library_dex_index
- desugared_library_dex_dst = os.path.join(temp, desugared_library_dex_name)
- if is_archive(desugared_library_dex):
- zip_utils.extract_member(
- desugared_library_dex, 'classes.dex', desugared_library_dex_dst)
- else:
- shutil.copyfile(desugared_library_dex, desugared_library_dex_dst)
-
- # Insert the new dex and resource files from `processed_out` into the APK.
- with utils.ChangedWorkingDirectory(processed_out, quiet=quiet):
- dex_files = glob.glob('*.dex')
- dex_files.sort()
- resource_files = glob.glob(resources) if resources else []
- cmd = ['zip', '-u', '-0', processed_apk] + dex_files + resource_files
- utils.RunCmd(cmd, quiet=quiet, logging=logging)
- return processed_apk
def sign(unsigned_apk, keystore, temp, quiet, logging):
- signed_apk = os.path.join(temp, 'unaligned.apk')
- return apk_utils.sign_with_apksigner(
- unsigned_apk, signed_apk, keystore, quiet=quiet, logging=logging)
+ signed_apk = os.path.join(temp, 'unaligned.apk')
+ return apk_utils.sign_with_apksigner(unsigned_apk,
+ signed_apk,
+ keystore,
+ quiet=quiet,
+ logging=logging)
+
def align(signed_apk, temp, quiet, logging):
- utils.Print('Aligning', quiet=quiet)
- aligned_apk = os.path.join(temp, 'aligned.apk')
- return apk_utils.align(signed_apk, aligned_apk)
+ utils.Print('Aligning', quiet=quiet)
+ aligned_apk = os.path.join(temp, 'aligned.apk')
+ return apk_utils.align(signed_apk, aligned_apk)
-def masseur(
- apk, clear_profile=False, dex=None, desugared_library_dex=None,
- resources=None, out=None, adb_options=None, sign_before_align=False,
- keystore=None, install=False, quiet=False, logging=True):
- if not out:
- out = os.path.basename(apk)
- if not keystore:
- keystore = apk_utils.default_keystore()
- with utils.TempDir() as temp:
- processed_apk = None
- if dex or clear_profile:
- processed_apk = repack(
- apk, clear_profile, dex, desugared_library_dex, resources, temp,
- quiet, logging)
- else:
- assert not desugared_library_dex
- utils.Print(
- 'Signing original APK without modifying apk', quiet=quiet)
- processed_apk = os.path.join(temp, 'processed.apk')
- shutil.copyfile(apk, processed_apk)
- if sign_before_align:
- signed_apk = sign(
- processed_apk, keystore, temp, quiet=quiet, logging=logging)
- aligned_apk = align(signed_apk, temp, quiet=quiet, logging=logging)
- utils.Print('Writing result to {}'.format(out), quiet=quiet)
- shutil.copyfile(aligned_apk, out)
- else:
- aligned_apk = align(processed_apk, temp, quiet=quiet, logging=logging)
- signed_apk = sign(
- aligned_apk, keystore, temp, quiet=quiet, logging=logging)
- utils.Print('Writing result to {}'.format(out), quiet=quiet)
- shutil.copyfile(signed_apk, out)
- if install:
- adb_cmd = ['adb']
- if adb_options:
- adb_cmd.extend(
- [option for option in adb_options.split(' ') if option])
- adb_cmd.extend(['install', '-t', '-r', '-d', out]);
- utils.RunCmd(adb_cmd, quiet=quiet, logging=logging)
+
+def masseur(apk,
+ clear_profile=False,
+ dex=None,
+ desugared_library_dex=None,
+ resources=None,
+ out=None,
+ adb_options=None,
+ sign_before_align=False,
+ keystore=None,
+ install=False,
+ quiet=False,
+ logging=True):
+ if not out:
+ out = os.path.basename(apk)
+ if not keystore:
+ keystore = apk_utils.default_keystore()
+ with utils.TempDir() as temp:
+ processed_apk = None
+ if dex or clear_profile:
+ processed_apk = repack(apk, clear_profile, dex,
+ desugared_library_dex, resources, temp,
+ quiet, logging)
+ else:
+ assert not desugared_library_dex
+ utils.Print('Signing original APK without modifying apk',
+ quiet=quiet)
+ processed_apk = os.path.join(temp, 'processed.apk')
+ shutil.copyfile(apk, processed_apk)
+ if sign_before_align:
+ signed_apk = sign(processed_apk,
+ keystore,
+ temp,
+ quiet=quiet,
+ logging=logging)
+ aligned_apk = align(signed_apk, temp, quiet=quiet, logging=logging)
+ utils.Print('Writing result to {}'.format(out), quiet=quiet)
+ shutil.copyfile(aligned_apk, out)
+ else:
+ aligned_apk = align(processed_apk,
+ temp,
+ quiet=quiet,
+ logging=logging)
+ signed_apk = sign(aligned_apk,
+ keystore,
+ temp,
+ quiet=quiet,
+ logging=logging)
+ utils.Print('Writing result to {}'.format(out), quiet=quiet)
+ shutil.copyfile(signed_apk, out)
+ if install:
+ adb_cmd = ['adb']
+ if adb_options:
+ adb_cmd.extend(
+ [option for option in adb_options.split(' ') if option])
+ adb_cmd.extend(['install', '-t', '-r', '-d', out])
+ utils.RunCmd(adb_cmd, quiet=quiet, logging=logging)
+
def main():
- (options, apk) = parse_options()
- masseur(apk, **vars(options))
- return 0
+ (options, apk) = parse_options()
+ masseur(apk, **vars(options))
+ return 0
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/apk_utils.py b/tools/apk_utils.py
index 1da8d47..8ced491 100755
--- a/tools/apk_utils.py
+++ b/tools/apk_utils.py
@@ -15,116 +15,118 @@
USAGE = 'usage: %prog [options] <apk>'
+
def parse_options():
- parser = optparse.OptionParser(usage=USAGE)
- parser.add_option('--keystore',
- help='keystore file (default ~/.android/app.keystore)',
- default='~/.android/app.keystore')
- parser.add_option('--sign',
- help='Sign the passed in apk.',
- default=False,
- action='store_true')
- parser.add_option('--use_apksigner',
- help='Use apksigner to sign.',
- default=False,
- action='store_true')
- parser.add_option('--output',
- help='Where to put the signed apk.)',
- default=None)
+ parser = optparse.OptionParser(usage=USAGE)
+ parser.add_option('--keystore',
+ help='keystore file (default ~/.android/app.keystore)',
+ default='~/.android/app.keystore')
+ parser.add_option('--sign',
+ help='Sign the passed in apk.',
+ default=False,
+ action='store_true')
+ parser.add_option('--use_apksigner',
+ help='Use apksigner to sign.',
+ default=False,
+ action='store_true')
+ parser.add_option('--output',
+ help='Where to put the signed apk.)',
+ default=None)
- (options, args) = parser.parse_args()
- if len(args) != 1:
- parser.error('Expected <apk> argument, got: ' + ' '.join(args))
- apk = args[0]
- return (options, apk)
+ (options, args) = parser.parse_args()
+ if len(args) != 1:
+ parser.error('Expected <apk> argument, got: ' + ' '.join(args))
+ apk = args[0]
+ return (options, apk)
-def add_baseline_profile_to_apk(
- apk, baseline_profile, baseline_profile_metadata, tmp_dir):
- if baseline_profile is None:
- return apk
- ts = time.time_ns()
- dest_apk = os.path.join(tmp_dir, 'app-%s.apk' % ts)
- dest_apk_aligned = os.path.join(tmp_dir, 'app-aligned-%s.apk' % ts)
- dest_apk_signed = os.path.join(tmp_dir, 'app-signed-%s.apk' % ts)
- shutil.copy2(apk, dest_apk)
- zip_utils.remove_files_from_zip(
- ['assets/dexopt/baseline.prof', 'assets/dexopt/baseline.profm'], dest_apk)
- zip_utils.add_file_to_zip(
- baseline_profile, 'assets/dexopt/baseline.prof', dest_apk)
- if baseline_profile_metadata is not None:
- zip_utils.add_file_to_zip(
- baseline_profile_metadata, 'assets/dexopt/baseline.profm', dest_apk)
- align(dest_apk, dest_apk_aligned)
- sign_with_apksigner(dest_apk_aligned, dest_apk_signed)
- return dest_apk_signed
+
+def add_baseline_profile_to_apk(apk, baseline_profile,
+ baseline_profile_metadata, tmp_dir):
+ if baseline_profile is None:
+ return apk
+ ts = time.time_ns()
+ dest_apk = os.path.join(tmp_dir, 'app-%s.apk' % ts)
+ dest_apk_aligned = os.path.join(tmp_dir, 'app-aligned-%s.apk' % ts)
+ dest_apk_signed = os.path.join(tmp_dir, 'app-signed-%s.apk' % ts)
+ shutil.copy2(apk, dest_apk)
+ zip_utils.remove_files_from_zip(
+ ['assets/dexopt/baseline.prof', 'assets/dexopt/baseline.profm'],
+ dest_apk)
+ zip_utils.add_file_to_zip(baseline_profile, 'assets/dexopt/baseline.prof',
+ dest_apk)
+ if baseline_profile_metadata is not None:
+ zip_utils.add_file_to_zip(baseline_profile_metadata,
+ 'assets/dexopt/baseline.profm', dest_apk)
+ align(dest_apk, dest_apk_aligned)
+ sign_with_apksigner(dest_apk_aligned, dest_apk_signed)
+ return dest_apk_signed
+
def align(apk, aligned_apk):
- zipalign_path = (
- 'zipalign' if 'build_tools' in os.environ.get('PATH')
- else os.path.join(utils.getAndroidBuildTools(), 'zipalign'))
- cmd = [zipalign_path, '-f', '-p', '4', apk, aligned_apk]
- utils.RunCmd(cmd, quiet=True, logging=False)
- return aligned_apk
+ zipalign_path = ('zipalign' if 'build_tools' in os.environ.get('PATH') else
+ os.path.join(utils.getAndroidBuildTools(), 'zipalign'))
+ cmd = [zipalign_path, '-f', '-p', '4', apk, aligned_apk]
+ utils.RunCmd(cmd, quiet=True, logging=False)
+ return aligned_apk
+
def default_keystore():
- return os.path.join(os.getenv('HOME'), '.android', 'app.keystore')
+ return os.path.join(os.getenv('HOME'), '.android', 'app.keystore')
+
def get_min_api(apk):
- aapt = os.path.join(utils.getAndroidBuildTools(), 'aapt')
- cmd = [aapt, 'dump', 'badging', apk]
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- for line in stdout.splitlines():
- if line.startswith('sdkVersion:\''):
- return int(line[len('sdkVersion:\''): -1])
- raise ValueError('Unexpected stdout: %s' % stdout)
+ aapt = os.path.join(utils.getAndroidBuildTools(), 'aapt')
+ cmd = [aapt, 'dump', 'badging', apk]
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ for line in stdout.splitlines():
+ if line.startswith('sdkVersion:\''):
+ return int(line[len('sdkVersion:\''):-1])
+ raise ValueError('Unexpected stdout: %s' % stdout)
+
def sign(unsigned_apk, signed_apk, keystore, quiet=False, logging=True):
- utils.Print('Signing (ignore the warnings)', quiet=quiet)
- cmd = ['zip', '-d', unsigned_apk, 'META-INF/*']
- utils.RunCmd(cmd, quiet=quiet, logging=logging, fail=False)
- cmd = [
- 'jarsigner',
- '-sigalg', 'SHA1withRSA',
- '-digestalg', 'SHA1',
- '-keystore', keystore,
- '-storepass', 'android',
- '-signedjar', signed_apk,
- unsigned_apk,
- 'androiddebugkey'
- ]
- utils.RunCmd(cmd, quiet=quiet)
+ utils.Print('Signing (ignore the warnings)', quiet=quiet)
+ cmd = ['zip', '-d', unsigned_apk, 'META-INF/*']
+ utils.RunCmd(cmd, quiet=quiet, logging=logging, fail=False)
+ cmd = [
+ 'jarsigner', '-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1',
+ '-keystore', keystore, '-storepass', 'android', '-signedjar',
+ signed_apk, unsigned_apk, 'androiddebugkey'
+ ]
+ utils.RunCmd(cmd, quiet=quiet)
-def sign_with_apksigner(
- unsigned_apk, signed_apk, keystore=None, password='android', quiet=False,
- logging=True):
- cmd = [
- os.path.join(utils.getAndroidBuildTools(), 'apksigner'),
- 'sign',
- '-v',
- '--ks', keystore or default_keystore(),
- '--ks-pass', 'pass:' + password,
- '--min-sdk-version', '19',
- '--out', signed_apk,
- '--v2-signing-enabled',
- unsigned_apk
- ]
- utils.RunCmd(cmd, quiet=quiet, logging=logging)
- return signed_apk
+
+def sign_with_apksigner(unsigned_apk,
+ signed_apk,
+ keystore=None,
+ password='android',
+ quiet=False,
+ logging=True):
+ cmd = [
+ os.path.join(utils.getAndroidBuildTools(), 'apksigner'), 'sign', '-v',
+ '--ks', keystore or default_keystore(), '--ks-pass', 'pass:' + password,
+ '--min-sdk-version', '19', '--out', signed_apk, '--v2-signing-enabled',
+ unsigned_apk
+ ]
+ utils.RunCmd(cmd, quiet=quiet, logging=logging)
+ return signed_apk
+
def main():
- (options, apk) = parse_options()
- if options.sign:
- if not options.output:
- print('When signing you must specify an output apk')
- return 1
- if not options.keystore:
- print('When signing you must specify a keystore')
- return 1
- if options.use_apksigner:
- sign_with_apksigner(apk, options.output, options.keystore)
- else:
- sign(apk, options.output, options.keystore)
- return 0
+ (options, apk) = parse_options()
+ if options.sign:
+ if not options.output:
+ print('When signing you must specify an output apk')
+ return 1
+ if not options.keystore:
+ print('When signing you must specify a keystore')
+ return 1
+ if options.use_apksigner:
+ sign_with_apksigner(apk, options.output, options.keystore)
+ else:
+ sign(apk, options.output, options.keystore)
+ return 0
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/archive.py b/tools/archive.py
index b868296..84c0da3 100755
--- a/tools/archive.py
+++ b/tools/archive.py
@@ -11,10 +11,10 @@
import gradle
try:
- import resource
+ import resource
except ImportError:
- # Not a Unix system. Do what Gandalf tells you not to.
- pass
+ # Not a Unix system. Do what Gandalf tells you not to.
+ pass
import shutil
import subprocess
import sys
@@ -23,319 +23,336 @@
ARCHIVE_BUCKET = 'r8-releases'
+
def ParseOptions():
- result = optparse.OptionParser()
- result.add_option('--dry-run', '--dry_run',
- help='Build only, no upload.',
- default=False, action='store_true')
- result.add_option('--dry-run-output', '--dry_run_output',
- help='Output directory for \'build only, no upload\'.',
- type="string", action="store")
- result.add_option('--skip-gradle-build', '--skip_gradle_build',
- help='Skip Gradle build. Can only be used for local testing.',
- default=False, action='store_true')
- return result.parse_args()
+ result = optparse.OptionParser()
+ result.add_option('--dry-run',
+ '--dry_run',
+ help='Build only, no upload.',
+ default=False,
+ action='store_true')
+ result.add_option('--dry-run-output',
+ '--dry_run_output',
+ help='Output directory for \'build only, no upload\'.',
+ type="string",
+ action="store")
+ result.add_option(
+ '--skip-gradle-build',
+ '--skip_gradle_build',
+ help='Skip Gradle build. Can only be used for local testing.',
+ default=False,
+ action='store_true')
+ return result.parse_args()
+
def GetVersion():
- output = subprocess.check_output([
- jdk.GetJavaExecutable(), '-cp', utils.R8_JAR, 'com.android.tools.r8.R8',
- '--version'
- ]).decode('utf-8')
- r8_version = output.splitlines()[0].strip()
- return r8_version.split()[1]
+ output = subprocess.check_output([
+ jdk.GetJavaExecutable(), '-cp', utils.R8_JAR, 'com.android.tools.r8.R8',
+ '--version'
+ ]).decode('utf-8')
+ r8_version = output.splitlines()[0].strip()
+ return r8_version.split()[1]
+
def GetGitBranches():
- return subprocess.check_output(['git', 'show', '-s', '--pretty=%d', 'HEAD'])
+ return subprocess.check_output(['git', 'show', '-s', '--pretty=%d', 'HEAD'])
+
def GetGitHash():
- return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip()
+ return subprocess.check_output(['git', 'rev-parse',
+ 'HEAD']).decode('utf-8').strip()
+
def IsMain(version):
- branches = subprocess.check_output(['git', 'branch', '-r', '--contains',
- 'HEAD']).decode('utf-8')
- # CL runs from gerrit does not have a branch, we always treat them as main
- # commits to archive these to the hash based location
- if len(branches) == 0:
+ branches = subprocess.check_output(
+ ['git', 'branch', '-r', '--contains', 'HEAD']).decode('utf-8')
+ # CL runs from gerrit does not have a branch, we always treat them as main
+ # commits to archive these to the hash based location
+ if len(branches) == 0:
+ return True
+ if not version == 'main':
+ # Sanity check, we don't want to archive on top of release builds EVER
+ # Note that even though we branch, we never push the bots to build the same
+ # commit as main on a branch since we always change the version to
+ # not be just 'main' (or we crash here :-)).
+ if 'origin/main' in branches:
+ raise Exception('We are seeing origin/main in a commit that '
+ 'don\'t have \'main\' as version')
+ return False
+ if not 'origin/main' in branches:
+ raise Exception('We are not seeing origin/main '
+ 'in a commit that have \'main\' as version')
return True
- if not version == 'main':
- # Sanity check, we don't want to archive on top of release builds EVER
- # Note that even though we branch, we never push the bots to build the same
- # commit as main on a branch since we always change the version to
- # not be just 'main' (or we crash here :-)).
- if 'origin/main' in branches:
- raise Exception('We are seeing origin/main in a commit that '
- 'don\'t have \'main\' as version')
- return False
- if not 'origin/main' in branches:
- raise Exception('We are not seeing origin/main '
- 'in a commit that have \'main\' as version')
- return True
-def GetStorageDestination(storage_prefix,
- version_or_path,
- file_name,
- is_main):
- # We archive main commits under raw/main instead of directly under raw
- version_dir = GetVersionDestination(storage_prefix,
- version_or_path,
- is_main)
- return '%s/%s' % (version_dir, file_name)
+
+def GetStorageDestination(storage_prefix, version_or_path, file_name, is_main):
+ # We archive main commits under raw/main instead of directly under raw
+ version_dir = GetVersionDestination(storage_prefix, version_or_path,
+ is_main)
+ return '%s/%s' % (version_dir, file_name)
+
def GetVersionDestination(storage_prefix, version_or_path, is_main):
- archive_dir = 'raw/main' if is_main else 'raw'
- return '%s%s/%s/%s' % (storage_prefix, ARCHIVE_BUCKET,
- archive_dir, version_or_path)
+ archive_dir = 'raw/main' if is_main else 'raw'
+ return '%s%s/%s/%s' % (storage_prefix, ARCHIVE_BUCKET, archive_dir,
+ version_or_path)
+
def GetUploadDestination(version_or_path, file_name, is_main):
- return GetStorageDestination('gs://', version_or_path, file_name, is_main)
+ return GetStorageDestination('gs://', version_or_path, file_name, is_main)
+
def GetUrl(version_or_path, file_name, is_main):
- return GetStorageDestination('https://storage.googleapis.com/',
- version_or_path, file_name, is_main)
+ return GetStorageDestination('https://storage.googleapis.com/',
+ version_or_path, file_name, is_main)
+
def GetMavenUrl(is_main):
- return GetVersionDestination('https://storage.googleapis.com/', '', is_main)
+ return GetVersionDestination('https://storage.googleapis.com/', '', is_main)
+
def SetRLimitToMax():
- (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
- resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+ (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
+ resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+
def PrintResourceInfo():
- (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
- print('INFO: Open files soft limit: %s' % soft)
- print('INFO: Open files hard limit: %s' % hard)
+ (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
+ print('INFO: Open files soft limit: %s' % soft)
+ print('INFO: Open files hard limit: %s' % hard)
def Main():
- (options, args) = ParseOptions()
- Run(options)
+ (options, args) = ParseOptions()
+ Run(options)
+
def Run(options):
- if not utils.is_bot() and not options.dry_run:
- raise Exception('You are not a bot, don\'t archive builds. '
- + 'Use --dry-run to test locally')
- if (options.dry_run_output and
- (not os.path.exists(options.dry_run_output) or
- not os.path.isdir(options.dry_run_output))):
- raise Exception(options.dry_run_output
- + ' does not exist or is not a directory')
- if (options.skip_gradle_build and not options.dry_run):
- raise Exception('Using --skip-gradle-build only supported with --dry-run')
+ if not utils.is_bot() and not options.dry_run:
+ raise Exception('You are not a bot, don\'t archive builds. ' +
+ 'Use --dry-run to test locally')
+ if (options.dry_run_output and
+ (not os.path.exists(options.dry_run_output) or
+ not os.path.isdir(options.dry_run_output))):
+ raise Exception(options.dry_run_output +
+ ' does not exist or is not a directory')
+ if (options.skip_gradle_build and not options.dry_run):
+ raise Exception(
+ 'Using --skip-gradle-build only supported with --dry-run')
- if utils.is_bot() and not utils.IsWindows():
- SetRLimitToMax()
- if not utils.IsWindows():
- PrintResourceInfo()
+ if utils.is_bot() and not utils.IsWindows():
+ SetRLimitToMax()
+ if not utils.IsWindows():
+ PrintResourceInfo()
- with utils.TempDir() as temp:
- version_file = os.path.join(temp, 'r8-version.properties')
- with open(version_file,'w') as version_writer:
- version_writer.write('version.sha=' + GetGitHash() + '\n')
- if not os.environ.get('SWARMING_BOT_ID') and not options.dry_run:
- raise Exception('Environment variable SWARMING_BOT_ID not set')
+ with utils.TempDir() as temp:
+ version_file = os.path.join(temp, 'r8-version.properties')
+ with open(version_file, 'w') as version_writer:
+ version_writer.write('version.sha=' + GetGitHash() + '\n')
+ if not os.environ.get('SWARMING_BOT_ID') and not options.dry_run:
+ raise Exception('Environment variable SWARMING_BOT_ID not set')
- releaser = \
- ("<local developer build>" if options.dry_run
- else 'releaser=go/r8bot ('
- + (os.environ.get('SWARMING_BOT_ID') or 'foo') + ')\n')
- version_writer.write(releaser)
- version_writer.write('version-file.version.code=1\n')
+ releaser = \
+ ("<local developer build>" if options.dry_run
+ else 'releaser=go/r8bot ('
+ + (os.environ.get('SWARMING_BOT_ID') or 'foo') + ')\n')
+ version_writer.write(releaser)
+ version_writer.write('version-file.version.code=1\n')
- create_maven_release.generate_r8_maven_zip(
- utils.MAVEN_ZIP_LIB,
- version_file=version_file,
- skip_gradle_build=options.skip_gradle_build)
+ create_maven_release.generate_r8_maven_zip(
+ utils.MAVEN_ZIP_LIB,
+ version_file=version_file,
+ skip_gradle_build=options.skip_gradle_build)
- # Ensure all archived artifacts has been built before archiving.
- # The target tasks postfixed by 'lib' depend on the actual target task so
- # building it invokes the original task first.
- # The '-Pno_internal' flag is important because we generate the lib based on uses in tests.
- if (not options.skip_gradle_build):
- gradle.RunGradle([
- utils.GRADLE_TASK_CONSOLIDATED_LICENSE,
- utils.GRADLE_TASK_KEEP_ANNO_JAR,
- utils.GRADLE_TASK_R8,
- utils.GRADLE_TASK_R8LIB,
- utils.GRADLE_TASK_R8LIB_NO_DEPS,
- utils.GRADLE_TASK_RETRACE,
- utils.GRADLE_TASK_RETRACE_NO_DEPS,
- utils.GRADLE_TASK_SOURCE_JAR,
- utils.GRADLE_TASK_SWISS_ARMY_KNIFE,
- '-Pno_internal'
- ])
+ # Ensure all archived artifacts has been built before archiving.
+ # The target tasks postfixed by 'lib' depend on the actual target task so
+ # building it invokes the original task first.
+ # The '-Pno_internal' flag is important because we generate the lib based on uses in tests.
+ if (not options.skip_gradle_build):
+ gradle.RunGradle([
+ utils.GRADLE_TASK_CONSOLIDATED_LICENSE,
+ utils.GRADLE_TASK_KEEP_ANNO_JAR, utils.GRADLE_TASK_R8,
+ utils.GRADLE_TASK_R8LIB, utils.GRADLE_TASK_R8LIB_NO_DEPS,
+ utils.GRADLE_TASK_RETRACE, utils.GRADLE_TASK_RETRACE_NO_DEPS,
+ utils.GRADLE_TASK_SOURCE_JAR,
+ utils.GRADLE_TASK_SWISS_ARMY_KNIFE, '-Pno_internal'
+ ])
- # Create maven release of the desuage_jdk_libs configuration. This require
- # an r8.jar with dependencies to have been built.
- create_maven_release.generate_desugar_configuration_maven_zip(
- utils.DESUGAR_CONFIGURATION_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION,
- utils.DESUGAR_IMPLEMENTATION,
- utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
- create_maven_release.generate_desugar_configuration_maven_zip(
- utils.DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
+ # Create maven release of the desuage_jdk_libs configuration. This require
+ # an r8.jar with dependencies to have been built.
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_MAVEN_ZIP, utils.DESUGAR_CONFIGURATION,
+ utils.DESUGAR_IMPLEMENTATION,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
- create_maven_release.generate_desugar_configuration_maven_zip(
- utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
- create_maven_release.generate_desugar_configuration_maven_zip(
- utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
- create_maven_release.generate_desugar_configuration_maven_zip(
- utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_NIO,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_NIO,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
- version = GetVersion()
- is_main = IsMain(version)
- if is_main:
- # On main we use the git hash to archive with
- print('On main, using git hash for archiving')
- version = GetGitHash()
+ version = GetVersion()
+ is_main = IsMain(version)
+ if is_main:
+ # On main we use the git hash to archive with
+ print('On main, using git hash for archiving')
+ version = GetGitHash()
- destination = GetVersionDestination('gs://', version, is_main)
- if utils.cloud_storage_exists(destination) and not options.dry_run:
- raise Exception('Target archive directory %s already exists' % destination)
+ destination = GetVersionDestination('gs://', version, is_main)
+ if utils.cloud_storage_exists(destination) and not options.dry_run:
+ raise Exception('Target archive directory %s already exists' %
+ destination)
- # Create pom file for our maven repository that we build for testing.
- default_pom_file = os.path.join(temp, 'r8.pom')
- create_maven_release.write_default_r8_pom_file(default_pom_file, version)
- for_archiving = [
- utils.R8_JAR,
- utils.R8LIB_JAR,
- utils.R8LIB_JAR + '.map',
- utils.R8LIB_JAR + '_map.zip',
- utils.R8_FULL_EXCLUDE_DEPS_JAR,
- utils.R8LIB_EXCLUDE_DEPS_JAR,
- utils.R8LIB_EXCLUDE_DEPS_JAR + '.map',
- utils.R8LIB_EXCLUDE_DEPS_JAR + '_map.zip',
- utils.MAVEN_ZIP_LIB,
- utils.DESUGAR_CONFIGURATION,
- utils.DESUGAR_CONFIGURATION_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
- utils.DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP,
- utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP,
- utils.R8_SRC_JAR,
- utils.R8RETRACE_JAR,
- utils.R8RETRACE_JAR + '.map',
- utils.R8RETRACE_JAR + '_map.zip',
- utils.R8RETRACE_EXCLUDE_DEPS_JAR,
- utils.R8RETRACE_EXCLUDE_DEPS_JAR + '.map',
- utils.R8RETRACE_EXCLUDE_DEPS_JAR + '_map.zip',
- utils.KEEPANNO_ANNOTATIONS_JAR,
- utils.GENERATED_LICENSE]
- for file in for_archiving:
- file_name = os.path.basename(file)
- tagged_jar = os.path.join(temp, file_name)
- shutil.copyfile(file, tagged_jar)
- if file_name.endswith('.jar') and not file_name.endswith('-src.jar'):
- with zipfile.ZipFile(tagged_jar, 'a') as zip:
- zip.write(version_file, os.path.basename(version_file))
- destination = GetUploadDestination(version, file_name, is_main)
- print('Uploading %s to %s' % (tagged_jar, destination))
- if options.dry_run:
- if options.dry_run_output:
- dry_run_destination = os.path.join(options.dry_run_output, file_name)
- print('Dry run, not actually uploading. Copying to '
- + dry_run_destination)
- shutil.copyfile(tagged_jar, dry_run_destination)
- else:
- print('Dry run, not actually uploading')
- else:
- utils.upload_file_to_cloud_storage(tagged_jar, destination)
- print('File available at: %s' % GetUrl(version, file_name, is_main))
+ # Create pom file for our maven repository that we build for testing.
+ default_pom_file = os.path.join(temp, 'r8.pom')
+ create_maven_release.write_default_r8_pom_file(default_pom_file,
+ version)
+ for_archiving = [
+ utils.R8_JAR, utils.R8LIB_JAR, utils.R8LIB_JAR + '.map',
+ utils.R8LIB_JAR + '_map.zip', utils.R8_FULL_EXCLUDE_DEPS_JAR,
+ utils.R8LIB_EXCLUDE_DEPS_JAR, utils.R8LIB_EXCLUDE_DEPS_JAR + '.map',
+ utils.R8LIB_EXCLUDE_DEPS_JAR + '_map.zip', utils.MAVEN_ZIP_LIB,
+ utils.DESUGAR_CONFIGURATION, utils.DESUGAR_CONFIGURATION_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
+ utils.DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP,
+ utils.DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP, utils.R8_SRC_JAR,
+ utils.R8RETRACE_JAR, utils.R8RETRACE_JAR + '.map',
+ utils.R8RETRACE_JAR + '_map.zip', utils.R8RETRACE_EXCLUDE_DEPS_JAR,
+ utils.R8RETRACE_EXCLUDE_DEPS_JAR + '.map',
+ utils.R8RETRACE_EXCLUDE_DEPS_JAR + '_map.zip',
+ utils.KEEPANNO_ANNOTATIONS_JAR, utils.GENERATED_LICENSE
+ ]
+ for file in for_archiving:
+ file_name = os.path.basename(file)
+ tagged_jar = os.path.join(temp, file_name)
+ shutil.copyfile(file, tagged_jar)
+ if file_name.endswith(
+ '.jar') and not file_name.endswith('-src.jar'):
+ with zipfile.ZipFile(tagged_jar, 'a') as zip:
+ zip.write(version_file, os.path.basename(version_file))
+ destination = GetUploadDestination(version, file_name, is_main)
+ print('Uploading %s to %s' % (tagged_jar, destination))
+ if options.dry_run:
+ if options.dry_run_output:
+ dry_run_destination = os.path.join(options.dry_run_output,
+ file_name)
+ print('Dry run, not actually uploading. Copying to ' +
+ dry_run_destination)
+ shutil.copyfile(tagged_jar, dry_run_destination)
+ else:
+ print('Dry run, not actually uploading')
+ else:
+ utils.upload_file_to_cloud_storage(tagged_jar, destination)
+ print('File available at: %s' %
+ GetUrl(version, file_name, is_main))
- # Upload R8 to a maven compatible location.
- if file == utils.R8_JAR:
- maven_dst = GetUploadDestination(utils.get_maven_path('r8', version),
- 'r8-%s.jar' % version, is_main)
- maven_pom_dst = GetUploadDestination(
- utils.get_maven_path('r8', version),
- 'r8-%s.pom' % version, is_main)
- if options.dry_run:
- print('Dry run, not actually creating maven repo for R8')
- else:
- utils.upload_file_to_cloud_storage(tagged_jar, maven_dst)
- utils.upload_file_to_cloud_storage(default_pom_file, maven_pom_dst)
- print('Maven repo root available at: %s' % GetMavenUrl(is_main))
+ # Upload R8 to a maven compatible location.
+ if file == utils.R8_JAR:
+ maven_dst = GetUploadDestination(
+ utils.get_maven_path('r8', version), 'r8-%s.jar' % version,
+ is_main)
+ maven_pom_dst = GetUploadDestination(
+ utils.get_maven_path('r8', version), 'r8-%s.pom' % version,
+ is_main)
+ if options.dry_run:
+ print('Dry run, not actually creating maven repo for R8')
+ else:
+ utils.upload_file_to_cloud_storage(tagged_jar, maven_dst)
+ utils.upload_file_to_cloud_storage(default_pom_file,
+ maven_pom_dst)
+ print('Maven repo root available at: %s' %
+ GetMavenUrl(is_main))
- # Upload desugar_jdk_libs configuration to a maven compatible location.
- if file == utils.DESUGAR_CONFIGURATION:
- jar_basename = 'desugar_jdk_libs_configuration.jar'
- jar_version_name = 'desugar_jdk_libs_configuration-%s.jar' % version
- maven_dst = GetUploadDestination(
- utils.get_maven_path('desugar_jdk_libs_configuration', version),
- jar_version_name, is_main)
+ # Upload desugar_jdk_libs configuration to a maven compatible location.
+ if file == utils.DESUGAR_CONFIGURATION:
+ jar_basename = 'desugar_jdk_libs_configuration.jar'
+ jar_version_name = 'desugar_jdk_libs_configuration-%s.jar' % version
+ maven_dst = GetUploadDestination(
+ utils.get_maven_path('desugar_jdk_libs_configuration',
+ version), jar_version_name, is_main)
- with utils.TempDir() as tmp_dir:
- desugar_jdk_libs_configuration_jar = os.path.join(tmp_dir,
- jar_version_name)
- create_maven_release.generate_jar_with_desugar_configuration(
- utils.DESUGAR_CONFIGURATION,
- utils.DESUGAR_IMPLEMENTATION,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP,
- desugar_jdk_libs_configuration_jar)
+ with utils.TempDir() as tmp_dir:
+ desugar_jdk_libs_configuration_jar = os.path.join(
+ tmp_dir, jar_version_name)
+ create_maven_release.generate_jar_with_desugar_configuration(
+ utils.DESUGAR_CONFIGURATION,
+ utils.DESUGAR_IMPLEMENTATION,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP,
+ desugar_jdk_libs_configuration_jar)
- if options.dry_run:
- print('Dry run, not actually creating maven repo for '
- + 'desugar configuration.')
- if options.dry_run_output:
- shutil.copyfile(
- desugar_jdk_libs_configuration_jar,
- os.path.join(options.dry_run_output, jar_version_name))
- else:
- utils.upload_file_to_cloud_storage(
- desugar_jdk_libs_configuration_jar, maven_dst)
- print('Maven repo root available at: %s' % GetMavenUrl(is_main))
- # Also archive the jar as non maven destination for Google3
- jar_destination = GetUploadDestination(
- version, jar_basename, is_main)
- utils.upload_file_to_cloud_storage(
- desugar_jdk_libs_configuration_jar, jar_destination)
+ if options.dry_run:
+ print('Dry run, not actually creating maven repo for ' +
+ 'desugar configuration.')
+ if options.dry_run_output:
+ shutil.copyfile(
+ desugar_jdk_libs_configuration_jar,
+ os.path.join(options.dry_run_output,
+ jar_version_name))
+ else:
+ utils.upload_file_to_cloud_storage(
+ desugar_jdk_libs_configuration_jar, maven_dst)
+ print('Maven repo root available at: %s' %
+ GetMavenUrl(is_main))
+ # Also archive the jar as non maven destination for Google3
+ jar_destination = GetUploadDestination(
+ version, jar_basename, is_main)
+ utils.upload_file_to_cloud_storage(
+ desugar_jdk_libs_configuration_jar, jar_destination)
- # TODO(b/237636871): Refactor this to avoid the duplication of what is above.
- # Upload desugar_jdk_libs JDK-11 legacyconfiguration to a maven compatible location.
- if file == utils.DESUGAR_CONFIGURATION_JDK11_LEGACY:
- jar_basename = 'desugar_jdk_libs_configuration.jar'
- jar_version_name = 'desugar_jdk_libs_configuration-%s-jdk11-legacy.jar' % version
- maven_dst = GetUploadDestination(
- utils.get_maven_path('desugar_jdk_libs_configuration', version),
- jar_version_name, is_main)
+ # TODO(b/237636871): Refactor this to avoid the duplication of what is above.
+ # Upload desugar_jdk_libs JDK-11 legacyconfiguration to a maven compatible location.
+ if file == utils.DESUGAR_CONFIGURATION_JDK11_LEGACY:
+ jar_basename = 'desugar_jdk_libs_configuration.jar'
+ jar_version_name = 'desugar_jdk_libs_configuration-%s-jdk11-legacy.jar' % version
+ maven_dst = GetUploadDestination(
+ utils.get_maven_path('desugar_jdk_libs_configuration',
+ version), jar_version_name, is_main)
- with utils.TempDir() as tmp_dir:
- desugar_jdk_libs_configuration_jar = os.path.join(tmp_dir,
- jar_version_name)
- create_maven_release.generate_jar_with_desugar_configuration(
- utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP,
- desugar_jdk_libs_configuration_jar)
+ with utils.TempDir() as tmp_dir:
+ desugar_jdk_libs_configuration_jar = os.path.join(
+ tmp_dir, jar_version_name)
+ create_maven_release.generate_jar_with_desugar_configuration(
+ utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP,
+ desugar_jdk_libs_configuration_jar)
- if options.dry_run:
- print('Dry run, not actually creating maven repo for '
- + 'desugar configuration.')
- if options.dry_run_output:
- shutil.copyfile(
- desugar_jdk_libs_configuration_jar,
- os.path.join(options.dry_run_output, jar_version_name))
- else:
- utils.upload_file_to_cloud_storage(
- desugar_jdk_libs_configuration_jar, maven_dst)
- print('Maven repo root available at: %s' % GetMavenUrl(is_main))
- # Also archive the jar as non maven destination for Google3
- jar_destination = GetUploadDestination(
- version, jar_basename, is_main)
- utils.upload_file_to_cloud_storage(
- desugar_jdk_libs_configuration_jar, jar_destination)
+ if options.dry_run:
+ print('Dry run, not actually creating maven repo for ' +
+ 'desugar configuration.')
+ if options.dry_run_output:
+ shutil.copyfile(
+ desugar_jdk_libs_configuration_jar,
+ os.path.join(options.dry_run_output,
+ jar_version_name))
+ else:
+ utils.upload_file_to_cloud_storage(
+ desugar_jdk_libs_configuration_jar, maven_dst)
+ print('Maven repo root available at: %s' %
+ GetMavenUrl(is_main))
+ # Also archive the jar as non maven destination for Google3
+ jar_destination = GetUploadDestination(
+ version, jar_basename, is_main)
+ utils.upload_file_to_cloud_storage(
+ desugar_jdk_libs_configuration_jar, jar_destination)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/archive_desugar_jdk_libs.py b/tools/archive_desugar_jdk_libs.py
index 30d40a4..ead0bdd 100755
--- a/tools/archive_desugar_jdk_libs.py
+++ b/tools/archive_desugar_jdk_libs.py
@@ -40,11 +40,11 @@
VERSION_FILE_JDK11_NIO = 'VERSION_JDK11_NIO.txt'
VERSION_MAP = {
- 'jdk8': VERSION_FILE_JDK8,
- 'jdk11_legacy': VERSION_FILE_JDK11_LEGACY,
- 'jdk11_minimal': VERSION_FILE_JDK11_MINIMAL,
- 'jdk11': VERSION_FILE_JDK11,
- 'jdk11_nio': VERSION_FILE_JDK11_NIO
+ 'jdk8': VERSION_FILE_JDK8,
+ 'jdk11_legacy': VERSION_FILE_JDK11_LEGACY,
+ 'jdk11_minimal': VERSION_FILE_JDK11_MINIMAL,
+ 'jdk11': VERSION_FILE_JDK11,
+ 'jdk11_nio': VERSION_FILE_JDK11_NIO
}
GITHUB_REPRO = 'desugar_jdk_libs'
@@ -52,327 +52,350 @@
BASE_LIBRARY_NAME = 'desugar_jdk_libs'
LIBRARY_NAME_MAP = {
- 'jdk8': BASE_LIBRARY_NAME,
- 'jdk11_legacy': BASE_LIBRARY_NAME,
- 'jdk11_minimal': BASE_LIBRARY_NAME + '_minimal',
- 'jdk11': BASE_LIBRARY_NAME,
- 'jdk11_nio': BASE_LIBRARY_NAME + '_nio'
+ 'jdk8': BASE_LIBRARY_NAME,
+ 'jdk11_legacy': BASE_LIBRARY_NAME,
+ 'jdk11_minimal': BASE_LIBRARY_NAME + '_minimal',
+ 'jdk11': BASE_LIBRARY_NAME,
+ 'jdk11_nio': BASE_LIBRARY_NAME + '_nio'
}
MAVEN_RELEASE_TARGET_MAP = {
- 'jdk8': 'maven_release',
- 'jdk11_legacy': 'maven_release_jdk11_legacy',
- 'jdk11_minimal': 'maven_release_jdk11_minimal',
- 'jdk11': 'maven_release_jdk11',
- 'jdk11_nio': 'maven_release_jdk11_nio'
+ 'jdk8': 'maven_release',
+ 'jdk11_legacy': 'maven_release_jdk11_legacy',
+ 'jdk11_minimal': 'maven_release_jdk11_minimal',
+ 'jdk11': 'maven_release_jdk11',
+ 'jdk11_nio': 'maven_release_jdk11_nio'
}
MAVEN_RELEASE_ZIP = {
- 'jdk8': BASE_LIBRARY_NAME + '.zip',
- 'jdk11_legacy': BASE_LIBRARY_NAME + '_jdk11_legacy.zip',
- 'jdk11_minimal': BASE_LIBRARY_NAME + '_jdk11_minimal.zip',
- 'jdk11': BASE_LIBRARY_NAME + '_jdk11.zip',
- 'jdk11_nio': BASE_LIBRARY_NAME + '_jdk11_nio.zip'
+ 'jdk8': BASE_LIBRARY_NAME + '.zip',
+ 'jdk11_legacy': BASE_LIBRARY_NAME + '_jdk11_legacy.zip',
+ 'jdk11_minimal': BASE_LIBRARY_NAME + '_jdk11_minimal.zip',
+ 'jdk11': BASE_LIBRARY_NAME + '_jdk11.zip',
+ 'jdk11_nio': BASE_LIBRARY_NAME + '_jdk11_nio.zip'
}
-DESUGAR_JDK_LIBS_HASH_FILE = os.path.join(
- defines.THIRD_PARTY, 'openjdk', 'desugar_jdk_libs_11', 'desugar_jdk_libs_hash')
+DESUGAR_JDK_LIBS_HASH_FILE = os.path.join(defines.THIRD_PARTY, 'openjdk',
+ 'desugar_jdk_libs_11',
+ 'desugar_jdk_libs_hash')
def ParseOptions(argv):
- result = optparse.OptionParser()
- result.add_option('--variant',
- help="Variant(s) to build",
- metavar=('<variants(s)>'),
- choices=['jdk8', 'jdk11_legacy', 'jdk11_minimal', 'jdk11', 'jdk11_nio'],
- default=[],
- action='append')
- result.add_option('--dry-run', '--dry_run',
- help='Running on bot, use third_party dependency.',
- default=False,
- action='store_true')
- result.add_option('--dry-run-output', '--dry_run_output',
- help='Output directory for dry run.',
- type="string", action="store")
- result.add_option('--github-account', '--github_account',
- help='GitHub account to clone from.',
- default="google",
- type="string", action="store")
- result.add_option('--build_only', '--build-only',
- help='Build desugared library without archiving.',
- type="string", action="store")
- (options, args) = result.parse_args(argv)
- return (options, args)
+ result = optparse.OptionParser()
+ result.add_option(
+ '--variant',
+ help="Variant(s) to build",
+ metavar=('<variants(s)>'),
+ choices=['jdk8', 'jdk11_legacy', 'jdk11_minimal', 'jdk11', 'jdk11_nio'],
+ default=[],
+ action='append')
+ result.add_option('--dry-run',
+ '--dry_run',
+ help='Running on bot, use third_party dependency.',
+ default=False,
+ action='store_true')
+ result.add_option('--dry-run-output',
+ '--dry_run_output',
+ help='Output directory for dry run.',
+ type="string",
+ action="store")
+ result.add_option('--github-account',
+ '--github_account',
+ help='GitHub account to clone from.',
+ default="google",
+ type="string",
+ action="store")
+ result.add_option('--build_only',
+ '--build-only',
+ help='Build desugared library without archiving.',
+ type="string",
+ action="store")
+ (options, args) = result.parse_args(argv)
+ return (options, args)
def GetVersion(version_file_name):
- with open(version_file_name, 'r') as version_file:
- lines = [line.strip() for line in version_file.readlines()]
- lines = [line for line in lines if not line.startswith('#')]
- if len(lines) != 1:
- raise Exception('Version file '
- + version_file + ' is expected to have exactly one line')
- version = lines[0].strip()
- utils.check_basic_semver_version(
- version, 'in version file ' + version_file_name, allowPrerelease = True)
- return version
+ with open(version_file_name, 'r') as version_file:
+ lines = [line.strip() for line in version_file.readlines()]
+ lines = [line for line in lines if not line.startswith('#')]
+ if len(lines) != 1:
+ raise Exception('Version file ' + version_file +
+ ' is expected to have exactly one line')
+ version = lines[0].strip()
+ utils.check_basic_semver_version(version,
+ 'in version file ' + version_file_name,
+ allowPrerelease=True)
+ return version
def Upload(options, file_name, storage_path, destination, is_main):
- print('Uploading %s to %s' % (file_name, destination))
- if options.dry_run:
- if options.dry_run_output:
- dry_run_destination = \
- os.path.join(options.dry_run_output, os.path.basename(file_name))
- print('Dry run, not actually uploading. Copying to '
- + dry_run_destination)
- shutil.copyfile(file_name, dry_run_destination)
+ print('Uploading %s to %s' % (file_name, destination))
+ if options.dry_run:
+ if options.dry_run_output:
+ dry_run_destination = \
+ os.path.join(options.dry_run_output, os.path.basename(file_name))
+ print('Dry run, not actually uploading. Copying to ' +
+ dry_run_destination)
+ shutil.copyfile(file_name, dry_run_destination)
+ else:
+ print('Dry run, not actually uploading')
else:
- print('Dry run, not actually uploading')
- else:
- utils.upload_file_to_cloud_storage(file_name, destination)
- print('File available at: %s' %
- destination.replace('gs://', 'https://storage.googleapis.com/', 1))
+ utils.upload_file_to_cloud_storage(file_name, destination)
+ print(
+ 'File available at: %s' %
+ destination.replace('gs://', 'https://storage.googleapis.com/', 1))
+
def CloneDesugaredLibrary(github_account, checkout_dir, desugar_jdk_libs_hash):
- git_utils.GitClone(
- 'https://github.com/'
- + github_account + '/' + GITHUB_REPRO, checkout_dir)
- git_utils.GitCheckout(desugar_jdk_libs_hash, checkout_dir)
+ git_utils.GitClone(
+ 'https://github.com/' + github_account + '/' + GITHUB_REPRO,
+ checkout_dir)
+ git_utils.GitCheckout(desugar_jdk_libs_hash, checkout_dir)
+
def GetJavaEnv(androidHomeTemp):
- java_env = dict(os.environ, JAVA_HOME = jdk.GetJdk11Home())
- java_env['PATH'] = java_env['PATH'] + os.pathsep + os.path.join(jdk.GetJdk11Home(), 'bin')
- java_env['GRADLE_OPTS'] = '-Xmx1g'
- java_env['ANDROID_HOME'] = androidHomeTemp
- return java_env
+ java_env = dict(os.environ, JAVA_HOME=jdk.GetJdk11Home())
+ java_env['PATH'] = java_env['PATH'] + os.pathsep + os.path.join(
+ jdk.GetJdk11Home(), 'bin')
+ java_env['GRADLE_OPTS'] = '-Xmx1g'
+ java_env['ANDROID_HOME'] = androidHomeTemp
+ return java_env
+
def setUpFakeAndroidHome(androidHomeTemp):
- # Bazel will check if 30 is present then extract android.jar from 32.
- # We copy android.jar from third_party to mimic repository structure.
- subpath = os.path.join(androidHomeTemp, "build-tools")
- cmd = ["mkdir", subpath]
- subprocess.check_call(cmd)
- subpath = os.path.join(subpath, "32.0.0")
- cmd = ["mkdir", subpath]
- subprocess.check_call(cmd)
- subpath = os.path.join(androidHomeTemp, "platforms")
- cmd = ["mkdir", subpath]
- subprocess.check_call(cmd)
- subpath30 = os.path.join(subpath, "android-30")
- cmd = ["mkdir", subpath30]
- subprocess.check_call(cmd)
- subpath = os.path.join(subpath, "android-32")
- cmd = ["mkdir", subpath]
- subprocess.check_call(cmd)
- dest = os.path.join(subpath, "android.jar")
- sha = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32.tar.gz.sha1")
- utils.DownloadFromGoogleCloudStorage(sha)
- src = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32", "android.jar")
- cmd = ["cp", src, dest]
- subprocess.check_call(cmd)
+ # Bazel will check if 30 is present then extract android.jar from 32.
+ # We copy android.jar from third_party to mimic repository structure.
+ subpath = os.path.join(androidHomeTemp, "build-tools")
+ cmd = ["mkdir", subpath]
+ subprocess.check_call(cmd)
+ subpath = os.path.join(subpath, "32.0.0")
+ cmd = ["mkdir", subpath]
+ subprocess.check_call(cmd)
+ subpath = os.path.join(androidHomeTemp, "platforms")
+ cmd = ["mkdir", subpath]
+ subprocess.check_call(cmd)
+ subpath30 = os.path.join(subpath, "android-30")
+ cmd = ["mkdir", subpath30]
+ subprocess.check_call(cmd)
+ subpath = os.path.join(subpath, "android-32")
+ cmd = ["mkdir", subpath]
+ subprocess.check_call(cmd)
+ dest = os.path.join(subpath, "android.jar")
+ sha = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32.tar.gz.sha1")
+ utils.DownloadFromGoogleCloudStorage(sha)
+ src = os.path.join(utils.THIRD_PARTY, "android_jar", "lib-v32",
+ "android.jar")
+ cmd = ["cp", src, dest]
+ subprocess.check_call(cmd)
-def BuildDesugaredLibrary(checkout_dir, variant, version = None):
- if not variant in MAVEN_RELEASE_TARGET_MAP:
- raise Exception('Variant ' + variant + ' is not supported')
- if variant != 'jdk8' and variant != 'jdk11_legacy' and version is None:
- raise Exception('Variant ' + variant + ' require version for undesugaring')
- if variant != 'jdk8':
- # Hack to workaround b/256723819.
- os.remove(
- os.path.join(
- checkout_dir,
- "jdk11",
- "src",
- "java.base",
- "share",
- "classes",
- "java",
- "time",
- "format",
- "DesugarDateTimeFormatterBuilder.java"))
- with utils.ChangedWorkingDirectory(checkout_dir):
- with utils.TempDir() as androidHomeTemp:
- setUpFakeAndroidHome(androidHomeTemp)
- javaEnv = GetJavaEnv(androidHomeTemp)
- bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel')
- cmd = [
- bazel,
- '--bazelrc=/dev/null',
- 'build',
- '--spawn_strategy=local',
- '--verbose_failures',
- MAVEN_RELEASE_TARGET_MAP[variant]]
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd, env=javaEnv)
- cmd = [bazel, 'shutdown']
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd, env=javaEnv)
- # Locate the library jar and the maven zip with the jar from the
- # bazel build.
- if variant == 'jdk8':
- library_jar = os.path.join(
- checkout_dir, 'bazel-bin', 'src', 'share', 'classes', 'java', 'libjava.jar')
- else:
- # All JDK11 variants use the same library code.
- library_jar = os.path.join(
- checkout_dir, 'bazel-bin', 'jdk11', 'src', 'd8_java_base_selected_with_addon.jar')
- maven_zip = os.path.join(
- checkout_dir,
- 'bazel-bin',
- MAVEN_RELEASE_ZIP[variant])
+def BuildDesugaredLibrary(checkout_dir, variant, version=None):
+ if not variant in MAVEN_RELEASE_TARGET_MAP:
+ raise Exception('Variant ' + variant + ' is not supported')
+ if variant != 'jdk8' and variant != 'jdk11_legacy' and version is None:
+ raise Exception('Variant ' + variant +
+ ' require version for undesugaring')
+ if variant != 'jdk8':
+ # Hack to workaround b/256723819.
+ os.remove(
+ os.path.join(checkout_dir, "jdk11", "src", "java.base", "share",
+ "classes", "java", "time", "format",
+ "DesugarDateTimeFormatterBuilder.java"))
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ with utils.TempDir() as androidHomeTemp:
+ setUpFakeAndroidHome(androidHomeTemp)
+ javaEnv = GetJavaEnv(androidHomeTemp)
+ bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin',
+ 'bazel')
+ cmd = [
+ bazel, '--bazelrc=/dev/null', 'build', '--spawn_strategy=local',
+ '--verbose_failures', MAVEN_RELEASE_TARGET_MAP[variant]
+ ]
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd, env=javaEnv)
+ cmd = [bazel, 'shutdown']
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd, env=javaEnv)
- if variant != 'jdk8' and variant != 'jdk11_legacy':
- # The undesugaring is temporary...
- undesugared_maven_zip = os.path.join(checkout_dir, 'undesugared_maven')
- Undesugar(variant, maven_zip, version, undesugared_maven_zip)
- undesugared_maven_zip = os.path.join(checkout_dir, 'undesugared_maven.zip')
- return (library_jar, undesugared_maven_zip)
- else:
- return (library_jar, maven_zip)
+ # Locate the library jar and the maven zip with the jar from the
+ # bazel build.
+ if variant == 'jdk8':
+ library_jar = os.path.join(checkout_dir, 'bazel-bin', 'src',
+ 'share', 'classes', 'java',
+ 'libjava.jar')
+ else:
+ # All JDK11 variants use the same library code.
+ library_jar = os.path.join(checkout_dir, 'bazel-bin', 'jdk11',
+ 'src',
+ 'd8_java_base_selected_with_addon.jar')
+ maven_zip = os.path.join(checkout_dir, 'bazel-bin',
+ MAVEN_RELEASE_ZIP[variant])
+
+ if variant != 'jdk8' and variant != 'jdk11_legacy':
+ # The undesugaring is temporary...
+ undesugared_maven_zip = os.path.join(checkout_dir,
+ 'undesugared_maven')
+ Undesugar(variant, maven_zip, version, undesugared_maven_zip)
+ undesugared_maven_zip = os.path.join(checkout_dir,
+ 'undesugared_maven.zip')
+ return (library_jar, undesugared_maven_zip)
+ else:
+ return (library_jar, maven_zip)
+
def hash_for(file, hash):
- with open(file, 'rb') as f:
- while True:
- # Read chunks of 1MB
- chunk = f.read(2 ** 20)
- if not chunk:
- break
- hash.update(chunk)
- return hash.hexdigest()
+ with open(file, 'rb') as f:
+ while True:
+ # Read chunks of 1MB
+ chunk = f.read(2**20)
+ if not chunk:
+ break
+ hash.update(chunk)
+ return hash.hexdigest()
+
def write_md5_for(file):
- hexdigest = hash_for(file, hashlib.md5())
- with (open(file + '.md5', 'w')) as file:
- file.write(hexdigest)
+ hexdigest = hash_for(file, hashlib.md5())
+ with (open(file + '.md5', 'w')) as file:
+ file.write(hexdigest)
+
def write_sha1_for(file):
- hexdigest = hash_for(file, hashlib.sha1())
- with (open(file + '.sha1', 'w')) as file:
- file.write(hexdigest)
+ hexdigest = hash_for(file, hashlib.sha1())
+ with (open(file + '.sha1', 'w')) as file:
+ file.write(hexdigest)
+
def Undesugar(variant, maven_zip, version, undesugared_maven_zip):
- gradle.RunGradle([utils.GRADLE_TASK_R8,
- utils.GRADLE_TASK_TEST_JAR,
- utils.GRADLE_TASK_TEST_DEPS_JAR,
- '-Pno_internal'])
- with utils.TempDir() as tmp:
- with zipfile.ZipFile(maven_zip, 'r') as zip_ref:
- zip_ref.extractall(tmp)
- desugar_jdk_libs_jar = os.path.join(
- tmp,
- 'com',
- 'android',
- 'tools',
- LIBRARY_NAME_MAP[variant],
- version,
- '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version))
- print(desugar_jdk_libs_jar)
- undesugared_jar = os.path.join(tmp, 'undesugared.jar')
- buildLibs = os.path.join(defines.REPO_ROOT, 'build', 'libs')
- cmd = [jdk.GetJavaExecutable(),
- '-cp',
- '%s:%s:%s' % (utils.R8_JAR, utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR),
- 'com.android.tools.r8.desugar.desugaredlibrary.jdk11.DesugaredLibraryJDK11Undesugarer',
- desugar_jdk_libs_jar,
- undesugared_jar]
- print(cmd)
- try:
- output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
- except subprocess.CalledProcessError as e:
- print(e)
- print(e.output)
- raise e
- print(output)
- # Copy the undesugared jar into place and update the checksums.
- shutil.copyfile(undesugared_jar, desugar_jdk_libs_jar)
- write_md5_for(desugar_jdk_libs_jar)
- write_sha1_for(desugar_jdk_libs_jar)
- shutil.make_archive(undesugared_maven_zip, 'zip', tmp)
- print(undesugared_maven_zip)
- output = subprocess.check_output(['ls', '-l', os.path.dirname(undesugared_maven_zip)], stderr = subprocess.STDOUT).decode('utf-8')
- print(output)
+ gradle.RunGradle([
+ utils.GRADLE_TASK_R8, utils.GRADLE_TASK_TEST_JAR,
+ utils.GRADLE_TASK_TEST_DEPS_JAR, '-Pno_internal'
+ ])
+ with utils.TempDir() as tmp:
+ with zipfile.ZipFile(maven_zip, 'r') as zip_ref:
+ zip_ref.extractall(tmp)
+ desugar_jdk_libs_jar = os.path.join(
+ tmp, 'com', 'android', 'tools', LIBRARY_NAME_MAP[variant], version,
+ '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version))
+ print(desugar_jdk_libs_jar)
+ undesugared_jar = os.path.join(tmp, 'undesugared.jar')
+ buildLibs = os.path.join(defines.REPO_ROOT, 'build', 'libs')
+ cmd = [
+ jdk.GetJavaExecutable(), '-cp',
+ '%s:%s:%s' %
+ (utils.R8_JAR, utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR),
+ 'com.android.tools.r8.desugar.desugaredlibrary.jdk11.DesugaredLibraryJDK11Undesugarer',
+ desugar_jdk_libs_jar, undesugared_jar
+ ]
+ print(cmd)
+ try:
+ output = subprocess.check_output(
+ cmd, stderr=subprocess.STDOUT).decode('utf-8')
+ except subprocess.CalledProcessError as e:
+ print(e)
+ print(e.output)
+ raise e
+ print(output)
+ # Copy the undesugared jar into place and update the checksums.
+ shutil.copyfile(undesugared_jar, desugar_jdk_libs_jar)
+ write_md5_for(desugar_jdk_libs_jar)
+ write_sha1_for(desugar_jdk_libs_jar)
+ shutil.make_archive(undesugared_maven_zip, 'zip', tmp)
+ print(undesugared_maven_zip)
+ output = subprocess.check_output(
+ ['ls', '-l', os.path.dirname(undesugared_maven_zip)],
+ stderr=subprocess.STDOUT).decode('utf-8')
+ print(output)
+
def MustBeExistingDirectory(path):
- if (not os.path.exists(path) or not os.path.isdir(path)):
- raise Exception(path + ' does not exist or is not a directory')
+ if (not os.path.exists(path) or not os.path.isdir(path)):
+ raise Exception(path + ' does not exist or is not a directory')
+
def BuildAndUpload(options, variant):
- desugar_jdk_libs_hash = ''
- with open(DESUGAR_JDK_LIBS_HASH_FILE, 'r') as input_hash:
- desugar_jdk_libs_hash = input_hash.readline()
- if options.build_only:
+ desugar_jdk_libs_hash = ''
+ with open(DESUGAR_JDK_LIBS_HASH_FILE, 'r') as input_hash:
+ desugar_jdk_libs_hash = input_hash.readline()
+ if options.build_only:
+ with utils.TempDir() as checkout_dir:
+ CloneDesugaredLibrary(options.github_account, checkout_dir,
+ desugar_jdk_libs_hash)
+ (library_jar,
+ maven_zip) = BuildDesugaredLibrary(checkout_dir, variant,
+ desugar_jdk_libs_hash)
+ shutil.copyfile(
+ library_jar,
+ os.path.join(options.build_only, os.path.basename(library_jar)))
+ shutil.copyfile(
+ maven_zip,
+ os.path.join(options.build_only, os.path.basename(maven_zip)))
+ return
+
+ # Only handling versioned desugar_jdk_libs.
+ is_main = False
+
with utils.TempDir() as checkout_dir:
- CloneDesugaredLibrary(options.github_account, checkout_dir, desugar_jdk_libs_hash)
- (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, variant, desugar_jdk_libs_hash)
- shutil.copyfile(
- library_jar,
- os.path.join(options.build_only, os.path.basename(library_jar)))
- shutil.copyfile(
- maven_zip,
- os.path.join(options.build_only, os.path.basename(maven_zip)))
- return
+ CloneDesugaredLibrary(options.github_account, checkout_dir,
+ desugar_jdk_libs_hash)
+ version = GetVersion(os.path.join(checkout_dir, VERSION_MAP[variant]))
- # Only handling versioned desugar_jdk_libs.
- is_main = False
+ destination = archive.GetVersionDestination(
+ 'gs://', LIBRARY_NAME_MAP[variant] + '/' + version, is_main)
+ if utils.cloud_storage_exists(destination) and not options.dry_run:
+ raise Exception('Target archive directory %s already exists' %
+ destination)
- with utils.TempDir() as checkout_dir:
- CloneDesugaredLibrary(options.github_account, checkout_dir, desugar_jdk_libs_hash)
- version = GetVersion(os.path.join(checkout_dir, VERSION_MAP[variant]))
+ (library_jar,
+ maven_zip) = BuildDesugaredLibrary(checkout_dir, variant, version)
- destination = archive.GetVersionDestination(
- 'gs://', LIBRARY_NAME_MAP[variant] + '/' + version, is_main)
- if utils.cloud_storage_exists(destination) and not options.dry_run:
- raise Exception(
- 'Target archive directory %s already exists' % destination)
+ storage_path = LIBRARY_NAME_MAP[variant] + '/' + version
+ # Upload the jar file with the library.
+ destination = archive.GetUploadDestination(
+ storage_path, LIBRARY_NAME_MAP[variant] + '.jar', is_main)
+ Upload(options, library_jar, storage_path, destination, is_main)
- (library_jar, maven_zip) = BuildDesugaredLibrary(checkout_dir, variant, version)
+ # Upload the maven zip file with the library.
+ destination = archive.GetUploadDestination(storage_path,
+ MAVEN_RELEASE_ZIP[variant],
+ is_main)
+ Upload(options, maven_zip, storage_path, destination, is_main)
- storage_path = LIBRARY_NAME_MAP[variant] + '/' + version
- # Upload the jar file with the library.
- destination = archive.GetUploadDestination(
- storage_path, LIBRARY_NAME_MAP[variant] + '.jar', is_main)
- Upload(options, library_jar, storage_path, destination, is_main)
+ # Upload the jar file for accessing GCS as a maven repro.
+ maven_destination = archive.GetUploadDestination(
+ utils.get_maven_path(LIBRARY_NAME_MAP[variant], version),
+ '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version), is_main)
+ if options.dry_run:
+ print('Dry run, not actually creating maven repo')
+ else:
+ utils.upload_file_to_cloud_storage(library_jar, maven_destination)
+ print('Maven repo root available at: %s' %
+ archive.GetMavenUrl(is_main))
- # Upload the maven zip file with the library.
- destination = archive.GetUploadDestination(
- storage_path, MAVEN_RELEASE_ZIP[variant], is_main)
- Upload(options, maven_zip, storage_path, destination, is_main)
-
- # Upload the jar file for accessing GCS as a maven repro.
- maven_destination = archive.GetUploadDestination(
- utils.get_maven_path(LIBRARY_NAME_MAP[variant], version),
- '%s-%s.jar' % (LIBRARY_NAME_MAP[variant], version),
- is_main)
- if options.dry_run:
- print('Dry run, not actually creating maven repo')
- else:
- utils.upload_file_to_cloud_storage(library_jar, maven_destination)
- print('Maven repo root available at: %s' % archive.GetMavenUrl(is_main))
def Main(argv):
- (options, args) = ParseOptions(argv)
- if (len(args) > 0):
- raise Exception('Unsupported arguments')
- if not utils.is_bot() and not (options.dry_run or options.build_only):
- raise Exception('You are not a bot, don\'t archive builds. '
- + 'Use --dry-run or --build-only to test locally')
- if options.dry_run_output:
- MustBeExistingDirectory(options.dry_run_output)
- if options.build_only:
- MustBeExistingDirectory(options.build_only)
- if utils.is_bot():
- archive.SetRLimitToMax()
+ (options, args) = ParseOptions(argv)
+ if (len(args) > 0):
+ raise Exception('Unsupported arguments')
+ if not utils.is_bot() and not (options.dry_run or options.build_only):
+ raise Exception('You are not a bot, don\'t archive builds. ' +
+ 'Use --dry-run or --build-only to test locally')
+ if options.dry_run_output:
+ MustBeExistingDirectory(options.dry_run_output)
+ if options.build_only:
+ MustBeExistingDirectory(options.build_only)
+ if utils.is_bot():
+ archive.SetRLimitToMax()
- # Make sure bazel is extracted in third_party.
- utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
- utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
- utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
- utils.DownloadFromGoogleCloudStorage(utils.DESUGAR_JDK_LIBS_11_SHA_FILE)
+ # Make sure bazel is extracted in third_party.
+ utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
+ utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
+ utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
+ utils.DownloadFromGoogleCloudStorage(utils.DESUGAR_JDK_LIBS_11_SHA_FILE)
- for v in options.variant:
- BuildAndUpload(options, v)
+ for v in options.variant:
+ BuildAndUpload(options, v)
+
if __name__ == '__main__':
- sys.exit(Main(sys.argv[1:]))
+ sys.exit(Main(sys.argv[1:]))
diff --git a/tools/archive_smali.py b/tools/archive_smali.py
index db7cb89..fac42c5 100755
--- a/tools/archive_smali.py
+++ b/tools/archive_smali.py
@@ -7,10 +7,10 @@
import os
import re
try:
- import resource
+ import resource
except ImportError:
- # Not a Unix system. Do what Gandalf tells you not to.
- pass
+ # Not a Unix system. Do what Gandalf tells you not to.
+ pass
import shutil
import subprocess
import sys
@@ -21,129 +21,155 @@
REPO = 'https://github.com/google/smali'
NO_DRYRUN_OUTPUT = object()
+
def checkout(temp):
- subprocess.check_call(['git', 'clone', REPO, temp])
- return temp
+ subprocess.check_call(['git', 'clone', REPO, temp])
+ return temp
+
def parse_options():
- result = argparse.ArgumentParser(description='Release Smali')
- result.add_argument('--version',
- metavar=('<version>'),
- help='The version of smali to archive.')
- result.add_argument('--dry-run', '--dry_run',
- nargs='?',
- help='Build only, no upload.',
- metavar='<output directory>',
- default=None,
- const=NO_DRYRUN_OUTPUT)
- result.add_argument('--checkout',
- help='Use existing checkout.')
- return result.parse_args()
+ result = argparse.ArgumentParser(description='Release Smali')
+ result.add_argument('--version',
+ metavar=('<version>'),
+ help='The version of smali to archive.')
+ result.add_argument('--dry-run',
+ '--dry_run',
+ nargs='?',
+ help='Build only, no upload.',
+ metavar='<output directory>',
+ default=None,
+ const=NO_DRYRUN_OUTPUT)
+ result.add_argument('--checkout', help='Use existing checkout.')
+ return result.parse_args()
def set_rlimit_to_max():
- (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
- resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+ (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
+ resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
def Main():
- options = parse_options()
- if not utils.is_bot() and not options.dry_run:
- raise Exception('You are not a bot, don\'t archive builds. '
- + 'Use --dry-run to test locally')
- if options.checkout and not options.dry_run:
- raise Exception('Using local checkout is only allowed with --dry-run')
- if not options.checkout and not options.version:
- raise Exception('Option --version is required (when not using local checkout)')
+ options = parse_options()
+ if not utils.is_bot() and not options.dry_run:
+ raise Exception('You are not a bot, don\'t archive builds. ' +
+ 'Use --dry-run to test locally')
+ if options.checkout and not options.dry_run:
+ raise Exception('Using local checkout is only allowed with --dry-run')
+ if not options.checkout and not options.version:
+ raise Exception(
+ 'Option --version is required (when not using local checkout)')
- if utils.is_bot() and not utils.IsWindows():
- set_rlimit_to_max()
+ if utils.is_bot() and not utils.IsWindows():
+ set_rlimit_to_max()
- with utils.TempDir() as temp:
- # Resolve dry run location to support relative directories.
- dry_run_output = None
- if options.dry_run and options.dry_run != NO_DRYRUN_OUTPUT:
- if not os.path.isdir(options.dry_run):
- os.mkdir(options.dry_run)
- dry_run_output = os.path.abspath(options.dry_run)
+ with utils.TempDir() as temp:
+ # Resolve dry run location to support relative directories.
+ dry_run_output = None
+ if options.dry_run and options.dry_run != NO_DRYRUN_OUTPUT:
+ if not os.path.isdir(options.dry_run):
+ os.mkdir(options.dry_run)
+ dry_run_output = os.path.abspath(options.dry_run)
- checkout_dir = options.checkout if options.checkout else checkout(temp)
- with utils.ChangedWorkingDirectory(checkout_dir):
- if options.version:
- output = subprocess.check_output(['git', 'tag', '-l', options.version])
- if len(output) == 0:
- raise Exception(
- 'Repository does not have a release tag for version %s' % options.version)
- subprocess.check_call(['git', 'checkout', options.version])
+ checkout_dir = options.checkout if options.checkout else checkout(temp)
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ if options.version:
+ output = subprocess.check_output(
+ ['git', 'tag', '-l', options.version])
+ if len(output) == 0:
+ raise Exception(
+ 'Repository does not have a release tag for version %s'
+ % options.version)
+ subprocess.check_call(['git', 'checkout', options.version])
- # Find version from `build.gradle`.
- for line in open(os.path.join('build.gradle'), 'r'):
- result = re.match(
- r'^version = \'(\d+)\.(\d+)\.(\d+)\'', line)
- if result:
- break
- version = '%s.%s.%s' % (result.group(1), result.group(2), result.group(3))
- if options.version and version != options.version:
- message = 'version %s, expected version %s' % (version, options.version)
- if (options.checkout):
- raise Exception('Checkout %s has %s' % (options.checkout, message))
- else:
- raise Exception('Tag % has %s' % (options.version, message))
+ # Find version from `build.gradle`.
+ for line in open(os.path.join('build.gradle'), 'r'):
+ result = re.match(r'^version = \'(\d+)\.(\d+)\.(\d+)\'', line)
+ if result:
+ break
+ version = '%s.%s.%s' % (result.group(1), result.group(2),
+ result.group(3))
+ if options.version and version != options.version:
+ message = 'version %s, expected version %s' % (version,
+ options.version)
+ if (options.checkout):
+ raise Exception('Checkout %s has %s' %
+ (options.checkout, message))
+ else:
+ raise Exception('Tag % has %s' % (options.version, message))
- print('Building version: %s' % version)
+ print('Building version: %s' % version)
- # Build release to local Maven repository.
- m2 = os.path.join(temp, 'm2')
- os.mkdir(m2)
- subprocess.check_call(
- ['./gradlew', '-Dmaven.repo.local=%s' % m2 , 'release', 'test', 'publishToMavenLocal'])
- base = os.path.join('com', 'android', 'tools', 'smali')
+ # Build release to local Maven repository.
+ m2 = os.path.join(temp, 'm2')
+ os.mkdir(m2)
+ subprocess.check_call([
+ './gradlew',
+ '-Dmaven.repo.local=%s' % m2, 'release', 'test',
+ 'publishToMavenLocal'
+ ])
+ base = os.path.join('com', 'android', 'tools', 'smali')
- # Check that the local maven repository only has the single version directory in
- # each artifact directory.
- for name in ['smali-util', 'smali-dexlib2', 'smali', 'smali-baksmali']:
- dirnames = next(os.walk(os.path.join(m2, base, name)), (None, None, []))[1]
- if not dirnames or len(dirnames) != 1 or dirnames[0] != version:
- raise Exception('Found unexpected directory %s in %s' % (dirnames, name))
+ # Check that the local maven repository only has the single version directory in
+ # each artifact directory.
+ for name in [
+ 'smali-util', 'smali-dexlib2', 'smali', 'smali-baksmali'
+ ]:
+ dirnames = next(os.walk(os.path.join(m2, base, name)),
+ (None, None, []))[1]
+ if not dirnames or len(dirnames) != 1 or dirnames[0] != version:
+ raise Exception('Found unexpected directory %s in %s' %
+ (dirnames, name))
- # Build an archive with the relevant content of the local maven repository.
- m2_filtered = os.path.join(temp, 'm2_filtered')
- shutil.copytree(m2, m2_filtered, ignore=shutil.ignore_patterns('maven-metadata-local.xml'))
- maven_release_archive = shutil.make_archive(
- 'smali-maven-release-%s' % version, 'zip', m2_filtered, base)
+ # Build an archive with the relevant content of the local maven repository.
+ m2_filtered = os.path.join(temp, 'm2_filtered')
+ shutil.copytree(
+ m2,
+ m2_filtered,
+ ignore=shutil.ignore_patterns('maven-metadata-local.xml'))
+ maven_release_archive = shutil.make_archive(
+ 'smali-maven-release-%s' % version, 'zip', m2_filtered, base)
- # Collect names of the fat jars.
- fat_jars = list(map(
- lambda prefix: '%s-%s-fat.jar' % (prefix, version),
- ['smali/build/libs/smali', 'baksmali/build/libs/baksmali']))
+ # Collect names of the fat jars.
+ fat_jars = list(
+ map(lambda prefix: '%s-%s-fat.jar' % (prefix, version),
+ ['smali/build/libs/smali', 'baksmali/build/libs/baksmali']))
- # Copy artifacts.
- files = [maven_release_archive]
- files.extend(fat_jars)
- if options.dry_run:
- if dry_run_output:
- print('Dry run, not actually uploading. Copying to %s:' % dry_run_output)
- for file in files:
- destination = os.path.join(dry_run_output, os.path.basename(file))
- shutil.copyfile(file, destination)
- print(" %s" % destination)
- else:
- print('Dry run, not actually uploading. Generated files:')
- for file in files:
- print(" %s" % os.path.basename(file))
- else:
- destination_prefix = 'gs://%s/smali/%s' % (ARCHIVE_BUCKET, version)
- if utils.cloud_storage_exists(destination_prefix):
- raise Exception('Target archive directory %s already exists' % destination_prefix)
- for file in files:
- destination = '%s/%s' % (destination_prefix, os.path.basename(file))
- if utils.cloud_storage_exists(destination):
- raise Exception('Target %s already exists' % destination)
- utils.upload_file_to_cloud_storage(file, destination)
- public_url = 'https://storage.googleapis.com/%s/smali/%s' % (ARCHIVE_BUCKET, version)
- print('Artifacts available at: %s' % public_url)
+ # Copy artifacts.
+ files = [maven_release_archive]
+ files.extend(fat_jars)
+ if options.dry_run:
+ if dry_run_output:
+ print('Dry run, not actually uploading. Copying to %s:' %
+ dry_run_output)
+ for file in files:
+ destination = os.path.join(dry_run_output,
+ os.path.basename(file))
+ shutil.copyfile(file, destination)
+ print(" %s" % destination)
+ else:
+ print('Dry run, not actually uploading. Generated files:')
+ for file in files:
+ print(" %s" % os.path.basename(file))
+ else:
+ destination_prefix = 'gs://%s/smali/%s' % (ARCHIVE_BUCKET,
+ version)
+ if utils.cloud_storage_exists(destination_prefix):
+ raise Exception(
+ 'Target archive directory %s already exists' %
+ destination_prefix)
+ for file in files:
+ destination = '%s/%s' % (destination_prefix,
+ os.path.basename(file))
+ if utils.cloud_storage_exists(destination):
+ raise Exception('Target %s already exists' %
+ destination)
+ utils.upload_file_to_cloud_storage(file, destination)
+ public_url = 'https://storage.googleapis.com/%s/smali/%s' % (
+ ARCHIVE_BUCKET, version)
+ print('Artifacts available at: %s' % public_url)
- print("Done!")
+ print("Done!")
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/as_utils.py b/tools/as_utils.py
index 3c69d10..69f7562 100644
--- a/tools/as_utils.py
+++ b/tools/as_utils.py
@@ -10,229 +10,246 @@
import utils
if utils.is_python3():
- from html.parser import HTMLParser
+ from html.parser import HTMLParser
else:
- from HTMLParser import HTMLParser
+ from HTMLParser import HTMLParser
def add_r8_dependency(checkout_dir, temp_dir, minified):
- build_file = os.path.join(checkout_dir, 'build.gradle')
- assert os.path.isfile(build_file), (
- 'Expected a file to be present at {}'.format(build_file))
+ build_file = os.path.join(checkout_dir, 'build.gradle')
+ assert os.path.isfile(build_file), (
+ 'Expected a file to be present at {}'.format(build_file))
- with open(build_file) as f:
- lines = f.readlines()
+ with open(build_file) as f:
+ lines = f.readlines()
- added_r8_dependency = False
- is_inside_dependencies = False
+ added_r8_dependency = False
+ is_inside_dependencies = False
- with open(build_file, 'w') as f:
- gradle_version = None
- for line in lines:
- stripped = line.strip()
- if stripped == 'dependencies {':
- assert not is_inside_dependencies, (
- 'Unexpected line with \'dependencies {\'')
- is_inside_dependencies = True
- if is_inside_dependencies:
- if '/r8.jar' in stripped or '/r8lib.jar' in stripped:
- # Skip line to avoid dependency on r8.jar
- continue
- elif 'com.android.tools.build:gradle:' in stripped:
- gradle_version = stripped[stripped.rindex(':')+1:-1]
- indent = ''.ljust(line.index('classpath'))
- jar = os.path.join(temp_dir, 'r8lib.jar' if minified else 'r8.jar')
- f.write('{}classpath files(\'{}\')\n'.format(indent, jar))
- added_r8_dependency = True
- elif stripped == '}':
- is_inside_dependencies = False
- f.write(line)
+ with open(build_file, 'w') as f:
+ gradle_version = None
+ for line in lines:
+ stripped = line.strip()
+ if stripped == 'dependencies {':
+ assert not is_inside_dependencies, (
+ 'Unexpected line with \'dependencies {\'')
+ is_inside_dependencies = True
+ if is_inside_dependencies:
+ if '/r8.jar' in stripped or '/r8lib.jar' in stripped:
+ # Skip line to avoid dependency on r8.jar
+ continue
+ elif 'com.android.tools.build:gradle:' in stripped:
+ gradle_version = stripped[stripped.rindex(':') + 1:-1]
+ indent = ''.ljust(line.index('classpath'))
+ jar = os.path.join(temp_dir,
+ 'r8lib.jar' if minified else 'r8.jar')
+ f.write('{}classpath files(\'{}\')\n'.format(indent, jar))
+ added_r8_dependency = True
+ elif stripped == '}':
+ is_inside_dependencies = False
+ f.write(line)
- assert added_r8_dependency, 'Unable to add R8 as a dependency'
- assert gradle_version
- assert LooseVersion(gradle_version) >= LooseVersion('3.2'), (
- 'Unsupported gradle version: {} (must use at least gradle '
- + 'version 3.2)').format(gradle_version)
+ assert added_r8_dependency, 'Unable to add R8 as a dependency'
+ assert gradle_version
+ assert LooseVersion(gradle_version) >= LooseVersion('3.2'), (
+ 'Unsupported gradle version: {} (must use at least gradle ' +
+ 'version 3.2)').format(gradle_version)
+
def add_settings_gradle(checkout_dir, name):
- settings_file = os.path.join(checkout_dir, 'settings.gradle')
- if os.path.isfile(settings_file):
- return
+ settings_file = os.path.join(checkout_dir, 'settings.gradle')
+ if os.path.isfile(settings_file):
+ return
- with open(settings_file, "w+") as f:
- f.write("rootProject.name = '{}'\n".format(name))
+ with open(settings_file, "w+") as f:
+ f.write("rootProject.name = '{}'\n".format(name))
+
def remove_r8_dependency(checkout_dir):
- build_file = os.path.join(checkout_dir, 'build.gradle')
- assert os.path.isfile(build_file), (
- 'Expected a file to be present at {}'.format(build_file))
- with open(build_file) as f:
- lines = f.readlines()
- with open(build_file, 'w') as f:
- for line in lines:
- if ('/r8.jar' not in line) and ('/r8lib.jar' not in line):
- f.write(line)
+ build_file = os.path.join(checkout_dir, 'build.gradle')
+ assert os.path.isfile(build_file), (
+ 'Expected a file to be present at {}'.format(build_file))
+ with open(build_file) as f:
+ lines = f.readlines()
+ with open(build_file, 'w') as f:
+ for line in lines:
+ if ('/r8.jar' not in line) and ('/r8lib.jar' not in line):
+ f.write(line)
+
def GetMinAndCompileSdk(app, checkout_dir, apk_reference):
- compile_sdk = app.compile_sdk
- min_sdk = app.min_sdk
+ compile_sdk = app.compile_sdk
+ min_sdk = app.min_sdk
- if not compile_sdk or not min_sdk:
- build_gradle_file = os.path.join(checkout_dir, app.module, 'build.gradle')
- assert os.path.isfile(build_gradle_file), (
- 'Expected to find build.gradle file at {}'.format(build_gradle_file))
+ if not compile_sdk or not min_sdk:
+ build_gradle_file = os.path.join(checkout_dir, app.module,
+ 'build.gradle')
+ assert os.path.isfile(build_gradle_file), (
+ 'Expected to find build.gradle file at {}'.format(build_gradle_file)
+ )
- # Attempt to find the sdk values from build.gradle.
- with open(build_gradle_file) as f:
- for line in f.readlines():
- stripped = line.strip()
- if stripped.startswith('compileSdkVersion '):
- if not app.compile_sdk:
- assert not compile_sdk
- compile_sdk = int(stripped[len('compileSdkVersion '):])
- elif stripped.startswith('minSdkVersion '):
- if not app.min_sdk:
- assert not min_sdk
- min_sdk = int(stripped[len('minSdkVersion '):])
+ # Attempt to find the sdk values from build.gradle.
+ with open(build_gradle_file) as f:
+ for line in f.readlines():
+ stripped = line.strip()
+ if stripped.startswith('compileSdkVersion '):
+ if not app.compile_sdk:
+ assert not compile_sdk
+ compile_sdk = int(stripped[len('compileSdkVersion '):])
+ elif stripped.startswith('minSdkVersion '):
+ if not app.min_sdk:
+ assert not min_sdk
+ min_sdk = int(stripped[len('minSdkVersion '):])
- assert min_sdk, (
- 'Expected to find `minSdkVersion` in {}'.format(build_gradle_file))
- assert compile_sdk, (
- 'Expected to find `compileSdkVersion` in {}'.format(build_gradle_file))
+ assert min_sdk, (
+ 'Expected to find `minSdkVersion` in {}'.format(build_gradle_file))
+ assert compile_sdk, (
+ 'Expected to find `compileSdkVersion` in {}'.format(build_gradle_file))
- return (min_sdk, compile_sdk)
+ return (min_sdk, compile_sdk)
+
def IsGradleTaskName(x):
- # Check that it is non-empty.
- if not x:
- return False
- # Check that there is no whitespace.
- for c in x:
- if c.isspace():
- return False
- # Check that the first character following an optional ':' is a lower-case
- # alphabetic character.
- c = x[0]
- if c == ':' and len(x) >= 2:
- c = x[1]
- return c.isalpha() and c.islower()
+ # Check that it is non-empty.
+ if not x:
+ return False
+ # Check that there is no whitespace.
+ for c in x:
+ if c.isspace():
+ return False
+ # Check that the first character following an optional ':' is a lower-case
+ # alphabetic character.
+ c = x[0]
+ if c == ':' and len(x) >= 2:
+ c = x[1]
+ return c.isalpha() and c.islower()
+
def IsGradleCompilerTask(x, shrinker):
- if 'r8' in shrinker:
- assert 'transformClassesWithDexBuilderFor' not in x
- assert 'transformDexArchiveWithDexMergerFor' not in x
- return 'transformClassesAndResourcesWithR8For' in x
+ if 'r8' in shrinker:
+ assert 'transformClassesWithDexBuilderFor' not in x
+ assert 'transformDexArchiveWithDexMergerFor' not in x
+ return 'transformClassesAndResourcesWithR8For' in x
- assert shrinker == 'pg'
- return ('transformClassesAndResourcesWithProguard' in x
- or 'transformClassesWithDexBuilderFor' in x
- or 'transformDexArchiveWithDexMergerFor' in x)
+ assert shrinker == 'pg'
+ return ('transformClassesAndResourcesWithProguard' in x or
+ 'transformClassesWithDexBuilderFor' in x or
+ 'transformDexArchiveWithDexMergerFor' in x)
+
def ListFiles(directory, predicate=None):
- files = []
- for root, directories, filenames in os.walk(directory):
- for filename in filenames:
- file = os.path.join(root, filename)
- if predicate is None or predicate(file):
- files.append(file)
- return files
+ files = []
+ for root, directories, filenames in os.walk(directory):
+ for filename in filenames:
+ file = os.path.join(root, filename)
+ if predicate is None or predicate(file):
+ files.append(file)
+ return files
+
def SetPrintConfigurationDirective(app, checkout_dir, destination):
- proguard_config_file = FindProguardConfigurationFile(app, checkout_dir)
- with open(proguard_config_file) as f:
- lines = f.readlines()
- with open(proguard_config_file, 'w') as f:
- for line in lines:
- if '-printconfiguration' not in line:
- f.write(line)
- # Check that there is a line-break at the end of the file or insert one.
- if len(lines) and lines[-1].strip():
- f.write('\n')
- f.write('-printconfiguration {}\n'.format(destination))
+ proguard_config_file = FindProguardConfigurationFile(app, checkout_dir)
+ with open(proguard_config_file) as f:
+ lines = f.readlines()
+ with open(proguard_config_file, 'w') as f:
+ for line in lines:
+ if '-printconfiguration' not in line:
+ f.write(line)
+ # Check that there is a line-break at the end of the file or insert one.
+ if len(lines) and lines[-1].strip():
+ f.write('\n')
+ f.write('-printconfiguration {}\n'.format(destination))
+
def FindProguardConfigurationFile(app, checkout_dir):
- candidates = [
- 'proguard.cfg',
- 'proguard-rules.pro',
- 'proguard-rules.txt',
- 'proguard-project.txt']
- for candidate in candidates:
- proguard_config_file = os.path.join(checkout_dir, app.module, candidate)
- if os.path.isfile(proguard_config_file):
- return proguard_config_file
- # Currently assuming that the Proguard configuration file can be found at
- # one of the predefined locations.
- assert False, 'Unable to find Proguard configuration file'
+ candidates = [
+ 'proguard.cfg', 'proguard-rules.pro', 'proguard-rules.txt',
+ 'proguard-project.txt'
+ ]
+ for candidate in candidates:
+ proguard_config_file = os.path.join(checkout_dir, app.module, candidate)
+ if os.path.isfile(proguard_config_file):
+ return proguard_config_file
+ # Currently assuming that the Proguard configuration file can be found at
+ # one of the predefined locations.
+ assert False, 'Unable to find Proguard configuration file'
+
def Move(src, dst, quiet=False):
- if not quiet:
- print('Moving `{}` to `{}`'.format(src, dst))
- dst_parent = os.path.dirname(dst)
- if not os.path.isdir(dst_parent):
- os.makedirs(dst_parent)
- elif os.path.isdir(dst):
- shutil.rmtree(dst)
- elif os.path.isfile(dst):
- os.remove(dst)
- shutil.move(src, dst)
+ if not quiet:
+ print('Moving `{}` to `{}`'.format(src, dst))
+ dst_parent = os.path.dirname(dst)
+ if not os.path.isdir(dst_parent):
+ os.makedirs(dst_parent)
+ elif os.path.isdir(dst):
+ shutil.rmtree(dst)
+ elif os.path.isfile(dst):
+ os.remove(dst)
+ shutil.move(src, dst)
+
def MoveDir(src, dst, quiet=False):
- assert os.path.isdir(src)
- Move(src, dst, quiet=quiet)
+ assert os.path.isdir(src)
+ Move(src, dst, quiet=quiet)
+
def MoveFile(src, dst, quiet=False):
- assert os.path.isfile(src), "Expected a file to be present at " + src
- Move(src, dst, quiet=quiet)
+ assert os.path.isfile(src), "Expected a file to be present at " + src
+ Move(src, dst, quiet=quiet)
+
def MoveProfileReportTo(dest_dir, build_stdout, quiet=False):
- html_file = None
- profile_message = 'See the profiling report at: '
- # We are not interested in the profiling report for buildSrc.
- for line in build_stdout:
- if (profile_message in line) and ('buildSrc' not in line):
- assert not html_file, "Only one report should be created"
- html_file = line[len(profile_message):]
- if html_file.startswith('file://'):
- html_file = html_file[len('file://'):]
+ html_file = None
+ profile_message = 'See the profiling report at: '
+ # We are not interested in the profiling report for buildSrc.
+ for line in build_stdout:
+ if (profile_message in line) and ('buildSrc' not in line):
+ assert not html_file, "Only one report should be created"
+ html_file = line[len(profile_message):]
+ if html_file.startswith('file://'):
+ html_file = html_file[len('file://'):]
- if not html_file:
- return
+ if not html_file:
+ return
- assert os.path.isfile(html_file), 'Expected to find HTML file at {}'.format(
- html_file)
- MoveFile(html_file, os.path.join(dest_dir, 'index.html'), quiet=quiet)
+ assert os.path.isfile(html_file), 'Expected to find HTML file at {}'.format(
+ html_file)
+ MoveFile(html_file, os.path.join(dest_dir, 'index.html'), quiet=quiet)
- html_dir = os.path.dirname(html_file)
- for dir_name in ['css', 'js']:
- MoveDir(os.path.join(html_dir, dir_name), os.path.join(dest_dir, dir_name),
- quiet=quiet)
+ html_dir = os.path.dirname(html_file)
+ for dir_name in ['css', 'js']:
+ MoveDir(os.path.join(html_dir, dir_name),
+ os.path.join(dest_dir, dir_name),
+ quiet=quiet)
+
def MoveXMLTestResultFileTo(xml_test_result_dest, test_stdout, quiet=False):
- xml_test_result_file = None
- xml_result_reporter_message = 'XML test result file generated at '
- for line in test_stdout:
- if xml_result_reporter_message in line:
- index_from = (
- line.index(xml_result_reporter_message)
- + len(xml_result_reporter_message))
- index_to = line.index('.xml') + len('.xml')
- xml_test_result_file = line[index_from:index_to]
- break
+ xml_test_result_file = None
+ xml_result_reporter_message = 'XML test result file generated at '
+ for line in test_stdout:
+ if xml_result_reporter_message in line:
+ index_from = (line.index(xml_result_reporter_message) +
+ len(xml_result_reporter_message))
+ index_to = line.index('.xml') + len('.xml')
+ xml_test_result_file = line[index_from:index_to]
+ break
- assert os.path.isfile(xml_test_result_file), (
- 'Expected to find XML file at {}'.format(xml_test_result_file))
+ assert os.path.isfile(xml_test_result_file), (
+ 'Expected to find XML file at {}'.format(xml_test_result_file))
- MoveFile(xml_test_result_file, xml_test_result_dest, quiet=quiet)
+ MoveFile(xml_test_result_file, xml_test_result_dest, quiet=quiet)
+
def ParseProfileReport(profile_dir):
- html_file = os.path.join(profile_dir, 'index.html')
- assert os.path.isfile(html_file)
+ html_file = os.path.join(profile_dir, 'index.html')
+ assert os.path.isfile(html_file)
- parser = ProfileReportParser()
- with open(html_file) as f:
- for line in f.readlines():
- parser.feed(line)
- return parser.result
+ parser = ProfileReportParser()
+ with open(html_file) as f:
+ for line in f.readlines():
+ parser.feed(line)
+ return parser.result
+
# A simple HTML parser that recognizes the following pattern:
#
@@ -242,51 +259,50 @@
# <td></td>
# </tr>
class ProfileReportParser(HTMLParser):
- def __init__(self):
- HTMLParser.__init__(self)
- self.entered_table_row = False
- self.entered_task_name_cell = False
- self.entered_duration_cell = False
- self.current_task_name = None
- self.current_duration = None
+ def __init__(self):
+ HTMLParser.__init__(self)
+ self.entered_table_row = False
+ self.entered_task_name_cell = False
+ self.entered_duration_cell = False
- self.result = {}
+ self.current_task_name = None
+ self.current_duration = None
- def handle_starttag(self, tag, attrs):
- entered_table_row_before = self.entered_table_row
- entered_task_name_cell_before = self.entered_task_name_cell
+ self.result = {}
- self.entered_table_row = (tag == 'tr')
- self.entered_task_name_cell = (tag == 'td' and entered_table_row_before)
- self.entered_duration_cell = (
- self.current_task_name
- and tag == 'td'
- and entered_task_name_cell_before)
+ def handle_starttag(self, tag, attrs):
+ entered_table_row_before = self.entered_table_row
+ entered_task_name_cell_before = self.entered_task_name_cell
- def handle_endtag(self, tag):
- if tag == 'tr':
- if self.current_task_name and self.current_duration:
- self.result[self.current_task_name] = self.current_duration
- self.current_task_name = None
- self.current_duration = None
- self.entered_table_row = False
+ self.entered_table_row = (tag == 'tr')
+ self.entered_task_name_cell = (tag == 'td' and entered_table_row_before)
+ self.entered_duration_cell = (self.current_task_name and tag == 'td' and
+ entered_task_name_cell_before)
- def handle_data(self, data):
- stripped = data.strip()
- if not stripped:
- return
- if self.entered_task_name_cell:
- if IsGradleTaskName(stripped):
- self.current_task_name = stripped
- elif self.entered_duration_cell and stripped.endswith('s'):
- duration = stripped[:-1]
- if 'm' in duration:
- tmp = duration.split('m')
- minutes = int(tmp[0])
- seconds = float(tmp[1])
- else:
- minutes = 0
- seconds = float(duration)
- self.current_duration = 60 * minutes + seconds
- self.entered_table_row = False
+ def handle_endtag(self, tag):
+ if tag == 'tr':
+ if self.current_task_name and self.current_duration:
+ self.result[self.current_task_name] = self.current_duration
+ self.current_task_name = None
+ self.current_duration = None
+ self.entered_table_row = False
+
+ def handle_data(self, data):
+ stripped = data.strip()
+ if not stripped:
+ return
+ if self.entered_task_name_cell:
+ if IsGradleTaskName(stripped):
+ self.current_task_name = stripped
+ elif self.entered_duration_cell and stripped.endswith('s'):
+ duration = stripped[:-1]
+ if 'm' in duration:
+ tmp = duration.split('m')
+ minutes = int(tmp[0])
+ seconds = float(tmp[1])
+ else:
+ minutes = 0
+ seconds = float(duration)
+ self.current_duration = 60 * minutes + seconds
+ self.entered_table_row = False
diff --git a/tools/asmifier.py b/tools/asmifier.py
index bfb1226..9af1499 100755
--- a/tools/asmifier.py
+++ b/tools/asmifier.py
@@ -13,45 +13,51 @@
ASM_VERSION = '9.5'
ASM_JAR = os.path.join(utils.DEPENDENCIES_DIR, 'org', 'ow2', 'asm', 'asm',
ASM_VERSION, 'asm-' + ASM_VERSION + '.jar')
-ASM_UTIL_JAR = os.path.join(utils.DEPENDENCIES_DIR, 'org', 'ow2', 'asm', 'asm-util',
- ASM_VERSION, 'asm-util-' + ASM_VERSION + '.jar')
+ASM_UTIL_JAR = os.path.join(utils.DEPENDENCIES_DIR, 'org', 'ow2', 'asm',
+ 'asm-util', ASM_VERSION,
+ 'asm-util-' + ASM_VERSION + '.jar')
+
def run(args):
- cmd = []
- cmd.append(jdk.GetJavaExecutable())
- cp = ":".join([ASM_JAR, ASM_UTIL_JAR])
- print(cp)
- cmd.extend(['-cp', cp])
- cmd.append('org.objectweb.asm.util.ASMifier')
- cmd.extend(args)
- utils.PrintCmd(cmd)
- result = subprocess.check_output(cmd).decode('utf-8')
- print(result)
- return result
+ cmd = []
+ cmd.append(jdk.GetJavaExecutable())
+ cp = ":".join([ASM_JAR, ASM_UTIL_JAR])
+ print(cp)
+ cmd.extend(['-cp', cp])
+ cmd.append('org.objectweb.asm.util.ASMifier')
+ cmd.extend(args)
+ utils.PrintCmd(cmd)
+ result = subprocess.check_output(cmd).decode('utf-8')
+ print(result)
+ return result
+
def main():
- help = True
- args = []
- for arg in sys.argv[1:]:
- if arg == "--no-debug":
- args.append("-debug")
- elif arg in ("-help", "--help", "-debug"):
- help = True
- break
- else:
- help = False
- args.append(arg)
- if help:
- print("asmifier.py [--no-debug] <classfile>*")
- print(" --no-debug Don't include local variable information in output.")
- return
- try:
- run(args)
- except subprocess.CalledProcessError as e:
- # In case anything relevant was printed to stdout, normally this is already
- # on stderr.
- print(e.output)
- return e.returncode
+ help = True
+ args = []
+ for arg in sys.argv[1:]:
+ if arg == "--no-debug":
+ args.append("-debug")
+ elif arg in ("-help", "--help", "-debug"):
+ help = True
+ break
+ else:
+ help = False
+ args.append(arg)
+ if help:
+ print("asmifier.py [--no-debug] <classfile>*")
+ print(
+ " --no-debug Don't include local variable information in output."
+ )
+ return
+ try:
+ run(args)
+ except subprocess.CalledProcessError as e:
+ # In case anything relevant was printed to stdout, normally this is already
+ # on stderr.
+ print(e.output)
+ return e.returncode
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/benchmarks/get_deps.py b/tools/benchmarks/get_deps.py
index 716db48..0e178ae 100644
--- a/tools/benchmarks/get_deps.py
+++ b/tools/benchmarks/get_deps.py
@@ -7,15 +7,16 @@
import sys
import main_utils
-utils = main_utils.GetUtils();
+
+utils = main_utils.GetUtils()
ANDROID_EMULATORS = os.path.join(utils.TOOLS_DIR, 'benchmarks',
'android-sdk-linux.tar.gz.sha1')
+
def Main():
- utils.DownloadFromGoogleCloudStorage(ANDROID_EMULATORS)
+ utils.DownloadFromGoogleCloudStorage(ANDROID_EMULATORS)
+
if __name__ == '__main__':
- sys.exit(Main())
-
-
+ sys.exit(Main())
diff --git a/tools/benchmarks/main_utils.py b/tools/benchmarks/main_utils.py
index b3ea703..f419004 100644
--- a/tools/benchmarks/main_utils.py
+++ b/tools/benchmarks/main_utils.py
@@ -5,9 +5,10 @@
import imp
import os
-TOOLS_DIR = os.path.abspath(
- os.path.normpath(os.path.join(__file__, '..', '..')))
+TOOLS_DIR = os.path.abspath(os.path.normpath(os.path.join(__file__, '..',
+ '..')))
+
def GetUtils():
- '''Dynamically load the tools/utils.py python module.'''
- return imp.load_source('utils', os.path.join(TOOLS_DIR, 'utils.py'))
+ '''Dynamically load the tools/utils.py python module.'''
+ return imp.load_source('utils', os.path.join(TOOLS_DIR, 'utils.py'))
diff --git a/tools/build_aosp.py b/tools/build_aosp.py
deleted file mode 100755
index 5475830..0000000
--- a/tools/build_aosp.py
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-from os.path import join
-from glob import glob
-from itertools import chain
-from stat import S_IRWXU
-import argparse
-import multiprocessing
-import os
-import re
-import sys
-
-import gradle
-import utils
-import utils_aosp
-
-J_DEFAULT = multiprocessing.cpu_count() - 2
-
-EXIT_FAILURE = 1
-
-def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Checkout the AOSP source tree.')
- utils_aosp.add_common_arguments(parser)
- parser.add_argument('--tool',
- choices = ['d8', 'r8', 'default'],
- default = 'd8',
- help='Compiler tool to use. Defaults to d8.')
- parser.add_argument('--mmm',
- action = 'store_true',
- help='Use mmm instead of make')
- parser.add_argument('--mmma',
- action = 'store_true',
- help='Use mmma instead of make')
- parser.add_argument('--show-commands',
- action = 'store_true',
- help='Show commands executed during build.')
- parser.add_argument('-j',
- help='Projects to fetch simultaneously. ' +
- 'Defaults to ' + str(J_DEFAULT) + '.',
- type=int,
- default=-1)
- parser.add_argument('target', nargs='?')
- return parser.parse_args()
-
-def build_aosp(aosp_root, lunch, make, tool,
- concurrency, target, show_commands):
- d8_option = 'USE_D8=false'
- if tool == 'd8' or tool == 'r8' :
- d8_option = 'USE_D8=true'
-
- r8_option = 'USE_R8=false'
- if tool == 'r8':
- r8_option = 'USE_R8=true'
-
- j_option = '-j'
- if concurrency > 0:
- j_option += str(concurrency)
-
- command = [make, j_option]
- if show_commands:
- command.append('showcommands')
- command.extend([d8_option, r8_option])
- if target:
- command.append(target)
-
- print 'Building using: ' + ' '.join(command)
- utils_aosp.run_through_aosp_helper(lunch, command, aosp_root)
-
-def Main():
- args = parse_arguments()
-
- make = 'm'
- if args.mmm:
- make = 'mmm'
- if args.mmma:
- make = 'mmma'
- build_aosp(args.aosp_root, args.lunch, make, args.tool,
- args.j, args.target, args.show_commands)
-
-if __name__ == '__main__':
- sys.exit(Main())
diff --git a/tools/build_sample_apk.py b/tools/build_sample_apk.py
index 2cc95aa..0d577d4 100755
--- a/tools/build_sample_apk.py
+++ b/tools/build_sample_apk.py
@@ -18,9 +18,8 @@
import utils
import uuid
-
-DEFAULT_AAPT = 'aapt' # Assume in path.
-DEFAULT_AAPT2 = 'aapt2' # Assume in path.
+DEFAULT_AAPT = 'aapt' # Assume in path.
+DEFAULT_AAPT2 = 'aapt2' # Assume in path.
DEFAULT_D8 = os.path.join(utils.REPO_ROOT, 'tools', 'd8.py')
DEFAULT_DEXSPLITTER = os.path.join(utils.REPO_ROOT, 'tools', 'dexsplitter.py')
DEFAULT_JAVAC = jdk.GetJavacExecutable()
@@ -30,308 +29,342 @@
STANDARD_ACTIVITY = "R8Activity"
BENCHMARK_ITERATIONS = 30
-SAMPLE_APKS = [
- 'simple',
- 'split'
-]
+SAMPLE_APKS = ['simple', 'split']
+
def parse_options():
- result = optparse.OptionParser()
- result.add_option('--aapt',
- help='aapt executable to use',
- default=DEFAULT_AAPT)
- result.add_option('--aapt2',
- help='aapt2 executable to use',
- default=DEFAULT_AAPT2)
- result.add_option('--api',
- help='Android api level',
- default=21,
- choices=['14', '15', '19', '21', '22', '23', '24', '25',
- '26'])
- result.add_option('--keystore',
- help='Keystore used for signing',
- default=DEFAULT_KEYSTORE)
- result.add_option('--split',
- help='Split the app using the split.spec file',
- default=False, action='store_true')
- result.add_option('--generate-proto-apk',
- help='Use aapt2 to generate the proto version of the apk.',
- default=False, action='store_true')
- result.add_option('--install',
- help='Install the app (including featuresplit)',
- default=False, action='store_true')
- result.add_option('--benchmark',
- help='Benchmark the app on the phone with specialized markers',
- default=False, action='store_true')
- result.add_option('--benchmark-output-dir',
- help='Store benchmark results here.',
- default=None)
- result.add_option('--app',
- help='Which app to build',
- default='simple',
- choices=SAMPLE_APKS)
- return result.parse_args()
+ result = optparse.OptionParser()
+ result.add_option('--aapt',
+ help='aapt executable to use',
+ default=DEFAULT_AAPT)
+ result.add_option('--aapt2',
+ help='aapt2 executable to use',
+ default=DEFAULT_AAPT2)
+ result.add_option(
+ '--api',
+ help='Android api level',
+ default=21,
+ choices=['14', '15', '19', '21', '22', '23', '24', '25', '26'])
+ result.add_option('--keystore',
+ help='Keystore used for signing',
+ default=DEFAULT_KEYSTORE)
+ result.add_option('--split',
+ help='Split the app using the split.spec file',
+ default=False,
+ action='store_true')
+ result.add_option(
+ '--generate-proto-apk',
+ help='Use aapt2 to generate the proto version of the apk.',
+ default=False,
+ action='store_true')
+ result.add_option('--install',
+ help='Install the app (including featuresplit)',
+ default=False,
+ action='store_true')
+ result.add_option(
+ '--benchmark',
+ help='Benchmark the app on the phone with specialized markers',
+ default=False,
+ action='store_true')
+ result.add_option('--benchmark-output-dir',
+ help='Store benchmark results here.',
+ default=None)
+ result.add_option('--app',
+ help='Which app to build',
+ default='simple',
+ choices=SAMPLE_APKS)
+ return result.parse_args()
+
def run_aapt(aapt, args):
- command = [aapt]
- command.extend(args)
- utils.PrintCmd(command)
- subprocess.check_call(command)
-
-def get_build_dir(app):
- return os.path.join(utils.BUILD, 'sampleApks', app)
-
-def get_gen_path(app):
- gen_path = os.path.join(get_build_dir(app), 'gen')
- utils.makedirs_if_needed(gen_path)
- return gen_path
-
-def get_bin_path(app):
- bin_path = os.path.join(get_build_dir(app), 'bin')
- utils.makedirs_if_needed(bin_path)
- return bin_path
-
-
-def get_guava_jar():
- return os.path.join(utils.REPO_ROOT,
- 'third_party/gradle-plugin/com/google/guava/guava/22.0/guava-22.0.jar')
-
-def get_sample_dir(app):
- return os.path.join(utils.REPO_ROOT, 'src', 'test', 'sampleApks', app)
-
-def get_src_path(app):
- return os.path.join(get_sample_dir(app), 'src')
-
-def get_dex_path(app):
- return os.path.join(get_bin_path(app), 'classes.dex')
-
-def get_split_path(app, split):
- return os.path.join(get_bin_path(app), split, 'classes.dex')
-
-def get_package_name(app):
- return '%s.%s' % (PACKAGE_PREFIX, app)
-
-def get_qualified_activity(app):
- # The activity specified to adb start is PACKAGE_NAME/.ACTIVITY
- return '%s/.%s' % (get_package_name(app), STANDARD_ACTIVITY)
-
-def run_aapt_pack(aapt, api, app):
- with utils.ChangedWorkingDirectory(get_sample_dir(app)):
- args = ['package',
- '-v', '-f',
- '-I', utils.get_android_jar(api),
- '-M', 'AndroidManifest.xml',
- '-A', 'assets',
- '-S', 'res',
- '-m',
- '-J', get_gen_path(app),
- '-F', os.path.join(get_bin_path(app), 'resources.ap_'),
- '-G', os.path.join(get_build_dir(app), 'proguard_options')]
- run_aapt(aapt, args)
-
-def run_aapt_split_pack(aapt, api, app):
- with utils.ChangedWorkingDirectory(get_sample_dir(app)):
- args = ['package',
- '-v', '-f',
- '-I', utils.get_android_jar(api),
- '-M', 'split_manifest/AndroidManifest.xml',
- '-S', 'res',
- '-F', os.path.join(get_bin_path(app), 'split_resources.ap_')]
- run_aapt(aapt, args)
-
-def compile_with_javac(api, app):
- with utils.ChangedWorkingDirectory(get_sample_dir(app)):
- files = glob.glob(SRC_LOCATION.format(app=app))
- classpath = '%s:%s' % (utils.get_android_jar(api), get_guava_jar())
- command = [DEFAULT_JAVAC,
- '-classpath', classpath,
- '-sourcepath', '%s:%s:%s' % (
- get_src_path(app),
- get_gen_path(app),
- get_guava_jar()),
- '-d', get_bin_path(app)]
- command.extend(files)
+ command = [aapt]
+ command.extend(args)
utils.PrintCmd(command)
subprocess.check_call(command)
+
+def get_build_dir(app):
+ return os.path.join(utils.BUILD, 'sampleApks', app)
+
+
+def get_gen_path(app):
+ gen_path = os.path.join(get_build_dir(app), 'gen')
+ utils.makedirs_if_needed(gen_path)
+ return gen_path
+
+
+def get_bin_path(app):
+ bin_path = os.path.join(get_build_dir(app), 'bin')
+ utils.makedirs_if_needed(bin_path)
+ return bin_path
+
+
+def get_guava_jar():
+ return os.path.join(
+ utils.REPO_ROOT,
+ 'third_party/gradle-plugin/com/google/guava/guava/22.0/guava-22.0.jar')
+
+
+def get_sample_dir(app):
+ return os.path.join(utils.REPO_ROOT, 'src', 'test', 'sampleApks', app)
+
+
+def get_src_path(app):
+ return os.path.join(get_sample_dir(app), 'src')
+
+
+def get_dex_path(app):
+ return os.path.join(get_bin_path(app), 'classes.dex')
+
+
+def get_split_path(app, split):
+ return os.path.join(get_bin_path(app), split, 'classes.dex')
+
+
+def get_package_name(app):
+ return '%s.%s' % (PACKAGE_PREFIX, app)
+
+
+def get_qualified_activity(app):
+ # The activity specified to adb start is PACKAGE_NAME/.ACTIVITY
+ return '%s/.%s' % (get_package_name(app), STANDARD_ACTIVITY)
+
+
+def run_aapt_pack(aapt, api, app):
+ with utils.ChangedWorkingDirectory(get_sample_dir(app)):
+ args = [
+ 'package', '-v', '-f', '-I',
+ utils.get_android_jar(api), '-M', 'AndroidManifest.xml', '-A',
+ 'assets', '-S', 'res', '-m', '-J',
+ get_gen_path(app), '-F',
+ os.path.join(get_bin_path(app), 'resources.ap_'), '-G',
+ os.path.join(get_build_dir(app), 'proguard_options')
+ ]
+ run_aapt(aapt, args)
+
+
+def run_aapt_split_pack(aapt, api, app):
+ with utils.ChangedWorkingDirectory(get_sample_dir(app)):
+ args = [
+ 'package', '-v', '-f', '-I',
+ utils.get_android_jar(api), '-M',
+ 'split_manifest/AndroidManifest.xml', '-S', 'res', '-F',
+ os.path.join(get_bin_path(app), 'split_resources.ap_')
+ ]
+ run_aapt(aapt, args)
+
+
+def compile_with_javac(api, app):
+ with utils.ChangedWorkingDirectory(get_sample_dir(app)):
+ files = glob.glob(SRC_LOCATION.format(app=app))
+ classpath = '%s:%s' % (utils.get_android_jar(api), get_guava_jar())
+ command = [
+ DEFAULT_JAVAC, '-classpath', classpath, '-sourcepath',
+ '%s:%s:%s' %
+ (get_src_path(app), get_gen_path(app), get_guava_jar()), '-d',
+ get_bin_path(app)
+ ]
+ command.extend(files)
+ utils.PrintCmd(command)
+ subprocess.check_call(command)
+
+
def dex(app, api):
- files = []
- for root, dirnames, filenames in os.walk(get_bin_path(app)):
- for filename in fnmatch.filter(filenames, '*.class'):
- files.append(os.path.join(root, filename))
- command = [DEFAULT_D8, '--',
- '--output', get_bin_path(app),
- '--classpath', utils.get_android_jar(api),
- '--min-api', str(api)]
- command.extend(files)
- if app != 'simple':
- command.append(get_guava_jar())
+ files = []
+ for root, dirnames, filenames in os.walk(get_bin_path(app)):
+ for filename in fnmatch.filter(filenames, '*.class'):
+ files.append(os.path.join(root, filename))
+ command = [
+ DEFAULT_D8, '--', '--output',
+ get_bin_path(app), '--classpath',
+ utils.get_android_jar(api), '--min-api',
+ str(api)
+ ]
+ command.extend(files)
+ if app != 'simple':
+ command.append(get_guava_jar())
- utils.PrintCmd(command)
- subprocess.check_call(command)
-
-def split(app):
- split_spec = os.path.join(get_sample_dir(app), 'split.spec')
- command = [DEFAULT_DEXSPLITTER,
- '--input', get_dex_path(app),
- '--output', get_bin_path(app),
- '--feature-splits', split_spec]
- utils.PrintCmd(command)
- subprocess.check_call(command)
-
-def run_adb(args, ignore_exit=False):
- command = ['adb']
- command.extend(args)
- utils.PrintCmd(command)
- # On M adb install-multiple exits 1 but succeed in installing.
- if ignore_exit:
- subprocess.call(command)
- else:
+ utils.PrintCmd(command)
subprocess.check_call(command)
+
+def split(app):
+ split_spec = os.path.join(get_sample_dir(app), 'split.spec')
+ command = [
+ DEFAULT_DEXSPLITTER, '--input',
+ get_dex_path(app), '--output',
+ get_bin_path(app), '--feature-splits', split_spec
+ ]
+ utils.PrintCmd(command)
+ subprocess.check_call(command)
+
+
+def run_adb(args, ignore_exit=False):
+ command = ['adb']
+ command.extend(args)
+ utils.PrintCmd(command)
+ # On M adb install-multiple exits 1 but succeed in installing.
+ if ignore_exit:
+ subprocess.call(command)
+ else:
+ subprocess.check_call(command)
+
+
def adb_install(apks):
- args = [
- 'install-multiple' if len(apks) > 1 else 'install',
- '-r',
- '-d']
- args.extend(apks)
- run_adb(args, ignore_exit=True)
+ args = ['install-multiple' if len(apks) > 1 else 'install', '-r', '-d']
+ args.extend(apks)
+ run_adb(args, ignore_exit=True)
+
def create_temp_apk(app, prefix):
- temp_apk_path = os.path.join(get_bin_path(app), '%s.ap_' % app)
- shutil.copyfile(os.path.join(get_bin_path(app), '%sresources.ap_' % prefix),
- temp_apk_path)
- return temp_apk_path
+ temp_apk_path = os.path.join(get_bin_path(app), '%s.ap_' % app)
+ shutil.copyfile(os.path.join(get_bin_path(app), '%sresources.ap_' % prefix),
+ temp_apk_path)
+ return temp_apk_path
+
def aapt_add_dex(aapt, dex, temp_apk_path):
- args = ['add',
- '-k', temp_apk_path,
- dex]
- run_aapt(aapt, args)
+ args = ['add', '-k', temp_apk_path, dex]
+ run_aapt(aapt, args)
+
def kill(app):
- args = ['shell', 'am', 'force-stop', get_package_name(app)]
- run_adb(args)
+ args = ['shell', 'am', 'force-stop', get_package_name(app)]
+ run_adb(args)
+
def start_logcat():
- return subprocess.Popen(['adb', 'logcat'], bufsize=1024*1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ return subprocess.Popen(['adb', 'logcat'],
+ bufsize=1024 * 1024,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
def start(app):
- args = ['shell', 'am', 'start', '-n', get_qualified_activity(app)]
- run_adb(args)
+ args = ['shell', 'am', 'start', '-n', get_qualified_activity(app)]
+ run_adb(args)
+
def clear_logcat():
- args = ['logcat', '-c']
- run_adb(args)
+ args = ['logcat', '-c']
+ run_adb(args)
+
def stop_logcat(popen):
- popen.terminate()
- lines = []
- for l in popen.stdout:
- if 'System.out' in l:
- lines.append(l)
- return lines
+ popen.terminate()
+ lines = []
+ for l in popen.stdout:
+ if 'System.out' in l:
+ lines.append(l)
+ return lines
+
def store_or_print_benchmarks(lines, output):
- results = {}
- overall_total = 0
- # We assume that the total times are
- # prefixed with 'NAME Total: '. The logcat lines looks like:
- # 06-28 12:22:00.991 13698 13698 I System.out: Call Total: 61614
- for l in lines:
- if 'Total: ' in l:
- split = l.split('Total: ')
- time = split[1]
- name = split[0].split()[-1]
- overall_total += int(time)
- print('%s: %s' % (name, time))
- results[name] = time
+ results = {}
+ overall_total = 0
+ # We assume that the total times are
+ # prefixed with 'NAME Total: '. The logcat lines looks like:
+ # 06-28 12:22:00.991 13698 13698 I System.out: Call Total: 61614
+ for l in lines:
+ if 'Total: ' in l:
+ split = l.split('Total: ')
+ time = split[1]
+ name = split[0].split()[-1]
+ overall_total += int(time)
+ print('%s: %s' % (name, time))
+ results[name] = time
- print('Total: %s' % overall_total)
- if not output:
+ print('Total: %s' % overall_total)
+ if not output:
+ return overall_total
+ results['total'] = str(overall_total)
+ output_dir = os.path.join(output, str(uuid.uuid4()))
+ os.makedirs(output_dir)
+ written_files = []
+ for name, time in results.iteritems():
+ total_file = os.path.join(output_dir, name)
+ written_files.append(total_file)
+ with open(total_file, 'w') as f:
+ f.write(time)
+
+ print('Result stored in: \n%s' % ('\n'.join(written_files)))
return overall_total
- results['total'] = str(overall_total)
- output_dir = os.path.join(output, str(uuid.uuid4()))
- os.makedirs(output_dir)
- written_files = []
- for name, time in results.iteritems():
- total_file = os.path.join(output_dir, name)
- written_files.append(total_file)
- with open(total_file, 'w') as f:
- f.write(time)
- print('Result stored in: \n%s' % ('\n'.join(written_files)))
- return overall_total
def benchmark(app, output_dir):
- # Ensure app is not running
- kill(app)
- clear_logcat()
- logcat = start_logcat()
- start(app)
- # We could do better here by continiously parsing the logcat for a marker, but
- # this works nicely with the current setup.
- time.sleep(12)
- kill(app)
- return float(store_or_print_benchmarks(stop_logcat(logcat), output_dir))
+ # Ensure app is not running
+ kill(app)
+ clear_logcat()
+ logcat = start_logcat()
+ start(app)
+ # We could do better here by continiously parsing the logcat for a marker, but
+ # this works nicely with the current setup.
+ time.sleep(12)
+ kill(app)
+ return float(store_or_print_benchmarks(stop_logcat(logcat), output_dir))
+
def ensure_no_logcat():
- output = subprocess.check_output(['ps', 'aux'])
- if 'adb logcat' in output:
- raise Exception('You have adb logcat running, please close it and rerun')
+ output = subprocess.check_output(['ps', 'aux'])
+ if 'adb logcat' in output:
+ raise Exception(
+ 'You have adb logcat running, please close it and rerun')
+
def generate_proto_apks(apks, options):
- proto_apks = []
- for apk in apks:
- proto_apk = apk + '.proto'
- cmd = [options.aapt2, 'convert',
- '-o', proto_apk,
- '--output-format', 'proto',
- apk]
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
- proto_apks.append(proto_apk)
- return proto_apks
+ proto_apks = []
+ for apk in apks:
+ proto_apk = apk + '.proto'
+ cmd = [
+ options.aapt2, 'convert', '-o', proto_apk, '--output-format',
+ 'proto', apk
+ ]
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ proto_apks.append(proto_apk)
+ return proto_apks
+
def Main():
- (options, args) = parse_options()
- apks = []
- is_split = options.split
- run_aapt_pack(options.aapt, options.api, options.app)
- if is_split:
- run_aapt_split_pack(options.aapt, options.api, options.app)
- compile_with_javac(options.api, options.app)
- dex(options.app, options.api)
- dex_files = { options.app: get_dex_path(options.app)}
- dex_path = get_dex_path(options.app)
- if is_split:
- split(options.app)
- dex_path = get_split_path(options.app, 'base')
- temp_apk_path = create_temp_apk(options.app, '')
- aapt_add_dex(options.aapt, dex_path, temp_apk_path)
- apk_path = os.path.join(get_bin_path(options.app), '%s.apk' % options.app)
- apk_utils.sign(temp_apk_path, apk_path, options.keystore)
- apks.append(apk_path)
- if is_split:
- split_temp_apk_path = create_temp_apk(options.app, 'split_')
- aapt_add_dex(options.aapt,
- get_split_path(options.app, 'split'),
- temp_apk_path)
- split_apk_path = os.path.join(get_bin_path(options.app), 'featuresplit.apk')
- apk_utils.sign(temp_apk_path, split_apk_path, options.keystore)
- apks.append(split_apk_path)
- if options.generate_proto_apk:
- proto_apks = generate_proto_apks(apks, options)
- print('Generated proto apks available at: %s' % ' '.join(proto_apks))
- print('Generated apks available at: %s' % ' '.join(apks))
- if options.install or options.benchmark:
- adb_install(apks)
- grand_total = 0
- if options.benchmark:
- ensure_no_logcat()
- for _ in range(BENCHMARK_ITERATIONS):
- grand_total += benchmark(options.app, options.benchmark_output_dir)
- print('Combined average: %s' % (grand_total/BENCHMARK_ITERATIONS))
+ (options, args) = parse_options()
+ apks = []
+ is_split = options.split
+ run_aapt_pack(options.aapt, options.api, options.app)
+ if is_split:
+ run_aapt_split_pack(options.aapt, options.api, options.app)
+ compile_with_javac(options.api, options.app)
+ dex(options.app, options.api)
+ dex_files = {options.app: get_dex_path(options.app)}
+ dex_path = get_dex_path(options.app)
+ if is_split:
+ split(options.app)
+ dex_path = get_split_path(options.app, 'base')
+ temp_apk_path = create_temp_apk(options.app, '')
+ aapt_add_dex(options.aapt, dex_path, temp_apk_path)
+ apk_path = os.path.join(get_bin_path(options.app), '%s.apk' % options.app)
+ apk_utils.sign(temp_apk_path, apk_path, options.keystore)
+ apks.append(apk_path)
+ if is_split:
+ split_temp_apk_path = create_temp_apk(options.app, 'split_')
+ aapt_add_dex(options.aapt, get_split_path(options.app, 'split'),
+ temp_apk_path)
+ split_apk_path = os.path.join(get_bin_path(options.app),
+ 'featuresplit.apk')
+ apk_utils.sign(temp_apk_path, split_apk_path, options.keystore)
+ apks.append(split_apk_path)
+ if options.generate_proto_apk:
+ proto_apks = generate_proto_apks(apks, options)
+ print('Generated proto apks available at: %s' % ' '.join(proto_apks))
+ print('Generated apks available at: %s' % ' '.join(apks))
+ if options.install or options.benchmark:
+ adb_install(apks)
+ grand_total = 0
+ if options.benchmark:
+ ensure_no_logcat()
+ for _ in range(BENCHMARK_ITERATIONS):
+ grand_total += benchmark(options.app, options.benchmark_output_dir)
+ print('Combined average: %s' % (grand_total / BENCHMARK_ITERATIONS))
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/check-cherry-picks.py b/tools/check-cherry-picks.py
index 3d66d3c..3b27b08 100755
--- a/tools/check-cherry-picks.py
+++ b/tools/check-cherry-picks.py
@@ -8,20 +8,23 @@
import re
import r8_release
+
class Branch:
- def __init__(self, name, first, last=None):
- self.name = name
- self.first = first
- self.last = last # optional last for testing purposes.
- def origin(self):
- return "origin/%s" % self.name
+ def __init__(self, name, first, last=None):
+ self.name = name
+ self.first = first
+ self.last = last # optional last for testing purposes.
- def last_or_origin(self):
- return self.last if self.last else self.origin()
+ def origin(self):
+ return "origin/%s" % self.name
- def __str__(self):
- return self.name
+ def last_or_origin(self):
+ return self.last if self.last else self.origin()
+
+ def __str__(self):
+ return self.name
+
# The initial commit is the furthest back we need to search on main.
# Currently, it is the merge point of main onto 4.0.23-dev
@@ -30,155 +33,167 @@
DEV_BRANCH_VERSION = [int(s) for s in r8_release.R8_DEV_BRANCH.split('.')]
# List of change ids that should not be reported.
-IGNORED = [
- 'I92d7bf3afbf609fdea21683941cfd15c90305cf2'
-]
+IGNORED = ['I92d7bf3afbf609fdea21683941cfd15c90305cf2']
VERBOSE = False
+
# Helper to call and decode a shell command.
def run_cmd(cmd):
- if VERBOSE:
- print(' '.join(cmd))
- return subprocess.check_output(cmd).decode('UTF-8')
+ if VERBOSE:
+ print(' '.join(cmd))
+ return subprocess.check_output(cmd).decode('UTF-8')
+
# Comparator on major and minor branch versions.
def branch_version_less_than(b1, b2):
- if b1[0] < b2[0]:
- return True
- if b1[0] == b2[0] and b1[1] < b2[1]:
- return True
- return False
+ if b1[0] < b2[0]:
+ return True
+ if b1[0] == b2[0] and b1[1] < b2[1]:
+ return True
+ return False
+
# Find all release branches between OLDEST_BRANCH and DEV_BRANCH
def get_release_branches():
- # Release branches are assumed to be of the form 'origin/X.Y'
- out = run_cmd(['git', 'branch', '-r', '-l'])
- pattern = re.compile('origin/(\d+).(\d+)')
- releases = []
- for line in out.split('\n'):
- m = pattern.search(line.strip())
- if m:
- major = m.group(1)
- minor = m.group(2)
- if major and minor:
- candidate = (int(major), int(minor))
- if branch_version_less_than(candidate, OLDEST_BRANCH_VERSION):
- continue
- if branch_version_less_than(candidate, DEV_BRANCH_VERSION):
- releases.extend(find_dev_cutoff(candidate))
- return releases
+ # Release branches are assumed to be of the form 'origin/X.Y'
+ out = run_cmd(['git', 'branch', '-r', '-l'])
+ pattern = re.compile('origin/(\d+).(\d+)')
+ releases = []
+ for line in out.split('\n'):
+ m = pattern.search(line.strip())
+ if m:
+ major = m.group(1)
+ minor = m.group(2)
+ if major and minor:
+ candidate = (int(major), int(minor))
+ if branch_version_less_than(candidate, OLDEST_BRANCH_VERSION):
+ continue
+ if branch_version_less_than(candidate, DEV_BRANCH_VERSION):
+ releases.extend(find_dev_cutoff(candidate))
+ return releases
+
# Find the most recent commit hash that is for a -dev version.
# This is the starting point for the map of commits after cutoff from main.
def find_dev_cutoff(branch_version):
- out = run_cmd([
- 'git',
- 'log',
- 'origin/%d.%d' % branch_version,
- '--grep', 'Version .*-dev',
- '--pretty=oneline',
- ])
- # Format of output is: <hash> Version <version>-dev
- try:
- hash = out[0:out.index(' ')]
- return [Branch('%d.%d' % branch_version, hash)]
- except ValueError:
- throw_error("Failed to find dev cutoff for branch %d.%d" % branch_version)
+ out = run_cmd([
+ 'git',
+ 'log',
+ 'origin/%d.%d' % branch_version,
+ '--grep',
+ 'Version .*-dev',
+ '--pretty=oneline',
+ ])
+ # Format of output is: <hash> Version <version>-dev
+ try:
+ hash = out[0:out.index(' ')]
+ return [Branch('%d.%d' % branch_version, hash)]
+ except ValueError:
+ throw_error("Failed to find dev cutoff for branch %d.%d" %
+ branch_version)
+
# Build a map from each "Change-Id" hash to the hash of its commit.
def get_change_id_map(branch):
- out = run_cmd([
- 'git',
- 'log',
- '%s..%s' % (branch.first, branch.last_or_origin())
- ])
- map = {}
- current_commit = None
- for line in out.split('\n'):
- if line.startswith('commit '):
- current_commit = line[len('commit '):]
- assert len(current_commit) == 40
- elif line.strip().startswith('Change-Id: '):
- change_id = line.strip()[len('Change-Id: '):]
- assert len(change_id) == 41
- map[change_id] = current_commit
- return map
+ out = run_cmd(
+ ['git', 'log',
+ '%s..%s' % (branch.first, branch.last_or_origin())])
+ map = {}
+ current_commit = None
+ for line in out.split('\n'):
+ if line.startswith('commit '):
+ current_commit = line[len('commit '):]
+ assert len(current_commit) == 40
+ elif line.strip().startswith('Change-Id: '):
+ change_id = line.strip()[len('Change-Id: '):]
+ assert len(change_id) == 41
+ map[change_id] = current_commit
+ return map
+
# Check if a specific commit is present on a specific branch.
def is_commit_in(commit, branch):
- out = run_cmd(['git', 'branch', '-r', branch.origin(), '--contains', commit])
- return out.strip() == branch.origin()
+ out = run_cmd(
+ ['git', 'branch', '-r',
+ branch.origin(), '--contains', commit])
+ return out.strip() == branch.origin()
+
def main():
- found_errors = False
- # The main map is all commits back to the "init" point.
- main_map = get_change_id_map(MAIN)
- # Compute the release branches.
- release_branches = get_release_branches()
- # Populate the release maps with all commits after the last -dev point.
- release_maps = {}
- for branch in release_branches:
- release_maps[branch.name] = get_change_id_map(branch)
- # Each branch is then compared forwards with each subsequent branch.
- for i in range(len(release_branches)):
- branch = release_branches[i]
- newer_branches = release_branches[i+1:]
- if (len(newer_branches) == 0):
- print('Last non-dev release branch is %s, nothing to check.' % branch)
- continue
- print('Checking branch %s.' % branch)
- changes = release_maps[branch.name]
- cherry_picks_count = 0
- for change in changes.keys():
- is_cherry_pick = False
- missing_from = None
- commit_on_main = main_map.get(change)
- for newer_branch in newer_branches:
- if change in release_maps[newer_branch.name]:
- is_cherry_pick = True
- # If the change is in the release mappings check for holes.
- if missing_from:
- found_errors |= change_error(
- change,
- 'Error: missing Change-Id %s on branch %s. '
- 'Is present on %s and again on %s.' % (
- change, missing_from, branch, newer_branch,
- ))
- elif commit_on_main:
- is_cherry_pick = True
- # The change is not in the non-dev part of the branch, so we need to
- # check that the fork from main included the change.
- if not is_commit_in(commit_on_main, newer_branch):
- found_errors |= change_error(
- change,
- 'Error: missing Change-Id %s on branch %s. '
- 'Is present on %s and on main as commit %s.' % (
- change, newer_branch, branch, commit_on_main
- ))
- else:
- # The change is not on "main" so we just record for holes on releases.
- missing_from = newer_branch
- if is_cherry_pick:
- cherry_picks_count += 1
- print('Found %d cherry-picks (out of %d commits).' % (
- cherry_picks_count, len(changes)))
+ found_errors = False
+ # The main map is all commits back to the "init" point.
+ main_map = get_change_id_map(MAIN)
+ # Compute the release branches.
+ release_branches = get_release_branches()
+ # Populate the release maps with all commits after the last -dev point.
+ release_maps = {}
+ for branch in release_branches:
+ release_maps[branch.name] = get_change_id_map(branch)
+ # Each branch is then compared forwards with each subsequent branch.
+ for i in range(len(release_branches)):
+ branch = release_branches[i]
+ newer_branches = release_branches[i + 1:]
+ if (len(newer_branches) == 0):
+ print('Last non-dev release branch is %s, nothing to check.' %
+ branch)
+ continue
+ print('Checking branch %s.' % branch)
+ changes = release_maps[branch.name]
+ cherry_picks_count = 0
+ for change in changes.keys():
+ is_cherry_pick = False
+ missing_from = None
+ commit_on_main = main_map.get(change)
+ for newer_branch in newer_branches:
+ if change in release_maps[newer_branch.name]:
+ is_cherry_pick = True
+ # If the change is in the release mappings check for holes.
+ if missing_from:
+ found_errors |= change_error(
+ change, 'Error: missing Change-Id %s on branch %s. '
+ 'Is present on %s and again on %s.' % (
+ change,
+ missing_from,
+ branch,
+ newer_branch,
+ ))
+ elif commit_on_main:
+ is_cherry_pick = True
+ # The change is not in the non-dev part of the branch, so we need to
+ # check that the fork from main included the change.
+ if not is_commit_in(commit_on_main, newer_branch):
+ found_errors |= change_error(
+ change, 'Error: missing Change-Id %s on branch %s. '
+ 'Is present on %s and on main as commit %s.' %
+ (change, newer_branch, branch, commit_on_main))
+ else:
+ # The change is not on "main" so we just record for holes on releases.
+ missing_from = newer_branch
+ if is_cherry_pick:
+ cherry_picks_count += 1
+ print('Found %d cherry-picks (out of %d commits).' %
+ (cherry_picks_count, len(changes)))
- if found_errors:
- return 1
- return 0
+ if found_errors:
+ return 1
+ return 0
+
def change_error(change, msg):
- if change in IGNORED:
- return False
- error(msg)
- return True
+ if change in IGNORED:
+ return False
+ error(msg)
+ return True
+
def throw_error(msg):
- raise ValueError(msg)
+ raise ValueError(msg)
+
def error(msg):
- print(msg, file=sys.stderr)
+ print(msg, file=sys.stderr)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/checkout_aosp.py b/tools/checkout_aosp.py
index 36a5333..bef9413 100755
--- a/tools/checkout_aosp.py
+++ b/tools/checkout_aosp.py
@@ -14,56 +14,59 @@
import utils
import utils_aosp
-AOSP_MANIFEST_XML = join(utils.REPO_ROOT, 'third_party',
- 'aosp_manifest.xml')
+AOSP_MANIFEST_XML = join(utils.REPO_ROOT, 'third_party', 'aosp_manifest.xml')
AOSP_MANIFEST_URL = 'https://android.googlesource.com/platform/manifest'
J_DEFAULT = multiprocessing.cpu_count() - 2
+
# Checkout AOSP source to the specified direcotry using the speficied manifest.
def checkout_aosp(aosp_root, url, branch, manifest_xml, concurrency, shallow):
- utils.makedirs_if_needed(aosp_root)
- command = ['repo', 'init', '-u', url]
- if (shallow):
- command.extend(['--depth=1'])
- if (branch):
- command.extend(['-b', branch])
- else:
- manifests_dir = join(aosp_root, '.repo', 'manifests')
- utils.makedirs_if_needed(manifests_dir)
- copy2(manifest_xml, manifests_dir)
- command.extend(['-m', basename(manifest_xml)])
- check_call(command, cwd = aosp_root)
+ utils.makedirs_if_needed(aosp_root)
+ command = ['repo', 'init', '-u', url]
+ if (shallow):
+ command.extend(['--depth=1'])
+ if (branch):
+ command.extend(['-b', branch])
+ else:
+ manifests_dir = join(aosp_root, '.repo', 'manifests')
+ utils.makedirs_if_needed(manifests_dir)
+ copy2(manifest_xml, manifests_dir)
+ command.extend(['-m', basename(manifest_xml)])
+ check_call(command, cwd=aosp_root)
- check_call(['repo', 'sync', '-dq', '-j' + concurrency], cwd = aosp_root)
+ check_call(['repo', 'sync', '-dq', '-j' + concurrency], cwd=aosp_root)
+
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Checkout the AOSP source tree.')
- utils_aosp.add_root_argument(parser)
- parser.add_argument('--url',
- help='URL the repo. ' +
- 'Defaults to ' + AOSP_MANIFEST_URL + '.',
- default=AOSP_MANIFEST_URL)
- parser.add_argument('--manifest',
- help='Manifest to use for the checkout. ' +
- 'Defaults to ' + AOSP_MANIFEST_XML + '.',
- default=AOSP_MANIFEST_XML)
- parser.add_argument('--branch',
- help='Branch to checkout. This overrides ' +
- 'passing --manifest')
- parser.add_argument('--shallow',
- action = 'store_true',
- help='Shallow checkout.')
- parser.add_argument('-j',
- help='Projects to fetch simultaneously. ' +
- 'Defaults to ' + str(J_DEFAULT) + '.',
- default=str(J_DEFAULT))
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='Checkout the AOSP source tree.')
+ utils_aosp.add_root_argument(parser)
+ parser.add_argument('--url',
+ help='URL the repo. ' + 'Defaults to ' +
+ AOSP_MANIFEST_URL + '.',
+ default=AOSP_MANIFEST_URL)
+ parser.add_argument('--manifest',
+ help='Manifest to use for the checkout. ' +
+ 'Defaults to ' + AOSP_MANIFEST_XML + '.',
+ default=AOSP_MANIFEST_XML)
+ parser.add_argument('--branch',
+ help='Branch to checkout. This overrides ' +
+ 'passing --manifest')
+ parser.add_argument('--shallow',
+ action='store_true',
+ help='Shallow checkout.')
+ parser.add_argument('-j',
+ help='Projects to fetch simultaneously. ' +
+ 'Defaults to ' + str(J_DEFAULT) + '.',
+ default=str(J_DEFAULT))
+ return parser.parse_args()
+
def Main():
- args = parse_arguments()
- checkout_aosp(args.aosp_root, args.url, args.branch, args.manifest,
- args.j, args.shallow)
+ args = parse_arguments()
+ checkout_aosp(args.aosp_root, args.url, args.branch, args.manifest, args.j,
+ args.shallow)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/cherry-pick.py b/tools/cherry-pick.py
index 6a7109c..6e505c5 100755
--- a/tools/cherry-pick.py
+++ b/tools/cherry-pick.py
@@ -12,137 +12,158 @@
VERSION_FILE = 'src/main/java/com/android/tools/r8/Version.java'
VERSION_PREFIX = 'String LABEL = "'
-def parse_options():
- parser = argparse.ArgumentParser(description='Release r8')
- parser.add_argument('--branch',
- metavar=('<branch>'),
- help='Branch to cherry-pick to')
- parser.add_argument('--current-checkout', '--current_checkout',
- default=False,
- action='store_true',
- help='Perform cherry picks into the current checkout')
- parser.add_argument('--no-upload', '--no_upload',
- default=False,
- action='store_true',
- help='Do not upload to Gerrit')
- parser.add_argument('hashes', metavar='<hash>', nargs='+',
- help='Hashed to merge')
- return parser.parse_args()
+def parse_options():
+ parser = argparse.ArgumentParser(description='Release r8')
+ parser.add_argument('--branch',
+ metavar=('<branch>'),
+ help='Branch to cherry-pick to')
+ parser.add_argument('--current-checkout',
+ '--current_checkout',
+ default=False,
+ action='store_true',
+ help='Perform cherry picks into the current checkout')
+ parser.add_argument('--no-upload',
+ '--no_upload',
+ default=False,
+ action='store_true',
+ help='Do not upload to Gerrit')
+ parser.add_argument('hashes',
+ metavar='<hash>',
+ nargs='+',
+ help='Hashed to merge')
+
+ return parser.parse_args()
def run(args):
- # Checkout the branch.
- subprocess.check_output(['git', 'checkout', args.branch])
+ # Checkout the branch.
+ subprocess.check_output(['git', 'checkout', args.branch])
- if (args.current_checkout):
- for i in range(len(args.hashes) + 1):
- branch = 'cherry-%d' % (i + 1)
- print('Deleting branch %s' % branch)
- subprocess.run(['git', 'branch', branch, '-D'])
+ if (args.current_checkout):
+ for i in range(len(args.hashes) + 1):
+ branch = 'cherry-%d' % (i + 1)
+ print('Deleting branch %s' % branch)
+ subprocess.run(['git', 'branch', branch, '-D'])
- bugs = set()
+ bugs = set()
- count = 1
- for hash in args.hashes:
+ count = 1
+ for hash in args.hashes:
+ branch = 'cherry-%d' % count
+ print('Cherry-picking %s in %s' % (hash, branch))
+ if (count == 1):
+ subprocess.run([
+ 'git', 'new-branch', branch, '--upstream',
+ 'origin/%s' % args.branch
+ ])
+ else:
+ subprocess.run(['git', 'new-branch', branch, '--upstream-current'])
+
+ subprocess.run(['git', 'cherry-pick', hash])
+ confirm_and_upload(branch, args, bugs)
+ count = count + 1
+
branch = 'cherry-%d' % count
- print('Cherry-picking %s in %s' % (hash, branch))
- if (count == 1):
- subprocess.run(['git', 'new-branch', branch, '--upstream', 'origin/%s' % args.branch])
+ subprocess.run(['git', 'new-branch', branch, '--upstream-current'])
+
+ old_version = 'unknown'
+ for line in open(VERSION_FILE, 'r'):
+ index = line.find(VERSION_PREFIX)
+ if index > 0:
+ index += len(VERSION_PREFIX)
+ subline = line[index:]
+ old_version = subline[:subline.index('"')]
+ break
+
+ new_version = 'unknown'
+ if old_version.find('.') > 0:
+ split_version = old_version.split('.')
+ new_version = '.'.join(split_version[:-1] +
+ [str(int(split_version[-1]) + 1)])
+ subprocess.run([
+ 'sed', '-i',
+ 's/%s/%s/' % (old_version, new_version), VERSION_FILE
+ ])
else:
- subprocess.run(['git', 'new-branch', branch, '--upstream-current'])
+ editor = os.environ.get('VISUAL')
+ if not editor:
+ editor = os.environ.get('EDITOR')
+ if not editor:
+ editor = 'vi'
+ else:
+ print("Opening %s for version update with %s" %
+ (VERSION_FILE, editor))
+ subprocess.run([editor, VERSION_FILE])
- subprocess.run(['git', 'cherry-pick', hash])
- confirm_and_upload(branch, args, bugs)
- count = count + 1
+ message = ("Version %s\n\n" % new_version)
+ for bug in sorted(bugs):
+ message += 'Bug: b/%s\n' % bug
- branch = 'cherry-%d' % count
- subprocess.run(['git', 'new-branch', branch, '--upstream-current'])
+ subprocess.run(['git', 'commit', '-a', '-m', message])
+ confirm_and_upload(branch, args, None)
+ if (not args.current_checkout):
+ while True:
+ try:
+ answer = input(
+ "Type 'delete' to finish and delete checkout in %s: " %
+ os.getcwd())
+ if answer == 'delete':
+ break
+ except KeyboardInterrupt:
+ pass
- old_version = 'unknown'
- for line in open(VERSION_FILE, 'r'):
- index = line.find(VERSION_PREFIX)
- if index > 0:
- index += len(VERSION_PREFIX)
- subline = line[index:]
- old_version = subline[:subline.index('"')]
- break
-
- new_version = 'unknown'
- if old_version.find('.') > 0:
- split_version = old_version.split('.')
- new_version = '.'.join(split_version[:-1] + [str(int(split_version[-1]) + 1)])
- subprocess.run(['sed', '-i', 's/%s/%s/' % (old_version, new_version), VERSION_FILE])
- else:
- editor = os.environ.get('VISUAL')
- if not editor:
- editor = os.environ.get('EDITOR')
- if not editor:
- editor = 'vi'
- else:
- print("Opening %s for version update with %s" % (VERSION_FILE, editor))
- subprocess.run([editor, VERSION_FILE])
-
- message = ("Version %s\n\n" % new_version)
- for bug in sorted(bugs):
- message += 'Bug: b/%s\n' % bug
-
- subprocess.run(['git', 'commit', '-a', '-m', message])
- confirm_and_upload(branch, args, None)
- if (not args.current_checkout):
- while True:
- try:
- answer = input("Type 'delete' to finish and delete checkout in %s: " % os.getcwd())
- if answer == 'delete':
- break
- except KeyboardInterrupt:
- pass
def confirm_and_upload(branch, args, bugs):
- question = ('Ready to continue (cwd %s, will not upload to Gerrit)' % os.getcwd()
- if args.no_upload else
- 'Ready to upload %s (cwd %s)' % (branch, os.getcwd()))
+ question = ('Ready to continue (cwd %s, will not upload to Gerrit)' %
+ os.getcwd() if args.no_upload else
+ 'Ready to upload %s (cwd %s)' % (branch, os.getcwd()))
- while True:
- try:
- answer = input(question + ' [yes/abort]? ')
- if answer == 'yes':
- break
- if answer == 'abort':
- print('Aborting new branch for %s' % branch)
- sys.exit(1)
- except KeyboardInterrupt:
- pass
+ while True:
+ try:
+ answer = input(question + ' [yes/abort]? ')
+ if answer == 'yes':
+ break
+ if answer == 'abort':
+ print('Aborting new branch for %s' % branch)
+ sys.exit(1)
+ except KeyboardInterrupt:
+ pass
- # Compute the set of bug refs from the commit message after confirmation.
- # If done before a conflicting cherry-pick status will potentially include
- # references that are orthogonal to the pick.
- if bugs != None:
- commit_message = subprocess.check_output(['git', 'log', '--format=%B', '-n', '1', 'HEAD'])
- commit_lines = [l.strip() for l in commit_message.decode('UTF-8').split('\n')]
- for line in commit_lines:
- if line.startswith('Bug: '):
- normalized = line.replace('Bug: ', '').replace('b/', '')
- if len(normalized) > 0:
- bugs.add(normalized)
+ # Compute the set of bug refs from the commit message after confirmation.
+ # If done before a conflicting cherry-pick status will potentially include
+ # references that are orthogonal to the pick.
+ if bugs != None:
+ commit_message = subprocess.check_output(
+ ['git', 'log', '--format=%B', '-n', '1', 'HEAD'])
+ commit_lines = [
+ l.strip() for l in commit_message.decode('UTF-8').split('\n')
+ ]
+ for line in commit_lines:
+ if line.startswith('Bug: '):
+ normalized = line.replace('Bug: ', '').replace('b/', '')
+ if len(normalized) > 0:
+ bugs.add(normalized)
- if (not args.no_upload):
- subprocess.run(['git', 'cl', 'upload', '--bypass-hooks'])
+ if (not args.no_upload):
+ subprocess.run(['git', 'cl', 'upload', '--bypass-hooks'])
+
def main():
- args = parse_options()
+ args = parse_options()
- if (not args.current_checkout):
- with utils.TempDir() as temp:
- print("Performing cherry-picking in %s" % temp)
- subprocess.check_call(['git', 'clone', utils.REPO_SOURCE, temp])
- with utils.ChangedWorkingDirectory(temp):
+ if (not args.current_checkout):
+ with utils.TempDir() as temp:
+ print("Performing cherry-picking in %s" % temp)
+ subprocess.check_call(['git', 'clone', utils.REPO_SOURCE, temp])
+ with utils.ChangedWorkingDirectory(temp):
+ run(args)
+ else:
+ # Run in current directory.
+ print("Performing cherry-picking in %s" % os.getcwd())
+ subprocess.check_output(['git', 'fetch', 'origin'])
run(args)
- else:
- # Run in current directory.
- print("Performing cherry-picking in %s" % os.getcwd())
- subprocess.check_output(['git', 'fetch', 'origin'])
- run(args)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/chrome_data.py b/tools/chrome_data.py
index 5a7b6c3..8499d72 100644
--- a/tools/chrome_data.py
+++ b/tools/chrome_data.py
@@ -9,50 +9,62 @@
BASE = os.path.join(utils.THIRD_PARTY, 'chrome')
V200430_BASE = os.path.join(BASE, 'chrome_200430')
-V200520_MINIMAL_BASE = os.path.join(
- BASE, 'monochrome_public_minimal_apks', 'chrome_200520')
+V200520_MINIMAL_BASE = os.path.join(BASE, 'monochrome_public_minimal_apks',
+ 'chrome_200520')
VERSIONS = {
- '200430': {
- 'deploy' : {
- 'inputs': [os.path.join(V200430_BASE, 'program.jar')],
- 'pgconf': [os.path.join(V200430_BASE, 'proguard.config')],
- 'libraries': [os.path.join(V200430_BASE, 'library.jar')],
- 'min-api': ANDROID_N_API,
+ '200430': {
+ 'deploy': {
+ 'inputs': [os.path.join(V200430_BASE, 'program.jar')],
+ 'pgconf': [os.path.join(V200430_BASE, 'proguard.config')],
+ 'libraries': [os.path.join(V200430_BASE, 'library.jar')],
+ 'min-api': ANDROID_N_API,
+ },
},
- },
- '200520-monochrome_public_minimal_apks': {
- 'deploy' : {
- 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'program.jar')],
- 'features': [
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-1.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-2.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-3.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-4.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-5.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-6.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-7.jar')] },
- { 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-8.jar')] }
- ],
- 'pgconf': [os.path.join(V200520_MINIMAL_BASE, 'proguard.config'),
- utils.IGNORE_WARNINGS_RULES],
- 'libraries': [os.path.join(V200520_MINIMAL_BASE, 'library.jar')],
- 'min-api': ANDROID_N_API
+ '200520-monochrome_public_minimal_apks': {
+ 'deploy': {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'program.jar')],
+ 'features': [{
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-1.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-2.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-3.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-4.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-5.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-6.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-7.jar')]
+ }, {
+ 'inputs': [os.path.join(V200520_MINIMAL_BASE, 'feature-8.jar')]
+ }],
+ 'pgconf': [
+ os.path.join(V200520_MINIMAL_BASE, 'proguard.config'),
+ utils.IGNORE_WARNINGS_RULES
+ ],
+ 'libraries': [os.path.join(V200520_MINIMAL_BASE, 'library.jar')],
+ 'min-api': ANDROID_N_API
+ },
},
- },
}
+
def GetLatestVersion():
- return '200520-monochrome_public_minimal_apks'
+ return '200520-monochrome_public_minimal_apks'
+
def GetName():
- return 'chrome'
+ return 'chrome'
+
def GetMemoryData(version):
- assert version == '200520-monochrome_public_minimal_apks'
- return {
- 'find-xmx-min': 600,
- 'find-xmx-max': 700,
- 'find-xmx-range': 16,
- 'oom-threshold': 625,
- }
+ assert version == '200520-monochrome_public_minimal_apks'
+ return {
+ 'find-xmx-min': 600,
+ 'find-xmx-max': 700,
+ 'find-xmx-range': 16,
+ 'oom-threshold': 625,
+ }
diff --git a/tools/compare_apk_sizes.py b/tools/compare_apk_sizes.py
index ad75024..6349689 100755
--- a/tools/compare_apk_sizes.py
+++ b/tools/compare_apk_sizes.py
@@ -21,177 +21,190 @@
USAGE = """%prog [options] app1 app2
NOTE: This only makes sense if minification is disabled"""
-MAX_THREADS=40
+MAX_THREADS = 40
+
def parse_options():
- result = optparse.OptionParser(usage=USAGE)
- result.add_option('--no-build',
- help='Run without building first',
- default=False,
- action='store_true')
- result.add_option('--temp',
- help='Temporary directory to store extracted classes in')
- result.add_option('--use_code_size',
- help='Use the size of code segments instead of the full size of the dex.',
- default=False, action='store_true')
- result.add_option('--report',
- help='Print comparison to this location instead of stdout')
- return result.parse_args()
+ result = optparse.OptionParser(usage=USAGE)
+ result.add_option('--no-build',
+ help='Run without building first',
+ default=False,
+ action='store_true')
+ result.add_option('--temp',
+ help='Temporary directory to store extracted classes in')
+ result.add_option(
+ '--use_code_size',
+ help=
+ 'Use the size of code segments instead of the full size of the dex.',
+ default=False,
+ action='store_true')
+ result.add_option(
+ '--report', help='Print comparison to this location instead of stdout')
+ return result.parse_args()
+
def extract_apk(apk, output):
- if os.path.exists(output):
- shutil.rmtree(output)
- zipfile.ZipFile(apk).extractall(output)
- with utils.ChangedWorkingDirectory(output):
- dex = glob.glob('*.dex')
- return [os.path.join(output, dexfile) for dexfile in dex]
+ if os.path.exists(output):
+ shutil.rmtree(output)
+ zipfile.ZipFile(apk).extractall(output)
+ with utils.ChangedWorkingDirectory(output):
+ dex = glob.glob('*.dex')
+ return [os.path.join(output, dexfile) for dexfile in dex]
+
def ensure_exists(files):
- for f in files:
- if not os.path.exists(f):
- raise Exception('%s does not exist' % f)
+ for f in files:
+ if not os.path.exists(f):
+ raise Exception('%s does not exist' % f)
+
def extract_classes(input, output, options):
- if os.path.exists(output):
- shutil.rmtree(output)
- os.makedirs(output)
- args = ['--file-per-class',
- '--output', output]
- if options.no_build:
- args.extend(['--no-build'])
- args.extend(input)
- if toolhelper.run('d8', args) is not 0:
- raise Exception('Failed running d8')
+ if os.path.exists(output):
+ shutil.rmtree(output)
+ os.makedirs(output)
+ args = ['--file-per-class', '--output', output]
+ if options.no_build:
+ args.extend(['--no-build'])
+ args.extend(input)
+ if toolhelper.run('d8', args) is not 0:
+ raise Exception('Failed running d8')
+
def get_code_size(path):
- segments = toolhelper.run('dexsegments',
- [path],
- build=False,
- return_stdout=True)
- for line in segments.splitlines():
- if 'Code' in line:
- # The code size line looks like:
- # - Code: 264 / 4
- splits = line.split(' ')
- return int(splits[3])
- # Some classes has no code.
- return 0
+ segments = toolhelper.run('dexsegments', [path],
+ build=False,
+ return_stdout=True)
+ for line in segments.splitlines():
+ if 'Code' in line:
+ # The code size line looks like:
+ # - Code: 264 / 4
+ splits = line.split(' ')
+ return int(splits[3])
+ # Some classes has no code.
+ return 0
+
class FileInfo:
- def __init__(self, path, root):
- self.path = path
- self.full_path = os.path.join(root, path)
- def __eq__(self, other):
- return self.full_path == other.full_path
+ def __init__(self, path, root):
+ self.path = path
+ self.full_path = os.path.join(root, path)
- def set_size(self, use_code_size):
- if use_code_size:
- self.size = get_code_size(self.full_path)
- else:
- self.size = os.path.getsize(self.full_path)
+ def __eq__(self, other):
+ return self.full_path == other.full_path
+
+ def set_size(self, use_code_size):
+ if use_code_size:
+ self.size = get_code_size(self.full_path)
+ else:
+ self.size = os.path.getsize(self.full_path)
+
def generate_file_info(path, options):
- file_info_map = {}
- with utils.ChangedWorkingDirectory(path):
- for root, dirs, files in os.walk('.'):
- for f in files:
- assert f.endswith('dex')
- file_path = os.path.join(root, f)
- entry = FileInfo(file_path, path)
- if not options.use_code_size:
- entry.set_size(False)
- file_info_map[file_path] = entry
- threads = []
- file_infos = file_info_map.values() if options.use_code_size else []
- while len(file_infos) > 0 or len(threads)> 0:
- for t in threads:
- if not t.is_alive():
- threads.remove(t)
- # sleep
- if len(threads) == MAX_THREADS or len(file_infos) == 0:
- time.sleep(0.5)
- while len(threads) < MAX_THREADS and len(file_infos) > 0:
- info = file_infos.pop()
- print('Added %s for size calculation' % info.full_path)
- t = threading.Thread(target=info.set_size, args=(options.use_code_size,))
- threads.append(t)
- t.start()
- print('Missing %s files, threads=%s ' % (len(file_infos), len(threads)))
+ file_info_map = {}
+ with utils.ChangedWorkingDirectory(path):
+ for root, dirs, files in os.walk('.'):
+ for f in files:
+ assert f.endswith('dex')
+ file_path = os.path.join(root, f)
+ entry = FileInfo(file_path, path)
+ if not options.use_code_size:
+ entry.set_size(False)
+ file_info_map[file_path] = entry
+ threads = []
+ file_infos = file_info_map.values() if options.use_code_size else []
+ while len(file_infos) > 0 or len(threads) > 0:
+ for t in threads:
+ if not t.is_alive():
+ threads.remove(t)
+ # sleep
+ if len(threads) == MAX_THREADS or len(file_infos) == 0:
+ time.sleep(0.5)
+ while len(threads) < MAX_THREADS and len(file_infos) > 0:
+ info = file_infos.pop()
+ print('Added %s for size calculation' % info.full_path)
+ t = threading.Thread(target=info.set_size,
+ args=(options.use_code_size,))
+ threads.append(t)
+ t.start()
+ print('Missing %s files, threads=%s ' % (len(file_infos), len(threads)))
- return file_info_map
+ return file_info_map
+
def print_info(app, app_files, only_in_app, bigger_in_app, output):
- output.write('Only in %s\n' % app)
- only_app_sorted = sorted(only_in_app,
- key=lambda a: app_files[a].size,
- reverse=True)
- output.write('\n'.join([' %s %s bytes' %
- (x, app_files[x].size) for x in only_app_sorted]))
- output.write('\n\n')
- output.write('Bigger in %s\n' % app)
- # Sort by the percentage diff compared to size
- percent = lambda a: (0.0 + bigger_in_app.get(a))/app_files.get(a).size * 100
- for bigger in sorted(bigger_in_app, key=percent, reverse=True):
- output.write(' {0:.3f}% {1} bytes {2}\n'.format(percent(bigger),
- bigger_in_app[bigger],
- bigger))
- output.write('\n\n')
+ output.write('Only in %s\n' % app)
+ only_app_sorted = sorted(only_in_app,
+ key=lambda a: app_files[a].size,
+ reverse=True)
+ output.write('\n'.join(
+ [' %s %s bytes' % (x, app_files[x].size) for x in only_app_sorted]))
+ output.write('\n\n')
+ output.write('Bigger in %s\n' % app)
+ # Sort by the percentage diff compared to size
+ percent = lambda a: (0.0 + bigger_in_app.get(a)) / app_files.get(a
+ ).size * 100
+ for bigger in sorted(bigger_in_app, key=percent, reverse=True):
+ output.write(' {0:.3f}% {1} bytes {2}\n'.format(
+ percent(bigger), bigger_in_app[bigger], bigger))
+ output.write('\n\n')
def compare(app1_classes_dir, app2_classes_dir, app1, app2, options):
- app1_files = generate_file_info(app1_classes_dir, options)
- app2_files = generate_file_info(app2_classes_dir, options)
- only_in_app1 = [k for k in app1_files if k not in app2_files]
- only_in_app2 = [k for k in app2_files if k not in app1_files]
- in_both = [k for k in app2_files if k in app1_files]
- assert len(app1_files) == len(only_in_app1) + len(in_both)
- assert len(app2_files) == len(only_in_app2) + len(in_both)
- bigger_in_app1 = {}
- bigger_in_app2 = {}
- same_size = []
- for f in in_both:
- app1_entry = app1_files[f]
- app2_entry = app2_files[f]
- if app1_entry.size > app2_entry.size:
- bigger_in_app1[f] = app1_entry.size - app2_entry.size
- elif app2_entry.size > app1_entry.size:
- bigger_in_app2[f] = app2_entry.size - app1_entry.size
- else:
- same_size.append(f)
- output = open(options.report, 'w') if options.report else sys.stdout
- print_info(app1, app1_files, only_in_app1, bigger_in_app1, output)
- print_info(app2, app2_files, only_in_app2, bigger_in_app2, output)
- output.write('Same size\n')
- output.write('\n'.join([' %s' % x for x in same_size]))
- if options.report:
- output.close()
+ app1_files = generate_file_info(app1_classes_dir, options)
+ app2_files = generate_file_info(app2_classes_dir, options)
+ only_in_app1 = [k for k in app1_files if k not in app2_files]
+ only_in_app2 = [k for k in app2_files if k not in app1_files]
+ in_both = [k for k in app2_files if k in app1_files]
+ assert len(app1_files) == len(only_in_app1) + len(in_both)
+ assert len(app2_files) == len(only_in_app2) + len(in_both)
+ bigger_in_app1 = {}
+ bigger_in_app2 = {}
+ same_size = []
+ for f in in_both:
+ app1_entry = app1_files[f]
+ app2_entry = app2_files[f]
+ if app1_entry.size > app2_entry.size:
+ bigger_in_app1[f] = app1_entry.size - app2_entry.size
+ elif app2_entry.size > app1_entry.size:
+ bigger_in_app2[f] = app2_entry.size - app1_entry.size
+ else:
+ same_size.append(f)
+ output = open(options.report, 'w') if options.report else sys.stdout
+ print_info(app1, app1_files, only_in_app1, bigger_in_app1, output)
+ print_info(app2, app2_files, only_in_app2, bigger_in_app2, output)
+ output.write('Same size\n')
+ output.write('\n'.join([' %s' % x for x in same_size]))
+ if options.report:
+ output.close()
+
def Main():
- (options, args) = parse_options()
- if len(args) is not 2:
- print(args)
- print('Takes exactly two arguments, the two apps to compare')
- return 1
- app1 = args[0]
- app2 = args[1]
- ensure_exists([app1, app2])
- with utils.TempDir() as temporary:
- # If a temp dir is passed in, use that instead of the generated temporary
- output = options.temp if options.temp else temporary
- ensure_exists([output])
- app1_input = [app1]
- app2_input = [app2]
- if app1.endswith('apk'):
- app1_input = extract_apk(app1, os.path.join(output, 'app1'))
- if app2.endswith('apk'):
- app2_input = extract_apk(app2, os.path.join(output, 'app2'))
- app1_classes_dir = os.path.join(output, 'app1_classes')
- app2_classes_dir = os.path.join(output, 'app2_classes')
+ (options, args) = parse_options()
+ if len(args) is not 2:
+ print(args)
+ print('Takes exactly two arguments, the two apps to compare')
+ return 1
+ app1 = args[0]
+ app2 = args[1]
+ ensure_exists([app1, app2])
+ with utils.TempDir() as temporary:
+ # If a temp dir is passed in, use that instead of the generated temporary
+ output = options.temp if options.temp else temporary
+ ensure_exists([output])
+ app1_input = [app1]
+ app2_input = [app2]
+ if app1.endswith('apk'):
+ app1_input = extract_apk(app1, os.path.join(output, 'app1'))
+ if app2.endswith('apk'):
+ app2_input = extract_apk(app2, os.path.join(output, 'app2'))
+ app1_classes_dir = os.path.join(output, 'app1_classes')
+ app2_classes_dir = os.path.join(output, 'app2_classes')
- extract_classes(app1_input, app1_classes_dir, options)
- extract_classes(app2_input, app2_classes_dir, options)
- compare(app1_classes_dir, app2_classes_dir, app1, app2, options)
+ extract_classes(app1_input, app1_classes_dir, options)
+ extract_classes(app2_input, app2_classes_dir, options)
+ compare(app1_classes_dir, app2_classes_dir, app1, app2, options)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/compare_cts_results.py b/tools/compare_cts_results.py
index 0da116d..eed3435 100755
--- a/tools/compare_cts_results.py
+++ b/tools/compare_cts_results.py
@@ -13,127 +13,142 @@
import utils
+
class Module:
- def __init__(self):
- self.test_cases = {}
- self.bf_covered_in_file = 0 # bitfield, one bit per file
- def get_test_case_maybe_create(self, test_case_name):
- return self.test_cases.setdefault(test_case_name, TestCase())
+ def __init__(self):
+ self.test_cases = {}
+ self.bf_covered_in_file = 0 # bitfield, one bit per file
- def set_file_index_present(self, file_idx):
- self.bf_covered_in_file |= (1 << file_idx)
+ def get_test_case_maybe_create(self, test_case_name):
+ return self.test_cases.setdefault(test_case_name, TestCase())
- def report(self, module_name, files, diff_only):
- bf_all_files = self.bf_covered_in_file
- for test_case_name, test_case in self.test_cases.iteritems():
- if test_case.bf_covered_in_file != bf_all_files:
- report_missing_thing('test_case', module_name + '/' + test_case_name,
- test_case.bf_covered_in_file, files)
- for test_case_name, test_case in self.test_cases.iteritems():
- test_case.report(module_name, test_case_name, files, diff_only)
+ def set_file_index_present(self, file_idx):
+ self.bf_covered_in_file |= (1 << file_idx)
+
+ def report(self, module_name, files, diff_only):
+ bf_all_files = self.bf_covered_in_file
+ for test_case_name, test_case in self.test_cases.iteritems():
+ if test_case.bf_covered_in_file != bf_all_files:
+ report_missing_thing('test_case',
+ module_name + '/' + test_case_name,
+ test_case.bf_covered_in_file, files)
+ for test_case_name, test_case in self.test_cases.iteritems():
+ test_case.report(module_name, test_case_name, files, diff_only)
+
class TestCase:
- def __init__(self):
- self.tests = {}
- self.bf_covered_in_file = 0 # bitfield, one bit per file
- def get_test_maybe_create(self, test_name):
- return self.tests.setdefault(test_name, Test())
+ def __init__(self):
+ self.tests = {}
+ self.bf_covered_in_file = 0 # bitfield, one bit per file
- def set_file_index_present(self, file_idx):
- self.bf_covered_in_file |= (1 << file_idx)
+ def get_test_maybe_create(self, test_name):
+ return self.tests.setdefault(test_name, Test())
- def report(self, module_name, test_case_name, files, diff_only):
- bf_all_files = self.bf_covered_in_file
- for test_name, test in self.tests.iteritems():
- do_report = test.bf_passing_in_file != bf_all_files
- if diff_only:
- do_report = do_report and test.bf_failing_in_file != bf_all_files
- if do_report:
- test.report(module_name, test_case_name, test_name, files)
+ def set_file_index_present(self, file_idx):
+ self.bf_covered_in_file |= (1 << file_idx)
+
+ def report(self, module_name, test_case_name, files, diff_only):
+ bf_all_files = self.bf_covered_in_file
+ for test_name, test in self.tests.iteritems():
+ do_report = test.bf_passing_in_file != bf_all_files
+ if diff_only:
+ do_report = do_report and test.bf_failing_in_file != bf_all_files
+ if do_report:
+ test.report(module_name, test_case_name, test_name, files)
+
class Test:
- def __init__(self):
- self.bf_failing_in_file = 0 # bitfields, one bit per file
- self.bf_passing_in_file = 0
- def set_file_index_outcome(self, outcome_is_passed, file_idx):
- bf_value = (1 << file_idx)
- if outcome_is_passed:
- self.bf_passing_in_file |= bf_value
- else:
- self.bf_failing_in_file |= bf_value
+ def __init__(self):
+ self.bf_failing_in_file = 0 # bitfields, one bit per file
+ self.bf_passing_in_file = 0
- # Report test's status in all files: pass/fail/missing
- def report(self, module_name, test_case_name, test_name, files):
- print('Test: {}/{}/{}:'.format(module_name, test_case_name, test_name))
- for file_idx, f in enumerate(files):
- bf_value = 1 << file_idx
- print('\t- {:20}'.format(basename(f)), end = '')
- if self.bf_passing_in_file & bf_value:
- print('PASS')
- elif self.bf_failing_in_file & bf_value:
- print(' FAIL')
- else:
- print(' -- -- (missing)')
+ def set_file_index_outcome(self, outcome_is_passed, file_idx):
+ bf_value = (1 << file_idx)
+ if outcome_is_passed:
+ self.bf_passing_in_file |= bf_value
+ else:
+ self.bf_failing_in_file |= bf_value
+
+ # Report test's status in all files: pass/fail/missing
+ def report(self, module_name, test_case_name, test_name, files):
+ print('Test: {}/{}/{}:'.format(module_name, test_case_name, test_name))
+ for file_idx, f in enumerate(files):
+ bf_value = 1 << file_idx
+ print('\t- {:20}'.format(basename(f)), end='')
+ if self.bf_passing_in_file & bf_value:
+ print('PASS')
+ elif self.bf_failing_in_file & bf_value:
+ print(' FAIL')
+ else:
+ print(' -- -- (missing)')
+
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Compare multiple Android CTS test_result.xml files.')
- parser.add_argument('files', nargs = '+',
- help = 'List of (possibly renamed) test_result.xml files')
- parser.add_argument('--diff-only',
- action = 'store_true',
- help = "Don't list tests that consistently fail in all result files,"
- " list only differences.")
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='Compare multiple Android CTS test_result.xml files.')
+ parser.add_argument('files',
+ nargs='+',
+ help='List of (possibly renamed) test_result.xml files')
+ parser.add_argument(
+ '--diff-only',
+ action='store_true',
+ help="Don't list tests that consistently fail in all result files,"
+ " list only differences.")
+ return parser.parse_args()
+
# Read CTS test_result.xml from file and merge into result_tree
def add_to_result_tree(result_tree, file_xml, file_idx):
- module = None
- test_case = None
- for x in utils.read_cts_test_result(file_xml):
- if type(x) is utils.CtsModule:
- module = result_tree.setdefault(x.name, Module())
- module.set_file_index_present(file_idx)
- elif type(x) is utils.CtsTestCase:
- test_case = module.get_test_case_maybe_create(x.name)
- test_case.set_file_index_present(file_idx)
- else:
- assert(type(x) is utils.CtsTest)
- v = test_case.get_test_maybe_create(x.name)
- v.set_file_index_outcome(x.outcome, file_idx)
+ module = None
+ test_case = None
+ for x in utils.read_cts_test_result(file_xml):
+ if type(x) is utils.CtsModule:
+ module = result_tree.setdefault(x.name, Module())
+ module.set_file_index_present(file_idx)
+ elif type(x) is utils.CtsTestCase:
+ test_case = module.get_test_case_maybe_create(x.name)
+ test_case.set_file_index_present(file_idx)
+ else:
+ assert (type(x) is utils.CtsTest)
+ v = test_case.get_test_maybe_create(x.name)
+ v.set_file_index_outcome(x.outcome, file_idx)
+
# main tree_report function
def tree_report(result_tree, files, diff_only):
- bf_all_files = (1 << len(files)) - 1
- for module_name, module in result_tree.iteritems():
- if module.bf_covered_in_file != bf_all_files:
- report_missing_thing('module', module_name, module.bf_covered_in_file,
- files)
- for module_name, module in result_tree.iteritems():
- module.report(module_name, files, diff_only)
+ bf_all_files = (1 << len(files)) - 1
+ for module_name, module in result_tree.iteritems():
+ if module.bf_covered_in_file != bf_all_files:
+ report_missing_thing('module', module_name,
+ module.bf_covered_in_file, files)
+ for module_name, module in result_tree.iteritems():
+ module.report(module_name, files, diff_only)
+
def report_missing_thing(thing_type, thing_name, bf_covered_in_file, files):
- print('Missing {}: {}, from:'.format(thing_type, thing_name))
- for file_idx, f in enumerate(files):
- if not (bf_covered_in_file & (1 << file_idx)):
- print('\t- ' + f)
+ print('Missing {}: {}, from:'.format(thing_type, thing_name))
+ for file_idx, f in enumerate(files):
+ if not (bf_covered_in_file & (1 << file_idx)):
+ print('\t- ' + f)
+
def Main():
- m = Module()
- m.get_test_case_maybe_create('qwe')
+ m = Module()
+ m.get_test_case_maybe_create('qwe')
- args = parse_arguments()
+ args = parse_arguments()
- result_tree = {}
- for file_idx, f in enumerate(args.files):
- add_to_result_tree(result_tree, f, file_idx)
+ result_tree = {}
+ for file_idx, f in enumerate(args.files):
+ add_to_result_tree(result_tree, f, file_idx)
- tree_report(result_tree, args.files, args.diff_only)
+ tree_report(result_tree, args.files, args.diff_only)
- return 0
+ return 0
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/compatdx.py b/tools/compatdx.py
index d58b907..777facd 100755
--- a/tools/compatdx.py
+++ b/tools/compatdx.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('compatdx', sys.argv[1:]))
+ sys.exit(toolhelper.run('compatdx', sys.argv[1:]))
diff --git a/tools/compatproguard.py b/tools/compatproguard.py
index 7c56549..4a2e462 100755
--- a/tools/compatproguard.py
+++ b/tools/compatproguard.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('compatproguard', sys.argv[1:]))
+ sys.exit(toolhelper.run('compatproguard', sys.argv[1:]))
diff --git a/tools/compiledump.py b/tools/compiledump.py
index 255f00b..2d1a3a8 100755
--- a/tools/compiledump.py
+++ b/tools/compiledump.py
@@ -17,683 +17,727 @@
def make_parser():
- parser = argparse.ArgumentParser(description = 'Compile a dump artifact.')
- parser.add_argument(
- '--summary',
- help='List a summary of the contents of the dumps.',
- default=False,
- action='store_true')
- parser.add_argument(
- '-d',
- '--dump',
- help='Dump file or directory to compile',
- default=None)
- parser.add_argument(
- '-o',
- '--output',
- help='File to output (defaults to out.jar in temp)',
- default=None)
- parser.add_argument(
- '--temp',
- help='Temp directory to extract the dump to, allows you to rerun the command'
- ' more easily in the terminal with changes',
- default=None)
- parser.add_argument(
- '-c',
- '--compiler',
- help='Compiler to use',
- default=None)
- parser.add_argument(
- '--minify',
- help='Force enable/disable minification'
- ' (defaults to app proguard config)',
- choices=['default', 'force-enable', 'force-disable'],
- default='default')
- parser.add_argument(
- '--optimize',
- help='Force enable/disable optimizations'
- ' (defaults to app proguard config)',
- choices=['default', 'force-enable', 'force-disable'],
- default='default')
- parser.add_argument(
- '--shrink',
- help='Force enable/disable shrinking'
- ' (defaults to app proguard config)',
- choices=['default', 'force-enable', 'force-disable'],
- default='default')
- parser.add_argument(
- '-v',
- '--version',
- help='Compiler version to use (default read from dump version file).'
- 'Valid arguments are:'
- ' "main" to run from your own tree,'
- ' "source" to run from build classes directly,'
- ' "X.Y.Z" to run a specific version, or'
- ' <hash> to run that hash from main.',
- default=None)
- parser.add_argument(
- '--r8-jar',
- help='Path to an R8 jar.',
- default=None)
- parser.add_argument(
- '--r8-flags', '--r8_flags',
- help='Additional option(s) for the compiler.')
- parser.add_argument(
- '--pg-conf', '--pg_conf',
- help='Keep rule file(s).',
- action='append')
- parser.add_argument(
- '--override',
- help='Do not override any extracted dump in temp-dir',
- default=False,
- action='store_true')
- parser.add_argument(
- '--nolib',
- help='Use the non-lib distribution (default uses the lib distribution)',
- default=False,
- action='store_true')
- parser.add_argument(
- '--print-times',
- help='Print timing information from r8',
- default=False,
- action='store_true')
- parser.add_argument(
- '--disable-assertions', '--disable_assertions', '-da',
- help='Disable Java assertions when running the compiler (default enabled)',
- default=False,
- action='store_true')
- parser.add_argument(
- '--classfile',
- help='Run with classfile output',
- default=False,
- action='store_true')
- parser.add_argument(
- '--debug-agent',
- help='Enable Java debug agent and suspend compilation (default disabled)',
- default=False,
- action='store_true')
- parser.add_argument(
- '--xmx',
- help='Set JVM max heap size (-Xmx)',
- default=None)
- parser.add_argument(
- '--threads',
- help='Set the number of threads to use',
- default=None)
- parser.add_argument(
- '--min-api',
- help='Set min-api (default read from dump properties file)',
- default=None)
- parser.add_argument(
- '--desugared-lib',
- help='Set desugared-library (default set from dump)',
- default=None)
- parser.add_argument(
- '--disable-desugared-lib',
- help='Disable desugared-libary if it will be set from dump',
- default=False,
- action='store_true'
- )
- parser.add_argument(
- '--loop',
- help='Run the compilation in a loop',
- default=False,
- action='store_true')
- parser.add_argument(
- '--enable-missing-library-api-modeling',
- help='Run with api modeling',
- default=False,
- action='store_true')
- parser.add_argument(
- '--android-platform-build',
- help='Run as a platform build',
- default=False,
- action='store_true')
- parser.add_argument(
- '--compilation-mode', '--compilation_mode',
- help='Run compilation in specified mode',
- choices=['debug', 'release'],
- default=None)
- return parser
+ parser = argparse.ArgumentParser(description='Compile a dump artifact.')
+ parser.add_argument('--summary',
+ help='List a summary of the contents of the dumps.',
+ default=False,
+ action='store_true')
+ parser.add_argument('-d',
+ '--dump',
+ help='Dump file or directory to compile',
+ default=None)
+ parser.add_argument('-o',
+ '--output',
+ help='File to output (defaults to out.jar in temp)',
+ default=None)
+ parser.add_argument(
+ '--temp',
+ help=
+ 'Temp directory to extract the dump to, allows you to rerun the command'
+ ' more easily in the terminal with changes',
+ default=None)
+ parser.add_argument('-c',
+ '--compiler',
+ help='Compiler to use',
+ default=None)
+ parser.add_argument('--minify',
+ help='Force enable/disable minification'
+ ' (defaults to app proguard config)',
+ choices=['default', 'force-enable', 'force-disable'],
+ default='default')
+ parser.add_argument('--optimize',
+ help='Force enable/disable optimizations'
+ ' (defaults to app proguard config)',
+ choices=['default', 'force-enable', 'force-disable'],
+ default='default')
+ parser.add_argument('--shrink',
+ help='Force enable/disable shrinking'
+ ' (defaults to app proguard config)',
+ choices=['default', 'force-enable', 'force-disable'],
+ default='default')
+ parser.add_argument(
+ '-v',
+ '--version',
+ help='Compiler version to use (default read from dump version file).'
+ 'Valid arguments are:'
+ ' "main" to run from your own tree,'
+ ' "source" to run from build classes directly,'
+ ' "X.Y.Z" to run a specific version, or'
+ ' <hash> to run that hash from main.',
+ default=None)
+ parser.add_argument('--r8-jar', help='Path to an R8 jar.', default=None)
+ parser.add_argument('--r8-flags',
+ '--r8_flags',
+ help='Additional option(s) for the compiler.')
+ parser.add_argument('--pg-conf',
+ '--pg_conf',
+ help='Keep rule file(s).',
+ action='append')
+ parser.add_argument('--override',
+ help='Do not override any extracted dump in temp-dir',
+ default=False,
+ action='store_true')
+ parser.add_argument(
+ '--nolib',
+ help='Use the non-lib distribution (default uses the lib distribution)',
+ default=False,
+ action='store_true')
+ parser.add_argument('--print-times',
+ help='Print timing information from r8',
+ default=False,
+ action='store_true')
+ parser.add_argument(
+ '--disable-assertions',
+ '--disable_assertions',
+ '-da',
+ help=
+ 'Disable Java assertions when running the compiler (default enabled)',
+ default=False,
+ action='store_true')
+ parser.add_argument('--classfile',
+ help='Run with classfile output',
+ default=False,
+ action='store_true')
+ parser.add_argument(
+ '--debug-agent',
+ help=
+ 'Enable Java debug agent and suspend compilation (default disabled)',
+ default=False,
+ action='store_true')
+ parser.add_argument('--xmx',
+ help='Set JVM max heap size (-Xmx)',
+ default=None)
+ parser.add_argument('--threads',
+ help='Set the number of threads to use',
+ default=None)
+ parser.add_argument(
+ '--min-api',
+ help='Set min-api (default read from dump properties file)',
+ default=None)
+ parser.add_argument('--desugared-lib',
+ help='Set desugared-library (default set from dump)',
+ default=None)
+ parser.add_argument(
+ '--disable-desugared-lib',
+ help='Disable desugared-libary if it will be set from dump',
+ default=False,
+ action='store_true')
+ parser.add_argument('--loop',
+ help='Run the compilation in a loop',
+ default=False,
+ action='store_true')
+ parser.add_argument('--enable-missing-library-api-modeling',
+ help='Run with api modeling',
+ default=False,
+ action='store_true')
+ parser.add_argument('--android-platform-build',
+ help='Run as a platform build',
+ default=False,
+ action='store_true')
+ parser.add_argument('--compilation-mode',
+ '--compilation_mode',
+ help='Run compilation in specified mode',
+ choices=['debug', 'release'],
+ default=None)
+ return parser
+
def error(msg):
- print(msg)
- sys.exit(1)
+ print(msg)
+ sys.exit(1)
+
class Dump(object):
- def __init__(self, directory):
- self.directory = directory
+ def __init__(self, directory):
+ self.directory = directory
- def if_exists(self, name):
- f = os.path.join(self.directory, name)
- if os.path.exists(f):
- return f
- return None
+ def if_exists(self, name):
+ f = os.path.join(self.directory, name)
+ if os.path.exists(f):
+ return f
+ return None
- def program_jar(self):
- return self.if_exists('program.jar')
+ def program_jar(self):
+ return self.if_exists('program.jar')
- def feature_jars(self):
- feature_jars = []
- i = 1
- while True:
- feature_jar = self.if_exists('feature-%s.jar' % i)
- if feature_jar:
- feature_jars.append(feature_jar)
- i = i + 1
- else:
- return feature_jars
+ def feature_jars(self):
+ feature_jars = []
+ i = 1
+ while True:
+ feature_jar = self.if_exists('feature-%s.jar' % i)
+ if feature_jar:
+ feature_jars.append(feature_jar)
+ i = i + 1
+ else:
+ return feature_jars
- def library_jar(self):
- return self.if_exists('library.jar')
+ def library_jar(self):
+ return self.if_exists('library.jar')
- def classpath_jar(self):
- return self.if_exists('classpath.jar')
+ def classpath_jar(self):
+ return self.if_exists('classpath.jar')
- def desugared_library_json(self):
- return self.if_exists('desugared-library.json')
+ def desugared_library_json(self):
+ return self.if_exists('desugared-library.json')
- def proguard_input_map(self):
- if self.if_exists('proguard_input.config'):
- print("Unimplemented: proguard_input configuration.")
+ def proguard_input_map(self):
+ if self.if_exists('proguard_input.config'):
+ print("Unimplemented: proguard_input configuration.")
- def main_dex_list_resource(self):
- return self.if_exists('main-dex-list.txt')
+ def main_dex_list_resource(self):
+ return self.if_exists('main-dex-list.txt')
- def main_dex_rules_resource(self):
- return self.if_exists('main-dex-rules.txt')
+ def main_dex_rules_resource(self):
+ return self.if_exists('main-dex-rules.txt')
- def art_profile_resources(self):
- art_profile_resources = []
- while True:
- current_art_profile_index = len(art_profile_resources) + 1
- art_profile_resource = self.if_exists(
- 'art-profile-%s.txt' % current_art_profile_index)
- if art_profile_resource is None:
- return art_profile_resources
- art_profile_resources.append(art_profile_resource)
+ def art_profile_resources(self):
+ art_profile_resources = []
+ while True:
+ current_art_profile_index = len(art_profile_resources) + 1
+ art_profile_resource = self.if_exists('art-profile-%s.txt' %
+ current_art_profile_index)
+ if art_profile_resource is None:
+ return art_profile_resources
+ art_profile_resources.append(art_profile_resource)
- def startup_profile_resources(self):
- startup_profile_resources = []
- while True:
- current_startup_profile_index = len(startup_profile_resources) + 1
- startup_profile_resource = self.if_exists(
- 'startup-profile-%s.txt' % current_startup_profile_index)
- if startup_profile_resource is None:
- return startup_profile_resources
- startup_profile_resources.append(startup_profile_resource)
+ def startup_profile_resources(self):
+ startup_profile_resources = []
+ while True:
+ current_startup_profile_index = len(startup_profile_resources) + 1
+ startup_profile_resource = self.if_exists(
+ 'startup-profile-%s.txt' % current_startup_profile_index)
+ if startup_profile_resource is None:
+ return startup_profile_resources
+ startup_profile_resources.append(startup_profile_resource)
- def build_properties_file(self):
- return self.if_exists('build.properties')
+ def build_properties_file(self):
+ return self.if_exists('build.properties')
- def config_file(self):
- return self.if_exists('proguard.config')
+ def config_file(self):
+ return self.if_exists('proguard.config')
- def version_file(self):
- return self.if_exists('r8-version')
+ def version_file(self):
+ return self.if_exists('r8-version')
- def version(self):
- f = self.version_file()
- if f:
- return open(f).read().split(' ')[0]
- return None
+ def version(self):
+ f = self.version_file()
+ if f:
+ return open(f).read().split(' ')[0]
+ return None
+
def read_dump_from_args(args, temp):
- if args.dump is None:
- error("A dump file or directory must be specified")
- return read_dump(args.dump, temp, args.override)
+ if args.dump is None:
+ error("A dump file or directory must be specified")
+ return read_dump(args.dump, temp, args.override)
+
def read_dump(dump, temp, override=False):
- if os.path.isdir(dump):
- return Dump(dump)
- dump_file = zipfile.ZipFile(os.path.abspath(dump), 'r')
- r8_version_file = os.path.join(temp, 'r8-version')
+ if os.path.isdir(dump):
+ return Dump(dump)
+ dump_file = zipfile.ZipFile(os.path.abspath(dump), 'r')
+ r8_version_file = os.path.join(temp, 'r8-version')
- if override or not os.path.isfile(r8_version_file):
- dump_file.extractall(temp)
- if not os.path.isfile(r8_version_file):
- error("Did not extract into %s. Either the zip file is invalid or the "
- "dump is missing files" % temp)
- return Dump(temp)
+ if override or not os.path.isfile(r8_version_file):
+ dump_file.extractall(temp)
+ if not os.path.isfile(r8_version_file):
+ error(
+ "Did not extract into %s. Either the zip file is invalid or the "
+ "dump is missing files" % temp)
+ return Dump(temp)
+
def determine_build_properties(args, dump):
- build_properties = {}
- build_properties_file = dump.build_properties_file()
- if build_properties_file:
- with open(build_properties_file) as f:
- build_properties_contents = f.readlines()
- for line in build_properties_contents:
- stripped = line.strip()
- if stripped:
- pair = stripped.split('=')
- build_properties[pair[0]] = pair[1]
- if 'mode' not in build_properties:
- build_properties['mode'] = 'release'
- return build_properties
+ build_properties = {}
+ build_properties_file = dump.build_properties_file()
+ if build_properties_file:
+ with open(build_properties_file) as f:
+ build_properties_contents = f.readlines()
+ for line in build_properties_contents:
+ stripped = line.strip()
+ if stripped:
+ pair = stripped.split('=')
+ build_properties[pair[0]] = pair[1]
+ if 'mode' not in build_properties:
+ build_properties['mode'] = 'release'
+ return build_properties
+
def determine_version(args, dump):
- if args.version is None:
- return dump.version()
- return args.version
+ if args.version is None:
+ return dump.version()
+ return args.version
+
def determine_compiler(args, build_properties):
- compilers = ['d8', 'r8', 'r8full', 'l8', 'l8d8', 'tracereferences']
- compiler = args.compiler
- if not compiler and 'tool' in build_properties:
- compiler = build_properties.get('tool').lower()
- if compiler == 'r8':
- if not 'force-proguard-compatibility' in build_properties:
- error("Unable to determine R8 compiler variant from build.properties."
- " No value for 'force-proguard-compatibility'.")
- if build_properties.get('force-proguard-compatibility').lower() == 'false':
- compiler = compiler + 'full'
- if compiler == 'TraceReferences':
- compiler = build_properties.get('tool').lower()
- if compiler not in compilers:
- error("Unable to determine a compiler to use. Specified %s,"
- " Valid options: %s" % (args.compiler, ', '.join(compilers)))
- return compiler
+ compilers = ['d8', 'r8', 'r8full', 'l8', 'l8d8', 'tracereferences']
+ compiler = args.compiler
+ if not compiler and 'tool' in build_properties:
+ compiler = build_properties.get('tool').lower()
+ if compiler == 'r8':
+ if not 'force-proguard-compatibility' in build_properties:
+ error(
+ "Unable to determine R8 compiler variant from build.properties."
+ " No value for 'force-proguard-compatibility'.")
+ if build_properties.get(
+ 'force-proguard-compatibility').lower() == 'false':
+ compiler = compiler + 'full'
+ if compiler == 'TraceReferences':
+ compiler = build_properties.get('tool').lower()
+ if compiler not in compilers:
+ error("Unable to determine a compiler to use. Specified %s,"
+ " Valid options: %s" % (args.compiler, ', '.join(compilers)))
+ return compiler
+
def determine_trace_references_commands(build_properties, output):
- trace_ref_consumer = build_properties.get('trace_references_consumer')
- if trace_ref_consumer == 'com.android.tools.r8.tracereferences.TraceReferencesCheckConsumer':
- return ["--check"]
- else:
- assert trace_ref_consumer == 'com.android.tools.r8.tracereferences.TraceReferencesKeepRules'
- args = ['--allowobfuscation'] if build_properties.get('minification') == 'true' else []
- args.extend(['--keep-rules', '--output', output])
- return args
+ trace_ref_consumer = build_properties.get('trace_references_consumer')
+ if trace_ref_consumer == 'com.android.tools.r8.tracereferences.TraceReferencesCheckConsumer':
+ return ["--check"]
+ else:
+ assert trace_ref_consumer == 'com.android.tools.r8.tracereferences.TraceReferencesKeepRules'
+ args = ['--allowobfuscation'
+ ] if build_properties.get('minification') == 'true' else []
+ args.extend(['--keep-rules', '--output', output])
+ return args
+
def is_l8_compiler(compiler):
- return compiler.startswith('l8')
+ return compiler.startswith('l8')
+
def is_r8_compiler(compiler):
- return compiler.startswith('r8')
+ return compiler.startswith('r8')
+
def determine_config_files(args, dump, temp):
- if args.pg_conf:
- config_files = []
- for config_file in args.pg_conf:
- dst = os.path.join(temp, 'proguard-%s.config' % len(config_files))
- shutil.copyfile(config_file, dst)
- config_files.append(dst)
- return config_files
- dump_config_file = dump.config_file()
- if dump_config_file:
- return [dump_config_file]
- return []
+ if args.pg_conf:
+ config_files = []
+ for config_file in args.pg_conf:
+ dst = os.path.join(temp, 'proguard-%s.config' % len(config_files))
+ shutil.copyfile(config_file, dst)
+ config_files.append(dst)
+ return config_files
+ dump_config_file = dump.config_file()
+ if dump_config_file:
+ return [dump_config_file]
+ return []
+
def determine_output(args, temp):
- if (args.output):
- return args.output
- return os.path.join(temp, 'out.jar')
+ if (args.output):
+ return args.output
+ return os.path.join(temp, 'out.jar')
+
def determine_min_api(args, build_properties):
- if args.min_api:
- return args.min_api
- if 'min-api' in build_properties:
- return build_properties.get('min-api')
- return None
-
-def determine_residual_art_profile_output(art_profile, temp):
- return os.path.join(temp, os.path.basename(art_profile)[:-4] + ".out.txt")
-
-def determine_desugared_lib_pg_conf_output(temp):
- return os.path.join(temp, 'desugared-library-keep-rules.config')
-
-def determine_feature_output(feature_jar, temp):
- return os.path.join(temp, os.path.basename(feature_jar)[:-4] + ".out.jar")
-
-def determine_program_jar(args, dump):
- if hasattr(args, 'program_jar') and args.program_jar:
- return args.program_jar
- return dump.program_jar()
-
-def determine_class_file(args, build_properties):
- return args.classfile \
- or build_properties.get('backend', 'dex').lower() == 'cf'
-
-def determine_android_platform_build(args, build_properties):
- if args.android_platform_build:
- return True
- return build_properties.get('android-platform-build') == 'true'
-
-def determine_enable_missing_library_api_modeling(args, build_properties):
- if args.enable_missing_library_api_modeling:
- return True
- return build_properties.get('enable-missing-library-api-modeling') == 'true'
-
-def determine_compilation_mode(args, build_properties):
- if args.compilation_mode:
- return args.compilation_mode
- return build_properties.get('mode')
-
-def determine_properties(build_properties):
- args = []
- for key, value in build_properties.items():
- # When writing dumps all system properties starting with com.android.tools.r8
- # are written to the build.properties file in the format
- # system-property-com.android.tools.r8.XXX=<value>
- if key.startswith('system-property-'):
- name = key[len('system-property-'):]
- if name.endswith('dumpinputtofile') or name.endswith('dumpinputtodirectory'):
- continue
- if len(value) == 0:
- args.append('-D' + name)
- else:
- args.append('-D' + name + '=' + value)
- return args
-
-def download_distribution(version, args, temp):
- nolib = args.nolib
- if version == 'main':
- return utils.R8_JAR if nolib else utils.R8LIB_JAR
- if version == 'source':
- return '%s:%s' % (utils.BUILD_JAVA_MAIN_DIR, utils.ALL_DEPS_JAR)
- name = 'r8.jar' if nolib else 'r8lib.jar'
- source = archive.GetUploadDestination(version, name, is_hash(version))
- dest = os.path.join(temp, 'r8.jar')
- utils.download_file_from_cloud_storage(source, dest)
- return dest
-
-def clean_configs(files, args):
- for file in files:
- clean_config(file, args)
-
-def clean_config(file, args):
- with open(file) as f:
- lines = f.readlines()
- minify = args.minify
- optimize = args.optimize
- shrink = args.shrink
- with open(file, 'w') as f:
- if minify == 'force-disable':
- print('Adding config line: -dontobfuscate')
- f.write('-dontobfuscate\n')
- if optimize == 'force-disable':
- print('Adding config line: -dontoptimize')
- f.write('-dontoptimize\n')
- if shrink == 'force-disable':
- print('Adding config line: -dontshrink')
- f.write('-dontshrink\n')
- for line in lines:
- if clean_config_line(line, minify, optimize, shrink):
- print('Removing from config line: \n%s' % line)
- else:
- f.write(line)
-
-def clean_config_line(line, minify, optimize, shrink):
- if line.lstrip().startswith('#'):
- return False
- if ('-injars' in line or '-libraryjars' in line or
- '-print' in line or '-applymapping' in line):
- return True
- if minify == 'force-enable' and '-dontobfuscate' in line:
- return True
- if optimize == 'force-enable' and '-dontoptimize' in line:
- return True
- if shrink == 'force-enable' and '-dontshrink' in line:
- return True
- return False
-
-def prepare_r8_wrapper(dist, temp, jdkhome):
- compile_wrapper_with_javac(
- dist,
- temp,
- jdkhome,
- os.path.join(
- utils.REPO_ROOT,
- 'src/main/java/com/android/tools/r8/utils/CompileDumpCompatR8.java'))
-
-def prepare_d8_wrapper(dist, temp, jdkhome):
- compile_wrapper_with_javac(
- dist,
- temp,
- jdkhome,
- os.path.join(
- utils.REPO_ROOT,
- 'src/main/java/com/android/tools/r8/utils/CompileDumpD8.java'))
-
-def compile_wrapper_with_javac(dist, temp, jdkhome, path):
- base_path = os.path.join(
- utils.REPO_ROOT,
- 'src/main/java/com/android/tools/r8/utils/CompileDumpBase.java')
- cmd = [
- jdk.GetJavacExecutable(jdkhome),
- path,
- base_path,
- '-d', temp,
- '-cp', dist,
- ]
- utils.PrintCmd(cmd)
- subprocess.check_output(cmd)
-
-def is_hash(version):
- return len(version) == 40
-
-def run1(out, args, otherargs, jdkhome=None, worker_id=None):
- jvmargs = []
- compilerargs = []
- for arg in otherargs:
- if arg.startswith('-D'):
- jvmargs.append(arg)
- else:
- compilerargs.append(arg)
- with utils.TempDir() as temp:
- if out:
- temp = out
- if not os.path.exists(temp):
- os.makedirs(temp)
- dump = read_dump_from_args(args, temp)
- if not dump.program_jar():
- error("Cannot compile dump with no program classes")
- if not dump.library_jar():
- print("WARNING: Unexpected lack of library classes in dump")
- build_properties = determine_build_properties(args, dump)
- version = determine_version(args, dump)
- compiler = determine_compiler(args, build_properties)
- config_files = determine_config_files(args, dump, temp)
- out = determine_output(args, temp)
- min_api = determine_min_api(args, build_properties)
- classfile = determine_class_file(args, build_properties)
- android_platform_build = determine_android_platform_build(args, build_properties)
- enable_missing_library_api_modeling = determine_enable_missing_library_api_modeling(args, build_properties)
- mode = determine_compilation_mode(args, build_properties)
- jar = args.r8_jar if args.r8_jar else download_distribution(version, args, temp)
- if ':' not in jar and not os.path.exists(jar):
- error("Distribution does not exist: " + jar)
- cmd = [jdk.GetJavaExecutable(jdkhome)]
- cmd.extend(jvmargs)
- if args.debug_agent:
- if not args.nolib:
- print("WARNING: Running debugging agent on r8lib is questionable...")
- cmd.append(
- '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005')
- if args.xmx:
- cmd.append('-Xmx' + args.xmx)
- if not args.disable_assertions:
- cmd.append('-ea')
- cmd.append('-Dcom.android.tools.r8.enableTestAssertions=1')
- if args.print_times:
- cmd.append('-Dcom.android.tools.r8.printtimes=1')
- if args.r8_flags:
- cmd.extend(args.r8_flags.split(' '))
- if hasattr(args, 'properties'):
- cmd.extend(args.properties)
- cmd.extend(determine_properties(build_properties))
- cmd.extend(['-cp', '%s:%s' % (temp, jar)])
- if compiler == 'd8':
- prepare_d8_wrapper(jar, temp, jdkhome)
- cmd.append('com.android.tools.r8.utils.CompileDumpD8')
- if is_l8_compiler(compiler):
- cmd.append('com.android.tools.r8.L8')
- if compiler == 'tracereferences':
- cmd.append('com.android.tools.r8.tracereferences.TraceReferences')
- cmd.extend(determine_trace_references_commands(build_properties, out))
- if compiler.startswith('r8'):
- prepare_r8_wrapper(jar, temp, jdkhome)
- cmd.append('com.android.tools.r8.utils.CompileDumpCompatR8')
- if compiler == 'r8':
- cmd.append('--compat')
- if compiler != 'tracereferences':
- assert mode == 'debug' or mode == 'release'
- cmd.append('--' + mode)
- # For recompilation of dumps run_on_app_dumps pass in a program jar.
- program_jar = determine_program_jar(args, dump)
- if compiler != 'tracereferences':
- cmd.append(program_jar)
- cmd.extend(['--output', out])
- else:
- cmd.extend(['--source', program_jar])
- for feature_jar in dump.feature_jars():
- cmd.extend(['--feature-jar', feature_jar,
- determine_feature_output(feature_jar, temp)])
- if dump.library_jar():
- cmd.extend(['--lib', dump.library_jar()])
- if dump.classpath_jar() and not is_l8_compiler(compiler):
- cmd.extend(
- ['--target' if compiler == 'tracereferences' else '--classpath',
- dump.classpath_jar()])
- if dump.desugared_library_json() and not args.disable_desugared_lib:
- cmd.extend(['--desugared-lib', dump.desugared_library_json()])
- if not is_l8_compiler(compiler):
- cmd.extend([
- '--desugared-lib-pg-conf-output',
- determine_desugared_lib_pg_conf_output(temp)])
- if (is_r8_compiler(compiler) or compiler == 'l8') and config_files:
- if hasattr(args, 'config_files_consumer') and args.config_files_consumer:
- args.config_files_consumer(config_files)
- else:
- # If we get a dump from the wild we can't use -injars, -libraryjars or
- # -print{mapping,usage}
- clean_configs(config_files, args)
- for config_file in config_files:
- cmd.extend(['--pg-conf', config_file])
- cmd.extend(['--pg-map-output', '%s.map' % out])
- if dump.main_dex_list_resource():
- cmd.extend(['--main-dex-list', dump.main_dex_list_resource()])
- if dump.main_dex_rules_resource():
- cmd.extend(['--main-dex-rules', dump.main_dex_rules_resource()])
- for art_profile_resource in dump.art_profile_resources():
- residual_art_profile_output = \
- determine_residual_art_profile_output(art_profile_resource, temp)
- cmd.extend([
- '--art-profile', art_profile_resource, residual_art_profile_output])
- for startup_profile_resource in dump.startup_profile_resources():
- cmd.extend(['--startup-profile', startup_profile_resource])
- if min_api:
- cmd.extend(['--min-api', min_api])
- if classfile:
- cmd.extend(['--classfile'])
- if android_platform_build:
- cmd.extend(['--android-platform-build'])
- if enable_missing_library_api_modeling:
- cmd.extend(['--enable-missing-library-api-modeling'])
- if args.threads:
- cmd.extend(['--threads', args.threads])
- cmd.extend(compilerargs)
- utils.PrintCmd(cmd, worker_id=worker_id)
- try:
- print(subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('utf-8'))
- return 0
- except subprocess.CalledProcessError as e:
- if args.nolib \
- or version == 'source' \
- or not try_retrace_output(e, version, temp):
- print(e.output.decode('UTF-8'))
- return 1
-
-def try_retrace_output(e, version, temp):
- try:
- stacktrace = os.path.join(temp, 'stacktrace')
- open(stacktrace, 'w+').write(e.output.decode('UTF-8'))
- print("=" * 80)
- print(" RETRACED OUTPUT")
- print("=" * 80)
- retrace.run(get_map_file(version, temp), stacktrace, None, no_r8lib=False)
- return True
- except Exception as e2:
- print("Failed to retrace for version: %s" % version)
- print(e2)
- return False
-
-def get_map_file(version, temp):
- if version == 'main':
- return utils.R8LIB_MAP
- download_path = archive.GetUploadDestination(
- version,
- 'r8lib.jar.map',
- is_hash(version))
- if utils.file_exists_on_cloud_storage(download_path):
- map_path = os.path.join(temp, 'mapping.map')
- utils.download_file_from_cloud_storage(download_path, map_path)
- return map_path
- else:
- print('Could not find map file from argument: %s.' % version)
+ if args.min_api:
+ return args.min_api
+ if 'min-api' in build_properties:
+ return build_properties.get('min-api')
return None
-def summarize_dump_files(dumpfiles):
- if len(dumpfiles) == 0:
- error('Summary command expects a list of dumps to summarize')
- for f in dumpfiles:
- print(f + ':')
+
+def determine_residual_art_profile_output(art_profile, temp):
+ return os.path.join(temp, os.path.basename(art_profile)[:-4] + ".out.txt")
+
+
+def determine_desugared_lib_pg_conf_output(temp):
+ return os.path.join(temp, 'desugared-library-keep-rules.config')
+
+
+def determine_feature_output(feature_jar, temp):
+ return os.path.join(temp, os.path.basename(feature_jar)[:-4] + ".out.jar")
+
+
+def determine_program_jar(args, dump):
+ if hasattr(args, 'program_jar') and args.program_jar:
+ return args.program_jar
+ return dump.program_jar()
+
+
+def determine_class_file(args, build_properties):
+ return args.classfile \
+ or build_properties.get('backend', 'dex').lower() == 'cf'
+
+
+def determine_android_platform_build(args, build_properties):
+ if args.android_platform_build:
+ return True
+ return build_properties.get('android-platform-build') == 'true'
+
+
+def determine_enable_missing_library_api_modeling(args, build_properties):
+ if args.enable_missing_library_api_modeling:
+ return True
+ return build_properties.get('enable-missing-library-api-modeling') == 'true'
+
+
+def determine_compilation_mode(args, build_properties):
+ if args.compilation_mode:
+ return args.compilation_mode
+ return build_properties.get('mode')
+
+
+def determine_properties(build_properties):
+ args = []
+ for key, value in build_properties.items():
+ # When writing dumps all system properties starting with com.android.tools.r8
+ # are written to the build.properties file in the format
+ # system-property-com.android.tools.r8.XXX=<value>
+ if key.startswith('system-property-'):
+ name = key[len('system-property-'):]
+ if name.endswith('dumpinputtofile') or name.endswith(
+ 'dumpinputtodirectory'):
+ continue
+ if len(value) == 0:
+ args.append('-D' + name)
+ else:
+ args.append('-D' + name + '=' + value)
+ return args
+
+
+def download_distribution(version, args, temp):
+ nolib = args.nolib
+ if version == 'main':
+ return utils.R8_JAR if nolib else utils.R8LIB_JAR
+ if version == 'source':
+ return '%s:%s' % (utils.BUILD_JAVA_MAIN_DIR, utils.ALL_DEPS_JAR)
+ name = 'r8.jar' if nolib else 'r8lib.jar'
+ source = archive.GetUploadDestination(version, name, is_hash(version))
+ dest = os.path.join(temp, 'r8.jar')
+ utils.download_file_from_cloud_storage(source, dest)
+ return dest
+
+
+def clean_configs(files, args):
+ for file in files:
+ clean_config(file, args)
+
+
+def clean_config(file, args):
+ with open(file) as f:
+ lines = f.readlines()
+ minify = args.minify
+ optimize = args.optimize
+ shrink = args.shrink
+ with open(file, 'w') as f:
+ if minify == 'force-disable':
+ print('Adding config line: -dontobfuscate')
+ f.write('-dontobfuscate\n')
+ if optimize == 'force-disable':
+ print('Adding config line: -dontoptimize')
+ f.write('-dontoptimize\n')
+ if shrink == 'force-disable':
+ print('Adding config line: -dontshrink')
+ f.write('-dontshrink\n')
+ for line in lines:
+ if clean_config_line(line, minify, optimize, shrink):
+ print('Removing from config line: \n%s' % line)
+ else:
+ f.write(line)
+
+
+def clean_config_line(line, minify, optimize, shrink):
+ if line.lstrip().startswith('#'):
+ return False
+ if ('-injars' in line or '-libraryjars' in line or '-print' in line or
+ '-applymapping' in line):
+ return True
+ if minify == 'force-enable' and '-dontobfuscate' in line:
+ return True
+ if optimize == 'force-enable' and '-dontoptimize' in line:
+ return True
+ if shrink == 'force-enable' and '-dontshrink' in line:
+ return True
+ return False
+
+
+def prepare_r8_wrapper(dist, temp, jdkhome):
+ compile_wrapper_with_javac(
+ dist, temp, jdkhome,
+ os.path.join(
+ utils.REPO_ROOT,
+ 'src/main/java/com/android/tools/r8/utils/CompileDumpCompatR8.java')
+ )
+
+
+def prepare_d8_wrapper(dist, temp, jdkhome):
+ compile_wrapper_with_javac(
+ dist, temp, jdkhome,
+ os.path.join(
+ utils.REPO_ROOT,
+ 'src/main/java/com/android/tools/r8/utils/CompileDumpD8.java'))
+
+
+def compile_wrapper_with_javac(dist, temp, jdkhome, path):
+ base_path = os.path.join(
+ utils.REPO_ROOT,
+ 'src/main/java/com/android/tools/r8/utils/CompileDumpBase.java')
+ cmd = [
+ jdk.GetJavacExecutable(jdkhome),
+ path,
+ base_path,
+ '-d',
+ temp,
+ '-cp',
+ dist,
+ ]
+ utils.PrintCmd(cmd)
+ subprocess.check_output(cmd)
+
+
+def is_hash(version):
+ return len(version) == 40
+
+
+def run1(out, args, otherargs, jdkhome=None, worker_id=None):
+ jvmargs = []
+ compilerargs = []
+ for arg in otherargs:
+ if arg.startswith('-D'):
+ jvmargs.append(arg)
+ else:
+ compilerargs.append(arg)
+ with utils.TempDir() as temp:
+ if out:
+ temp = out
+ if not os.path.exists(temp):
+ os.makedirs(temp)
+ dump = read_dump_from_args(args, temp)
+ if not dump.program_jar():
+ error("Cannot compile dump with no program classes")
+ if not dump.library_jar():
+ print("WARNING: Unexpected lack of library classes in dump")
+ build_properties = determine_build_properties(args, dump)
+ version = determine_version(args, dump)
+ compiler = determine_compiler(args, build_properties)
+ config_files = determine_config_files(args, dump, temp)
+ out = determine_output(args, temp)
+ min_api = determine_min_api(args, build_properties)
+ classfile = determine_class_file(args, build_properties)
+ android_platform_build = determine_android_platform_build(
+ args, build_properties)
+ enable_missing_library_api_modeling = determine_enable_missing_library_api_modeling(
+ args, build_properties)
+ mode = determine_compilation_mode(args, build_properties)
+ jar = args.r8_jar if args.r8_jar else download_distribution(
+ version, args, temp)
+ if ':' not in jar and not os.path.exists(jar):
+ error("Distribution does not exist: " + jar)
+ cmd = [jdk.GetJavaExecutable(jdkhome)]
+ cmd.extend(jvmargs)
+ if args.debug_agent:
+ if not args.nolib:
+ print(
+ "WARNING: Running debugging agent on r8lib is questionable..."
+ )
+ cmd.append(
+ '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+ )
+ if args.xmx:
+ cmd.append('-Xmx' + args.xmx)
+ if not args.disable_assertions:
+ cmd.append('-ea')
+ cmd.append('-Dcom.android.tools.r8.enableTestAssertions=1')
+ if args.print_times:
+ cmd.append('-Dcom.android.tools.r8.printtimes=1')
+ if args.r8_flags:
+ cmd.extend(args.r8_flags.split(' '))
+ if hasattr(args, 'properties'):
+ cmd.extend(args.properties)
+ cmd.extend(determine_properties(build_properties))
+ cmd.extend(['-cp', '%s:%s' % (temp, jar)])
+ if compiler == 'd8':
+ prepare_d8_wrapper(jar, temp, jdkhome)
+ cmd.append('com.android.tools.r8.utils.CompileDumpD8')
+ if is_l8_compiler(compiler):
+ cmd.append('com.android.tools.r8.L8')
+ if compiler == 'tracereferences':
+ cmd.append('com.android.tools.r8.tracereferences.TraceReferences')
+ cmd.extend(
+ determine_trace_references_commands(build_properties, out))
+ if compiler.startswith('r8'):
+ prepare_r8_wrapper(jar, temp, jdkhome)
+ cmd.append('com.android.tools.r8.utils.CompileDumpCompatR8')
+ if compiler == 'r8':
+ cmd.append('--compat')
+ if compiler != 'tracereferences':
+ assert mode == 'debug' or mode == 'release'
+ cmd.append('--' + mode)
+ # For recompilation of dumps run_on_app_dumps pass in a program jar.
+ program_jar = determine_program_jar(args, dump)
+ if compiler != 'tracereferences':
+ cmd.append(program_jar)
+ cmd.extend(['--output', out])
+ else:
+ cmd.extend(['--source', program_jar])
+ for feature_jar in dump.feature_jars():
+ cmd.extend([
+ '--feature-jar', feature_jar,
+ determine_feature_output(feature_jar, temp)
+ ])
+ if dump.library_jar():
+ cmd.extend(['--lib', dump.library_jar()])
+ if dump.classpath_jar() and not is_l8_compiler(compiler):
+ cmd.extend([
+ '--target' if compiler == 'tracereferences' else '--classpath',
+ dump.classpath_jar()
+ ])
+ if dump.desugared_library_json() and not args.disable_desugared_lib:
+ cmd.extend(['--desugared-lib', dump.desugared_library_json()])
+ if not is_l8_compiler(compiler):
+ cmd.extend([
+ '--desugared-lib-pg-conf-output',
+ determine_desugared_lib_pg_conf_output(temp)
+ ])
+ if (is_r8_compiler(compiler) or compiler == 'l8') and config_files:
+ if hasattr(args,
+ 'config_files_consumer') and args.config_files_consumer:
+ args.config_files_consumer(config_files)
+ else:
+ # If we get a dump from the wild we can't use -injars, -libraryjars or
+ # -print{mapping,usage}
+ clean_configs(config_files, args)
+ for config_file in config_files:
+ cmd.extend(['--pg-conf', config_file])
+ cmd.extend(['--pg-map-output', '%s.map' % out])
+ if dump.main_dex_list_resource():
+ cmd.extend(['--main-dex-list', dump.main_dex_list_resource()])
+ if dump.main_dex_rules_resource():
+ cmd.extend(['--main-dex-rules', dump.main_dex_rules_resource()])
+ for art_profile_resource in dump.art_profile_resources():
+ residual_art_profile_output = \
+ determine_residual_art_profile_output(art_profile_resource, temp)
+ cmd.extend([
+ '--art-profile', art_profile_resource,
+ residual_art_profile_output
+ ])
+ for startup_profile_resource in dump.startup_profile_resources():
+ cmd.extend(['--startup-profile', startup_profile_resource])
+ if min_api:
+ cmd.extend(['--min-api', min_api])
+ if classfile:
+ cmd.extend(['--classfile'])
+ if android_platform_build:
+ cmd.extend(['--android-platform-build'])
+ if enable_missing_library_api_modeling:
+ cmd.extend(['--enable-missing-library-api-modeling'])
+ if args.threads:
+ cmd.extend(['--threads', args.threads])
+ cmd.extend(compilerargs)
+ utils.PrintCmd(cmd, worker_id=worker_id)
+ try:
+ print(
+ subprocess.check_output(
+ cmd, stderr=subprocess.STDOUT).decode('utf-8'))
+ return 0
+ except subprocess.CalledProcessError as e:
+ if args.nolib \
+ or version == 'source' \
+ or not try_retrace_output(e, version, temp):
+ print(e.output.decode('UTF-8'))
+ return 1
+
+
+def try_retrace_output(e, version, temp):
try:
- with utils.TempDir() as temp:
- dump = read_dump(f, temp)
- summarize_dump(dump)
- except IOError as e:
- print("Error: " + str(e))
- except zipfile.BadZipfile as e:
- print("Error: " + str(e))
+ stacktrace = os.path.join(temp, 'stacktrace')
+ open(stacktrace, 'w+').write(e.output.decode('UTF-8'))
+ print("=" * 80)
+ print(" RETRACED OUTPUT")
+ print("=" * 80)
+ retrace.run(get_map_file(version, temp),
+ stacktrace,
+ None,
+ no_r8lib=False)
+ return True
+ except Exception as e2:
+ print("Failed to retrace for version: %s" % version)
+ print(e2)
+ return False
+
+
+def get_map_file(version, temp):
+ if version == 'main':
+ return utils.R8LIB_MAP
+ download_path = archive.GetUploadDestination(version, 'r8lib.jar.map',
+ is_hash(version))
+ if utils.file_exists_on_cloud_storage(download_path):
+ map_path = os.path.join(temp, 'mapping.map')
+ utils.download_file_from_cloud_storage(download_path, map_path)
+ return map_path
+ else:
+ print('Could not find map file from argument: %s.' % version)
+ return None
+
+
+def summarize_dump_files(dumpfiles):
+ if len(dumpfiles) == 0:
+ error('Summary command expects a list of dumps to summarize')
+ for f in dumpfiles:
+ print(f + ':')
+ try:
+ with utils.TempDir() as temp:
+ dump = read_dump(f, temp)
+ summarize_dump(dump)
+ except IOError as e:
+ print("Error: " + str(e))
+ except zipfile.BadZipfile as e:
+ print("Error: " + str(e))
+
def summarize_dump(dump):
- version = dump.version()
- if not version:
- print('No dump version info')
- return
- print('version=' + version)
- props = dump.build_properties_file()
- if props:
- with open(props) as props_file:
- print(props_file.read())
- if dump.library_jar():
- print('library.jar present')
- if dump.classpath_jar():
- print('classpath.jar present')
- prog = dump.program_jar()
- if prog:
- print('program.jar content:')
- summarize_jar(prog)
+ version = dump.version()
+ if not version:
+ print('No dump version info')
+ return
+ print('version=' + version)
+ props = dump.build_properties_file()
+ if props:
+ with open(props) as props_file:
+ print(props_file.read())
+ if dump.library_jar():
+ print('library.jar present')
+ if dump.classpath_jar():
+ print('classpath.jar present')
+ prog = dump.program_jar()
+ if prog:
+ print('program.jar content:')
+ summarize_jar(prog)
+
def summarize_jar(jar):
- with zipfile.ZipFile(jar) as zip:
- pkgs = {}
- for info in zip.infolist():
- if info.filename.endswith('.class'):
- pkg, clazz = os.path.split(info.filename)
- count = pkgs.get(pkg, 0)
- pkgs[pkg] = count + 1
- sorted = list(pkgs.keys())
- sorted.sort()
- for p in sorted:
- print(' ' + p + ': ' + str(pkgs[p]))
+ with zipfile.ZipFile(jar) as zip:
+ pkgs = {}
+ for info in zip.infolist():
+ if info.filename.endswith('.class'):
+ pkg, clazz = os.path.split(info.filename)
+ count = pkgs.get(pkg, 0)
+ pkgs[pkg] = count + 1
+ sorted = list(pkgs.keys())
+ sorted.sort()
+ for p in sorted:
+ print(' ' + p + ': ' + str(pkgs[p]))
+
def run(args, otherargs):
- if args.summary:
- summarize_dump_files(otherargs)
- elif args.loop:
- count = 1
- while True:
- print('Iteration {:03d}'.format(count))
- out = args.temp
- if out:
- out = os.path.join(out, '{:03d}'.format(count))
- run1(out, args, otherargs)
- count += 1
- else:
- run1(args.temp, args, otherargs)
+ if args.summary:
+ summarize_dump_files(otherargs)
+ elif args.loop:
+ count = 1
+ while True:
+ print('Iteration {:03d}'.format(count))
+ out = args.temp
+ if out:
+ out = os.path.join(out, '{:03d}'.format(count))
+ run1(out, args, otherargs)
+ count += 1
+ else:
+ run1(args.temp, args, otherargs)
+
if __name__ == '__main__':
- (args, otherargs) = make_parser().parse_known_args(sys.argv[1:])
- sys.exit(run(args, otherargs))
+ (args, otherargs) = make_parser().parse_known_args(sys.argv[1:])
+ sys.exit(run(args, otherargs))
diff --git a/tools/create_art_tests.py b/tools/create_art_tests.py
index 0c59ae9..99e9085 100755
--- a/tools/create_art_tests.py
+++ b/tools/create_art_tests.py
@@ -15,7 +15,7 @@
]
TOOLS = ["r8", "d8", "r8cf"]
TEMPLATE = Template(
-"""// Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
+ """// Copyright (c) 2016, the R8 project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.art.$testGeneratingToolchain.$compilerUnderTest;
@@ -61,43 +61,47 @@
def get_test_configurations():
- for toolchain, source_dir in TOOLCHAINS:
- for tool in TOOLS:
- if tool == "d8" and toolchain == "none":
- tool_enum = 'R8_AFTER_D8'
- else:
- tool_enum = tool.upper()
- if tool == "r8cf":
- if toolchain != "none":
- continue
- tool_enum = 'D8_AFTER_R8CF'
- output_dir = os.path.join(OUTPUT_DIR, toolchain, tool)
- yield (tool_enum, tool, toolchain, source_dir, output_dir)
+ for toolchain, source_dir in TOOLCHAINS:
+ for tool in TOOLS:
+ if tool == "d8" and toolchain == "none":
+ tool_enum = 'R8_AFTER_D8'
+ else:
+ tool_enum = tool.upper()
+ if tool == "r8cf":
+ if toolchain != "none":
+ continue
+ tool_enum = 'D8_AFTER_R8CF'
+ output_dir = os.path.join(OUTPUT_DIR, toolchain, tool)
+ yield (tool_enum, tool, toolchain, source_dir, output_dir)
def create_tests():
- for tool_enum, tool, toolchain, source_dir, output_dir in get_test_configurations():
- test_cases = [d for d in os.listdir(source_dir)
- if os.path.isdir(os.path.join(source_dir, d))]
- if os.path.exists(output_dir):
- shutil.rmtree(output_dir)
- os.makedirs(output_dir)
- for test_case in test_cases:
- class_name = "Art" + test_case.replace("-", "_") + "Test"
- contents = TEMPLATE.substitute(
- name=test_case,
- compilerUnderTestEnum=tool_enum,
- compilerUnderTest=tool,
- testGeneratingToolchain=toolchain,
- testGeneratingToolchainEnum=toolchain.upper(),
- testClassName=class_name)
- with open(os.path.join(output_dir, class_name + ".java"), "w") as fp:
- fp.write(contents)
+ for tool_enum, tool, toolchain, source_dir, output_dir in get_test_configurations(
+ ):
+ test_cases = [
+ d for d in os.listdir(source_dir)
+ if os.path.isdir(os.path.join(source_dir, d))
+ ]
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+ os.makedirs(output_dir)
+ for test_case in test_cases:
+ class_name = "Art" + test_case.replace("-", "_") + "Test"
+ contents = TEMPLATE.substitute(
+ name=test_case,
+ compilerUnderTestEnum=tool_enum,
+ compilerUnderTest=tool,
+ testGeneratingToolchain=toolchain,
+ testGeneratingToolchainEnum=toolchain.upper(),
+ testClassName=class_name)
+ with open(os.path.join(output_dir, class_name + ".java"),
+ "w") as fp:
+ fp.write(contents)
def main():
- create_tests()
+ create_tests()
if __name__ == "__main__":
- main()
+ main()
diff --git a/tools/create_dx_replay.py b/tools/create_dx_replay.py
index 8dfb0a0..1648eb0 100755
--- a/tools/create_dx_replay.py
+++ b/tools/create_dx_replay.py
@@ -27,61 +27,66 @@
import utils
-IN_SUBDIR = 'in' # subdirectory for the local copy of the input files
-OUT_SUBDIR = 'out' # subdirectory prefix for the output of DX
+IN_SUBDIR = 'in' # subdirectory for the local copy of the input files
+OUT_SUBDIR = 'out' # subdirectory prefix for the output of DX
REPLAY_SCRIPT_NAME = 'replay_script.py'
+
# This function will be called with arguments of the original DX invocation. It
# copies the original input files into the local input directory and replaces
# the references in orig_args to the local input files.
# Returns the new line to be appended to the replay script.
def process_line(out_dir, input_counter, orig_args):
- args = []
- inputs = []
- for arg in orig_args:
- if arg.startswith('--output='):
- continue # nothing to do, just skip this arg
- if arg.startswith('--'):
- args.append(arg)
- else:
- # 'arg' is the path of an input file: copy arg to local dir with
- # a new, unique name
- if isdir(arg):
- raise IOError("Adding directories ('{}') to the replay script is not"
- " implemented.".format(arg))
- elif not exists(arg):
- print("The input file to DX does not exist: '{}'.".format(arg))
+ args = []
+ inputs = []
+ for arg in orig_args:
+ if arg.startswith('--output='):
+ continue # nothing to do, just skip this arg
+ if arg.startswith('--'):
+ args.append(arg)
+ else:
+ # 'arg' is the path of an input file: copy arg to local dir with
+ # a new, unique name
+ if isdir(arg):
+ raise IOError(
+ "Adding directories ('{}') to the replay script is not"
+ " implemented.".format(arg))
+ elif not exists(arg):
+ print("The input file to DX does not exist: '{}'.".format(arg))
- input_file = '{}_{}'.format(input_counter, basename(arg))
+ input_file = '{}_{}'.format(input_counter, basename(arg))
- copy2(arg, join(out_dir, join(IN_SUBDIR, input_file)))
- inputs.append(input_file)
+ copy2(arg, join(out_dir, join(IN_SUBDIR, input_file)))
+ inputs.append(input_file)
- return 'call_dx({}, {}, {})\n'.format(input_counter, args, inputs)
+ return 'call_dx({}, {}, {})\n'.format(input_counter, args, inputs)
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Creates a self-contained directory for playing back a '
- ' sequence of DX calls.')
- parser.add_argument('dx_call_log',
- help = 'File containing tab-separated arguments for a DX call on each'
- ' line.')
- parser.add_argument('output_dir',
- help = 'Target path the create the self-contained directory at.')
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='Creates a self-contained directory for playing back a '
+ ' sequence of DX calls.')
+ parser.add_argument(
+ 'dx_call_log',
+ help='File containing tab-separated arguments for a DX call on each'
+ ' line.')
+ parser.add_argument(
+ 'output_dir',
+ help='Target path the create the self-contained directory at.')
+ return parser.parse_args()
+
def Main():
- args = parse_arguments()
+ args = parse_arguments()
- if isdir(args.output_dir):
- rmdir(args.output_dir) # make sure to write only to empty out dir
+ if isdir(args.output_dir):
+ rmdir(args.output_dir) # make sure to write only to empty out dir
- utils.makedirs_if_needed(join(args.output_dir, IN_SUBDIR))
+ utils.makedirs_if_needed(join(args.output_dir, IN_SUBDIR))
- # create the first lines of the replay script
- replay_script = \
-"""#!/usr/bin/env python3
+ # create the first lines of the replay script
+ replay_script = \
+ """#!/usr/bin/env python3
import os
import shutil
import subprocess
@@ -108,22 +113,24 @@
""".format(IN_SUBDIR, OUT_SUBDIR)
- with open(args.dx_call_log) as f:
- lines = f.read().splitlines()
+ with open(args.dx_call_log) as f:
+ lines = f.read().splitlines()
- input_counter = 1
- for line in lines:
- replay_script += \
- process_line(args.output_dir, input_counter, line.split('\t'))
- input_counter += 1
+ input_counter = 1
+ for line in lines:
+ replay_script += \
+ process_line(args.output_dir, input_counter, line.split('\t'))
+ input_counter += 1
- script_file = join(args.output_dir, REPLAY_SCRIPT_NAME)
- with open(script_file, 'w') as f:
- f.write(replay_script)
+ script_file = join(args.output_dir, REPLAY_SCRIPT_NAME)
+ with open(script_file, 'w') as f:
+ f.write(replay_script)
- # chmod +x for script_file
- st = os.stat(script_file)
- os.chmod(script_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+ # chmod +x for script_file
+ st = os.stat(script_file)
+ os.chmod(script_file,
+ st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/create_local_maven_with_dependencies.py b/tools/create_local_maven_with_dependencies.py
index 76caf6b..4ae9d25 100755
--- a/tools/create_local_maven_with_dependencies.py
+++ b/tools/create_local_maven_with_dependencies.py
@@ -15,9 +15,9 @@
# prefix with X- to control the order, as many dependencies are present
# in several repositories.
REPOSITORIES = [
- 'A-Google=https://maven.google.com/',
- 'B-Maven Central=https://repo1.maven.org/maven2/',
- "C-Gradle Plugins=https://plugins.gradle.org/m2/",
+ 'A-Google=https://maven.google.com/',
+ 'B-Maven Central=https://repo1.maven.org/maven2/',
+ "C-Gradle Plugins=https://plugins.gradle.org/m2/",
]
ANDRDID_SUPPORT_VERSION = '25.4.0'
@@ -42,125 +42,134 @@
STUDIO_SDK_VERSION = '31.2.0-alpha10'
BUILD_DEPENDENCIES = [
- 'com.google.code.gson:gson:{version}'.format(version = GSON_VERSION),
- 'com.google.guava:guava:{version}'.format(version = GUAVA_VERSION),
- 'it.unimi.dsi:fastutil:{version}'.format(version = FASTUTIL_VERSION),
- 'org.jetbrains.kotlinx:kotlinx-metadata-jvm:{version}'.format(version = KOTLIN_METADATA_VERSION),
- 'org.ow2.asm:asm:{version}'.format(version = ASM_VERSION),
- 'org.ow2.asm:asm-util:{version}'.format(version = ASM_VERSION),
- 'org.ow2.asm:asm-commons:{version}'.format(version = ASM_VERSION),
+ 'com.google.code.gson:gson:{version}'.format(version=GSON_VERSION),
+ 'com.google.guava:guava:{version}'.format(version=GUAVA_VERSION),
+ 'it.unimi.dsi:fastutil:{version}'.format(version=FASTUTIL_VERSION),
+ 'org.jetbrains.kotlinx:kotlinx-metadata-jvm:{version}'.format(
+ version=KOTLIN_METADATA_VERSION),
+ 'org.ow2.asm:asm:{version}'.format(version=ASM_VERSION),
+ 'org.ow2.asm:asm-util:{version}'.format(version=ASM_VERSION),
+ 'org.ow2.asm:asm-commons:{version}'.format(version=ASM_VERSION),
]
TEST_DEPENDENCIES = [
- 'junit:junit:{version}'.format(version = JUNIT_VERSION),
- 'com.android.tools.smali:smali:{version}'.format(version = SMALI_VERSION),
- 'com.android.tools.smali:smali-util:{version}'.format(version = SMALI_VERSION),
- 'com.google.errorprone:error_prone_core:{version}'.format(version = ERROR_PRONE_VERSION),
- 'org.javassist:javassist:{version}'.format(version = JAVASSIST_VERSION),
- 'org.jetbrains.kotlin:kotlin-stdlib:{version}'.format(version = KOTLIN_VERSION),
- 'org.jetbrains.kotlin:kotlin-reflect:{version}'.format(version = KOTLIN_VERSION),
- 'org.mockito:mockito-core:{version}'.format(version = MOCKITO_VERSION),
- 'org.testng:testng:{version}'.format(version = TESTNG_VERSION),
+ 'junit:junit:{version}'.format(version=JUNIT_VERSION),
+ 'com.android.tools.smali:smali:{version}'.format(version=SMALI_VERSION),
+ 'com.android.tools.smali:smali-util:{version}'.format(
+ version=SMALI_VERSION),
+ 'com.google.errorprone:error_prone_core:{version}'.format(
+ version=ERROR_PRONE_VERSION),
+ 'org.javassist:javassist:{version}'.format(version=JAVASSIST_VERSION),
+ 'org.jetbrains.kotlin:kotlin-stdlib:{version}'.format(
+ version=KOTLIN_VERSION),
+ 'org.jetbrains.kotlin:kotlin-reflect:{version}'.format(
+ version=KOTLIN_VERSION),
+ 'org.mockito:mockito-core:{version}'.format(version=MOCKITO_VERSION),
+ 'org.testng:testng:{version}'.format(version=TESTNG_VERSION),
]
NEW_DEPENDENCIES = [
- 'com.google.guava:guava:{version}'.format(version = GUAVA_VERSION_NEW),
- 'org.gradle.kotlin.kotlin-dsl:org.gradle.kotlin.kotlin-dsl.gradle.plugin:4.0.6',
- 'org.jetbrains.kotlin:kotlin-gradle-plugin-api:1.8.10',
- 'org.jetbrains.kotlin:kotlin-gradle-plugin-idea:1.8.10',
- 'org.jetbrains.kotlin:kotlin-reflect:1.6.10',
- 'org.jetbrains.kotlin:kotlin-reflect:1.8.10',
- 'org.jetbrains.kotlin:kotlin-script-runtime:1.8.10',
- 'org.jetbrains.kotlin:kotlin-tooling-core:1.8.10',
- 'net.ltgt.errorprone:net.ltgt.errorprone.gradle.plugin:3.0.1',
- 'com.google.errorprone:javac:9+181-r4173-1',
- # Gradle 8.3
- 'org.gradle.kotlin.kotlin-dsl:org.gradle.kotlin.kotlin-dsl.gradle.plugin:4.1.0',
- 'org.jetbrains.kotlin:kotlin-assignment-compiler-plugin-embeddable:1.9.0',
- 'org.jetbrains.kotlin:kotlin-gradle-plugin-api:1.9.0',
- 'org.jetbrains.kotlin:kotlin-reflect:1.9.0',
- 'org.jetbrains.kotlin:kotlin-script-runtime:1.9.0',
- 'org.jetbrains.kotlin:kotlin-sam-with-receiver-compiler-plugin-embeddable:1.9.0',
- # Resource shrinker
- 'com.android.tools.build:aapt2-proto:{version}'.format(version = AAPT2_PROTO_VERSION),
- 'com.android.tools.layoutlib:layoutlib-api:{version}'.format(version = STUDIO_SDK_VERSION),
- 'com.android.tools:common:{version}'.format(version = STUDIO_SDK_VERSION),
- 'com.android.tools:sdk-common:{version}'.format(version = STUDIO_SDK_VERSION),
- 'com.google.protobuf:protobuf-java:{version}'.format(version = PROTOBUF_VERSION),
+ 'com.google.guava:guava:{version}'.format(version=GUAVA_VERSION_NEW),
+ 'org.gradle.kotlin.kotlin-dsl:org.gradle.kotlin.kotlin-dsl.gradle.plugin:4.0.6',
+ 'org.jetbrains.kotlin:kotlin-gradle-plugin-api:1.8.10',
+ 'org.jetbrains.kotlin:kotlin-gradle-plugin-idea:1.8.10',
+ 'org.jetbrains.kotlin:kotlin-reflect:1.6.10',
+ 'org.jetbrains.kotlin:kotlin-reflect:1.8.10',
+ 'org.jetbrains.kotlin:kotlin-script-runtime:1.8.10',
+ 'org.jetbrains.kotlin:kotlin-tooling-core:1.8.10',
+ 'net.ltgt.errorprone:net.ltgt.errorprone.gradle.plugin:3.0.1',
+ 'com.google.errorprone:javac:9+181-r4173-1',
+ # Gradle 8.3
+ 'org.gradle.kotlin.kotlin-dsl:org.gradle.kotlin.kotlin-dsl.gradle.plugin:4.1.0',
+ 'org.jetbrains.kotlin:kotlin-assignment-compiler-plugin-embeddable:1.9.0',
+ 'org.jetbrains.kotlin:kotlin-gradle-plugin-api:1.9.0',
+ 'org.jetbrains.kotlin:kotlin-reflect:1.9.0',
+ 'org.jetbrains.kotlin:kotlin-script-runtime:1.9.0',
+ 'org.jetbrains.kotlin:kotlin-sam-with-receiver-compiler-plugin-embeddable:1.9.0',
+ # Resource shrinker
+ 'com.android.tools.build:aapt2-proto:{version}'.format(
+ version=AAPT2_PROTO_VERSION),
+ 'com.android.tools.layoutlib:layoutlib-api:{version}'.format(
+ version=STUDIO_SDK_VERSION),
+ 'com.android.tools:common:{version}'.format(version=STUDIO_SDK_VERSION),
+ 'com.android.tools:sdk-common:{version}'.format(version=STUDIO_SDK_VERSION),
+ 'com.google.protobuf:protobuf-java:{version}'.format(
+ version=PROTOBUF_VERSION),
]
+
def dependencies_tar(dependencies_path):
- return os.path.join(
- os.path.dirname(dependencies_path),
- os.path.basename(dependencies_path) + '.tar.gz')
+ return os.path.join(os.path.dirname(dependencies_path),
+ os.path.basename(dependencies_path) + '.tar.gz')
+
def dependencies_tar_sha1(dependencies_path):
- return os.path.join(
- os.path.dirname(dependencies_path),
- os.path.basename(dependencies_path) + '.tar.gz.sha1')
+ return os.path.join(os.path.dirname(dependencies_path),
+ os.path.basename(dependencies_path) + '.tar.gz.sha1')
+
def remove_local_maven_repository(dependencies_path):
- if os.path.exists(dependencies_path):
- shutil.rmtree(dependencies_path)
- tar = dependencies_tar(dependencies_path)
- if os.path.exists(tar):
- os.remove(tar)
- sha1 = dependencies_tar_sha1(dependencies_path)
- if os.path.exists(sha1):
- os.remove(sha1)
+ if os.path.exists(dependencies_path):
+ shutil.rmtree(dependencies_path)
+ tar = dependencies_tar(dependencies_path)
+ if os.path.exists(tar):
+ os.remove(tar)
+ sha1 = dependencies_tar_sha1(dependencies_path)
+ if os.path.exists(sha1):
+ os.remove(sha1)
-def create_local_maven_repository(args, dependencies_path, repositories, dependencies):
- with utils.ChangedWorkingDirectory(args.studio):
- cmd = [
- os.path.join('tools', 'base', 'bazel', 'bazel'),
- 'run',
- '//tools/base/bazel:local_maven_repository_generator_cli',
- '--',
- '--repo-path',
- dependencies_path,
- '--fetch']
- for repository in repositories:
- cmd.extend(['--remote-repo', repository])
- for dependency in dependencies:
- cmd.append(dependency)
- subprocess.check_call(cmd)
+
+def create_local_maven_repository(args, dependencies_path, repositories,
+ dependencies):
+ with utils.ChangedWorkingDirectory(args.studio):
+ cmd = [
+ os.path.join('tools', 'base', 'bazel', 'bazel'), 'run',
+ '//tools/base/bazel:local_maven_repository_generator_cli', '--',
+ '--repo-path', dependencies_path, '--fetch'
+ ]
+ for repository in repositories:
+ cmd.extend(['--remote-repo', repository])
+ for dependency in dependencies:
+ cmd.append(dependency)
+ subprocess.check_call(cmd)
+
def parse_options():
- result = argparse.ArgumentParser(
- description='Create local Maven repository woth dependencies')
- result.add_argument('--studio',
- metavar=('<path>'),
- required=True,
- help='Path to a studio-main checkout (to get the tool '
- '//tools/base/bazel:local_maven_repository_generator_cli)')
- return result.parse_args()
+ result = argparse.ArgumentParser(
+ description='Create local Maven repository woth dependencies')
+ result.add_argument(
+ '--studio',
+ metavar=('<path>'),
+ required=True,
+ help='Path to a studio-main checkout (to get the tool '
+ '//tools/base/bazel:local_maven_repository_generator_cli)')
+ return result.parse_args()
def main():
- args = parse_options()
+ args = parse_options()
- dependencies_path = os.path.join(utils.THIRD_PARTY, 'dependencies')
- print("Downloading to " + dependencies_path)
- remove_local_maven_repository(dependencies_path)
- create_local_maven_repository(
- args, dependencies_path, REPOSITORIES, BUILD_DEPENDENCIES + TEST_DEPENDENCIES)
+ dependencies_path = os.path.join(utils.THIRD_PARTY, 'dependencies')
+ print("Downloading to " + dependencies_path)
+ remove_local_maven_repository(dependencies_path)
+ create_local_maven_repository(args, dependencies_path, REPOSITORIES,
+ BUILD_DEPENDENCIES + TEST_DEPENDENCIES)
- dependencies_new_path = os.path.join(utils.THIRD_PARTY, 'dependencies_new')
- print("Downloading to " + dependencies_new_path)
- remove_local_maven_repository(dependencies_new_path)
- create_local_maven_repository(
- args, dependencies_new_path, REPOSITORIES, NEW_DEPENDENCIES)
+ dependencies_new_path = os.path.join(utils.THIRD_PARTY, 'dependencies_new')
+ print("Downloading to " + dependencies_new_path)
+ remove_local_maven_repository(dependencies_new_path)
+ create_local_maven_repository(args, dependencies_new_path, REPOSITORIES,
+ NEW_DEPENDENCIES)
- print("Uploading to Google Cloud Storage:")
- with utils.ChangedWorkingDirectory(utils.THIRD_PARTY):
- for dependency in ['dependencies', 'dependencies_new']:
- cmd = [
- 'upload_to_google_storage.py',
- '-a',
- '--bucket',
- 'r8-deps',
- dependency]
- subprocess.check_call(cmd)
+ print("Uploading to Google Cloud Storage:")
+ with utils.ChangedWorkingDirectory(utils.THIRD_PARTY):
+ for dependency in ['dependencies', 'dependencies_new']:
+ cmd = [
+ 'upload_to_google_storage.py', '-a', '--bucket', 'r8-deps',
+ dependency
+ ]
+ subprocess.check_call(cmd)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/create_maven_release.py b/tools/create_maven_release.py
index ac7aa85..d90ea42 100755
--- a/tools/create_maven_release.py
+++ b/tools/create_maven_release.py
@@ -20,16 +20,14 @@
import gradle
import utils
-LICENSETEMPLATE = Template(
-"""
+LICENSETEMPLATE = Template("""
<license>
<name>$name</name>
<url>$url</url>
<distribution>repo</distribution>
</license>""")
-R8_POMTEMPLATE = Template(
-"""<project
+R8_POMTEMPLATE = Template("""<project
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -68,8 +66,7 @@
</project>
""")
-DESUGAR_CONFIGUATION_POMTEMPLATE = Template(
-"""<project
+DESUGAR_CONFIGUATION_POMTEMPLATE = Template("""<project
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -106,174 +103,189 @@
</project>
""")
+
def parse_options(argv):
- result = argparse.ArgumentParser()
- result.add_argument('--out', help='The zip file to output')
- group = result.add_mutually_exclusive_group()
- group.add_argument('--desugar-configuration', action='store_true',
- help='Build desugar library configuration (original JDK-8)')
- group.add_argument('--desugar-configuration-jdk8', action='store_true',
- help='Build desugar library configuration (original JDK-8)')
- group.add_argument('--desugar-configuration-jdk11-legacy', action='store_true',
- help='Build desugar library configuration (JDK-11 legacy)')
- group.add_argument('--desugar-configuration-jdk11-minimal', action='store_true',
- help='Build desugar library configuration (JDK-11 minimal)')
- group.add_argument('--desugar-configuration-jdk11', action='store_true',
- help='Build desugar library configuration (JDK-11)')
- group.add_argument('--desugar-configuration-jdk11-nio', action='store_true',
- help='Build desugar library configuration (JDK-11 nio)')
- return result.parse_args(argv)
+ result = argparse.ArgumentParser()
+ result.add_argument('--out', help='The zip file to output')
+ group = result.add_mutually_exclusive_group()
+ group.add_argument(
+ '--desugar-configuration',
+ action='store_true',
+ help='Build desugar library configuration (original JDK-8)')
+ group.add_argument(
+ '--desugar-configuration-jdk8',
+ action='store_true',
+ help='Build desugar library configuration (original JDK-8)')
+ group.add_argument(
+ '--desugar-configuration-jdk11-legacy',
+ action='store_true',
+ help='Build desugar library configuration (JDK-11 legacy)')
+ group.add_argument(
+ '--desugar-configuration-jdk11-minimal',
+ action='store_true',
+ help='Build desugar library configuration (JDK-11 minimal)')
+ group.add_argument('--desugar-configuration-jdk11',
+ action='store_true',
+ help='Build desugar library configuration (JDK-11)')
+ group.add_argument('--desugar-configuration-jdk11-nio',
+ action='store_true',
+ help='Build desugar library configuration (JDK-11 nio)')
+ return result.parse_args(argv)
+
def determine_version():
- version_file = join(
- utils.SRC_ROOT, 'com', 'android', 'tools', 'r8', 'Version.java')
- with open(version_file, 'r') as file:
- for line in file:
- if 'final String LABEL ' in line:
- result = line[line.find('"') + 1:]
- result = result[:result.find('"')]
- return result
- raise Exception('Unable to determine version.')
+ version_file = join(utils.SRC_ROOT, 'com', 'android', 'tools', 'r8',
+ 'Version.java')
+ with open(version_file, 'r') as file:
+ for line in file:
+ if 'final String LABEL ' in line:
+ result = line[line.find('"') + 1:]
+ result = result[:result.find('"')]
+ return result
+ raise Exception('Unable to determine version.')
+
def generate_library_licenses():
- artifact_prefix = '- artifact: '
- license_prefix = 'license: '
- licenses = []
- license_url_prefix = 'licenseUrl: '
- license_urls = []
- # The ./LIBRARY-LICENSE file is a simple yaml file, which for each dependency
- # has the following information:
- #
- # - artifact: <maven artifact> // in the form <group-id>:<artifact-id>:+
- # name: <name of dependency>
- # copyrightHolder: <name of copyright holder>
- # license: <license name>
- # licenseUrl: <url to license test>
- #
- # E.g. for Guava:
- #
- # - artifact: com.google.guava:guava:+
- # name: Guava Google Core Libraries for Java
- # copyrightHolder: The Guava Authors
- # license: The Apache Software License, Version 2.0
- # licenseUrl: http://www.apache.org/licenses/LICENSE-2.0.txt
- #
- # This file should always be up to date as the build will fail if it
- # is does not have information for all dependencies.
- with open('LIBRARY-LICENSE', 'r') as file:
- name = None
- url = None
- for line in file:
- trimmed = line.strip()
- # Collect license name and url for each artifact. They must come in
- # pairs for each artifact.
- if trimmed.startswith(artifact_prefix):
- assert not name
- assert not url
- if trimmed.startswith(license_prefix):
- name = trimmed[len(license_prefix):]
- if trimmed.startswith(license_url_prefix):
- url = trimmed[len(license_url_prefix):]
- # Licenses come in name/url pairs. When both are present add pair
- # to collected licenses if either name or url has not been recorded yet,
- # as some licenses with slightly different names point to the same url.
- if name and url:
- if (not name in licenses) or (not url in license_urls):
- licenses.append(name)
- license_urls.append(url)
+ artifact_prefix = '- artifact: '
+ license_prefix = 'license: '
+ licenses = []
+ license_url_prefix = 'licenseUrl: '
+ license_urls = []
+ # The ./LIBRARY-LICENSE file is a simple yaml file, which for each dependency
+ # has the following information:
+ #
+ # - artifact: <maven artifact> // in the form <group-id>:<artifact-id>:+
+ # name: <name of dependency>
+ # copyrightHolder: <name of copyright holder>
+ # license: <license name>
+ # licenseUrl: <url to license test>
+ #
+ # E.g. for Guava:
+ #
+ # - artifact: com.google.guava:guava:+
+ # name: Guava Google Core Libraries for Java
+ # copyrightHolder: The Guava Authors
+ # license: The Apache Software License, Version 2.0
+ # licenseUrl: http://www.apache.org/licenses/LICENSE-2.0.txt
+ #
+ # This file should always be up to date as the build will fail if it
+ # is does not have information for all dependencies.
+ with open('LIBRARY-LICENSE', 'r') as file:
name = None
url = None
- assert len(licenses) == len(license_urls)
- result = ''
- for i in range(len(licenses)):
- name = licenses[i]
- url = license_urls[i]
- result += LICENSETEMPLATE.substitute(name=name, url=url)
- return result
+ for line in file:
+ trimmed = line.strip()
+ # Collect license name and url for each artifact. They must come in
+ # pairs for each artifact.
+ if trimmed.startswith(artifact_prefix):
+ assert not name
+ assert not url
+ if trimmed.startswith(license_prefix):
+ name = trimmed[len(license_prefix):]
+ if trimmed.startswith(license_url_prefix):
+ url = trimmed[len(license_url_prefix):]
+ # Licenses come in name/url pairs. When both are present add pair
+ # to collected licenses if either name or url has not been recorded yet,
+ # as some licenses with slightly different names point to the same url.
+ if name and url:
+ if (not name in licenses) or (not url in license_urls):
+ licenses.append(name)
+ license_urls.append(url)
+ name = None
+ url = None
+ assert len(licenses) == len(license_urls)
+ result = ''
+ for i in range(len(licenses)):
+ name = licenses[i]
+ url = license_urls[i]
+ result += LICENSETEMPLATE.substitute(name=name, url=url)
+ return result
+
def write_default_r8_pom_file(pom_file, version):
- write_pom_file(R8_POMTEMPLATE, pom_file, version)
+ write_pom_file(R8_POMTEMPLATE, pom_file, version)
-def write_pom_file(
- template, pom_file, version, artifact_id=None, library_licenses=''):
- version_pom = (
- template.substitute(
- artifactId=artifact_id,
- version=version,
- library_licenses=library_licenses)
- if artifact_id else
- template.substitute(
- version=version, library_licenses=library_licenses))
- with open(pom_file, 'w') as file:
- file.write(version_pom)
+
+def write_pom_file(template,
+ pom_file,
+ version,
+ artifact_id=None,
+ library_licenses=''):
+ version_pom = (template.substitute(artifactId=artifact_id,
+ version=version,
+ library_licenses=library_licenses)
+ if artifact_id else template.substitute(
+ version=version, library_licenses=library_licenses))
+ with open(pom_file, 'w') as file:
+ file.write(version_pom)
+
def hash_for(file, hash):
- with open(file, 'rb') as f:
- while True:
- # Read chunks of 1MB
- chunk = f.read(2 ** 20)
- if not chunk:
- break
- hash.update(chunk)
- return hash.hexdigest()
+ with open(file, 'rb') as f:
+ while True:
+ # Read chunks of 1MB
+ chunk = f.read(2**20)
+ if not chunk:
+ break
+ hash.update(chunk)
+ return hash.hexdigest()
+
def write_md5_for(file):
- hexdigest = hash_for(file, hashlib.md5())
- with (open(file + '.md5', 'w')) as file:
- file.write(hexdigest)
+ hexdigest = hash_for(file, hashlib.md5())
+ with (open(file + '.md5', 'w')) as file:
+ file.write(hexdigest)
+
def write_sha1_for(file):
- hexdigest = hash_for(file, hashlib.sha1())
- with (open(file + '.sha1', 'w')) as file:
- file.write(hexdigest)
+ hexdigest = hash_for(file, hashlib.sha1())
+ with (open(file + '.sha1', 'w')) as file:
+ file.write(hexdigest)
+
def generate_maven_zip(name, version, pom_file, jar_file, out):
- with utils.TempDir() as tmp_dir:
- # Create the base maven version directory
- version_dir = join(tmp_dir, utils.get_maven_path(name, version))
- makedirs(version_dir)
- # Write the pom file.
- pom_file_location = join(version_dir, name + '-' + version + '.pom')
- copyfile(pom_file, pom_file_location)
- # Write the jar file.
- jar_file_location = join(version_dir, name + '-' + version + '.jar')
- copyfile(jar_file, jar_file_location)
- # Create check sums.
- write_md5_for(jar_file_location)
- write_md5_for(pom_file_location)
- write_sha1_for(jar_file_location)
- write_sha1_for(pom_file_location)
- # Zip it up - make_archive will append zip to the file, so remove.
- assert out.endswith('.zip')
- base_no_zip = out[0:len(out)-4]
- make_archive(base_no_zip, 'zip', tmp_dir)
+ with utils.TempDir() as tmp_dir:
+ # Create the base maven version directory
+ version_dir = join(tmp_dir, utils.get_maven_path(name, version))
+ makedirs(version_dir)
+ # Write the pom file.
+ pom_file_location = join(version_dir, name + '-' + version + '.pom')
+ copyfile(pom_file, pom_file_location)
+ # Write the jar file.
+ jar_file_location = join(version_dir, name + '-' + version + '.jar')
+ copyfile(jar_file, jar_file_location)
+ # Create check sums.
+ write_md5_for(jar_file_location)
+ write_md5_for(pom_file_location)
+ write_sha1_for(jar_file_location)
+ write_sha1_for(pom_file_location)
+ # Zip it up - make_archive will append zip to the file, so remove.
+ assert out.endswith('.zip')
+ base_no_zip = out[0:len(out) - 4]
+ make_archive(base_no_zip, 'zip', tmp_dir)
+
def generate_r8_maven_zip(out, version_file=None, skip_gradle_build=False):
- if not skip_gradle_build:
- gradle.RunGradle([utils.GRADLE_TASK_R8LIB, '-Pno_internal'])
- version = determine_version()
- with utils.TempDir() as tmp_dir:
- file_copy = join(tmp_dir, 'copy_of_jar.jar')
- copyfile(utils.R8LIB_JAR, file_copy)
+ if not skip_gradle_build:
+ gradle.RunGradle([utils.GRADLE_TASK_R8LIB, '-Pno_internal'])
+ version = determine_version()
+ with utils.TempDir() as tmp_dir:
+ file_copy = join(tmp_dir, 'copy_of_jar.jar')
+ copyfile(utils.R8LIB_JAR, file_copy)
- if version_file:
- with zipfile.ZipFile(file_copy, 'a') as zip:
- zip.write(version_file, basename(version_file))
+ if version_file:
+ with zipfile.ZipFile(file_copy, 'a') as zip:
+ zip.write(version_file, basename(version_file))
- # Generate the pom file.
- pom_file = join(tmp_dir, 'r8.pom')
- write_pom_file(
- R8_POMTEMPLATE,
- pom_file,
- version,
- library_licenses=generate_library_licenses())
- # Write the maven zip file.
- generate_maven_zip(
- 'r8',
- version,
- pom_file,
- file_copy,
- out)
+ # Generate the pom file.
+ pom_file = join(tmp_dir, 'r8.pom')
+ write_pom_file(R8_POMTEMPLATE,
+ pom_file,
+ version,
+ library_licenses=generate_library_licenses())
+ # Write the maven zip file.
+ generate_maven_zip('r8', version, pom_file, file_copy, out)
+
# Write the desugaring configuration of a jar file with the following content:
# java/
@@ -287,113 +299,107 @@
# desugar.json
# lint/
# <lint files>
-def generate_jar_with_desugar_configuration(
- configuration, implementation, conversions, destination):
- with utils.TempDir() as tmp_dir:
- # Add conversion classes.
- with zipfile.ZipFile(conversions, 'r') as conversions_zip:
- conversions_zip.extractall(tmp_dir)
+def generate_jar_with_desugar_configuration(configuration, implementation,
+ conversions, destination):
+ with utils.TempDir() as tmp_dir:
+ # Add conversion classes.
+ with zipfile.ZipFile(conversions, 'r') as conversions_zip:
+ conversions_zip.extractall(tmp_dir)
- # Add configuration.
- configuration_dir = join(tmp_dir, 'META-INF', 'desugar', 'd8')
- makedirs(configuration_dir)
- copyfile(configuration, join(configuration_dir, 'desugar.json'))
+ # Add configuration.
+ configuration_dir = join(tmp_dir, 'META-INF', 'desugar', 'd8')
+ makedirs(configuration_dir)
+ copyfile(configuration, join(configuration_dir, 'desugar.json'))
- # Add lint configuartion.
- lint_dir = join(configuration_dir, 'lint')
- makedirs(lint_dir)
+ # Add lint configuartion.
+ lint_dir = join(configuration_dir, 'lint')
+ makedirs(lint_dir)
+ cmd = [
+ jdk.GetJavaExecutable(), '-cp', utils.R8_JAR,
+ 'com.android.tools.r8.ir.desugar.desugaredlibrary.lint.GenerateDesugaredLibraryLintFiles',
+ configuration, implementation, lint_dir,
+ utils.get_android_jar(34)
+ ]
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+
+ # Add LICENSE file.
+ copyfile(join(utils.REPO_ROOT, 'LICENSE'), join(tmp_dir, 'LICENSE'))
+
+ make_archive(destination, 'zip', tmp_dir)
+ move(destination + '.zip', destination)
+
+
+def convert_desugar_configuration(configuration, conversions, implementation,
+ machine_configuration):
cmd = [
- jdk.GetJavaExecutable(),
- '-cp',
- utils.R8_JAR,
- 'com.android.tools.r8.ir.desugar.desugaredlibrary.lint.GenerateDesugaredLibraryLintFiles',
- configuration,
- implementation,
- lint_dir,
- utils.get_android_jar(34)]
- utils.PrintCmd(cmd)
+ jdk.GetJavaExecutable(), '-cp', utils.R8_JAR,
+ 'com.android.tools.r8.ir.desugar.desugaredlibrary.specificationconversion.DesugaredLibraryConverter',
+ configuration, implementation, conversions,
+ utils.get_android_jar(33), machine_configuration
+ ]
subprocess.check_call(cmd)
- # Add LICENSE file.
- copyfile(join(utils.REPO_ROOT, 'LICENSE'), join(tmp_dir, 'LICENSE'))
-
- make_archive(destination, 'zip', tmp_dir)
- move(destination + '.zip', destination)
-
-def convert_desugar_configuration(
- configuration, conversions, implementation, machine_configuration):
- cmd = [jdk.GetJavaExecutable(),
- '-cp',
- utils.R8_JAR,
- 'com.android.tools.r8.ir.desugar.desugaredlibrary.specificationconversion.DesugaredLibraryConverter',
- configuration,
- implementation,
- conversions,
- utils.get_android_jar(33),
- machine_configuration]
- subprocess.check_call(cmd)
# Generate the maven zip for the configuration to desugar desugar_jdk_libs.
-def generate_desugar_configuration_maven_zip(
- out, configuration, implementation, conversions):
- with utils.TempDir() as tmp_dir:
- (name, version) = utils.desugar_configuration_name_and_version(configuration, False)
+def generate_desugar_configuration_maven_zip(out, configuration, implementation,
+ conversions):
+ with utils.TempDir() as tmp_dir:
+ (name, version) = utils.desugar_configuration_name_and_version(
+ configuration, False)
- if (not version.startswith("1.")):
- machine_configuration = join(tmp_dir, "machine.json")
- convert_desugar_configuration(configuration, conversions, implementation, machine_configuration)
- configuration = machine_configuration
+ if (not version.startswith("1.")):
+ machine_configuration = join(tmp_dir, "machine.json")
+ convert_desugar_configuration(configuration, conversions,
+ implementation, machine_configuration)
+ configuration = machine_configuration
- # Generate the pom file.
- pom_file = join(tmp_dir, 'desugar_configuration.pom')
- write_pom_file(DESUGAR_CONFIGUATION_POMTEMPLATE, pom_file, version, artifact_id=name)
- # Generate the jar with the configuration file.
- jar_file = join(tmp_dir, 'desugar_configuration.jar')
- generate_jar_with_desugar_configuration(
- configuration,
- implementation,
- conversions,
- jar_file)
- # Write the maven zip file.
- generate_maven_zip(name, version, pom_file, jar_file, out)
+ # Generate the pom file.
+ pom_file = join(tmp_dir, 'desugar_configuration.pom')
+ write_pom_file(DESUGAR_CONFIGUATION_POMTEMPLATE,
+ pom_file,
+ version,
+ artifact_id=name)
+ # Generate the jar with the configuration file.
+ jar_file = join(tmp_dir, 'desugar_configuration.jar')
+ generate_jar_with_desugar_configuration(configuration, implementation,
+ conversions, jar_file)
+ # Write the maven zip file.
+ generate_maven_zip(name, version, pom_file, jar_file, out)
+
def main(argv):
- options = parse_options(argv)
- if options.out == None:
- raise Exception(
- 'Need to supply output zip with --out.')
- if options.desugar_configuration or options.desugar_configuration_jdk8:
- generate_desugar_configuration_maven_zip(
- options.out,
- utils.DESUGAR_CONFIGURATION,
- utils.DESUGAR_IMPLEMENTATION,
- utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
- elif options.desugar_configuration_jdk11_legacy:
- generate_desugar_configuration_maven_zip(
- options.out,
- utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
- elif options.desugar_configuration_jdk11_minimal:
- generate_desugar_configuration_maven_zip(
- options.out,
- utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
- elif options.desugar_configuration_jdk11:
- generate_desugar_configuration_maven_zip(
- options.out,
- utils.DESUGAR_CONFIGURATION_JDK11,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
- elif options.desugar_configuration_jdk11_nio:
- generate_desugar_configuration_maven_zip(
- options.out,
- utils.DESUGAR_CONFIGURATION_JDK11_NIO,
- utils.DESUGAR_IMPLEMENTATION_JDK11,
- utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
- else:
- generate_r8_maven_zip(options.out)
+ options = parse_options(argv)
+ if options.out == None:
+ raise Exception('Need to supply output zip with --out.')
+ if options.desugar_configuration or options.desugar_configuration_jdk8:
+ generate_desugar_configuration_maven_zip(
+ options.out, utils.DESUGAR_CONFIGURATION,
+ utils.DESUGAR_IMPLEMENTATION,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
+ elif options.desugar_configuration_jdk11_legacy:
+ generate_desugar_configuration_maven_zip(
+ options.out, utils.DESUGAR_CONFIGURATION_JDK11_LEGACY,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP)
+ elif options.desugar_configuration_jdk11_minimal:
+ generate_desugar_configuration_maven_zip(
+ options.out, utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ elif options.desugar_configuration_jdk11:
+ generate_desugar_configuration_maven_zip(
+ options.out, utils.DESUGAR_CONFIGURATION_JDK11,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ elif options.desugar_configuration_jdk11_nio:
+ generate_desugar_configuration_maven_zip(
+ options.out, utils.DESUGAR_CONFIGURATION_JDK11_NIO,
+ utils.DESUGAR_IMPLEMENTATION_JDK11,
+ utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP)
+ else:
+ generate_r8_maven_zip(options.out)
+
if __name__ == "__main__":
- exit(main(sys.argv[1:]))
+ exit(main(sys.argv[1:]))
diff --git a/tools/create_r8lib.py b/tools/create_r8lib.py
index a778ab3..93bfcbd 100755
--- a/tools/create_r8lib.py
+++ b/tools/create_r8lib.py
@@ -20,112 +20,105 @@
}
"""
+
def parse_options():
- parser = argparse.ArgumentParser(description='Tag R8 Versions')
- parser.add_argument(
- '--classpath',
- action='append',
- help='Dependencies to add to classpath')
- parser.add_argument(
- '--debug-agent',
- action='store_true',
- default=False,
- help='Create a socket for debugging')
- parser.add_argument(
- '--excldeps-variant',
- action='store_true',
- default=False,
- help='Mark this artifact as an "excldeps" variant of the compiler')
- parser.add_argument(
- '--debug-variant',
- action='store_true',
- default=False,
- help='Compile with debug flag')
- parser.add_argument(
- '--lib',
- action='append',
- help='Additional libraries (JDK 1.8 rt.jar already included)')
- parser.add_argument(
- '--output',
- required=True,
- help='The output path for the r8lib')
- parser.add_argument(
- '--pg-conf',
- action='append',
- help='Keep configuration')
- parser.add_argument(
- '--pg-map',
- default=None,
- help='Input map for distribution and composition')
- parser.add_argument(
- '--r8jar',
- required=True,
- help='The R8 jar to compile')
- parser.add_argument(
- '--r8compiler',
- default='build/libs/r8_with_deps.jar',
- help='The R8 compiler to use')
- return parser.parse_args()
+ parser = argparse.ArgumentParser(description='Tag R8 Versions')
+ parser.add_argument('--classpath',
+ action='append',
+ help='Dependencies to add to classpath')
+ parser.add_argument('--debug-agent',
+ action='store_true',
+ default=False,
+ help='Create a socket for debugging')
+ parser.add_argument(
+ '--excldeps-variant',
+ action='store_true',
+ default=False,
+ help='Mark this artifact as an "excldeps" variant of the compiler')
+ parser.add_argument('--debug-variant',
+ action='store_true',
+ default=False,
+ help='Compile with debug flag')
+ parser.add_argument(
+ '--lib',
+ action='append',
+ help='Additional libraries (JDK 1.8 rt.jar already included)')
+ parser.add_argument('--output',
+ required=True,
+ help='The output path for the r8lib')
+ parser.add_argument('--pg-conf', action='append', help='Keep configuration')
+ parser.add_argument('--pg-map',
+ default=None,
+ help='Input map for distribution and composition')
+ parser.add_argument('--r8jar', required=True, help='The R8 jar to compile')
+ parser.add_argument('--r8compiler',
+ default='build/libs/r8_with_deps.jar',
+ help='The R8 compiler to use')
+ return parser.parse_args()
+
def get_r8_version(r8jar):
- with utils.TempDir() as temp:
- name = os.path.join(temp, "VersionExtractor.java")
- fd = open(name, 'w')
- fd.write(VERSION_EXTRACTOR)
- fd.close()
- cmd = [jdk.GetJavacExecutable(), '-cp', r8jar, name]
- print(' '.join(cmd))
- cp_separator = ';' if utils.IsWindows() else ':'
- subprocess.check_call(cmd)
- output = subprocess.check_output([
- jdk.GetJavaExecutable(),
- '-cp',
- cp_separator.join([r8jar, os.path.dirname(name)]),
- 'VersionExtractor'
- ]).decode('UTF-8').strip()
- if output == 'main':
- return subprocess.check_output(
- ['git', 'rev-parse', 'HEAD']).decode('UTF-8').strip()
- else:
- return output
+ with utils.TempDir() as temp:
+ name = os.path.join(temp, "VersionExtractor.java")
+ fd = open(name, 'w')
+ fd.write(VERSION_EXTRACTOR)
+ fd.close()
+ cmd = [jdk.GetJavacExecutable(), '-cp', r8jar, name]
+ print(' '.join(cmd))
+ cp_separator = ';' if utils.IsWindows() else ':'
+ subprocess.check_call(cmd)
+ output = subprocess.check_output([
+ jdk.GetJavaExecutable(), '-cp',
+ cp_separator.join([r8jar, os.path.dirname(name)]),
+ 'VersionExtractor'
+ ]).decode('UTF-8').strip()
+ if output == 'main':
+ return subprocess.check_output(['git', 'rev-parse',
+ 'HEAD']).decode('UTF-8').strip()
+ else:
+ return output
+
def main():
- args = parse_options()
- if not os.path.exists(args.r8jar):
- print("Could not find jar: " + args.r8jar)
- return 1
- version = get_r8_version(args.r8jar)
- variant = '+excldeps' if args.excldeps_variant else ''
- map_id_template = version + variant
- source_file_template = 'R8_%MAP_ID_%MAP_HASH'
- # TODO(b/139725780): See if we can remove or lower the heap size (-Xmx8g).
- cmd = [jdk.GetJavaExecutable(), '-Xmx8g', '-ea']
- if args.debug_agent:
- cmd.extend(['-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'])
- cmd.extend(['-cp', args.r8compiler, 'com.android.tools.r8.R8'])
- cmd.append(args.r8jar)
- if args.debug_variant:
- cmd.append('--debug')
- cmd.append('--classfile')
- cmd.extend(['--map-id-template', map_id_template])
- cmd.extend(['--source-file-template', source_file_template])
- cmd.extend(['--output', args.output])
- cmd.extend(['--pg-map-output', args.output + '.map'])
- cmd.extend(['--partition-map-output', args.output + '_map.zip'])
- cmd.extend(['--lib', jdk.GetJdkHome()])
- if args.pg_conf:
- for pgconf in args.pg_conf:
- cmd.extend(['--pg-conf', pgconf])
- if args.lib:
- for lib in args.lib:
- cmd.extend(['--lib', lib])
- if args.classpath:
- for cp in args.classpath:
- cmd.extend(['--classpath', cp])
- if args.pg_map:
- cmd.extend(['--pg-map', args.pg_map])
- print(' '.join(cmd))
- subprocess.check_call(cmd)
+ args = parse_options()
+ if not os.path.exists(args.r8jar):
+ print("Could not find jar: " + args.r8jar)
+ return 1
+ version = get_r8_version(args.r8jar)
+ variant = '+excldeps' if args.excldeps_variant else ''
+ map_id_template = version + variant
+ source_file_template = 'R8_%MAP_ID_%MAP_HASH'
+ # TODO(b/139725780): See if we can remove or lower the heap size (-Xmx8g).
+ cmd = [jdk.GetJavaExecutable(), '-Xmx8g', '-ea']
+ if args.debug_agent:
+ cmd.extend([
+ '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+ ])
+ cmd.extend(['-cp', args.r8compiler, 'com.android.tools.r8.R8'])
+ cmd.append(args.r8jar)
+ if args.debug_variant:
+ cmd.append('--debug')
+ cmd.append('--classfile')
+ cmd.extend(['--map-id-template', map_id_template])
+ cmd.extend(['--source-file-template', source_file_template])
+ cmd.extend(['--output', args.output])
+ cmd.extend(['--pg-map-output', args.output + '.map'])
+ cmd.extend(['--partition-map-output', args.output + '_map.zip'])
+ cmd.extend(['--lib', jdk.GetJdkHome()])
+ if args.pg_conf:
+ for pgconf in args.pg_conf:
+ cmd.extend(['--pg-conf', pgconf])
+ if args.lib:
+ for lib in args.lib:
+ cmd.extend(['--lib', lib])
+ if args.classpath:
+ for cp in args.classpath:
+ cmd.extend(['--classpath', cp])
+ if args.pg_map:
+ cmd.extend(['--pg-map', args.pg_map])
+ print(' '.join(cmd))
+ subprocess.check_call(cmd)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/d8.py b/tools/d8.py
index c2dfe87..aab63ed 100755
--- a/tools/d8.py
+++ b/tools/d8.py
@@ -9,45 +9,41 @@
import sys
import toolhelper
+
def ParseOptions(argv):
- parser = optparse.OptionParser(usage='%prog [options] -- [D8 options]')
- parser.add_option(
- '-c',
- '--commit-hash',
- '--commit_hash',
- help='Commit hash of D8 to use.',
- default=None)
- parser.add_option(
- '--print-runtimeraw', '--print_runtimeraw',
- metavar='BENCHMARKNAME',
- help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
- ' <elapsed> ms\' at the end where <elapsed> is' +
- ' the elapsed time in milliseconds.')
- parser.add_option(
- '--version',
- help='Version of D8 to use.',
- default=None)
- parser.add_option(
- '--tag',
- help='Tag of D8 to use.',
- default=None)
- return parser.parse_args(argv)
+ parser = optparse.OptionParser(usage='%prog [options] -- [D8 options]')
+ parser.add_option('-c',
+ '--commit-hash',
+ '--commit_hash',
+ help='Commit hash of D8 to use.',
+ default=None)
+ parser.add_option('--print-runtimeraw',
+ '--print_runtimeraw',
+ metavar='BENCHMARKNAME',
+ help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
+ ' <elapsed> ms\' at the end where <elapsed> is' +
+ ' the elapsed time in milliseconds.')
+ parser.add_option('--version', help='Version of D8 to use.', default=None)
+ parser.add_option('--tag', help='Tag of D8 to use.', default=None)
+ return parser.parse_args(argv)
+
def main(argv):
- (options, args) = ParseOptions(sys.argv)
- d8_args = args[1:]
- time_consumer = lambda duration : print_duration(duration, options)
- return toolhelper.run(
- 'd8',
- d8_args,
- jar=utils.find_r8_jar_from_options(options),
- main='com.android.tools.r8.D8',
- time_consumer=time_consumer)
+ (options, args) = ParseOptions(sys.argv)
+ d8_args = args[1:]
+ time_consumer = lambda duration: print_duration(duration, options)
+ return toolhelper.run('d8',
+ d8_args,
+ jar=utils.find_r8_jar_from_options(options),
+ main='com.android.tools.r8.D8',
+ time_consumer=time_consumer)
+
def print_duration(duration, options):
- benchmark_name = options.print_runtimeraw
- if benchmark_name:
- print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration))
+ benchmark_name = options.print_runtimeraw
+ if benchmark_name:
+ print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration))
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/d8logger.py b/tools/d8logger.py
index b523294..73d4a9e 100755
--- a/tools/d8logger.py
+++ b/tools/d8logger.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('d8logger', sys.argv[1:]))
+ sys.exit(toolhelper.run('d8logger', sys.argv[1:]))
diff --git a/tools/defines.py b/tools/defines.py
index 6fdd24f..8f2cecc 100644
--- a/tools/defines.py
+++ b/tools/defines.py
@@ -12,11 +12,14 @@
REPO_ROOT = os.path.realpath(os.path.join(TOOLS_DIR, '..'))
THIRD_PARTY = os.path.join(REPO_ROOT, 'third_party')
+
def IsWindows():
- return sys.platform.startswith('win')
+ return sys.platform.startswith('win')
+
def IsLinux():
- return sys.platform.startswith('linux')
+ return sys.platform.startswith('linux')
+
def IsOsX():
- return sys.platform.startswith('darwin')
\ No newline at end of file
+ return sys.platform.startswith('darwin')
diff --git a/tools/desugar_jdk_libs_repository.py b/tools/desugar_jdk_libs_repository.py
index d128bc6..d2ebeac 100755
--- a/tools/desugar_jdk_libs_repository.py
+++ b/tools/desugar_jdk_libs_repository.py
@@ -17,6 +17,7 @@
import create_maven_release
import archive_desugar_jdk_libs
+
class Variant(Enum):
jdk8 = 'jdk8'
jdk11_legacy = 'jdk11_legacy'
@@ -27,259 +28,274 @@
def __str__(self):
return self.value
+
def parse_options():
- parser = argparse.ArgumentParser(
- description='Local desugared library repository for desugared library configurations')
- parser.add_argument('--repo-root', '--repo_root',
- default='/tmp/repo',
- metavar=('<path>'),
- help='Location for Maven repository.')
- parser.add_argument('--clear-repo', '--clear_repo',
- default=False,
- action='store_true',
- help='Clear the Maven repository so it only has one version present')
- parser.add_argument('--variant', type=Variant, choices=list(Variant))
- parser.add_argument('--desugar-jdk-libs-checkout', '--desugar_jdk_libs_checkout',
- default=None,
- metavar=('<path>'),
- help='Use existing checkout of github.com/google/desugar_jdk_libs.')
- parser.add_argument('--desugar-jdk-libs-revision', '--desugar_jdk_libs_revision',
- default=None,
- metavar=('<revision>'),
- help='Revision of github.com/google/desugar_jdk_libs to use.')
- parser.add_argument('--release-version', '--release_version',
- metavar=('<version>'),
- help='The desugared library release version to use. This will pull from the archived releases')
- args = parser.parse_args()
- return args
+ parser = argparse.ArgumentParser(
+ description=
+ 'Local desugared library repository for desugared library configurations'
+ )
+ parser.add_argument('--repo-root',
+ '--repo_root',
+ default='/tmp/repo',
+ metavar=('<path>'),
+ help='Location for Maven repository.')
+ parser.add_argument(
+ '--clear-repo',
+ '--clear_repo',
+ default=False,
+ action='store_true',
+ help='Clear the Maven repository so it only has one version present')
+ parser.add_argument('--variant', type=Variant, choices=list(Variant))
+ parser.add_argument(
+ '--desugar-jdk-libs-checkout',
+ '--desugar_jdk_libs_checkout',
+ default=None,
+ metavar=('<path>'),
+ help='Use existing checkout of github.com/google/desugar_jdk_libs.')
+ parser.add_argument(
+ '--desugar-jdk-libs-revision',
+ '--desugar_jdk_libs_revision',
+ default=None,
+ metavar=('<revision>'),
+ help='Revision of github.com/google/desugar_jdk_libs to use.')
+ parser.add_argument(
+ '--release-version',
+ '--release_version',
+ metavar=('<version>'),
+ help=
+ 'The desugared library release version to use. This will pull from the archived releases'
+ )
+ args = parser.parse_args()
+ return args
+
def jar_or_pom_file(unzip_dir, artifact, version, extension):
- return join(
- unzip_dir,
- 'com',
- 'android',
- 'tools',
- artifact,
- version,
- artifact + '-' + version + '.' + extension)
+ return join(unzip_dir, 'com', 'android', 'tools', artifact, version,
+ artifact + '-' + version + '.' + extension)
+
def jar_file(unzip_dir, artifact, version):
- return jar_or_pom_file(unzip_dir, artifact, version, 'jar')
+ return jar_or_pom_file(unzip_dir, artifact, version, 'jar')
+
def pom_file(unzip_dir, artifact, version):
- return jar_or_pom_file(unzip_dir, artifact, version, 'pom')
+ return jar_or_pom_file(unzip_dir, artifact, version, 'pom')
+
def run(args):
- artifact = None
- configuration_artifact = None
- configuration = None
- conversions = None
- implementation = None
- version_file = None
- implementation_build_target = None
- implementation_maven_zip = None
- release_archive_location = None
- match args.variant:
- case Variant.jdk8:
- artifact = 'desugar_jdk_libs'
- configuration_artifact = 'desugar_jdk_libs_configuration'
- configuration = utils.DESUGAR_CONFIGURATION
- conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP
- implementation = utils.DESUGAR_IMPLEMENTATION
- version_file = 'VERSION.txt'
- implementation_build_target = ':maven_release'
- implementation_maven_zip = 'desugar_jdk_libs.zip'
- release_archive_location = 'desugar_jdk_libs'
- case Variant.jdk11_legacy:
- artifact = 'desugar_jdk_libs'
- configuration_artifact = 'desugar_jdk_libs_configuration'
- configuration = utils.DESUGAR_CONFIGURATION_JDK11_LEGACY
- conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP
- implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
- version_file = 'VERSION_JDK11_LEGACY.txt'
- implementation_build_target = ':maven_release_jdk11_legacy'
- implementation_maven_zip = 'desugar_jdk_libs_jdk11_legacy.zip'
- release_archive_location = 'desugar_jdk_libs'
- case Variant.jdk11_minimal:
- artifact = 'desugar_jdk_libs_minimal'
- configuration_artifact = 'desugar_jdk_libs_configuration_minimal'
- configuration = utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL
- conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP
- implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
- version_file = 'VERSION_JDK11_MINIMAL.txt'
- implementation_build_target = ':maven_release_jdk11_minimal'
- implementation_maven_zip = 'desugar_jdk_libs_jdk11_minimal.zip'
- release_archive_location = 'desugar_jdk_libs_minimal'
- case Variant.jdk11:
- artifact = 'desugar_jdk_libs'
- configuration_artifact = 'desugar_jdk_libs_configuration'
- configuration = utils.DESUGAR_CONFIGURATION_JDK11
- conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP
- implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
- version_file = 'VERSION_JDK11.txt'
- implementation_build_target = ':maven_release_jdk11'
- implementation_maven_zip = 'desugar_jdk_libs_jdk11.zip'
- release_archive_location = 'desugar_jdk_libs'
- case Variant.jdk11_nio:
- artifact = 'desugar_jdk_libs_nio'
- configuration_artifact = 'desugar_jdk_libs_configuration_nio'
- configuration = utils.DESUGAR_CONFIGURATION_JDK11_NIO
- conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP
- implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
- version_file = 'VERSION_JDK11_NIO.txt'
- implementation_build_target = ':maven_release_jdk11_nio'
- implementation_maven_zip = 'desugar_jdk_libs_jdk11_nio.zip'
- release_archive_location = 'desugar_jdk_libs_nio'
- implementation_build_output = join('bazel-bin', implementation_maven_zip)
- gradle.RunGradle([utils.GRADLE_TASK_R8])
- with utils.TempDir() as tmp_dir:
- (name, configuration_version) = utils.desugar_configuration_name_and_version(configuration, False)
- if (args.release_version != None and args.release_version != configuration_version):
- raise Exception(
- 'Configuration version %s is different for specified version %s'
- % (configuration_version, version))
- version = configuration_version
- print("Name: %s" % name)
- print("Version: %s" % version)
- # Checkout desugar_jdk_libs from GitHub
- use_existing_checkout = args.desugar_jdk_libs_checkout != None
- checkout_dir = (args.desugar_jdk_libs_checkout
- if use_existing_checkout
- else join(tmp_dir, 'desugar_jdk_libs'))
- if (not args.release_version and not use_existing_checkout):
- subprocess.check_call(
- ['git', 'clone', 'https://github.com/google/desugar_jdk_libs.git', checkout_dir])
- if (args.desugar_jdk_libs_revision):
- subprocess.check_call(
- ['git', '-C', checkout_dir, 'checkout', args.desugar_jdk_libs_revision])
- with utils.ChangedWorkingDirectory(checkout_dir):
- with open(version_file) as version_file:
- version_file_lines = version_file.readlines()
- for line in version_file_lines:
- if not line.startswith('#'):
- desugar_jdk_libs_version = line.strip()
- if (version != desugar_jdk_libs_version):
- raise Exception(
- "Version mismatch. Configuration has version '"
- + version
- + "', and desugar_jdk_libs has version '"
- + desugar_jdk_libs_version
- + "'")
+ artifact = None
+ configuration_artifact = None
+ configuration = None
+ conversions = None
+ implementation = None
+ version_file = None
+ implementation_build_target = None
+ implementation_maven_zip = None
+ release_archive_location = None
+ match args.variant:
+ case Variant.jdk8:
+ artifact = 'desugar_jdk_libs'
+ configuration_artifact = 'desugar_jdk_libs_configuration'
+ configuration = utils.DESUGAR_CONFIGURATION
+ conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP
+ implementation = utils.DESUGAR_IMPLEMENTATION
+ version_file = 'VERSION.txt'
+ implementation_build_target = ':maven_release'
+ implementation_maven_zip = 'desugar_jdk_libs.zip'
+ release_archive_location = 'desugar_jdk_libs'
+ case Variant.jdk11_legacy:
+ artifact = 'desugar_jdk_libs'
+ configuration_artifact = 'desugar_jdk_libs_configuration'
+ configuration = utils.DESUGAR_CONFIGURATION_JDK11_LEGACY
+ conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_LEGACY_ZIP
+ implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
+ version_file = 'VERSION_JDK11_LEGACY.txt'
+ implementation_build_target = ':maven_release_jdk11_legacy'
+ implementation_maven_zip = 'desugar_jdk_libs_jdk11_legacy.zip'
+ release_archive_location = 'desugar_jdk_libs'
+ case Variant.jdk11_minimal:
+ artifact = 'desugar_jdk_libs_minimal'
+ configuration_artifact = 'desugar_jdk_libs_configuration_minimal'
+ configuration = utils.DESUGAR_CONFIGURATION_JDK11_MINIMAL
+ conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP
+ implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
+ version_file = 'VERSION_JDK11_MINIMAL.txt'
+ implementation_build_target = ':maven_release_jdk11_minimal'
+ implementation_maven_zip = 'desugar_jdk_libs_jdk11_minimal.zip'
+ release_archive_location = 'desugar_jdk_libs_minimal'
+ case Variant.jdk11:
+ artifact = 'desugar_jdk_libs'
+ configuration_artifact = 'desugar_jdk_libs_configuration'
+ configuration = utils.DESUGAR_CONFIGURATION_JDK11
+ conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP
+ implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
+ version_file = 'VERSION_JDK11.txt'
+ implementation_build_target = ':maven_release_jdk11'
+ implementation_maven_zip = 'desugar_jdk_libs_jdk11.zip'
+ release_archive_location = 'desugar_jdk_libs'
+ case Variant.jdk11_nio:
+ artifact = 'desugar_jdk_libs_nio'
+ configuration_artifact = 'desugar_jdk_libs_configuration_nio'
+ configuration = utils.DESUGAR_CONFIGURATION_JDK11_NIO
+ conversions = utils.LIBRARY_DESUGAR_CONVERSIONS_ZIP
+ implementation = utils.DESUGAR_IMPLEMENTATION_JDK11
+ version_file = 'VERSION_JDK11_NIO.txt'
+ implementation_build_target = ':maven_release_jdk11_nio'
+ implementation_maven_zip = 'desugar_jdk_libs_jdk11_nio.zip'
+ release_archive_location = 'desugar_jdk_libs_nio'
+ implementation_build_output = join('bazel-bin', implementation_maven_zip)
+ gradle.RunGradle([utils.GRADLE_TASK_R8])
+ with utils.TempDir() as tmp_dir:
+ (name,
+ configuration_version) = utils.desugar_configuration_name_and_version(
+ configuration, False)
+ if (args.release_version != None and
+ args.release_version != configuration_version):
+ raise Exception(
+ 'Configuration version %s is different for specified version %s'
+ % (configuration_version, version))
+ version = configuration_version
+ print("Name: %s" % name)
+ print("Version: %s" % version)
+ # Checkout desugar_jdk_libs from GitHub
+ use_existing_checkout = args.desugar_jdk_libs_checkout != None
+ checkout_dir = (args.desugar_jdk_libs_checkout if use_existing_checkout
+ else join(tmp_dir, 'desugar_jdk_libs'))
+ if (not args.release_version and not use_existing_checkout):
+ subprocess.check_call([
+ 'git', 'clone',
+ 'https://github.com/google/desugar_jdk_libs.git', checkout_dir
+ ])
+ if (args.desugar_jdk_libs_revision):
+ subprocess.check_call([
+ 'git', '-C', checkout_dir, 'checkout',
+ args.desugar_jdk_libs_revision
+ ])
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ with open(version_file) as version_file:
+ version_file_lines = version_file.readlines()
+ for line in version_file_lines:
+ if not line.startswith('#'):
+ desugar_jdk_libs_version = line.strip()
+ if (version != desugar_jdk_libs_version):
+ raise Exception(
+ "Version mismatch. Configuration has version '"
+ + version +
+ "', and desugar_jdk_libs has version '" +
+ desugar_jdk_libs_version + "'")
- # Build desugared library configuration.
- print("Building desugared library configuration " + version)
- maven_zip = join(tmp_dir, 'desugar_configuration.zip')
- create_maven_release.generate_desugar_configuration_maven_zip(
- maven_zip,
- configuration,
- implementation,
- conversions)
- unzip_dir = join(tmp_dir, 'desugar_jdk_libs_configuration_unzipped')
- cmd = ['unzip', '-q', maven_zip, '-d', unzip_dir]
- subprocess.check_call(cmd)
- cmd = [
- 'mvn',
- 'deploy:deploy-file',
- '-Durl=file:' + args.repo_root,
- '-DrepositoryId=someName',
- '-Dfile=' + jar_file(unzip_dir, configuration_artifact, version),
- '-DpomFile=' + pom_file(unzip_dir, configuration_artifact, version)]
- subprocess.check_call(cmd)
+ # Build desugared library configuration.
+ print("Building desugared library configuration " + version)
+ maven_zip = join(tmp_dir, 'desugar_configuration.zip')
+ create_maven_release.generate_desugar_configuration_maven_zip(
+ maven_zip, configuration, implementation, conversions)
+ unzip_dir = join(tmp_dir, 'desugar_jdk_libs_configuration_unzipped')
+ cmd = ['unzip', '-q', maven_zip, '-d', unzip_dir]
+ subprocess.check_call(cmd)
+ cmd = [
+ 'mvn', 'deploy:deploy-file', '-Durl=file:' + args.repo_root,
+ '-DrepositoryId=someName',
+ '-Dfile=' + jar_file(unzip_dir, configuration_artifact, version),
+ '-DpomFile=' + pom_file(unzip_dir, configuration_artifact, version)
+ ]
+ subprocess.check_call(cmd)
- undesugared_if_needed = None
- if not args.release_version:
- # Build desugared library.
- print("Building desugared library " + version)
- with utils.ChangedWorkingDirectory(checkout_dir):
- subprocess.check_call([
- 'bazel',
- '--bazelrc=/dev/null',
- 'build',
- '--spawn_strategy=local',
- '--verbose_failures',
- implementation_build_target])
+ undesugared_if_needed = None
+ if not args.release_version:
+ # Build desugared library.
+ print("Building desugared library " + version)
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ subprocess.check_call([
+ 'bazel', '--bazelrc=/dev/null', 'build',
+ '--spawn_strategy=local', '--verbose_failures',
+ implementation_build_target
+ ])
- # Undesugar desugared library if needed.
- undesugared_if_needed = join(checkout_dir, implementation_build_output)
- if (args.variant == Variant.jdk11_minimal
- or args.variant == Variant.jdk11
- or args.variant == Variant.jdk11_nio):
- undesugared_if_needed = join(tmp_dir, 'undesugared.zip')
- archive_desugar_jdk_libs.Undesugar(
- str(args.variant),
- join(checkout_dir, implementation_build_output),
- version,
- undesugared_if_needed)
- else:
- # Download the already built and undesugared library from release archive.
- undesugared_if_needed = join(tmp_dir, implementation_maven_zip)
- urllib.request.urlretrieve(
- ('https://storage.googleapis.com/r8-releases/raw/%s/%s/%s'
- % (release_archive_location, version, implementation_maven_zip)),
- undesugared_if_needed)
+ # Undesugar desugared library if needed.
+ undesugared_if_needed = join(checkout_dir,
+ implementation_build_output)
+ if (args.variant == Variant.jdk11_minimal or
+ args.variant == Variant.jdk11 or
+ args.variant == Variant.jdk11_nio):
+ undesugared_if_needed = join(tmp_dir, 'undesugared.zip')
+ archive_desugar_jdk_libs.Undesugar(
+ str(args.variant),
+ join(checkout_dir, implementation_build_output), version,
+ undesugared_if_needed)
+ else:
+ # Download the already built and undesugared library from release archive.
+ undesugared_if_needed = join(tmp_dir, implementation_maven_zip)
+ urllib.request.urlretrieve(
+ ('https://storage.googleapis.com/r8-releases/raw/%s/%s/%s' %
+ (release_archive_location, version, implementation_maven_zip)),
+ undesugared_if_needed)
- unzip_dir = join(tmp_dir, 'desugar_jdk_libs_unzipped')
- cmd = [
- 'unzip',
- '-q',
- undesugared_if_needed,
- '-d',
- unzip_dir]
- subprocess.check_call(cmd)
- cmd = [
- 'mvn',
- 'deploy:deploy-file',
- '-Durl=file:' + args.repo_root,
- '-DrepositoryId=someName',
- '-Dfile=' + jar_file(unzip_dir, artifact, version),
- '-DpomFile=' + pom_file(unzip_dir, artifact, version)]
- subprocess.check_call(cmd)
+ unzip_dir = join(tmp_dir, 'desugar_jdk_libs_unzipped')
+ cmd = ['unzip', '-q', undesugared_if_needed, '-d', unzip_dir]
+ subprocess.check_call(cmd)
+ cmd = [
+ 'mvn', 'deploy:deploy-file', '-Durl=file:' + args.repo_root,
+ '-DrepositoryId=someName',
+ '-Dfile=' + jar_file(unzip_dir, artifact, version),
+ '-DpomFile=' + pom_file(unzip_dir, artifact, version)
+ ]
+ subprocess.check_call(cmd)
- print()
- print("Artifacts:")
- print(" com.android.tools:%s:%s" % (configuration_artifact, version))
- print(" com.android.tools:%s:%s" % (artifact, version))
- print()
- print("deployed to Maven repository at " + args.repo_root + ".")
- print()
- print("Add")
- print()
- print(" maven {")
- print(" url uri('file://" + args.repo_root + "')")
- print(" }")
- print()
- print("to dependencyResolutionManagement.repositories in settings.gradle, and use")
- print('the "changing" property of the coreLibraryDesugaring dependency:')
- print()
- print(" coreLibraryDesugaring('com.android.tools:%s:%s') {" % (artifact, version))
- print(" changing = true")
- print(" }")
- print()
- print('If not using the "changing" propertyRemember to run gradle with '
- + " --refresh-dependencies (./gradlew --refresh-dependencies ...) "
- + "to ensure the cache is not used when the same version is published."
- + "multiple times.")
+ print()
+ print("Artifacts:")
+ print(" com.android.tools:%s:%s" % (configuration_artifact, version))
+ print(" com.android.tools:%s:%s" % (artifact, version))
+ print()
+ print("deployed to Maven repository at " + args.repo_root + ".")
+ print()
+ print("Add")
+ print()
+ print(" maven {")
+ print(" url uri('file://" + args.repo_root + "')")
+ print(" }")
+ print()
+ print(
+ "to dependencyResolutionManagement.repositories in settings.gradle, and use"
+ )
+ print(
+ 'the "changing" property of the coreLibraryDesugaring dependency:')
+ print()
+ print(" coreLibraryDesugaring('com.android.tools:%s:%s') {" %
+ (artifact, version))
+ print(" changing = true")
+ print(" }")
+ print()
+ print(
+ 'If not using the "changing" propertyRemember to run gradle with ' +
+ " --refresh-dependencies (./gradlew --refresh-dependencies ...) " +
+ "to ensure the cache is not used when the same version is published."
+ + "multiple times.")
+
def main():
- args = parse_options()
- if args.desugar_jdk_libs_checkout and args.release_version:
- raise Exception(
- 'Options --desugar-jdk-libs-checkout and --release-version are mutually exclusive')
- if args.desugar_jdk_libs_revision and args.release_version:
- raise Exception(
- 'Options --desugar-jdk-libs-revision and --release-version are mutually exclusive')
- if args.desugar_jdk_libs_checkout and args.desugar_jdk_libs_revision:
- raise Exception(
- 'Options --desugar-jdk-libs-checkout and --desugar-jdk-libs-revision are mutually exclusive')
- if args.clear_repo:
- shutil.rmtree(args.repo_root, ignore_errors=True)
- utils.makedirs_if_needed(args.repo_root)
- if (args.variant):
- run(args)
- else:
- for v in Variant:
- args.variant = v
- run(args)
+ args = parse_options()
+ if args.desugar_jdk_libs_checkout and args.release_version:
+ raise Exception(
+ 'Options --desugar-jdk-libs-checkout and --release-version are mutually exclusive'
+ )
+ if args.desugar_jdk_libs_revision and args.release_version:
+ raise Exception(
+ 'Options --desugar-jdk-libs-revision and --release-version are mutually exclusive'
+ )
+ if args.desugar_jdk_libs_checkout and args.desugar_jdk_libs_revision:
+ raise Exception(
+ 'Options --desugar-jdk-libs-checkout and --desugar-jdk-libs-revision are mutually exclusive'
+ )
+ if args.clear_repo:
+ shutil.rmtree(args.repo_root, ignore_errors=True)
+ utils.makedirs_if_needed(args.repo_root)
+ if (args.variant):
+ run(args)
+ else:
+ for v in Variant:
+ args.variant = v
+ run(args)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/desugar_jdk_libs_update.py b/tools/desugar_jdk_libs_update.py
index c1aa5df..3514cf8 100755
--- a/tools/desugar_jdk_libs_update.py
+++ b/tools/desugar_jdk_libs_update.py
@@ -14,108 +14,107 @@
import utils
+
def sed(pattern, replace, path):
- with open(path, "r") as sources:
- lines = sources.readlines()
- with open(path, "w") as sources:
- for line in lines:
- sources.write(re.sub(pattern, replace, line))
+ with open(path, "r") as sources:
+ lines = sources.readlines()
+ with open(path, "w") as sources:
+ for line in lines:
+ sources.write(re.sub(pattern, replace, line))
+
def GetGitHash(checkout_dir):
- return subprocess.check_output(
- ['git', '-C', checkout_dir, 'rev-parse', 'HEAD']).decode('utf-8').strip()
+ return subprocess.check_output(
+ ['git', '-C', checkout_dir, 'rev-parse',
+ 'HEAD']).decode('utf-8').strip()
+
def run(args):
- with utils.TempDir() as tmp_dir:
- use_existing_checkout = args.desugar_jdk_libs_checkout != None
- checkout_dir = (args.desugar_jdk_libs_checkout
- if use_existing_checkout
- else join(tmp_dir, 'desugar_jdk_libs'))
- if (not use_existing_checkout):
- subprocess.check_call(
- ['git', 'clone', 'https://github.com/google/desugar_jdk_libs.git', checkout_dir])
- if (args.desugar_jdk_libs_revision):
- subprocess.check_call(
- ['git', '-C', checkout_dir, 'checkout', args.desugar_jdk_libs_revision])
- print("Hack to workaround b/256723819")
- os.remove(
- join(
- checkout_dir,
- "jdk11",
- "src",
- "java.base",
- "share",
- "classes",
- "java",
- "time",
- "format",
- "DesugarDateTimeFormatterBuilder.java"))
- print("Building desugared library")
- bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel')
- with utils.ChangedWorkingDirectory(checkout_dir):
- for target in [':desugar_jdk_libs_jdk11', '//jdk11/src:java_base_chm_only']:
- subprocess.check_call([
- bazel,
- '--bazelrc=/dev/null',
- 'build',
- '--spawn_strategy=local',
- '--verbose_failures',
- target])
+ with utils.TempDir() as tmp_dir:
+ use_existing_checkout = args.desugar_jdk_libs_checkout != None
+ checkout_dir = (args.desugar_jdk_libs_checkout if use_existing_checkout
+ else join(tmp_dir, 'desugar_jdk_libs'))
+ if (not use_existing_checkout):
+ subprocess.check_call([
+ 'git', 'clone',
+ 'https://github.com/google/desugar_jdk_libs.git', checkout_dir
+ ])
+ if (args.desugar_jdk_libs_revision):
+ subprocess.check_call([
+ 'git', '-C', checkout_dir, 'checkout',
+ args.desugar_jdk_libs_revision
+ ])
+ print("Hack to workaround b/256723819")
+ os.remove(
+ join(checkout_dir, "jdk11", "src", "java.base", "share", "classes",
+ "java", "time", "format",
+ "DesugarDateTimeFormatterBuilder.java"))
+ print("Building desugared library")
+ bazel = os.path.join(utils.BAZEL_TOOL, 'lib', 'bazel', 'bin', 'bazel')
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ for target in [
+ ':desugar_jdk_libs_jdk11', '//jdk11/src:java_base_chm_only'
+ ]:
+ subprocess.check_call([
+ bazel, '--bazelrc=/dev/null', 'build',
+ '--spawn_strategy=local', '--verbose_failures', target
+ ])
- openjdk_dir = join('third_party', 'openjdk')
- openjdk_subdir = 'desugar_jdk_libs_11'
- dest_dir = join(openjdk_dir, openjdk_subdir)
- src_dir = join(checkout_dir, 'bazel-bin', 'jdk11', 'src')
+ openjdk_dir = join('third_party', 'openjdk')
+ openjdk_subdir = 'desugar_jdk_libs_11'
+ dest_dir = join(openjdk_dir, openjdk_subdir)
+ src_dir = join(checkout_dir, 'bazel-bin', 'jdk11', 'src')
- metadata_files = ('LICENSE', 'README.google')
- for f in metadata_files:
- shutil.copyfile(join(dest_dir, f), join(tmp_dir, f))
- shutil.rmtree(dest_dir)
- os.remove(join(openjdk_dir, openjdk_subdir + '.tar.gz'))
- os.remove(join(openjdk_dir, openjdk_subdir + '.tar.gz.sha1'))
- os.mkdir(dest_dir)
- for s in [
- (join(src_dir, 'd8_java_base_selected_with_addon.jar'),
- join(dest_dir, 'desugar_jdk_libs.jar')),
- (join(src_dir, 'java_base_chm_only.jar'),
- join(dest_dir, 'desugar_jdk_libs_chm_only.jar'))]:
- shutil.copyfile(s[0], s[1])
- for f in metadata_files:
- shutil.copyfile(join(tmp_dir, f), join(dest_dir, f))
- desugar_jdk_libs_hash = os.path.join(dest_dir, 'desugar_jdk_libs_hash')
- with open(desugar_jdk_libs_hash, 'w') as desugar_jdk_libs_hash_writer:
- desugar_jdk_libs_hash_writer.write(GetGitHash(checkout_dir))
- sed('^Version: [0-9a-f]{40}$',
- 'Version: %s' % GetGitHash(checkout_dir),
- join(dest_dir, 'README.google'))
- sed('^Date: .*$',
- 'Date: %s' % datetime.today().strftime('%Y-%m-%d'),
- join(dest_dir, 'README.google'))
+ metadata_files = ('LICENSE', 'README.google')
+ for f in metadata_files:
+ shutil.copyfile(join(dest_dir, f), join(tmp_dir, f))
+ shutil.rmtree(dest_dir)
+ os.remove(join(openjdk_dir, openjdk_subdir + '.tar.gz'))
+ os.remove(join(openjdk_dir, openjdk_subdir + '.tar.gz.sha1'))
+ os.mkdir(dest_dir)
+ for s in [(join(src_dir, 'd8_java_base_selected_with_addon.jar'),
+ join(dest_dir, 'desugar_jdk_libs.jar')),
+ (join(src_dir, 'java_base_chm_only.jar'),
+ join(dest_dir, 'desugar_jdk_libs_chm_only.jar'))]:
+ shutil.copyfile(s[0], s[1])
+ for f in metadata_files:
+ shutil.copyfile(join(tmp_dir, f), join(dest_dir, f))
+ desugar_jdk_libs_hash = os.path.join(dest_dir, 'desugar_jdk_libs_hash')
+ with open(desugar_jdk_libs_hash, 'w') as desugar_jdk_libs_hash_writer:
+ desugar_jdk_libs_hash_writer.write(GetGitHash(checkout_dir))
+ sed('^Version: [0-9a-f]{40}$', 'Version: %s' % GetGitHash(checkout_dir),
+ join(dest_dir, 'README.google'))
+ sed('^Date: .*$', 'Date: %s' % datetime.today().strftime('%Y-%m-%d'),
+ join(dest_dir, 'README.google'))
- print('Now run')
- print(' (cd %s; upload_to_google_storage.py -a --bucket r8-deps %s)'
- % (openjdk_dir, openjdk_subdir))
-
+ print('Now run')
+ print(' (cd %s; upload_to_google_storage.py -a --bucket r8-deps %s)' %
+ (openjdk_dir, openjdk_subdir))
def main():
- args = parse_options()
- run(args)
+ args = parse_options()
+ run(args)
+
def parse_options():
- parser = argparse.ArgumentParser(
- description='Script for updating third_party/openjdk/desugar_jdk_libs*')
- parser.add_argument('--desugar-jdk-libs-checkout', '--desugar_jdk_libs_checkout',
- default=None,
- metavar=('<path>'),
- help='Use existing checkout of github.com/google/desugar_jdk_libs.')
- parser.add_argument('--desugar-jdk-libs-revision', '--desugar_jdk_libs_revision',
- default=None,
- metavar=('<revision>'),
- help='Revision of github.com/google/desugar_jdk_libs to use.')
- args = parser.parse_args()
- return args
+ parser = argparse.ArgumentParser(
+ description='Script for updating third_party/openjdk/desugar_jdk_libs*')
+ parser.add_argument(
+ '--desugar-jdk-libs-checkout',
+ '--desugar_jdk_libs_checkout',
+ default=None,
+ metavar=('<path>'),
+ help='Use existing checkout of github.com/google/desugar_jdk_libs.')
+ parser.add_argument(
+ '--desugar-jdk-libs-revision',
+ '--desugar_jdk_libs_revision',
+ default=None,
+ metavar=('<revision>'),
+ help='Revision of github.com/google/desugar_jdk_libs to use.')
+ args = parser.parse_args()
+ return args
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/dex2oat.py b/tools/dex2oat.py
index 52082cc..8df8f86 100755
--- a/tools/dex2oat.py
+++ b/tools/dex2oat.py
@@ -15,172 +15,179 @@
LATEST = '12.0.0'
VERSIONS = [
- '12.0.0',
- # TODO(b/258170524): Fix the broken dex2oat versions.
- # 'default',
- # '9.0.0',
- # '8.1.0',
- # '7.0.0',
- '6.0.1',
- # '5.1.1',
+ '12.0.0',
+ # TODO(b/258170524): Fix the broken dex2oat versions.
+ # 'default',
+ # '9.0.0',
+ # '8.1.0',
+ # '7.0.0',
+ '6.0.1',
+ # '5.1.1',
]
DIRS = {
- '12.0.0': 'host/art-12.0.0-beta4',
- 'default': 'art',
- '9.0.0': 'art-9.0.0',
- '8.1.0': 'art-8.1.0',
- '7.0.0': 'art-7.0.0',
- '6.0.1': 'art-6.0.1',
- '5.1.1': 'art-5.1.1',
+ '12.0.0': 'host/art-12.0.0-beta4',
+ 'default': 'art',
+ '9.0.0': 'art-9.0.0',
+ '8.1.0': 'art-8.1.0',
+ '7.0.0': 'art-7.0.0',
+ '6.0.1': 'art-6.0.1',
+ '5.1.1': 'art-5.1.1',
}
PRODUCTS = {
- '12.0.0': 'redfin',
- 'default': 'angler',
- '9.0.0': 'marlin',
- '8.1.0': 'marlin',
- '7.0.0': 'angler',
- '6.0.1': 'angler',
- '5.1.1': 'mako',
+ '12.0.0': 'redfin',
+ 'default': 'angler',
+ '9.0.0': 'marlin',
+ '8.1.0': 'marlin',
+ '7.0.0': 'angler',
+ '6.0.1': 'angler',
+ '5.1.1': 'mako',
}
ARCHS = {
- '12.0.0': 'x86_64',
- 'default': 'arm64',
- '9.0.0': 'arm64',
- '8.1.0': 'arm64',
- '7.0.0': 'arm64',
- '6.0.1': 'arm64',
- '5.1.1': 'arm',
+ '12.0.0': 'x86_64',
+ 'default': 'arm64',
+ '9.0.0': 'arm64',
+ '8.1.0': 'arm64',
+ '7.0.0': 'arm64',
+ '6.0.1': 'arm64',
+ '5.1.1': 'arm',
}
VERBOSE_OPTIONS = [
- 'verifier',
- 'compiler',
- 'gc',
- 'jit',
- 'jni',
- 'class',
- 'all',
+ 'verifier',
+ 'compiler',
+ 'gc',
+ 'jit',
+ 'jni',
+ 'class',
+ 'all',
]
-BOOT_IMAGE = {
- '12.0.0': 'apex/art_boot_images/javalib/boot.art'
-}
+BOOT_IMAGE = {'12.0.0': 'apex/art_boot_images/javalib/boot.art'}
+
def ParseOptions():
- parser = optparse.OptionParser()
- parser.add_option('--version',
- help='Version of dex2oat. (defaults to latest: ' + LATEST + ').',
- choices=VERSIONS,
- default=LATEST)
- parser.add_option('--device',
- help='Run dex2oat on this device (this is passed as the -s SERIAL.')
- parser.add_option('--all',
- help='Run dex2oat on all possible versions',
- default=False,
- action='store_true')
- parser.add_option('--output',
- help='Where to place the output oat (defaults to no output / temp file).',
- default=None)
- parser.add_option('--verbose',
- help='Enable verbose dex2oat logging.',
- choices=VERBOSE_OPTIONS,
- default=None)
- return parser.parse_args()
+ parser = optparse.OptionParser()
+ parser.add_option('--version',
+ help='Version of dex2oat. (defaults to latest: ' +
+ LATEST + ').',
+ choices=VERSIONS,
+ default=LATEST)
+ parser.add_option(
+ '--device',
+ help='Run dex2oat on this device (this is passed as the -s SERIAL.')
+ parser.add_option('--all',
+ help='Run dex2oat on all possible versions',
+ default=False,
+ action='store_true')
+ parser.add_option(
+ '--output',
+ help=
+ 'Where to place the output oat (defaults to no output / temp file).',
+ default=None)
+ parser.add_option('--verbose',
+ help='Enable verbose dex2oat logging.',
+ choices=VERBOSE_OPTIONS,
+ default=None)
+ return parser.parse_args()
+
def Main():
- (options, args) = ParseOptions()
- if len(args) != 1:
- print("Can only take a single dex/zip/jar/apk file as input.")
- return 1
- if (options.device):
- return run_device_dex2oat(options, args)
- else:
- return run_host_dex2oat(options, args)
+ (options, args) = ParseOptions()
+ if len(args) != 1:
+ print("Can only take a single dex/zip/jar/apk file as input.")
+ return 1
+ if (options.device):
+ return run_device_dex2oat(options, args)
+ else:
+ return run_host_dex2oat(options, args)
+
def run_host_dex2oat(options, args):
- if options.all and options.output:
- print("Can't write output when running all versions.")
- return 1
- dexfile = args[0]
- oatfile = options.output
- versions = VERSIONS if options.all else [options.version]
- for version in versions:
- run(options, dexfile, oatfile, version)
- print("")
- return 0
+ if options.all and options.output:
+ print("Can't write output when running all versions.")
+ return 1
+ dexfile = args[0]
+ oatfile = options.output
+ versions = VERSIONS if options.all else [options.version]
+ for version in versions:
+ run(options, dexfile, oatfile, version)
+ print("")
+ return 0
+
def adb_cmd(serial, *args):
- cmd = ['adb', '-s', serial]
- cmd.extend(args)
- return cmd
+ cmd = ['adb', '-s', serial]
+ cmd.extend(args)
+ return cmd
+
def append_dex2oat_verbose_flags(options, cmd):
- verbose = [options.verbose] if options.verbose else []
- if 'all' in verbose:
- verbose = [x for x in VERBOSE_OPTIONS if x != 'all']
- for flag in verbose:
- cmd += ['--runtime-arg', '-verbose:' + flag]
- return cmd
+ verbose = [options.verbose] if options.verbose else []
+ if 'all' in verbose:
+ verbose = [x for x in VERBOSE_OPTIONS if x != 'all']
+ for flag in verbose:
+ cmd += ['--runtime-arg', '-verbose:' + flag]
+ return cmd
+
def run_device_dex2oat(options, args):
- serial = options.device
- dexfile = args[0]
- device_dexfile = '/data/local/tmp/' + os.path.basename(dexfile)
- device_oatfile = '/data/local/tmp/unused.oat'
- cmd = adb_cmd(serial, 'shell', 'rm', '-f', device_dexfile, device_oatfile)
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
- cmd = adb_cmd(serial, 'push', dexfile, device_dexfile)
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
- cmd = adb_cmd(serial, 'logcat', '-c')
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
- cmd = adb_cmd(
- serial,
- 'shell',
- 'dex2oat',
- '--dex-file=' + device_dexfile,
- '--oat-file=/data/local/tmp/unused.oat')
- append_dex2oat_verbose_flags(options, cmd)
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
- cmd = adb_cmd(serial, 'logcat', '-d', '-s', 'dex2oat')
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
- return 0
+ serial = options.device
+ dexfile = args[0]
+ device_dexfile = '/data/local/tmp/' + os.path.basename(dexfile)
+ device_oatfile = '/data/local/tmp/unused.oat'
+ cmd = adb_cmd(serial, 'shell', 'rm', '-f', device_dexfile, device_oatfile)
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ cmd = adb_cmd(serial, 'push', dexfile, device_dexfile)
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ cmd = adb_cmd(serial, 'logcat', '-c')
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ cmd = adb_cmd(serial, 'shell', 'dex2oat', '--dex-file=' + device_dexfile,
+ '--oat-file=/data/local/tmp/unused.oat')
+ append_dex2oat_verbose_flags(options, cmd)
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ cmd = adb_cmd(serial, 'logcat', '-d', '-s', 'dex2oat')
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ return 0
+
def run(options, dexfile, oatfile=None, version=None):
- if not version:
- version = LATEST
- # dex2oat accepts non-existent dex files, check here instead
- if not os.path.exists(dexfile):
- raise Exception('DEX file not found: "{}"'.format(dexfile))
- with utils.TempDir() as temp:
- if not oatfile:
- oatfile = os.path.join(temp, "out.oat")
- base = os.path.join(LINUX_DIR, DIRS[version])
- product = PRODUCTS[version]
- arch = ARCHS[version]
- cmd = [
- os.path.join(base, 'bin', 'dex2oat'),
- '--android-root=' + os.path.join(base, 'product', product, 'system'),
- '--runtime-arg',
- '-Xnorelocate',
- '--dex-file=' + dexfile,
- '--oat-file=' + oatfile,
- '--instruction-set=' + arch,
- ]
- append_dex2oat_verbose_flags(options, cmd)
- if version in BOOT_IMAGE:
- cmd += ['--boot-image=' + BOOT_IMAGE[version]]
- env = {"LD_LIBRARY_PATH": os.path.join(base, 'lib')}
- utils.PrintCmd(cmd)
- with utils.ChangedWorkingDirectory(base):
- subprocess.check_call(cmd, env = env)
+ if not version:
+ version = LATEST
+ # dex2oat accepts non-existent dex files, check here instead
+ if not os.path.exists(dexfile):
+ raise Exception('DEX file not found: "{}"'.format(dexfile))
+ with utils.TempDir() as temp:
+ if not oatfile:
+ oatfile = os.path.join(temp, "out.oat")
+ base = os.path.join(LINUX_DIR, DIRS[version])
+ product = PRODUCTS[version]
+ arch = ARCHS[version]
+ cmd = [
+ os.path.join(base, 'bin', 'dex2oat'),
+ '--android-root=' +
+ os.path.join(base, 'product', product, 'system'),
+ '--runtime-arg',
+ '-Xnorelocate',
+ '--dex-file=' + dexfile,
+ '--oat-file=' + oatfile,
+ '--instruction-set=' + arch,
+ ]
+ append_dex2oat_verbose_flags(options, cmd)
+ if version in BOOT_IMAGE:
+ cmd += ['--boot-image=' + BOOT_IMAGE[version]]
+ env = {"LD_LIBRARY_PATH": os.path.join(base, 'lib')}
+ utils.PrintCmd(cmd)
+ with utils.ChangedWorkingDirectory(base):
+ subprocess.check_call(cmd, env=env)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/dexfilemerger.py b/tools/dexfilemerger.py
index de4e6ae..c14a65b 100755
--- a/tools/dexfilemerger.py
+++ b/tools/dexfilemerger.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('dexfilemerger', sys.argv[1:]))
+ sys.exit(toolhelper.run('dexfilemerger', sys.argv[1:]))
diff --git a/tools/dexsegments.py b/tools/dexsegments.py
index 7b901de..737688b 100755
--- a/tools/dexsegments.py
+++ b/tools/dexsegments.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('dexsegments', sys.argv[1:]))
+ sys.exit(toolhelper.run('dexsegments', sys.argv[1:]))
diff --git a/tools/disasm.py b/tools/disasm.py
index e37ac4c..e341557 100755
--- a/tools/disasm.py
+++ b/tools/disasm.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('disasm', sys.argv[1:], debug=False))
+ sys.exit(toolhelper.run('disasm', sys.argv[1:], debug=False))
diff --git a/tools/download_from_x20.py b/tools/download_from_x20.py
index 5796823..b8eba4e 100755
--- a/tools/download_from_x20.py
+++ b/tools/download_from_x20.py
@@ -16,36 +16,40 @@
GMSCORE_DEPS = '/google/data/ro/teams/r8/deps'
+
def parse_options():
- return optparse.OptionParser().parse_args()
+ return optparse.OptionParser().parse_args()
+
def download(src, dest):
- print('Downloading %s to %s' % (src, dest))
- shutil.copyfile(src, dest)
- utils.unpack_archive(dest)
+ print('Downloading %s to %s' % (src, dest))
+ shutil.copyfile(src, dest)
+ utils.unpack_archive(dest)
+
def Main():
- (options, args) = parse_options()
- assert len(args) == 1
- sha1_file = args[0]
- dest = sha1_file[:-5]
- print('Ensuring %s' % dest)
- with open(sha1_file, 'r') as input_sha:
- sha1 = input_sha.readline()
- if os.path.exists(dest) and utils.get_sha1(dest) == sha1:
- print('sha1 matches, not downloading')
- dest_dir = utils.extract_dir(dest)
- if os.path.exists(dest_dir):
- print('destination directory exists, no extraction')
- else:
- utils.unpack_archive(dest)
- return
- src = os.path.join(GMSCORE_DEPS, sha1)
- if not os.path.exists(src):
- print('File (%s) does not exist on x20' % src)
- print('Maybe pass -Pno_internal to your gradle invocation')
- return 42
- download(src, dest)
+ (options, args) = parse_options()
+ assert len(args) == 1
+ sha1_file = args[0]
+ dest = sha1_file[:-5]
+ print('Ensuring %s' % dest)
+ with open(sha1_file, 'r') as input_sha:
+ sha1 = input_sha.readline()
+ if os.path.exists(dest) and utils.get_sha1(dest) == sha1:
+ print('sha1 matches, not downloading')
+ dest_dir = utils.extract_dir(dest)
+ if os.path.exists(dest_dir):
+ print('destination directory exists, no extraction')
+ else:
+ utils.unpack_archive(dest)
+ return
+ src = os.path.join(GMSCORE_DEPS, sha1)
+ if not os.path.exists(src):
+ print('File (%s) does not exist on x20' % src)
+ print('Maybe pass -Pno_internal to your gradle invocation')
+ return 42
+ download(src, dest)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/download_kotlin_dev.py b/tools/download_kotlin_dev.py
index 3bcf9b7..aa73ba8 100755
--- a/tools/download_kotlin_dev.py
+++ b/tools/download_kotlin_dev.py
@@ -5,13 +5,13 @@
import utils
if utils.is_python3():
- from html.parser import HTMLParser
- import urllib.request
- url_request = urllib.request
+ from html.parser import HTMLParser
+ import urllib.request
+ url_request = urllib.request
else:
- from HTMLParser import HTMLParser
- import urllib
- url_request = urllib
+ from HTMLParser import HTMLParser
+ import urllib
+ url_request = urllib
import os
import sys
@@ -19,69 +19,75 @@
"kotlin/bootstrap/org/jetbrains/kotlin/"
KOTLIN_RELEASE_URL = JETBRAINS_KOTLIN_MAVEN_URL + "kotlin-compiler/"
+
def download_newest():
- response = url_request.urlopen(KOTLIN_RELEASE_URL)
- if response.getcode() != 200:
- raise Exception('Url: %s \n returned %s'
- % (KOTLIN_RELEASE_URL, response.getcode()))
- content = str(response.read())
- release_candidates = []
+ response = url_request.urlopen(KOTLIN_RELEASE_URL)
+ if response.getcode() != 200:
+ raise Exception('Url: %s \n returned %s' %
+ (KOTLIN_RELEASE_URL, response.getcode()))
+ content = str(response.read())
+ release_candidates = []
- class HTMLContentParser(HTMLParser):
- def handle_data(self, data):
- if ('-dev-' in data):
- release_candidates.append(data)
+ class HTMLContentParser(HTMLParser):
- parser = HTMLContentParser()
- parser.feed(content)
+ def handle_data(self, data):
+ if ('-dev-' in data):
+ release_candidates.append(data)
- top_most_version = (0, 0, 0, 0)
- top_most_version_and_build = None
+ parser = HTMLContentParser()
+ parser.feed(content)
- for version in release_candidates:
- # The compiler version is on the form <major>.<minor>.<revision>-dev-<build>/
- version = version.replace('/', '')
- version_build_args = version.split('-')
- version_components = version_build_args[0].split('.')
- version_components.append(version_build_args[2])
- current_version = tuple(map(int, version_components))
- if (current_version > top_most_version):
- top_most_version = current_version
- top_most_version_and_build = version
+ top_most_version = (0, 0, 0, 0)
+ top_most_version_and_build = None
- if (top_most_version_and_build is None):
- raise Exception('Url: %s \n returned %s'
- % (KOTLIN_RELEASE_URL, response.getcode()))
+ for version in release_candidates:
+ # The compiler version is on the form <major>.<minor>.<revision>-dev-<build>/
+ version = version.replace('/', '')
+ version_build_args = version.split('-')
+ version_components = version_build_args[0].split('.')
+ version_components.append(version_build_args[2])
+ current_version = tuple(map(int, version_components))
+ if (current_version > top_most_version):
+ top_most_version = current_version
+ top_most_version_and_build = version
- # We can now download all files related to the kotlin compiler version.
- print("Downloading version: " + top_most_version_and_build)
+ if (top_most_version_and_build is None):
+ raise Exception('Url: %s \n returned %s' %
+ (KOTLIN_RELEASE_URL, response.getcode()))
- kotlinc_lib = os.path.join(
- utils.THIRD_PARTY, "kotlin", "kotlin-compiler-dev", "kotlinc", "lib")
+ # We can now download all files related to the kotlin compiler version.
+ print("Downloading version: " + top_most_version_and_build)
- utils.DownloadFromGoogleCloudStorage(
- os.path.join(
- utils.THIRD_PARTY, "kotlin", "kotlin-compiler-dev.tar.gz.sha1"))
+ kotlinc_lib = os.path.join(utils.THIRD_PARTY, "kotlin",
+ "kotlin-compiler-dev", "kotlinc", "lib")
- download_and_save(
- JETBRAINS_KOTLIN_MAVEN_URL + "kotlin-compiler/{0}/kotlin-compiler-{0}.jar"
- .format(top_most_version_and_build), kotlinc_lib, "kotlin-compiler.jar")
- download_and_save(
- JETBRAINS_KOTLIN_MAVEN_URL + "kotlin-stdlib/{0}/kotlin-stdlib-{0}.jar"
- .format(top_most_version_and_build), kotlinc_lib, "kotlin-stdlib.jar")
- download_and_save(
- JETBRAINS_KOTLIN_MAVEN_URL + "kotlin-reflect/{0}/kotlin-reflect-{0}.jar"
- .format(top_most_version_and_build), kotlinc_lib, "kotlin-reflect.jar")
- download_and_save(
- JETBRAINS_KOTLIN_MAVEN_URL + "kotlin-script-runtime/{0}/kotlin-script-runtime-{0}.jar"
- .format(top_most_version_and_build), kotlinc_lib, "kotlin-script-runtime.jar")
+ utils.DownloadFromGoogleCloudStorage(
+ os.path.join(utils.THIRD_PARTY, "kotlin",
+ "kotlin-compiler-dev.tar.gz.sha1"))
+
+ download_and_save(
+ JETBRAINS_KOTLIN_MAVEN_URL +
+ "kotlin-compiler/{0}/kotlin-compiler-{0}.jar".format(
+ top_most_version_and_build), kotlinc_lib, "kotlin-compiler.jar")
+ download_and_save(
+ JETBRAINS_KOTLIN_MAVEN_URL +
+ "kotlin-stdlib/{0}/kotlin-stdlib-{0}.jar".format(
+ top_most_version_and_build), kotlinc_lib, "kotlin-stdlib.jar")
+ download_and_save(
+ JETBRAINS_KOTLIN_MAVEN_URL +
+ "kotlin-reflect/{0}/kotlin-reflect-{0}.jar".format(
+ top_most_version_and_build), kotlinc_lib, "kotlin-reflect.jar")
+ download_and_save(
+ JETBRAINS_KOTLIN_MAVEN_URL +
+ "kotlin-script-runtime/{0}/kotlin-script-runtime-{0}.jar".format(
+ top_most_version_and_build), kotlinc_lib,
+ "kotlin-script-runtime.jar")
def download_and_save(url, path, name):
- print('Downloading: ' + url)
- url_request.urlretrieve(url, os.path.join(path, name))
+ print('Downloading: ' + url)
+ url_request.urlretrieve(url, os.path.join(path, name))
if __name__ == '__main__':
- sys.exit(download_newest())
-
+ sys.exit(download_newest())
diff --git a/tools/emulator_aosp.py b/tools/emulator_aosp.py
deleted file mode 100755
index d5f80dc..0000000
--- a/tools/emulator_aosp.py
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-import argparse
-import sys
-
-import utils
-import utils_aosp
-
-def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Checkout the AOSP source tree.')
- utils_aosp.add_common_arguments(parser)
- return parser.parse_args()
-
-def emulator_aosp(aosp_root, lunch):
- print "Running AOSP emulator in " + aosp_root
-
- utils_aosp.run_through_aosp_helper(lunch, ['emulator_fg',
- '-partition-size', '4096', '-wipe-data'], cwd = aosp_root)
-
-def Main():
- args = parse_arguments()
-
- emulator_aosp(args.aosp_root, args.lunch)
-
-if __name__ == '__main__':
- sys.exit(Main())
diff --git a/tools/extractmarker.py b/tools/extractmarker.py
index d8348dd..522be29 100755
--- a/tools/extractmarker.py
+++ b/tools/extractmarker.py
@@ -7,26 +7,32 @@
import sys
import toolhelper
+
def extractmarker(apk_or_dex, build=True):
- stdout = toolhelper.run('extractmarker', [apk_or_dex], build=build, return_stdout=True)
- return stdout
+ stdout = toolhelper.run('extractmarker', [apk_or_dex],
+ build=build,
+ return_stdout=True)
+ return stdout
+
def parse_options(argv):
- result = argparse.ArgumentParser(
- description='Relayout a given APK using a startup profile.')
- result.add_argument('--no-build',
- action='store_true',
- default=False,
- help='To disable building using gradle')
- options, args = result.parse_known_args(argv)
- return options, args
+ result = argparse.ArgumentParser(
+ description='Relayout a given APK using a startup profile.')
+ result.add_argument('--no-build',
+ action='store_true',
+ default=False,
+ help='To disable building using gradle')
+ options, args = result.parse_known_args(argv)
+ return options, args
+
def main(argv):
- options, args = parse_options(argv)
- build = not options.no_build
- for apk_or_dex in args:
- print(extractmarker(apk_or_dex, build=build))
- build = False
+ options, args = parse_options(argv)
+ build = not options.no_build
+ for apk_or_dex in args:
+ print(extractmarker(apk_or_dex, build=build))
+ build = False
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/find_haning_test.py b/tools/find_haning_test.py
deleted file mode 100755
index 3c65317..0000000
--- a/tools/find_haning_test.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-import argparse
-import sys
-
-
-def ParseOptions():
- parser = argparse.ArgumentParser(
- description = 'Find tests started but not done from bot stdout.')
- return parser.parse_known_args()
-
-def get_started(stdout):
- # Lines look like:
- # Start executing test runBigInteger_ZERO_A01 [com.android.tools.r8.x.r8cf.math.BigInteger.ZERO.BigInteger_ZERO_A01]
- start_lines = []
- for line in stdout:
- if line.startswith('Start executing test'):
- split = line.split(' ')
- start_lines.append('%s %s' % (split[3], split[4].strip()))
- return start_lines
-
-def get_ended(stdout):
- # Lines look like:
- # Done executing test runBigInteger_subtract_A01 [com.android.tools.r8.x.r8cf.math.BigInteger.subtractLjava_math_BigInteger.BigInteger_subtract_A01] with result: SUCCESS
- done_lines = []
- for line in stdout:
- if line.startswith('Done executing test'):
- split = line.split(' ')
- done_lines.append('%s %s' % (split[3], split[4].strip()))
- return done_lines
-
-def Main():
- (options, args) = ParseOptions()
- if len(args) != 1:
- raise "fail"
-
- with open(args[0], 'r') as f:
- lines = f.readlines()
- started = get_started(lines)
- ended = get_ended(lines)
- for test in started:
- if not test in ended:
- print 'Test %s started but did not end' % test
-
-
-if __name__ == '__main__':
- sys.exit(Main())
diff --git a/tools/fmt-diff.py b/tools/fmt-diff.py
index cda33f9..0e4ada7 100755
--- a/tools/fmt-diff.py
+++ b/tools/fmt-diff.py
@@ -3,6 +3,7 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
+import argparse
import os
import subprocess
import sys
@@ -11,23 +12,60 @@
from subprocess import Popen, PIPE
-GOOGLE_JAVA_FORMAT_DIFF = os.path.join(
- utils.THIRD_PARTY,
- 'google',
- 'google-java-format',
- '1.14.0',
- 'google-java-format-1.14.0',
- 'scripts',
- 'google-java-format-diff.py')
+GOOGLE_JAVA_FORMAT_DIFF = os.path.join(utils.THIRD_PARTY, 'google',
+ 'google-java-format', '1.14.0',
+ 'google-java-format-1.14.0', 'scripts',
+ 'google-java-format-diff.py')
+
+GOOGLE_YAPF = os.path.join(utils.THIRD_PARTY, 'google/yapf/20231013')
+
+
+def ParseOptions():
+ result = argparse.ArgumentParser()
+ result.add_argument('--no-java',
+ help='Run google-java-format.',
+ action='store_true',
+ default=False)
+ result.add_argument('--python',
+ help='Run YAPF.',
+ action='store_true',
+ default=False)
+ return result.parse_known_args()
+
+
+def FormatJava(upstream):
+ git_diff_process = Popen(['git', 'diff', '-U0', upstream], stdout=PIPE)
+ fmt_process = Popen([sys.executable, GOOGLE_JAVA_FORMAT_DIFF, '-p1', '-i'],
+ stdin=git_diff_process.stdout)
+ git_diff_process.stdout.close()
+ fmt_process.communicate()
+
+
+def FormatPython(upstream):
+ changed_files_cmd = ['git', 'diff', '--name-only', upstream, '*.py']
+ changed_files = subprocess.check_output(changed_files_cmd).decode(
+ 'utf-8').splitlines()
+ if not changed_files:
+ return
+ format_cmd = [
+ sys.executable,
+ os.path.join(GOOGLE_YAPF, 'yapf'), '--in-place', '--style', 'google'
+ ]
+ format_cmd.extend(changed_files)
+ yapf_python_path = [GOOGLE_YAPF, os.path.join(GOOGLE_YAPF, 'third_party')]
+ subprocess.check_call(format_cmd,
+ env={'PYTHONPATH': ':'.join(yapf_python_path)})
+
def main():
- upstream = subprocess.check_output(['git', 'cl', 'upstream']).strip()
- git_diff_process = Popen(['git', 'diff', '-U0', upstream], stdout=PIPE)
- fmt_process = Popen(
- [sys.executable, GOOGLE_JAVA_FORMAT_DIFF, '-p1', '-i'],
- stdin=git_diff_process.stdout)
- git_diff_process.stdout.close()
- fmt_process.communicate()
+ (options, args) = ParseOptions()
+ upstream = subprocess.check_output(['git', 'cl',
+ 'upstream']).decode('utf-8').strip()
+ if not options.no_java:
+ FormatJava(upstream)
+ if options.python:
+ FormatPython(upstream)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/git_sync_cl_chain.py b/tools/git_sync_cl_chain.py
index 0fd65c6..f89bde6 100755
--- a/tools/git_sync_cl_chain.py
+++ b/tools/git_sync_cl_chain.py
@@ -32,165 +32,189 @@
REPO_ROOT = defines.REPO_ROOT
+
class Repo(object):
- def __init__(self, name, is_current, upstream):
- self.name = name
- self.is_current = is_current
- self.upstream = upstream
+
+ def __init__(self, name, is_current, upstream):
+ self.name = name
+ self.is_current = is_current
+ self.upstream = upstream
+
def ParseOptions(argv):
- result = optparse.OptionParser()
- result.add_option('--bypass-hooks',
- help='Bypass presubmit hooks',
- action='store_true')
- result.add_option('--delete', '-d',
- help='Delete closed branches',
- choices=['y', 'n', 'ask'],
- default='ask')
- result.add_option('--from_branch', '-f',
- help='Uppermost upstream to sync from',
- default='main')
- result.add_option('--leave_upstream', '--leave-upstream',
- help='To not update the upstream of the first open branch',
- action='store_true')
- result.add_option('--message', '-m',
- help='Message for patchset', default='Sync')
- result.add_option('--rebase',
- help='To use `git pull --rebase` instead of `git pull`',
- action='store_true')
- result.add_option('--no_upload', '--no-upload',
- help='Disable uploading to Gerrit', action='store_true')
- result.add_option('--skip_main', '--skip-main',
- help='Disable syncing for main',
- action='store_true')
- (options, args) = result.parse_args(argv)
- options.upload = not options.no_upload
- assert options.delete != 'y' or not options.leave_upstream, (
- 'Inconsistent options: cannot leave the upstream of the first open ' +
- 'branch (--leave_upstream) and delete the closed branches at the same ' +
- 'time (--delete).')
- assert options.message, 'A message for the patchset is required.'
- assert len(args) == 0
- return options
+ result = optparse.OptionParser()
+ result.add_option('--bypass-hooks',
+ help='Bypass presubmit hooks',
+ action='store_true')
+ result.add_option('--delete',
+ '-d',
+ help='Delete closed branches',
+ choices=['y', 'n', 'ask'],
+ default='ask')
+ result.add_option('--from_branch',
+ '-f',
+ help='Uppermost upstream to sync from',
+ default='main')
+ result.add_option(
+ '--leave_upstream',
+ '--leave-upstream',
+ help='To not update the upstream of the first open branch',
+ action='store_true')
+ result.add_option('--message',
+ '-m',
+ help='Message for patchset',
+ default='Sync')
+ result.add_option('--rebase',
+ help='To use `git pull --rebase` instead of `git pull`',
+ action='store_true')
+ result.add_option('--no_upload',
+ '--no-upload',
+ help='Disable uploading to Gerrit',
+ action='store_true')
+ result.add_option('--skip_main',
+ '--skip-main',
+ help='Disable syncing for main',
+ action='store_true')
+ (options, args) = result.parse_args(argv)
+ options.upload = not options.no_upload
+ assert options.delete != 'y' or not options.leave_upstream, (
+ 'Inconsistent options: cannot leave the upstream of the first open ' +
+ 'branch (--leave_upstream) and delete the closed branches at the same '
+ + 'time (--delete).')
+ assert options.message, 'A message for the patchset is required.'
+ assert len(args) == 0
+ return options
+
def main(argv):
- options = ParseOptions(argv)
- with utils.ChangedWorkingDirectory(REPO_ROOT, quiet=True):
- branches = [
- parse(line)
- for line in utils.RunCmd(['git', 'branch', '-vv'], quiet=True)]
+ options = ParseOptions(argv)
+ with utils.ChangedWorkingDirectory(REPO_ROOT, quiet=True):
+ branches = [
+ parse(line)
+ for line in utils.RunCmd(['git', 'branch', '-vv'], quiet=True)
+ ]
- current_branch = None
- for branch in branches:
- if branch.is_current:
- current_branch = branch
- break
- assert current_branch is not None
+ current_branch = None
+ for branch in branches:
+ if branch.is_current:
+ current_branch = branch
+ break
+ assert current_branch is not None
- if is_root_branch(current_branch, options):
- print('Nothing to sync')
- return
+ if is_root_branch(current_branch, options):
+ print('Nothing to sync')
+ return
- stack = []
- while current_branch:
- stack.append(current_branch)
- if is_root_branch(current_branch, options):
- break
- current_branch = get_branch_with_name(current_branch.upstream, branches)
+ stack = []
+ while current_branch:
+ stack.append(current_branch)
+ if is_root_branch(current_branch, options):
+ break
+ current_branch = get_branch_with_name(current_branch.upstream,
+ branches)
- closed_branches = []
- has_seen_local_branch = False # A branch that is not uploaded.
- has_seen_open_branch = False # A branch that is not closed.
- while len(stack) > 0:
- branch = stack.pop()
+ closed_branches = []
+ has_seen_local_branch = False # A branch that is not uploaded.
+ has_seen_open_branch = False # A branch that is not closed.
+ while len(stack) > 0:
+ branch = stack.pop()
- utils.RunCmd(['git', 'checkout', branch.name], quiet=True)
+ utils.RunCmd(['git', 'checkout', branch.name], quiet=True)
- status = get_status_for_current_branch(branch)
- print('Syncing %s (status: %s)' % (branch.name, status))
+ status = get_status_for_current_branch(branch)
+ print('Syncing %s (status: %s)' % (branch.name, status))
- pull_for_current_branch(branch, options)
+ pull_for_current_branch(branch, options)
- if branch.name == 'main':
- continue
+ if branch.name == 'main':
+ continue
- if status == 'closed':
- assert not has_seen_local_branch, (
- 'Unexpected closed branch %s after new branch' % branch.name)
- assert not has_seen_open_branch, (
- 'Unexpected closed branch %s after open branch' % branch.name)
- closed_branches.append(branch.name)
- continue
+ if status == 'closed':
+ assert not has_seen_local_branch, (
+ 'Unexpected closed branch %s after new branch' %
+ branch.name)
+ assert not has_seen_open_branch, (
+ 'Unexpected closed branch %s after open branch' %
+ branch.name)
+ closed_branches.append(branch.name)
+ continue
- if not options.leave_upstream:
- if not has_seen_open_branch and len(closed_branches) > 0:
- print(
- 'Setting upstream for first open branch %s to main'
- % branch.name)
- set_upstream_for_current_branch_to_main()
+ if not options.leave_upstream:
+ if not has_seen_open_branch and len(closed_branches) > 0:
+ print('Setting upstream for first open branch %s to main' %
+ branch.name)
+ set_upstream_for_current_branch_to_main()
- has_seen_open_branch = True
- has_seen_local_branch = has_seen_local_branch or (status == 'None')
+ has_seen_open_branch = True
+ has_seen_local_branch = has_seen_local_branch or (status == 'None')
- if options.upload and status != 'closed':
- if has_seen_local_branch:
- print(
- 'Cannot upload branch %s since it comes after a local branch'
- % branch.name)
- else:
- upload_cmd = ['git', 'cl', 'upload', '-m', options.message]
- if options.bypass_hooks:
- upload_cmd.append('--bypass-hooks')
- utils.RunCmd(upload_cmd, quiet=True)
+ if options.upload and status != 'closed':
+ if has_seen_local_branch:
+ print(
+ 'Cannot upload branch %s since it comes after a local branch'
+ % branch.name)
+ else:
+ upload_cmd = ['git', 'cl', 'upload', '-m', options.message]
+ if options.bypass_hooks:
+ upload_cmd.append('--bypass-hooks')
+ utils.RunCmd(upload_cmd, quiet=True)
- if get_delete_branches_option(closed_branches, options):
- delete_branches(closed_branches)
+ if get_delete_branches_option(closed_branches, options):
+ delete_branches(closed_branches)
- utils.RunCmd(['git', 'cl', 'issue'])
+ utils.RunCmd(['git', 'cl', 'issue'])
+
def delete_branches(branches):
- assert len(branches) > 0
- cmd = ['git', 'branch', '-D']
- cmd.extend(branches)
- utils.RunCmd(cmd, quiet=True)
+ assert len(branches) > 0
+ cmd = ['git', 'branch', '-D']
+ cmd.extend(branches)
+ utils.RunCmd(cmd, quiet=True)
+
def get_branch_with_name(name, branches):
- for branch in branches:
- if branch.name == name:
- return branch
- return None
+ for branch in branches:
+ if branch.name == name:
+ return branch
+ return None
+
def get_delete_branches_option(closed_branches, options):
- if len(closed_branches) == 0:
- return False
- if options.leave_upstream:
- return False
- if options.delete == 'y':
- return True
- if options.delete == 'n':
- return False
- assert options.delete == 'ask'
- print('Delete closed branches: %s (Y/N)?' % ", ".join(closed_branches))
- answer = sys.stdin.read(1)
- return answer.lower() == 'y'
+ if len(closed_branches) == 0:
+ return False
+ if options.leave_upstream:
+ return False
+ if options.delete == 'y':
+ return True
+ if options.delete == 'n':
+ return False
+ assert options.delete == 'ask'
+ print('Delete closed branches: %s (Y/N)?' % ", ".join(closed_branches))
+ answer = sys.stdin.read(1)
+ return answer.lower() == 'y'
+
def get_status_for_current_branch(current_branch):
- if current_branch.name == 'main':
- return 'main'
- return utils.RunCmd(['git', 'cl', 'status', '--field', 'status'], quiet=True)[0].strip()
+ if current_branch.name == 'main':
+ return 'main'
+ return utils.RunCmd(['git', 'cl', 'status', '--field', 'status'],
+ quiet=True)[0].strip()
+
def is_root_branch(branch, options):
- return branch.name == options.from_branch or branch.upstream is None
+ return branch.name == options.from_branch or branch.upstream is None
+
def pull_for_current_branch(branch, options):
- if branch.name == 'main' and options.skip_main:
- return
- rebase_args = ['--rebase'] if options.rebase else []
- utils.RunCmd(['git', 'pull'] + rebase_args, quiet=True)
+ if branch.name == 'main' and options.skip_main:
+ return
+ rebase_args = ['--rebase'] if options.rebase else []
+ utils.RunCmd(['git', 'pull'] + rebase_args, quiet=True)
def set_upstream_for_current_branch_to_main():
- utils.RunCmd(['git', 'cl', 'upstream', 'main'], quiet=True)
+ utils.RunCmd(['git', 'cl', 'upstream', 'main'], quiet=True)
+
# Parses a line from the output of `git branch -vv`.
#
@@ -203,29 +227,30 @@
# feature_prereq_a xxxxxxxxx [main: ...] ...
# main xxxxxxxxx [origin/main] ...
def parse(line):
- is_current = False
- if line.startswith('*'):
- is_current = True
- line = line[1:].lstrip()
- else:
- line = line.lstrip()
+ is_current = False
+ if line.startswith('*'):
+ is_current = True
+ line = line[1:].lstrip()
+ else:
+ line = line.lstrip()
- name_end_index = line.index(' ')
- name = line[:name_end_index]
- line = line[name_end_index:].lstrip()
+ name_end_index = line.index(' ')
+ name = line[:name_end_index]
+ line = line[name_end_index:].lstrip()
- if '[' in line:
- line = line[line.index('[')+1:]
+ if '[' in line:
+ line = line[line.index('[') + 1:]
- if ':' in line:
- upstream = line[:line.index(':')]
- return Repo(name, is_current, upstream)
+ if ':' in line:
+ upstream = line[:line.index(':')]
+ return Repo(name, is_current, upstream)
- if ']' in line:
- upstream = line[:line.index(']')]
- return Repo(name, is_current, upstream)
+ if ']' in line:
+ upstream = line[:line.index(']')]
+ return Repo(name, is_current, upstream)
- return Repo(name, is_current, None)
+ return Repo(name, is_current, None)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/git_utils.py b/tools/git_utils.py
index 542947a..764039d 100644
--- a/tools/git_utils.py
+++ b/tools/git_utils.py
@@ -6,20 +6,23 @@
import utils
import subprocess
-def GitClone(url, checkout_dir):
- cmd = ['git', 'clone', url, checkout_dir]
- utils.PrintCmd(cmd)
- return subprocess.check_call(cmd)
-def GitCheckout(revision, checkout_dir):
- with utils.ChangedWorkingDirectory(checkout_dir):
- cmd = ['git', 'checkout', revision]
+def GitClone(url, checkout_dir):
+ cmd = ['git', 'clone', url, checkout_dir]
utils.PrintCmd(cmd)
return subprocess.check_call(cmd)
+
+def GitCheckout(revision, checkout_dir):
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ cmd = ['git', 'checkout', revision]
+ utils.PrintCmd(cmd)
+ return subprocess.check_call(cmd)
+
+
def GetHeadRevision(checkout_dir, use_main=False):
- revision_from = 'origin/main' if use_main else 'HEAD'
- cmd = ['git', 'rev-parse', revision_from]
- utils.PrintCmd(cmd)
- with utils.ChangedWorkingDirectory(checkout_dir):
- return subprocess.check_output(cmd).strip().decode('utf-8')
+ revision_from = 'origin/main' if use_main else 'HEAD'
+ cmd = ['git', 'rev-parse', revision_from]
+ utils.PrintCmd(cmd)
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ return subprocess.check_output(cmd).strip().decode('utf-8')
diff --git a/tools/gmail_data.py b/tools/gmail_data.py
index 6cb0322..6170a66 100644
--- a/tools/gmail_data.py
+++ b/tools/gmail_data.py
@@ -16,29 +16,32 @@
ANDROID_JAR = utils.get_android_jar(25)
VERSIONS = {
- '180826.15': {
- 'dex' : {
- 'flags': '--no-desugaring',
- 'inputs': [os.path.join(V180826_15_BASE, 'Gmail_release_unsigned.apk')],
- 'main-dex-list': os.path.join(V180826_15_BASE, 'main_dex_list.txt') ,
- 'pgmap': '%s_proguard.map' % V180826_15_PREFIX,
- 'libraries' : [ANDROID_JAR],
+ '180826.15': {
+ 'dex': {
+ 'flags': '--no-desugaring',
+ 'inputs': [
+ os.path.join(V180826_15_BASE, 'Gmail_release_unsigned.apk')
+ ],
+ 'main-dex-list': os.path.join(V180826_15_BASE, 'main_dex_list.txt'),
+ 'pgmap': '%s_proguard.map' % V180826_15_PREFIX,
+ 'libraries': [ANDROID_JAR],
+ },
+ 'deploy': {
+ 'flags': '--no-desugaring',
+ 'inputs': ['%s_deploy.jar' % V180826_15_PREFIX],
+ 'pgconf': [
+ '%s_proguard.config' % V180826_15_PREFIX,
+ '%s/proguardsettings/Gmail_proguard.config' % utils.THIRD_PARTY,
+ utils.IGNORE_WARNINGS_RULES
+ ],
+ 'min-api': ANDROID_L_API,
+ 'allow-type-errors': 1,
+ },
+ 'proguarded': {
+ 'flags': '--no-desugaring',
+ 'inputs': ['%s_proguard.jar' % V180826_15_PREFIX],
+ 'main-dex-list': os.path.join(V180826_15_BASE, 'main_dex_list.txt'),
+ 'pgmap': '%s_proguard.map' % V180826_15_PREFIX,
+ }
},
- 'deploy' : {
- 'flags': '--no-desugaring',
- 'inputs': ['%s_deploy.jar' % V180826_15_PREFIX],
- 'pgconf': [
- '%s_proguard.config' % V180826_15_PREFIX,
- '%s/proguardsettings/Gmail_proguard.config' % utils.THIRD_PARTY,
- utils.IGNORE_WARNINGS_RULES],
- 'min-api' : ANDROID_L_API,
- 'allow-type-errors' : 1,
- },
- 'proguarded' : {
- 'flags': '--no-desugaring',
- 'inputs': ['%s_proguard.jar' % V180826_15_PREFIX],
- 'main-dex-list': os.path.join(V180826_15_BASE, 'main_dex_list.txt') ,
- 'pgmap': '%s_proguard.map' % V180826_15_PREFIX,
- }
- },
}
diff --git a/tools/gmaven.py b/tools/gmaven.py
index 1ad96c0..ce52e48 100644
--- a/tools/gmaven.py
+++ b/tools/gmaven.py
@@ -7,53 +7,55 @@
import subprocess
GMAVEN_PUBLISHER = '/google/bin/releases/android-devtools/gmaven/publisher/gmaven-publisher'
-GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN = re.compile('Release ID = ([0-9a-f\-]+)')
+GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN = re.compile(
+ 'Release ID = ([0-9a-f\-]+)')
-def publisher_stage(gfiles, dry_run = False):
- if dry_run:
- print('Dry-run, would have staged %s' % gfiles)
- return 'dry-run-release-id'
+def publisher_stage(gfiles, dry_run=False):
+ if dry_run:
+ print('Dry-run, would have staged %s' % gfiles)
+ return 'dry-run-release-id'
- print("Staging: %s" % ', '.join(gfiles))
- print("")
+ print("Staging: %s" % ', '.join(gfiles))
+ print("")
- cmd = [GMAVEN_PUBLISHER, 'stage', '--gfile', ','.join(gfiles)]
- output = subprocess.check_output(cmd)
+ cmd = [GMAVEN_PUBLISHER, 'stage', '--gfile', ','.join(gfiles)]
+ output = subprocess.check_output(cmd)
- # Expect output to contain:
- # [INFO] 06/19/2020 09:35:12 CEST: >>>>>>>>>> Staged
- # [INFO] 06/19/2020 09:35:12 CEST: Release ID = 9171d015-18f6-4a90-9984-1c362589dc1b
- # [INFO] 06/19/2020 09:35:12 CEST: Stage Path = /bigstore/studio_staging/maven2/sgjesse/9171d015-18f6-4a90-9984-1c362589dc1b
+ # Expect output to contain:
+ # [INFO] 06/19/2020 09:35:12 CEST: >>>>>>>>>> Staged
+ # [INFO] 06/19/2020 09:35:12 CEST: Release ID = 9171d015-18f6-4a90-9984-1c362589dc1b
+ # [INFO] 06/19/2020 09:35:12 CEST: Stage Path = /bigstore/studio_staging/maven2/sgjesse/9171d015-18f6-4a90-9984-1c362589dc1b
- matches = GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN.findall(output.decode("utf-8"))
- if matches == None or len(matches) > 1:
- print("Could not determine the release ID from the gmaven_publisher " +
- "output. Expected a line with 'Release ID = <release id>'.")
- print("Output was:")
+ matches = GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN.findall(
+ output.decode("utf-8"))
+ if matches == None or len(matches) > 1:
+ print("Could not determine the release ID from the gmaven_publisher " +
+ "output. Expected a line with 'Release ID = <release id>'.")
+ print("Output was:")
+ print(output)
+ sys.exit(1)
+
print(output)
- sys.exit(1)
- print(output)
-
- release_id = matches[0]
- return release_id
+ release_id = matches[0]
+ return release_id
def publisher_stage_redir_test_info(release_id, artifact, dst):
- redir_command = ("/google/data/ro/teams/android-devtools-infra/tools/redir "
- + "--alsologtostderr "
- + "--gcs_bucket_path=/bigstore/gmaven-staging/${USER}/%s "
- + "--port=1480") % release_id
+ redir_command = ("/google/data/ro/teams/android-devtools-infra/tools/redir "
+ + "--alsologtostderr " +
+ "--gcs_bucket_path=/bigstore/gmaven-staging/${USER}/%s " +
+ "--port=1480") % release_id
- get_command = ("mvn org.apache.maven.plugins:maven-dependency-plugin:2.4:get "
- + "-Dmaven.repo.local=/tmp/maven_repo_local "
- + "-DremoteRepositories=http://localhost:1480 "
- + "-Dartifact=%s "
- + "-Ddest=%s") % (artifact, dst)
+ get_command = (
+ "mvn org.apache.maven.plugins:maven-dependency-plugin:2.4:get " +
+ "-Dmaven.repo.local=/tmp/maven_repo_local " +
+ "-DremoteRepositories=http://localhost:1480 " + "-Dartifact=%s " +
+ "-Ddest=%s") % (artifact, dst)
- print("""To test the staged content with 'redir' run:
+ print("""To test the staged content with 'redir' run:
%s
@@ -84,10 +86,10 @@
""" % (redir_command, artifact, get_command))
-def publisher_publish(release_id, dry_run = False):
- if dry_run:
- print('Dry-run, would have published %s' % release_id)
- return
+def publisher_publish(release_id, dry_run=False):
+ if dry_run:
+ print('Dry-run, would have published %s' % release_id)
+ return
- cmd = [GMAVEN_PUBLISHER, 'publish', release_id]
- output = subprocess.check_output(cmd)
+ cmd = [GMAVEN_PUBLISHER, 'publish', release_id]
+ output = subprocess.check_output(cmd)
diff --git a/tools/golem_build.py b/tools/golem_build.py
index 4f668d6..ae09558 100755
--- a/tools/golem_build.py
+++ b/tools/golem_build.py
@@ -13,15 +13,18 @@
GRADLE_ARGS = ['--no-daemon', '-Pno_internal']
+
def lower(items):
- return [ item.lower() for item in items ]
+ return [item.lower() for item in items]
+
def Main():
- targets = set()
- targets.update(lower(run_benchmark.GOLEM_BUILD_TARGETS))
- targets.update(lower(run_on_app_dump.GOLEM_BUILD_TARGETS))
- cmd = GRADLE_ARGS + [target for target in targets]
- gradle.RunGradle(cmd)
+ targets = set()
+ targets.update(lower(run_benchmark.GOLEM_BUILD_TARGETS))
+ targets.update(lower(run_on_app_dump.GOLEM_BUILD_TARGETS))
+ cmd = GRADLE_ARGS + [target for target in targets]
+ gradle.RunGradle(cmd)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/google-java-format-diff.py b/tools/google-java-format-diff.py
index c9c3dc3..c3d89ca 100755
--- a/tools/google-java-format-diff.py
+++ b/tools/google-java-format-diff.py
@@ -8,7 +8,6 @@
# License. See LICENSE.TXT for details.
#
#===------------------------------------------------------------------------===#
-
"""
google-java-format Diff Reformatter
============================
@@ -36,122 +35,145 @@
import sys
from shutil import which
+
def main():
- parser = argparse.ArgumentParser(description=
- 'Reformat changed lines in diff. Without -i '
- 'option just output the diff that would be '
- 'introduced.')
- parser.add_argument('-i', action='store_true', default=False,
- help='apply edits to files instead of displaying a diff')
+ parser = argparse.ArgumentParser(
+ description='Reformat changed lines in diff. Without -i '
+ 'option just output the diff that would be '
+ 'introduced.')
+ parser.add_argument(
+ '-i',
+ action='store_true',
+ default=False,
+ help='apply edits to files instead of displaying a diff')
- parser.add_argument('-p', metavar='NUM', default=0,
- help='strip the smallest prefix containing P slashes')
- parser.add_argument('-regex', metavar='PATTERN', default=None,
- help='custom pattern selecting file paths to reformat '
- '(case sensitive, overrides -iregex)')
- parser.add_argument('-iregex', metavar='PATTERN', default=r'.*\.java',
- help='custom pattern selecting file paths to reformat '
- '(case insensitive, overridden by -regex)')
- parser.add_argument('-v', '--verbose', action='store_true',
- help='be more verbose, ineffective without -i')
- parser.add_argument('-a', '--aosp', action='store_true',
- help='use AOSP style instead of Google Style (4-space indentation)')
- parser.add_argument('--skip-sorting-imports', action='store_true',
- help='do not fix the import order')
- parser.add_argument('--skip-removing-unused-imports', action='store_true',
- help='do not remove ununsed imports')
- parser.add_argument(
- '--skip-javadoc-formatting',
- action='store_true',
- default=False,
- help='do not reformat javadoc')
- parser.add_argument('-b', '--binary', help='path to google-java-format binary')
- parser.add_argument('--google-java-format-jar', metavar='ABSOLUTE_PATH', default=None,
- help='use a custom google-java-format jar')
+ parser.add_argument('-p',
+ metavar='NUM',
+ default=0,
+ help='strip the smallest prefix containing P slashes')
+ parser.add_argument('-regex',
+ metavar='PATTERN',
+ default=None,
+ help='custom pattern selecting file paths to reformat '
+ '(case sensitive, overrides -iregex)')
+ parser.add_argument('-iregex',
+ metavar='PATTERN',
+ default=r'.*\.java',
+ help='custom pattern selecting file paths to reformat '
+ '(case insensitive, overridden by -regex)')
+ parser.add_argument('-v',
+ '--verbose',
+ action='store_true',
+ help='be more verbose, ineffective without -i')
+ parser.add_argument(
+ '-a',
+ '--aosp',
+ action='store_true',
+ help='use AOSP style instead of Google Style (4-space indentation)')
+ parser.add_argument('--skip-sorting-imports',
+ action='store_true',
+ help='do not fix the import order')
+ parser.add_argument('--skip-removing-unused-imports',
+ action='store_true',
+ help='do not remove ununsed imports')
+ parser.add_argument('--skip-javadoc-formatting',
+ action='store_true',
+ default=False,
+ help='do not reformat javadoc')
+ parser.add_argument('-b',
+ '--binary',
+ help='path to google-java-format binary')
+ parser.add_argument('--google-java-format-jar',
+ metavar='ABSOLUTE_PATH',
+ default=None,
+ help='use a custom google-java-format jar')
- args = parser.parse_args()
+ args = parser.parse_args()
- # Extract changed lines for each file.
- filename = None
- lines_by_file = {}
+ # Extract changed lines for each file.
+ filename = None
+ lines_by_file = {}
- for line in sys.stdin:
- match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
- if match:
- filename = match.group(2)
- if filename == None:
- continue
+ for line in sys.stdin:
+ match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
+ if match:
+ filename = match.group(2)
+ if filename == None:
+ continue
- if args.regex is not None:
- if not re.match('^%s$' % args.regex, filename):
- continue
+ if args.regex is not None:
+ if not re.match('^%s$' % args.regex, filename):
+ continue
+ else:
+ if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
+ continue
+
+ match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
+ if match:
+ start_line = int(match.group(1))
+ line_count = 1
+ if match.group(3):
+ line_count = int(match.group(3))
+ if line_count == 0:
+ continue
+ end_line = start_line + line_count - 1
+ lines_by_file.setdefault(filename, []).extend(
+ ['-lines', str(start_line) + ':' + str(end_line)])
+
+ if args.binary:
+ base_command = [args.binary]
+ elif args.google_java_format_jar:
+ base_command = [
+ os.path.join('third_party', 'openjdk', 'jdk-17', 'linux', 'bin',
+ 'java'), '-jar',
+ '--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED',
+ '--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED',
+ '--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED',
+ '--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED',
+ '--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED',
+ '--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED',
+ args.google_java_format_jar
+ ]
else:
- if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
- continue
+ binary = which('google-java-format') or '/usr/bin/google-java-format'
+ base_command = [binary]
- match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
- if match:
- start_line = int(match.group(1))
- line_count = 1
- if match.group(3):
- line_count = int(match.group(3))
- if line_count == 0:
- continue
- end_line = start_line + line_count - 1;
- lines_by_file.setdefault(filename, []).extend(
- ['-lines', str(start_line) + ':' + str(end_line)])
+ # Reformat files containing changes in place.
+ for filename, lines in lines_by_file.items():
+ if args.i and args.verbose:
+ print('Formatting', filename)
+ command = base_command[:]
+ if args.i:
+ command.append('-i')
+ if args.aosp:
+ command.append('--aosp')
+ if args.skip_sorting_imports:
+ command.append('--skip-sorting-imports')
+ if args.skip_removing_unused_imports:
+ command.append('--skip-removing-unused-imports')
+ if args.skip_javadoc_formatting:
+ command.append('--skip-javadoc-formatting')
+ command.extend(lines)
+ command.append(filename)
+ p = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=None,
+ stdin=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ if p.returncode != 0:
+ sys.exit(p.returncode)
- if args.binary:
- base_command = [args.binary]
- elif args.google_java_format_jar:
- base_command = [
- os.path.join(
- 'third_party', 'openjdk', 'jdk-17', 'linux', 'bin', 'java'),
- '-jar',
- '--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED',
- '--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED',
- '--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED',
- '--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED',
- '--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED',
- '--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED',
- args.google_java_format_jar]
- else:
- binary = which('google-java-format') or '/usr/bin/google-java-format'
- base_command = [binary]
+ if not args.i:
+ with open(filename) as f:
+ code = f.readlines()
+ formatted_code = io.StringIO(stdout.decode('utf-8')).readlines()
+ diff = difflib.unified_diff(code, formatted_code, filename,
+ filename, '(before formatting)',
+ '(after formatting)')
+ diff_string = ''.join(diff)
+ if len(diff_string) > 0:
+ sys.stdout.write(diff_string)
- # Reformat files containing changes in place.
- for filename, lines in lines_by_file.items():
- if args.i and args.verbose:
- print('Formatting', filename)
- command = base_command[:]
- if args.i:
- command.append('-i')
- if args.aosp:
- command.append('--aosp')
- if args.skip_sorting_imports:
- command.append('--skip-sorting-imports')
- if args.skip_removing_unused_imports:
- command.append('--skip-removing-unused-imports')
- if args.skip_javadoc_formatting:
- command.append('--skip-javadoc-formatting')
- command.extend(lines)
- command.append(filename)
- p = subprocess.Popen(command, stdout=subprocess.PIPE,
- stderr=None, stdin=subprocess.PIPE)
- stdout, stderr = p.communicate()
- if p.returncode != 0:
- sys.exit(p.returncode);
-
- if not args.i:
- with open(filename) as f:
- code = f.readlines()
- formatted_code = io.StringIO(stdout.decode('utf-8')).readlines()
- diff = difflib.unified_diff(code, formatted_code,
- filename, filename,
- '(before formatting)', '(after formatting)')
- diff_string = ''.join(diff)
- if len(diff_string) > 0:
- sys.stdout.write(diff_string)
if __name__ == '__main__':
- main()
+ main()
diff --git a/tools/gradle.py b/tools/gradle.py
index 8ce123f..76039f9 100755
--- a/tools/gradle.py
+++ b/tools/gradle.py
@@ -19,116 +19,134 @@
GRADLE8_SHA1 = os.path.join(GRADLE_DIR, 'gradle-8.3.tar.gz.sha1')
GRADLE8_TGZ = os.path.join(GRADLE_DIR, 'gradle-8.3.tar.gz')
+
def get_gradle():
- gradle_dir = 'gradle-8.3'
- if utils.IsWindows():
- return os.path.join(GRADLE_DIR, gradle_dir, 'bin', 'gradle.bat')
- else:
- return os.path.join(GRADLE_DIR, gradle_dir, 'bin', 'gradle')
+ gradle_dir = 'gradle-8.3'
+ if utils.IsWindows():
+ return os.path.join(GRADLE_DIR, gradle_dir, 'bin', 'gradle.bat')
+ else:
+ return os.path.join(GRADLE_DIR, gradle_dir, 'bin', 'gradle')
+
def ParseOptions():
- parser = argparse.ArgumentParser(description = 'Call gradle.')
- parser.add_argument('--exclude-deps', '--exclude_deps',
- help='Build without internalized dependencies.',
- default=False, action='store_true')
- parser.add_argument('--no-internal', '--no_internal',
- help='Do not build with support for Google internal tests.',
- default=False, action='store_true')
- parser.add_argument('--java-home', '--java_home',
- help='Use a custom java version to run gradle.')
- parser.add_argument('--worktree',
- help='Gradle is running in a worktree and may lock up '
- 'the gradle caches.',
- action='store_true',
- default=False)
- return parser.parse_known_args()
+ parser = argparse.ArgumentParser(description='Call gradle.')
+ parser.add_argument('--exclude-deps',
+ '--exclude_deps',
+ help='Build without internalized dependencies.',
+ default=False,
+ action='store_true')
+ parser.add_argument(
+ '--no-internal',
+ '--no_internal',
+ help='Do not build with support for Google internal tests.',
+ default=False,
+ action='store_true')
+ parser.add_argument('--java-home',
+ '--java_home',
+ help='Use a custom java version to run gradle.')
+ parser.add_argument('--worktree',
+ help='Gradle is running in a worktree and may lock up '
+ 'the gradle caches.',
+ action='store_true',
+ default=False)
+ return parser.parse_known_args()
+
def GetJavaEnv(env):
- java_env = dict(env if env else os.environ, JAVA_HOME = jdk.GetJdkHome())
- java_env['PATH'] = java_env['PATH'] + os.pathsep + os.path.join(jdk.GetJdkHome(), 'bin')
- java_env['GRADLE_OPTS'] = '-Xmx1g'
- return java_env
+ java_env = dict(env if env else os.environ, JAVA_HOME=jdk.GetJdkHome())
+ java_env['PATH'] = java_env['PATH'] + os.pathsep + os.path.join(
+ jdk.GetJdkHome(), 'bin')
+ java_env['GRADLE_OPTS'] = '-Xmx1g'
+ return java_env
+
def PrintCmd(s):
- if type(s) is list:
- s = ' '.join(s)
- print('Running: %s' % s)
- # I know this will hit os on windows eventually if we don't do this.
- sys.stdout.flush()
+ if type(s) is list:
+ s = ' '.join(s)
+ print('Running: %s' % s)
+ # I know this will hit os on windows eventually if we don't do this.
+ sys.stdout.flush()
+
def EnsureGradle():
- utils.EnsureDepFromGoogleCloudStorage(
- get_gradle(), GRADLE8_TGZ, GRADLE8_SHA1, 'Gradle binary')
+ utils.EnsureDepFromGoogleCloudStorage(get_gradle(), GRADLE8_TGZ,
+ GRADLE8_SHA1, 'Gradle binary')
+
def EnsureJdk():
- # Gradle in the new setup will use the jdks in the evaluation - fetch
- # all beforehand.
- for root in jdk.GetAllJdkDirs():
- jdkTgz = root + '.tar.gz'
- jdkSha1 = jdkTgz + '.sha1'
- utils.EnsureDepFromGoogleCloudStorage(root, jdkTgz, jdkSha1, root)
+ # Gradle in the new setup will use the jdks in the evaluation - fetch
+ # all beforehand.
+ for root in jdk.GetAllJdkDirs():
+ jdkTgz = root + '.tar.gz'
+ jdkSha1 = jdkTgz + '.sha1'
+ utils.EnsureDepFromGoogleCloudStorage(root, jdkTgz, jdkSha1, root)
+
def EnsureDeps():
- EnsureGradle()
- EnsureJdk()
+ EnsureGradle()
+ EnsureJdk()
+
def RunGradleIn(gradleCmd, args, cwd, throw_on_failure=True, env=None):
- EnsureDeps()
- cmd = [gradleCmd]
- args.extend(['--offline', '-c=d8_r8/settings.gradle.kts'])
- cmd.extend(args)
- utils.PrintCmd(cmd)
- with utils.ChangedWorkingDirectory(cwd):
- return_value = subprocess.call(cmd, env=GetJavaEnv(env))
- if throw_on_failure and return_value != 0:
- raise Exception('Failed to execute gradle')
- return return_value
+ EnsureDeps()
+ cmd = [gradleCmd]
+ args.extend(['--offline', '-c=d8_r8/settings.gradle.kts'])
+ cmd.extend(args)
+ utils.PrintCmd(cmd)
+ with utils.ChangedWorkingDirectory(cwd):
+ return_value = subprocess.call(cmd, env=GetJavaEnv(env))
+ if throw_on_failure and return_value != 0:
+ raise Exception('Failed to execute gradle')
+ return return_value
+
def RunGradleWrapperIn(args, cwd, throw_on_failure=True, env=None):
- return RunGradleIn('./gradlew', args, cwd, throw_on_failure, env=env)
+ return RunGradleIn('./gradlew', args, cwd, throw_on_failure, env=env)
+
def RunGradle(args, throw_on_failure=True, env=None):
- return RunGradleIn(
- get_gradle(),
- args,
- utils.REPO_ROOT,
- throw_on_failure,
- env=env)
+ return RunGradleIn(get_gradle(),
+ args,
+ utils.REPO_ROOT,
+ throw_on_failure,
+ env=env)
+
def RunGradleExcludeDeps(args, throw_on_failure=True, env=None):
- EnsureDeps()
- args.append('-Pexclude_deps')
- return RunGradle(args, throw_on_failure, env=env)
+ EnsureDeps()
+ args.append('-Pexclude_deps')
+ return RunGradle(args, throw_on_failure, env=env)
+
def RunGradleInGetOutput(gradleCmd, args, cwd, env=None):
- EnsureDeps()
- cmd = [gradleCmd]
- cmd.extend(args)
- utils.PrintCmd(cmd)
- with utils.ChangedWorkingDirectory(cwd):
- return subprocess.check_output(cmd, env=GetJavaEnv(env)).decode('utf-8')
+ EnsureDeps()
+ cmd = [gradleCmd]
+ cmd.extend(args)
+ utils.PrintCmd(cmd)
+ with utils.ChangedWorkingDirectory(cwd):
+ return subprocess.check_output(cmd, env=GetJavaEnv(env)).decode('utf-8')
+
def RunGradleWrapperInGetOutput(args, cwd, env=None):
- return RunGradleInGetOutput('./gradlew', args, cwd, env=env)
+ return RunGradleInGetOutput('./gradlew', args, cwd, env=env)
+
def RunGradleGetOutput(args, env=None):
- return RunGradleInGetOutput(
- get_gradle(),
- args,
- utils.REPO_ROOT,
- env=env)
+ return RunGradleInGetOutput(get_gradle(), args, utils.REPO_ROOT, env=env)
+
def Main():
- (options, args) = ParseOptions()
- if options.java_home:
- args.append('-Dorg.gradle.java.home=' + options.java_home)
- if options.no_internal:
- args.append('-Pno_internal')
- if options.exclude_deps:
- args.append('-Pexclude_deps')
- if options.worktree:
- args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
- return RunGradle(args)
+ (options, args) = ParseOptions()
+ if options.java_home:
+ args.append('-Dorg.gradle.java.home=' + options.java_home)
+ if options.no_internal:
+ args.append('-Pno_internal')
+ if options.exclude_deps:
+ args.append('-Pexclude_deps')
+ if options.worktree:
+ args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
+ return RunGradle(args)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/historic_memory_usage.py b/tools/historic_memory_usage.py
index 3ef2268..a40cf2a 100755
--- a/tools/historic_memory_usage.py
+++ b/tools/historic_memory_usage.py
@@ -20,60 +20,64 @@
APPS = ['gmscore', 'nest', 'youtube', 'gmail', 'chrome']
COMPILERS = ['d8', 'r8']
+
def ParseOptions(argv):
- result = optparse.OptionParser()
- result.add_option('--compiler',
- help='The compiler to use',
- default='d8',
- choices=COMPILERS)
- result.add_option('--app',
- help='What app to run on',
- default='gmail',
- choices=APPS)
- result.add_option('--top',
- default=historic_run.top_or_default(),
- help='The most recent commit to test')
- result.add_option('--bottom',
- help='The oldest commit to test')
- result.add_option('--output',
- default='build',
- help='Directory where to output results')
- result.add_option('--timeout',
- type=int,
- default=0,
- help='Set timeout instead of waiting for OOM.')
- return result.parse_args(argv)
+ result = optparse.OptionParser()
+ result.add_option('--compiler',
+ help='The compiler to use',
+ default='d8',
+ choices=COMPILERS)
+ result.add_option('--app',
+ help='What app to run on',
+ default='gmail',
+ choices=APPS)
+ result.add_option('--top',
+ default=historic_run.top_or_default(),
+ help='The most recent commit to test')
+ result.add_option('--bottom', help='The oldest commit to test')
+ result.add_option('--output',
+ default='build',
+ help='Directory where to output results')
+ result.add_option('--timeout',
+ type=int,
+ default=0,
+ help='Set timeout instead of waiting for OOM.')
+ return result.parse_args(argv)
+
def make_run_on_app_command(options):
- return lambda commit: run_on_app(options, commit)
+ return lambda commit: run_on_app(options, commit)
+
def run_on_app(options, commit):
- app = options.app
- compiler = options.compiler
- cmd = ['tools/run_on_app.py',
- '--app', app,
- '--compiler', compiler,
- '--timeout', str(options.timeout),
- '--no-build', '--find-min-xmx']
- stdout = subprocess.check_output(cmd)
- output_path = options.output or 'build'
- time_commit = '%s_%s' % (commit.timestamp, commit.git_hash)
- time_commit_path = os.path.join(output_path, time_commit)
- if not os.path.exists(time_commit_path):
- os.makedirs(time_commit_path)
- stdout_path = os.path.join(time_commit_path, 'stdout')
- with open(stdout_path, 'w') as f:
- f.write(stdout)
- print('Wrote stdout to: %s' % stdout_path)
+ app = options.app
+ compiler = options.compiler
+ cmd = [
+ 'tools/run_on_app.py', '--app', app, '--compiler', compiler,
+ '--timeout',
+ str(options.timeout), '--no-build', '--find-min-xmx'
+ ]
+ stdout = subprocess.check_output(cmd)
+ output_path = options.output or 'build'
+ time_commit = '%s_%s' % (commit.timestamp, commit.git_hash)
+ time_commit_path = os.path.join(output_path, time_commit)
+ if not os.path.exists(time_commit_path):
+ os.makedirs(time_commit_path)
+ stdout_path = os.path.join(time_commit_path, 'stdout')
+ with open(stdout_path, 'w') as f:
+ f.write(stdout)
+ print('Wrote stdout to: %s' % stdout_path)
+
def main(argv):
- (options, args) = ParseOptions(argv)
- if not options.app:
- raise Exception('Please specify an app')
- top = historic_run.top_or_default(options.top)
- bottom = historic_run.bottom_or_default(options.bottom)
- command = make_run_on_app_command(options)
- historic_run.run(command, top, bottom)
+ (options, args) = ParseOptions(argv)
+ if not options.app:
+ raise Exception('Please specify an app')
+ top = historic_run.top_or_default(options.top)
+ bottom = historic_run.bottom_or_default(options.bottom)
+ command = make_run_on_app_command(options)
+ historic_run.run(command, top, bottom)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/historic_run.py b/tools/historic_run.py
index 75819bb..9a035b0 100755
--- a/tools/historic_run.py
+++ b/tools/historic_run.py
@@ -17,171 +17,183 @@
MASTER_COMMITS = 'gs://r8-releases/raw/master'
+
def ParseOptions(argv):
- result = optparse.OptionParser()
- result.add_option(
- '--cmd',
- help='Command to run')
- result.add_option(
- '--top',
- default=top_or_default(),
- help='The most recent commit to test')
- result.add_option(
- '--bottom',
- help='The oldest commit to test')
- result.add_option(
- '--dry-run',
- help='Do not download or run the command, but print the actions',
- default=False,
- action='store_true')
- result.add_option(
- '--output',
- default='build',
- help='Directory where to output results')
- return result.parse_args(argv)
+ result = optparse.OptionParser()
+ result.add_option('--cmd', help='Command to run')
+ result.add_option('--top',
+ default=top_or_default(),
+ help='The most recent commit to test')
+ result.add_option('--bottom', help='The oldest commit to test')
+ result.add_option(
+ '--dry-run',
+ help='Do not download or run the command, but print the actions',
+ default=False,
+ action='store_true')
+ result.add_option('--output',
+ default='build',
+ help='Directory where to output results')
+ return result.parse_args(argv)
class GitCommit(object):
- def __init__(self, git_hash, destination_dir, destination, timestamp):
- self.git_hash = git_hash
- self.destination_dir = destination_dir
- self.destination = destination
- self.timestamp = timestamp
- def __str__(self):
- return '%s : %s (%s)' % (self.git_hash, self.destination, self.timestamp)
+ def __init__(self, git_hash, destination_dir, destination, timestamp):
+ self.git_hash = git_hash
+ self.destination_dir = destination_dir
+ self.destination = destination
+ self.timestamp = timestamp
- def __repr__(self):
- return self.__str__()
+ def __str__(self):
+ return '%s : %s (%s)' % (self.git_hash, self.destination,
+ self.timestamp)
+
+ def __repr__(self):
+ return self.__str__()
+
def git_commit_from_hash(hash):
- commit_timestamp = subprocess.check_output(['git', 'show', '--no-patch',
- '--no-notes', '--pretty=\'%ct\'',
- hash]).strip().strip('\'')
- destination_dir = '%s/%s/' % (MASTER_COMMITS, hash)
- destination = '%s%s' % (destination_dir, 'r8.jar')
- commit = GitCommit(hash, destination_dir, destination, commit_timestamp)
- return commit
+ commit_timestamp = subprocess.check_output(
+ ['git', 'show', '--no-patch', '--no-notes', '--pretty=\'%ct\'',
+ hash]).strip().strip('\'')
+ destination_dir = '%s/%s/' % (MASTER_COMMITS, hash)
+ destination = '%s%s' % (destination_dir, 'r8.jar')
+ commit = GitCommit(hash, destination_dir, destination, commit_timestamp)
+ return commit
+
def enumerate_git_commits(top, bottom):
- output = subprocess.check_output(['git', 'rev-list', '--first-parent', top])
- found_bottom = False
- commits = []
- for c in output.splitlines():
- commit_hash = c.strip()
- commits.append(git_commit_from_hash(commit_hash))
- if commit_hash == bottom:
- found_bottom = True
- break
- if not found_bottom:
- raise Exception('Bottom not found, did you not use a merge commit')
- return commits
+ output = subprocess.check_output(['git', 'rev-list', '--first-parent', top])
+ found_bottom = False
+ commits = []
+ for c in output.splitlines():
+ commit_hash = c.strip()
+ commits.append(git_commit_from_hash(commit_hash))
+ if commit_hash == bottom:
+ found_bottom = True
+ break
+ if not found_bottom:
+ raise Exception('Bottom not found, did you not use a merge commit')
+ return commits
+
def get_available_commits(commits):
- cloud_commits = subprocess.check_output(
- ['gsutil.py', 'ls', MASTER_COMMITS]).splitlines()
- available_commits = []
- for commit in commits:
- if commit.destination_dir in cloud_commits:
- available_commits.append(commit)
- return available_commits
+ cloud_commits = subprocess.check_output(['gsutil.py', 'ls',
+ MASTER_COMMITS]).splitlines()
+ available_commits = []
+ for commit in commits:
+ if commit.destination_dir in cloud_commits:
+ available_commits.append(commit)
+ return available_commits
+
def print_commits(commits):
- for commit in commits:
- print(commit)
+ for commit in commits:
+ print(commit)
+
def permutate_range(start, end):
- diff = end - start
- assert diff >= 0
- if diff == 1:
- return [start, end]
- if diff == 0:
- return [start]
- half = end - (diff / 2)
- numbers = [half]
- first_half = permutate_range(start, half - 1)
- second_half = permutate_range(half + 1, end)
- for index in range(len(first_half)):
- numbers.append(first_half[index])
- if index < len(second_half):
- numbers.append(second_half[index])
- return numbers
+ diff = end - start
+ assert diff >= 0
+ if diff == 1:
+ return [start, end]
+ if diff == 0:
+ return [start]
+ half = end - (diff / 2)
+ numbers = [half]
+ first_half = permutate_range(start, half - 1)
+ second_half = permutate_range(half + 1, end)
+ for index in range(len(first_half)):
+ numbers.append(first_half[index])
+ if index < len(second_half):
+ numbers.append(second_half[index])
+ return numbers
+
def permutate(number_of_commits):
- assert number_of_commits > 0
- numbers = permutate_range(0, number_of_commits - 1)
- assert all(n in numbers for n in range(number_of_commits))
- return numbers
+ assert number_of_commits > 0
+ numbers = permutate_range(0, number_of_commits - 1)
+ assert all(n in numbers for n in range(number_of_commits))
+ return numbers
+
def pull_r8_from_cloud(commit):
- utils.download_file_from_cloud_storage(commit.destination, utils.R8_JAR)
+ utils.download_file_from_cloud_storage(commit.destination, utils.R8_JAR)
+
def benchmark(commits, command, dryrun=False):
- commit_permutations = permutate(len(commits))
- count = 0
- for index in commit_permutations:
- count += 1
- print('Running commit %s out of %s' % (count, len(commits)))
- commit = commits[index]
- if not utils.cloud_storage_exists(commit.destination):
- # We may have a directory, but no r8.jar
- continue
- if not dryrun:
- pull_r8_from_cloud(commit)
- print('Running for commit: %s' % commit.git_hash)
- command(commit)
+ commit_permutations = permutate(len(commits))
+ count = 0
+ for index in commit_permutations:
+ count += 1
+ print('Running commit %s out of %s' % (count, len(commits)))
+ commit = commits[index]
+ if not utils.cloud_storage_exists(commit.destination):
+ # We may have a directory, but no r8.jar
+ continue
+ if not dryrun:
+ pull_r8_from_cloud(commit)
+ print('Running for commit: %s' % commit.git_hash)
+ command(commit)
+
def top_or_default(top=None):
- return top if top else utils.get_HEAD_sha1()
+ return top if top else utils.get_HEAD_sha1()
+
def bottom_or_default(bottom=None):
- # TODO(ricow): if not set, search back 1000
- if not bottom:
- raise Exception('No bottom specified')
- return bottom
+ # TODO(ricow): if not set, search back 1000
+ if not bottom:
+ raise Exception('No bottom specified')
+ return bottom
+
def run(command, top, bottom, dryrun=False):
- commits = enumerate_git_commits(top, bottom)
- available_commits = get_available_commits(commits)
- print('Running for:')
- print_commits(available_commits)
- benchmark(available_commits, command, dryrun=dryrun)
+ commits = enumerate_git_commits(top, bottom)
+ available_commits = get_available_commits(commits)
+ print('Running for:')
+ print_commits(available_commits)
+ benchmark(available_commits, command, dryrun=dryrun)
+
def make_cmd(options):
- return lambda commit: run_cmd(options, commit)
+ return lambda commit: run_cmd(options, commit)
+
def run_cmd(options, commit):
- cmd = [options.cmd, commit.git_hash]
- output_path = options.output or 'build'
- time_commit = '%s_%s' % (commit.timestamp, commit.git_hash)
- time_commit_path = os.path.join(output_path, time_commit)
- print(' '.join(cmd))
- if not options.dry_run:
- if not os.path.exists(time_commit_path):
- os.makedirs(time_commit_path)
- stdout_path = os.path.join(time_commit_path, 'stdout')
- stderr_path = os.path.join(time_commit_path, 'stderr')
- with open(stdout_path, 'w') as stdout:
- with open(stderr_path, 'w') as stderr:
- process = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
- timeout = 1000
- while process.poll() is None and timeout > 0:
- time.sleep(1)
- timeout -= 1
- if process.poll() is None:
- process.kill()
- print("Task timed out")
- stderr.write("timeout\n")
- print('Wrote outputs to: %s' % time_commit_path)
+ cmd = [options.cmd, commit.git_hash]
+ output_path = options.output or 'build'
+ time_commit = '%s_%s' % (commit.timestamp, commit.git_hash)
+ time_commit_path = os.path.join(output_path, time_commit)
+ print(' '.join(cmd))
+ if not options.dry_run:
+ if not os.path.exists(time_commit_path):
+ os.makedirs(time_commit_path)
+ stdout_path = os.path.join(time_commit_path, 'stdout')
+ stderr_path = os.path.join(time_commit_path, 'stderr')
+ with open(stdout_path, 'w') as stdout:
+ with open(stderr_path, 'w') as stderr:
+ process = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
+ timeout = 1000
+ while process.poll() is None and timeout > 0:
+ time.sleep(1)
+ timeout -= 1
+ if process.poll() is None:
+ process.kill()
+ print("Task timed out")
+ stderr.write("timeout\n")
+ print('Wrote outputs to: %s' % time_commit_path)
+
def main(argv):
- (options, args) = ParseOptions(argv)
- if not options.cmd:
- raise Exception('Please specify a command')
- top = top_or_default(options.top)
- bottom = bottom_or_default(options.bottom)
- command = make_cmd(options)
- run(command, top, bottom, dryrun=options.dry_run)
+ (options, args) = ParseOptions(argv)
+ if not options.cmd:
+ raise Exception('Please specify a command')
+ top = top_or_default(options.top)
+ bottom = bottom_or_default(options.bottom)
+ command = make_cmd(options)
+ run(command, top, bottom, dryrun=options.dry_run)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/internal_test.py b/tools/internal_test.py
index 47844c8..129b087 100755
--- a/tools/internal_test.py
+++ b/tools/internal_test.py
@@ -62,52 +62,47 @@
DEPENDENT_PYTHON_FILES = [gradle, utils, run_on_app]
+
def find_min_xmx_command(app_data):
- record = app_data.GetMemoryData(app_data.GetLatestVersion())
- assert record['find-xmx-min'] < record['find-xmx-max']
- assert record['find-xmx-range'] < record['find-xmx-max'] - record['find-xmx-min']
- return [
- 'tools/run_on_app.py',
- '--compiler=r8',
- '--compiler-build=lib',
- '--app=%s' % app_data.GetName(),
- '--version=%s' % app_data.GetLatestVersion(),
- '--no-debug',
- '--no-build',
- '--find-min-xmx',
- '--find-min-xmx-min-memory=%s' % record['find-xmx-min'],
- '--find-min-xmx-max-memory=%s' % record['find-xmx-max'],
- '--find-min-xmx-range-size=%s' % record['find-xmx-range'],
- '--find-min-xmx-archive']
+ record = app_data.GetMemoryData(app_data.GetLatestVersion())
+ assert record['find-xmx-min'] < record['find-xmx-max']
+ assert record[
+ 'find-xmx-range'] < record['find-xmx-max'] - record['find-xmx-min']
+ return [
+ 'tools/run_on_app.py', '--compiler=r8', '--compiler-build=lib',
+ '--app=%s' % app_data.GetName(),
+ '--version=%s' % app_data.GetLatestVersion(), '--no-debug',
+ '--no-build', '--find-min-xmx',
+ '--find-min-xmx-min-memory=%s' % record['find-xmx-min'],
+ '--find-min-xmx-max-memory=%s' % record['find-xmx-max'],
+ '--find-min-xmx-range-size=%s' % record['find-xmx-range'],
+ '--find-min-xmx-archive'
+ ]
+
def compile_with_memory_max_command(app_data):
- # TODO(b/152939233): Remove this special handling when fixed.
- factor = 1.25 if app_data.GetName() == 'chrome' else 1.15
- record = app_data.GetMemoryData(app_data.GetLatestVersion())
- return [] if 'skip-find-xmx-max' in record else [
- 'tools/run_on_app.py',
- '--compiler=r8',
- '--compiler-build=lib',
- '--app=%s' % app_data.GetName(),
- '--version=%s' % app_data.GetLatestVersion(),
- '--no-debug',
- '--no-build',
- '--max-memory=%s' % int(record['oom-threshold'] * factor)
- ]
+ # TODO(b/152939233): Remove this special handling when fixed.
+ factor = 1.25 if app_data.GetName() == 'chrome' else 1.15
+ record = app_data.GetMemoryData(app_data.GetLatestVersion())
+ return [] if 'skip-find-xmx-max' in record else [
+ 'tools/run_on_app.py', '--compiler=r8', '--compiler-build=lib',
+ '--app=%s' % app_data.GetName(),
+ '--version=%s' %
+ app_data.GetLatestVersion(), '--no-debug', '--no-build',
+ '--max-memory=%s' % int(record['oom-threshold'] * factor)
+ ]
+
def compile_with_memory_min_command(app_data):
- record = app_data.GetMemoryData(app_data.GetLatestVersion())
- return [
- 'tools/run_on_app.py',
- '--compiler=r8',
- '--compiler-build=lib',
- '--app=%s' % app_data.GetName(),
- '--version=%s' % app_data.GetLatestVersion(),
- '--no-debug',
- '--no-build',
- '--expect-oom',
- '--max-memory=%s' % int(record['oom-threshold'] * 0.85)
- ]
+ record = app_data.GetMemoryData(app_data.GetLatestVersion())
+ return [
+ 'tools/run_on_app.py', '--compiler=r8', '--compiler-build=lib',
+ '--app=%s' % app_data.GetName(),
+ '--version=%s' % app_data.GetLatestVersion(), '--no-debug',
+ '--no-build', '--expect-oom',
+ '--max-memory=%s' % int(record['oom-threshold'] * 0.85)
+ ]
+
CLEAN_COMMANDS = [
# Make sure we have a clean build to not be polluted by old test files
@@ -117,8 +112,10 @@
# TODO(b/210982978): Enable testing of min xmx again
TEST_COMMANDS = [
# Run test.py internal testing.
- ['tools/test.py', '--only_internal', '--slow_tests',
- '--java_max_memory_size=8G'],
+ [
+ 'tools/test.py', '--only_internal', '--slow_tests',
+ '--java_max_memory_size=8G'
+ ],
# Ensure that all internal apps compile.
['tools/run_on_app.py', '--run-all', '--out=out', '--workers', '4'],
]
@@ -127,259 +124,292 @@
RUN_TIMEOUT = 3600 * 7
BOT_RUN_TIMEOUT = RUN_TIMEOUT * len(TEST_COMMANDS)
+
def log(str):
- print("%s: %s" % (time.strftime("%c"), str))
- sys.stdout.flush()
+ print("%s: %s" % (time.strftime("%c"), str))
+ sys.stdout.flush()
+
def ParseOptions():
- result = optparse.OptionParser()
- result.add_option('--continuous',
- help='Continuously run internal tests and post results to GCS.',
- default=False, action='store_true')
- result.add_option('--print_logs',
- help='Fetch logs from gcs and print them, takes the commit to print for.',
- default=None)
- result.add_option('--bot',
- help='Run in bot mode, i.e., scheduling runs.',
- default=False, action='store_true')
- result.add_option('--archive',
- help='Post result to GCS, implied by --continuous',
- default=False, action='store_true')
- return result.parse_args()
+ result = optparse.OptionParser()
+ result.add_option(
+ '--continuous',
+ help='Continuously run internal tests and post results to GCS.',
+ default=False,
+ action='store_true')
+ result.add_option(
+ '--print_logs',
+ help=
+ 'Fetch logs from gcs and print them, takes the commit to print for.',
+ default=None)
+ result.add_option('--bot',
+ help='Run in bot mode, i.e., scheduling runs.',
+ default=False,
+ action='store_true')
+ result.add_option('--archive',
+ help='Post result to GCS, implied by --continuous',
+ default=False,
+ action='store_true')
+ return result.parse_args()
+
def ensure_git_clean():
- # Ensure clean git repo.
- diff = subprocess.check_output(['git', 'diff']).decode('utf-8')
- if len(diff) > 0:
- log('Local modifications to the git repo, exiting')
- sys.exit(1)
+ # Ensure clean git repo.
+ diff = subprocess.check_output(['git', 'diff']).decode('utf-8')
+ if len(diff) > 0:
+ log('Local modifications to the git repo, exiting')
+ sys.exit(1)
+
def git_pull():
- ensure_git_clean()
- subprocess.check_call(['git', 'checkout', 'main'])
- subprocess.check_call(['git', 'pull'])
- return utils.get_HEAD_sha1()
+ ensure_git_clean()
+ subprocess.check_call(['git', 'checkout', 'main'])
+ subprocess.check_call(['git', 'pull'])
+ return utils.get_HEAD_sha1()
+
def git_checkout(git_hash):
- ensure_git_clean()
- # Ensure that we are up to date to get the commit.
- git_pull()
- exitcode = subprocess.call(['git', 'checkout', git_hash])
- if exitcode != 0:
- return None
- return utils.get_HEAD_sha1()
+ ensure_git_clean()
+ # Ensure that we are up to date to get the commit.
+ git_pull()
+ exitcode = subprocess.call(['git', 'checkout', git_hash])
+ if exitcode != 0:
+ return None
+ return utils.get_HEAD_sha1()
+
def get_test_result_dir():
- return os.path.join(utils.R8_INTERNAL_TEST_RESULTS_BUCKET, TEST_RESULT_DIR)
+ return os.path.join(utils.R8_INTERNAL_TEST_RESULTS_BUCKET, TEST_RESULT_DIR)
+
def get_sha_destination(sha):
- return os.path.join(get_test_result_dir(), sha)
+ return os.path.join(get_test_result_dir(), sha)
+
def archive_status(failed):
- gs_destination = 'gs://%s' % get_sha_destination(utils.get_HEAD_sha1())
- utils.archive_value('status', gs_destination, failed)
+ gs_destination = 'gs://%s' % get_sha_destination(utils.get_HEAD_sha1())
+ utils.archive_value('status', gs_destination, failed)
+
def get_status(sha):
- gs_destination = 'gs://%s/status' % get_sha_destination(sha)
- return utils.cat_file_on_cloud_storage(gs_destination)
+ gs_destination = 'gs://%s/status' % get_sha_destination(sha)
+ return utils.cat_file_on_cloud_storage(gs_destination)
+
def archive_log(stdout, stderr, exitcode, timed_out, cmd):
- sha = utils.get_HEAD_sha1()
- cmd_dir = cmd.replace(' ', '_').replace('/', '_')
- destination = os.path.join(get_sha_destination(sha), cmd_dir)
- gs_destination = 'gs://%s' % destination
- url = 'https://storage.cloud.google.com/%s' % destination
- log('Archiving logs to: %s' % gs_destination)
- utils.archive_value(EXITCODE, gs_destination, exitcode)
- utils.archive_value(TIMED_OUT, gs_destination, timed_out)
- utils.archive_file(STDOUT, gs_destination, stdout)
- utils.archive_file(STDERR, gs_destination, stderr)
- log('Logs available at: %s' % url)
+ sha = utils.get_HEAD_sha1()
+ cmd_dir = cmd.replace(' ', '_').replace('/', '_')
+ destination = os.path.join(get_sha_destination(sha), cmd_dir)
+ gs_destination = 'gs://%s' % destination
+ url = 'https://storage.cloud.google.com/%s' % destination
+ log('Archiving logs to: %s' % gs_destination)
+ utils.archive_value(EXITCODE, gs_destination, exitcode)
+ utils.archive_value(TIMED_OUT, gs_destination, timed_out)
+ utils.archive_file(STDOUT, gs_destination, stdout)
+ utils.archive_file(STDERR, gs_destination, stderr)
+ log('Logs available at: %s' % url)
+
def get_magic_file_base_path():
- return 'gs://%s/magic' % get_test_result_dir()
+ return 'gs://%s/magic' % get_test_result_dir()
+
def get_magic_file_gs_path(name):
- return '%s/%s' % (get_magic_file_base_path(), name)
+ return '%s/%s' % (get_magic_file_base_path(), name)
+
def get_magic_file_exists(name):
- return utils.file_exists_on_cloud_storage(get_magic_file_gs_path(name))
+ return utils.file_exists_on_cloud_storage(get_magic_file_gs_path(name))
+
def delete_magic_file(name):
- utils.delete_file_from_cloud_storage(get_magic_file_gs_path(name))
+ utils.delete_file_from_cloud_storage(get_magic_file_gs_path(name))
+
def put_magic_file(name, sha):
- utils.archive_value(name, get_magic_file_base_path(), sha)
+ utils.archive_value(name, get_magic_file_base_path(), sha)
+
def get_magic_file_content(name, ignore_errors=False):
- return utils.cat_file_on_cloud_storage(get_magic_file_gs_path(name),
- ignore_errors=ignore_errors)
+ return utils.cat_file_on_cloud_storage(get_magic_file_gs_path(name),
+ ignore_errors=ignore_errors)
+
def print_magic_file_state():
- log('Magic file status:')
- for magic in ALL_MAGIC:
- if get_magic_file_exists(magic):
- content = get_magic_file_content(magic, ignore_errors=True)
- log('%s content: %s' % (magic, content))
+ log('Magic file status:')
+ for magic in ALL_MAGIC:
+ if get_magic_file_exists(magic):
+ content = get_magic_file_content(magic, ignore_errors=True)
+ log('%s content: %s' % (magic, content))
+
def fetch_and_print_logs(hash):
- gs_base = 'gs://%s' % get_sha_destination(hash)
- listing = utils.ls_files_on_cloud_storage(gs_base).strip().split('\n')
- for entry in listing:
- if not entry.endswith('/status'): # Ignore the overall status file
- for to_print in [EXITCODE, TIMED_OUT, STDERR, STDOUT]:
- gs_location = '%s%s' % (entry, to_print)
- value = utils.cat_file_on_cloud_storage(gs_location)
- print('\n\n%s had value:\n%s' % (to_print, value))
- print("\n\nPrinting find-min-xmx ranges for apps")
- run_on_app.print_min_xmx_ranges_for_hash(hash, 'r8', 'lib')
+ gs_base = 'gs://%s' % get_sha_destination(hash)
+ listing = utils.ls_files_on_cloud_storage(gs_base).strip().split('\n')
+ for entry in listing:
+ if not entry.endswith('/status'): # Ignore the overall status file
+ for to_print in [EXITCODE, TIMED_OUT, STDERR, STDOUT]:
+ gs_location = '%s%s' % (entry, to_print)
+ value = utils.cat_file_on_cloud_storage(gs_location)
+ print('\n\n%s had value:\n%s' % (to_print, value))
+ print("\n\nPrinting find-min-xmx ranges for apps")
+ run_on_app.print_min_xmx_ranges_for_hash(hash, 'r8', 'lib')
+
def run_bot():
- print_magic_file_state()
- # Ensure that there is nothing currently scheduled (broken/stopped run)
- for magic in ALL_MAGIC:
- if get_magic_file_exists(magic):
- log('ERROR: Synchronizing file %s exists, cleaning up' % magic)
- delete_magic_file(magic)
- print_magic_file_state()
- assert not get_magic_file_exists(READY_FOR_TESTING)
- git_hash = utils.get_HEAD_sha1()
- put_magic_file(READY_FOR_TESTING, git_hash)
- begin = time.time()
- while True:
- if time.time() - begin > BOT_RUN_TIMEOUT:
- log('Timeout exceeded: http://go/internal-r8-doc')
- raise Exception('Bot timeout')
- if get_magic_file_exists(TESTING_COMPLETE):
- if get_magic_file_content(TESTING_COMPLETE) == git_hash:
- break
- else:
- raise Exception('Non matching git hashes %s and %s' % (
- get_magic_file_content(TESTING_COMPLETE), git_hash))
- log('Still waiting for test result')
print_magic_file_state()
- time.sleep(PULL_DELAY)
- total_time = time.time()-begin
- log('Done running test for %s in %ss' % (git_hash, total_time))
- test_status = get_status(git_hash)
- delete_magic_file(TESTING_COMPLETE)
- log('Test status is: %s' % test_status)
- if test_status != '0':
- print('Tests failed, you can print the logs by running(googlers only):')
- print(' tools/internal_test.py --print_logs %s' % git_hash)
- return 1
+ # Ensure that there is nothing currently scheduled (broken/stopped run)
+ for magic in ALL_MAGIC:
+ if get_magic_file_exists(magic):
+ log('ERROR: Synchronizing file %s exists, cleaning up' % magic)
+ delete_magic_file(magic)
+ print_magic_file_state()
+ assert not get_magic_file_exists(READY_FOR_TESTING)
+ git_hash = utils.get_HEAD_sha1()
+ put_magic_file(READY_FOR_TESTING, git_hash)
+ begin = time.time()
+ while True:
+ if time.time() - begin > BOT_RUN_TIMEOUT:
+ log('Timeout exceeded: http://go/internal-r8-doc')
+ raise Exception('Bot timeout')
+ if get_magic_file_exists(TESTING_COMPLETE):
+ if get_magic_file_content(TESTING_COMPLETE) == git_hash:
+ break
+ else:
+ raise Exception(
+ 'Non matching git hashes %s and %s' %
+ (get_magic_file_content(TESTING_COMPLETE), git_hash))
+ log('Still waiting for test result')
+ print_magic_file_state()
+ time.sleep(PULL_DELAY)
+ total_time = time.time() - begin
+ log('Done running test for %s in %ss' % (git_hash, total_time))
+ test_status = get_status(git_hash)
+ delete_magic_file(TESTING_COMPLETE)
+ log('Test status is: %s' % test_status)
+ if test_status != '0':
+ print('Tests failed, you can print the logs by running(googlers only):')
+ print(' tools/internal_test.py --print_logs %s' % git_hash)
+ return 1
+
def run_continuously():
- while True:
- print_magic_file_state()
- if get_magic_file_exists(READY_FOR_TESTING):
- git_hash = get_magic_file_content(READY_FOR_TESTING)
- checked_out = git_checkout(git_hash)
- if not checked_out:
- # Gerrit change, we don't run these on internal.
- archive_status(0)
- put_magic_file(TESTING_COMPLETE, git_hash)
- delete_magic_file(READY_FOR_TESTING)
- continue
- # Sanity check, if this does not succeed stop.
- if checked_out != git_hash:
- log('Inconsistent state: %s %s' % (git_hash, checked_out))
- sys.exit(1)
- put_magic_file(TESTING, git_hash)
- delete_magic_file(READY_FOR_TESTING)
- log('Running with hash: %s' % git_hash)
- exitcode = run_external()
- log('Running finished with exit code %s' % exitcode)
- # If the bot timed out or something else triggered the bot to fail, don't
- # put up the result (it will not be displayed anywhere, and we can't
- # remove the magic file if the bot cleaned up).
- if get_magic_file_exists(TESTING):
- put_magic_file(TESTING_COMPLETE, git_hash)
- # There is still a potential race here (we check, bot deletes, we try to
- # delete) - this is unlikely and we ignore it (restart if it happens).
- delete_magic_file(TESTING)
- time.sleep(PULL_DELAY)
+ while True:
+ print_magic_file_state()
+ if get_magic_file_exists(READY_FOR_TESTING):
+ git_hash = get_magic_file_content(READY_FOR_TESTING)
+ checked_out = git_checkout(git_hash)
+ if not checked_out:
+ # Gerrit change, we don't run these on internal.
+ archive_status(0)
+ put_magic_file(TESTING_COMPLETE, git_hash)
+ delete_magic_file(READY_FOR_TESTING)
+ continue
+ # Sanity check, if this does not succeed stop.
+ if checked_out != git_hash:
+ log('Inconsistent state: %s %s' % (git_hash, checked_out))
+ sys.exit(1)
+ put_magic_file(TESTING, git_hash)
+ delete_magic_file(READY_FOR_TESTING)
+ log('Running with hash: %s' % git_hash)
+ exitcode = run_external()
+ log('Running finished with exit code %s' % exitcode)
+ # If the bot timed out or something else triggered the bot to fail, don't
+ # put up the result (it will not be displayed anywhere, and we can't
+ # remove the magic file if the bot cleaned up).
+ if get_magic_file_exists(TESTING):
+ put_magic_file(TESTING_COMPLETE, git_hash)
+ # There is still a potential race here (we check, bot deletes, we try to
+ # delete) - this is unlikely and we ignore it (restart if it happens).
+ delete_magic_file(TESTING)
+ time.sleep(PULL_DELAY)
+
def run_external():
- return subprocess.call([sys.executable, "tools/internal_test.py", "--archive"])
+ return subprocess.call(
+ [sys.executable, "tools/internal_test.py", "--archive"])
+
def handle_output(archive, stderr, stdout, exitcode, timed_out, cmd):
- if archive:
- archive_log(stdout, stderr, exitcode, timed_out, cmd)
- else:
- print('Execution of %s resulted in:' % cmd)
- print('exit code: %s ' % exitcode)
- print('timeout: %s ' % timed_out)
- with open(stderr, 'r') as f:
- print('stderr: %s' % f.read())
- with open(stdout, 'r') as f:
- print('stdout: %s' % f.read())
+ if archive:
+ archive_log(stdout, stderr, exitcode, timed_out, cmd)
+ else:
+ print('Execution of %s resulted in:' % cmd)
+ print('exit code: %s ' % exitcode)
+ print('timeout: %s ' % timed_out)
+ with open(stderr, 'r') as f:
+ print('stderr: %s' % f.read())
+ with open(stdout, 'r') as f:
+ print('stdout: %s' % f.read())
+
def execute(cmd, archive, env=None):
- if cmd == []:
- return
+ if cmd == []:
+ return
- assert(cmd[0].endswith('.py'))
- cmd = [sys.executable] + cmd
+ assert (cmd[0].endswith('.py'))
+ cmd = [sys.executable] + cmd
+ utils.PrintCmd(cmd)
+ with utils.TempDir() as temp:
+ try:
+ stderr_fd = None
+ stdout_fd = None
+ exitcode = 0
+ stderr = os.path.join(temp, 'stderr')
+ stderr_fd = open(stderr, 'w')
+ stdout = os.path.join(temp, 'stdout')
+ stdout_fd = open(stdout, 'w')
+ popen = subprocess.Popen(cmd,
+ bufsize=1024 * 1024 * 10,
+ stdout=stdout_fd,
+ stderr=stderr_fd,
+ env=env)
+ begin = time.time()
+ timed_out = False
+ while popen.poll() == None:
+ if time.time() - begin > RUN_TIMEOUT:
+ popen.terminate()
+ timed_out = True
+ time.sleep(2)
+ exitcode = popen.returncode
+ finally:
+ if stderr_fd:
+ stderr_fd.close()
+ if stdout_fd:
+ stdout_fd.close()
+ if exitcode != 0:
+ handle_output(archive, stderr, stdout, popen.returncode,
+ timed_out, ' '.join(cmd))
+ return exitcode
- utils.PrintCmd(cmd)
- with utils.TempDir() as temp:
- try:
- stderr_fd = None
- stdout_fd = None
- exitcode = 0
- stderr = os.path.join(temp, 'stderr')
- stderr_fd = open(stderr, 'w')
- stdout = os.path.join(temp, 'stdout')
- stdout_fd = open(stdout, 'w')
- popen = subprocess.Popen(cmd,
- bufsize=1024*1024*10,
- stdout=stdout_fd,
- stderr=stderr_fd,
- env=env)
- begin = time.time()
- timed_out = False
- while popen.poll() == None:
- if time.time() - begin > RUN_TIMEOUT:
- popen.terminate()
- timed_out = True
- time.sleep(2)
- exitcode = popen.returncode
- finally:
- if stderr_fd:
- stderr_fd.close()
- if stdout_fd:
- stdout_fd.close()
- if exitcode != 0:
- handle_output(archive, stderr, stdout, popen.returncode,
- timed_out, ' '.join(cmd))
- return exitcode
def run_once(archive):
- git_hash = utils.get_HEAD_sha1()
- log('Running once with hash %s' % git_hash)
- env = os.environ.copy()
- # Bot does not have a lot of memory.
- env['R8_GRADLE_CORES_PER_FORK'] = '5'
- if archive:
- [execute(cmd, archive, env) for cmd in CLEAN_COMMANDS]
- failed = any([execute(cmd, archive, env) for cmd in TEST_COMMANDS])
- # Gradle daemon occasionally leaks memory, stop it.
- gradle.RunGradle(['--stop'])
- archive_status(1 if failed else 0)
- return failed
+ git_hash = utils.get_HEAD_sha1()
+ log('Running once with hash %s' % git_hash)
+ env = os.environ.copy()
+ # Bot does not have a lot of memory.
+ env['R8_GRADLE_CORES_PER_FORK'] = '5'
+ if archive:
+ [execute(cmd, archive, env) for cmd in CLEAN_COMMANDS]
+ failed = any([execute(cmd, archive, env) for cmd in TEST_COMMANDS])
+ # Gradle daemon occasionally leaks memory, stop it.
+ gradle.RunGradle(['--stop'])
+ archive_status(1 if failed else 0)
+ return failed
+
def Main():
- (options, args) = ParseOptions()
- if options.continuous:
- run_continuously()
- elif options.bot:
- return run_bot()
- elif options.print_logs:
- return fetch_and_print_logs(options.print_logs)
- else:
- return run_once(options.archive)
+ (options, args) = ParseOptions()
+ if options.continuous:
+ run_continuously()
+ elif options.bot:
+ return run_bot()
+ elif options.print_logs:
+ return fetch_and_print_logs(options.print_logs)
+ else:
+ return run_once(options.archive)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/jardiff.py b/tools/jardiff.py
index 18587ab..afb462e 100755
--- a/tools/jardiff.py
+++ b/tools/jardiff.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('jardiff', sys.argv[1:]))
+ sys.exit(toolhelper.run('jardiff', sys.argv[1:]))
diff --git a/tools/java.py b/tools/java.py
deleted file mode 100755
index fb0ef13..0000000
--- a/tools/java.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-import jdk
-import utils
-import subprocess
-import sys
-
-def run(args):
- cmd = [jdk.GetJavaExecutable()] + args
- utils.PrintCmd(cmd)
- result = subprocess.check_output(cmd)
- print result
- return result
-
-def main():
- try:
- run(sys.argv[1:])
- except subprocess.CalledProcessError as e:
- # In case anything relevant was printed to stdout, normally this is already
- # on stderr.
- print e.output
- return e.returncode
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/tools/javac.py b/tools/javac.py
deleted file mode 100755
index 242fac0..0000000
--- a/tools/javac.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-import jdk
-import utils
-import subprocess
-import sys
-
-def run(args):
- cmd = [jdk.GetJavacExecutable()] + args
- utils.PrintCmd(cmd)
- result = subprocess.check_output(cmd)
- print result
- return result
-
-def main():
- try:
- run(sys.argv[1:])
- except subprocess.CalledProcessError as e:
- # In case anything relevant was printed to stdout, normally this is already
- # on stderr.
- print e.output
- return e.returncode
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/tools/jdk.py b/tools/jdk.py
index f37d272..ba7ddfb 100755
--- a/tools/jdk.py
+++ b/tools/jdk.py
@@ -10,80 +10,92 @@
JDK_DIR = os.path.join(defines.THIRD_PARTY, 'openjdk')
-ALL_JDKS = ['openjdk-9.0.4', 'jdk-11', 'jdk-15', 'jdk-16', 'jdk-17',
- 'jdk-18', 'jdk-20']
+ALL_JDKS = [
+ 'openjdk-9.0.4', 'jdk-11', 'jdk-15', 'jdk-16', 'jdk-17', 'jdk-18', 'jdk-20'
+]
def GetJdkHome():
- return GetJdk11Home()
+ return GetJdk11Home()
+
def GetJdkRoot():
- return GetJdk11Root()
+ return GetJdk11Root()
+
def GetJdk11Root():
- root = os.path.join(JDK_DIR, 'jdk-11')
- os_root = GetOSPath(root)
- return os_root if os_root else os.environ['JAVA_HOME']
+ root = os.path.join(JDK_DIR, 'jdk-11')
+ os_root = GetOSPath(root)
+ return os_root if os_root else os.environ['JAVA_HOME']
+
def GetOSPath(root):
- if defines.IsLinux():
- return os.path.join(root, 'linux')
- elif defines.IsOsX():
- return os.path.join(root, 'osx')
- elif defines.IsWindows():
- return os.path.join(root, 'windows')
- else:
- return None
+ if defines.IsLinux():
+ return os.path.join(root, 'linux')
+ elif defines.IsOsX():
+ return os.path.join(root, 'osx')
+ elif defines.IsWindows():
+ return os.path.join(root, 'windows')
+ else:
+ return None
+
def GetAllJdkDirs():
- dirs = []
- for jdk in ALL_JDKS:
- root = GetOSPath(os.path.join(JDK_DIR, jdk))
- # Some jdks are not available on windows, don't try to get these.
- if os.path.exists(root + '.tar.gz.sha1'):
- dirs.append(root)
- return dirs
+ dirs = []
+ for jdk in ALL_JDKS:
+ root = GetOSPath(os.path.join(JDK_DIR, jdk))
+ # Some jdks are not available on windows, don't try to get these.
+ if os.path.exists(root + '.tar.gz.sha1'):
+ dirs.append(root)
+ return dirs
+
def GetJdk11Home():
- root = GetJdk11Root()
- # osx has the home inside Contents/Home in the bundle
- if defines.IsOsX():
- return os.path.join(root,'Contents', 'Home')
- else:
- return root
+ root = GetJdk11Root()
+ # osx has the home inside Contents/Home in the bundle
+ if defines.IsOsX():
+ return os.path.join(root, 'Contents', 'Home')
+ else:
+ return root
+
def GetJdk9Home():
- root = os.path.join(JDK_DIR, 'openjdk-9.0.4')
- if defines.IsLinux():
- return os.path.join(root, 'linux')
- elif defines.IsOsX():
- return os.path.join(root, 'osx')
- elif defines.IsWindows():
- return os.path.join(root, 'windows')
- else:
- return os.environ['JAVA_HOME']
+ root = os.path.join(JDK_DIR, 'openjdk-9.0.4')
+ if defines.IsLinux():
+ return os.path.join(root, 'linux')
+ elif defines.IsOsX():
+ return os.path.join(root, 'osx')
+ elif defines.IsWindows():
+ return os.path.join(root, 'windows')
+ else:
+ return os.environ['JAVA_HOME']
+
def GetJdk8Home():
- root = os.path.join(JDK_DIR, 'jdk8')
- if defines.IsLinux():
- return os.path.join(root, 'linux-x86')
- elif defines.IsOsX():
- return os.path.join(root, 'darwin-x86')
- else:
- return os.environ['JAVA_HOME']
+ root = os.path.join(JDK_DIR, 'jdk8')
+ if defines.IsLinux():
+ return os.path.join(root, 'linux-x86')
+ elif defines.IsOsX():
+ return os.path.join(root, 'darwin-x86')
+ else:
+ return os.environ['JAVA_HOME']
+
def GetJavaExecutable(jdkHome=None):
- jdkHome = jdkHome if jdkHome else GetJdkHome()
- executable = 'java.exe' if defines.IsWindows() else 'java'
- return os.path.join(jdkHome, 'bin', executable) if jdkHome else executable
+ jdkHome = jdkHome if jdkHome else GetJdkHome()
+ executable = 'java.exe' if defines.IsWindows() else 'java'
+ return os.path.join(jdkHome, 'bin', executable) if jdkHome else executable
+
def GetJavacExecutable(jdkHome=None):
- jdkHome = jdkHome if jdkHome else GetJdkHome()
- executable = 'javac.exe' if defines.IsWindows() else 'javac'
- return os.path.join(jdkHome, 'bin', executable) if jdkHome else executable
+ jdkHome = jdkHome if jdkHome else GetJdkHome()
+ executable = 'javac.exe' if defines.IsWindows() else 'javac'
+ return os.path.join(jdkHome, 'bin', executable) if jdkHome else executable
+
def Main():
- print(GetJdkHome())
+ print(GetJdkHome())
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/keeprule_benchmark.py b/tools/keeprule_benchmark.py
deleted file mode 100755
index 1401084..0000000
--- a/tools/keeprule_benchmark.py
+++ /dev/null
@@ -1,241 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-import argparse
-import os
-import subprocess
-import sys
-import time
-
-import jdk
-import proguard
-import toolhelper
-import utils
-
-SHRINKERS = ['r8'] + proguard.getVersions()
-
-INPUT_PROGRAM = utils.PINNED_R8_JAR
-
-ANNO = 'com.android.tools.r8.com.google.common.annotations.VisibleForTesting'
-
-R8_OPTIONS = [
- 'printTimes',
- 'passthroughDexCode',
- 'enableClassMerging',
- 'enableDevirtualization',
- 'enableNonNullTracking',
- 'enableInlining',
- 'enableSwitchMapRemoval',
- 'enableValuePropagation',
- 'useSmaliSyntax',
- 'verbose',
- 'quiet',
- 'invalidDebugInfoFatal',
- 'intermediate',
- 'enableLambdaMerging',
- 'enableDesugaring',
- 'enableMainDexListCheck',
- 'enableTreeShaking',
- 'printCfg',
- 'ignoreMissingClasses',
- 'forceProguardCompatibility',
- 'enableMinification',
- 'disableAssertions',
- 'debugKeepRules',
- 'debug',
- 'minimalMainDex',
- 'skipReadingDexCode',
-]
-
-R8_CLASSES = [
- 'com.android.tools.r8.code.Format11x',
- 'com.android.tools.r8.code.MoveFrom16',
- 'com.android.tools.r8.code.AddLong2Addr',
- 'com.android.tools.r8.code.AgetByte',
- 'com.android.tools.r8.code.SubDouble',
- 'com.android.tools.r8.code.Sput',
- 'com.android.tools.r8.code.Format10x',
- 'com.android.tools.r8.code.RemInt',
- 'com.android.tools.r8.code.ConstWide',
- 'com.android.tools.r8.code.SgetWide',
- 'com.android.tools.r8.code.OrInt2Addr',
- 'com.android.tools.r8.code.Iget',
- 'com.android.tools.r8.code.Instruction',
- 'com.android.tools.r8.code.SubInt2Addr',
- 'com.android.tools.r8.code.SwitchPayload',
- 'com.android.tools.r8.code.Const4',
- 'com.android.tools.r8.code.ShrIntLit8',
- 'com.android.tools.r8.code.ConstWide16',
- 'com.android.tools.r8.code.NegInt',
- 'com.android.tools.r8.code.SgetBoolean',
- 'com.android.tools.r8.code.Format22x',
- 'com.android.tools.r8.code.InvokeVirtualRange',
- 'com.android.tools.r8.code.Format45cc',
- 'com.android.tools.r8.code.DivFloat2Addr',
- 'com.android.tools.r8.code.MulIntLit16',
- 'com.android.tools.r8.code.BytecodeStream',
-]
-
-KEEP_MAIN = \
- '-keep class com.android.tools.r8.R8 { void main(java.lang.String[]); }'
-
-BENCHMARKS = [
- # Baseline compile just keeps R8.main (implicitly kept for all benchmarks).
- ('KeepBaseline', ''),
-
- # Mirror default keep getters/setters, but independent of hierarchy.
- ('KeepGetters',
- '-keepclassmembers class * { *** get*(); }'),
- ('KeepGettersIf',
- '-if class * { *** get*(); } -keep class <1> { *** get<2>(); }'),
-
- # Mirror default keep getters/setters below View (here a class with a B).
- ('KeepSubGetters',
- '-keepclassmembers class * extends **.*B* { *** get*(); }'),
- ('KeepSubGettersIf',
- '-if class * extends **.*B* -keep class <1> { *** get*(); }'),
-
- # General keep rule to keep annotated members.
- ('KeepAnnoMethod',
- '-keepclasseswithmembers class * { @%s *** *(...); }' % ANNO),
- ('KeepAnnoMethodCond',
- '-keepclassmembers class * { @%s *** *(...); }' % ANNO),
- ('KeepAnnoMethodIf',
- '-if class * { @%s *** *(...); } -keep class <1> { @%s *** <2>(...); }' \
- % (ANNO, ANNO)),
-
- # Large collection of rules mirroring AAPT conditional rules on R fields.
- ('KeepAaptFieldIf',
- '\n'.join([
- '-if class **.InternalOptions { boolean %s; }'
- ' -keep class %s { <init>(...); }' % (f, c)
- for (f, c) in zip(R8_OPTIONS, R8_CLASSES) * 1 #100
- ])),
-
- # If rules with predicates that will never by true, but will need
- # consideration. The CodeSize of these should be equal to the baseline run.
- ('KeepIfNonExistingClass',
- '-if class **.*A*B*C*D*E*F* -keep class %s' % ANNO),
- ('KeepIfNonExistingMember',
- '-if class **.*A* { *** *a*b*c*d*e*f*(...); } -keep class %s' % ANNO)
-]
-
-def parse_arguments(argv):
- parser = argparse.ArgumentParser(
- description = 'Run keep-rule benchmarks.')
- parser.add_argument('--ignore-java-version',
- help='Do not check java version',
- default=False,
- action='store_true')
- parser.add_argument('--shrinker',
- help='The shrinker to use',
- choices=SHRINKERS,
- default=SHRINKERS[0])
- parser.add_argument('--runs',
- help='Number of runs to average out time on',
- type=int,
- default=3)
- parser.add_argument('--benchmark',
- help='Benchmark to run (default all)',
- choices=map(lambda (x,y): x, BENCHMARKS),
- default=None)
- options = parser.parse_args(argv)
- return options
-
-class BenchmarkResult:
- def __init__(self, name, size, runs):
- self.name = name
- self.size = size
- self.runs = runs
-
-def isPG(shrinker):
- return proguard.isValidVersion(shrinker)
-
-def shrinker_args(shrinker, keepfile, output):
- if shrinker == 'r8':
- return [
- jdk.GetJavaExecutable(),
- '-cp', utils.R8LIB_JAR,
- 'com.android.tools.r8.R8',
- INPUT_PROGRAM,
- '--lib', utils.RT_JAR,
- '--output', output,
- '--min-api', '10000',
- '--pg-conf', keepfile,
- ]
- elif isPG(shrinker):
- return proguard.getCmd([
- '-injars', INPUT_PROGRAM,
- '-libraryjars', utils.RT_JAR,
- '-outjars', output,
- '-dontwarn', '**',
- '-optimizationpasses', '2',
- '@' + keepfile,
- ],
- version=shrinker)
- else:
- assert False, "Unexpected shrinker " + shrinker
-
-def dex(input, output):
- toolhelper.run(
- 'd8',
- [
- input,
- '--lib', utils.RT_JAR,
- '--min-api', '10000',
- '--output', output
- ],
- build=False,
- debug=False)
-
-def run_shrinker(options, temp):
- benchmarks = BENCHMARKS
- if options.benchmark:
- for (name, rules) in BENCHMARKS:
- if name == options.benchmark:
- benchmarks = [(name, rules)]
- break
- assert len(benchmarks) == 1, "Unexpected benchmark " + options.benchmark
-
- run_count = options.runs
- benchmark_results = []
- for (name, rule) in benchmarks:
- benchmark_keep = os.path.join(temp, '%s-keep.txt' % name)
- with open(benchmark_keep, 'w') as fp:
- fp.write(KEEP_MAIN)
- fp.write('\n')
- fp.write(rule)
-
- benchmark_runs = []
- benchmark_size = 0
- for i in range(run_count):
- out = os.path.join(temp, '%s-out%d.jar' % (name, i))
- cmd = shrinker_args(options.shrinker, benchmark_keep, out)
- utils.PrintCmd(cmd)
- t0 = time.time()
- subprocess.check_output(cmd)
- t1 = time.time()
- benchmark_runs.append(t1 - t0)
- if isPG(options.shrinker):
- dexout = os.path.join(temp, '%s-out%d-dex.jar' % (name, i))
- dex(out, dexout)
- benchmark_size = utils.uncompressed_size(dexout)
- else:
- benchmark_size = utils.uncompressed_size(out)
- benchmark_results.append(
- BenchmarkResult(name, benchmark_size, benchmark_runs))
-
- print 'Runs:', options.runs
- for result in benchmark_results:
- benchmark_avg = sum(result.runs) / run_count
- print '%s(CodeSize): %d' % (result.name, result.size)
- print '%s(RunTimeRaw): %d ms' % (result.name, 1000.0 * benchmark_avg)
-
-if __name__ == '__main__':
- options = parse_arguments(sys.argv[1:])
- if not options.ignore_java_version:
- utils.check_java_version()
- with utils.TempDir() as temp:
- run_shrinker(options, temp)
diff --git a/tools/maindex.py b/tools/maindex.py
index 735ced5..f157498 100755
--- a/tools/maindex.py
+++ b/tools/maindex.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('maindex', sys.argv[1:]))
+ sys.exit(toolhelper.run('maindex', sys.argv[1:]))
diff --git a/tools/minify_tool.py b/tools/minify_tool.py
index cef72e5..2356ac1 100755
--- a/tools/minify_tool.py
+++ b/tools/minify_tool.py
@@ -2,7 +2,6 @@
# Copyright (c) 2018, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
-
'''
Run R8 (with the class-file backend) to optimize a command-line program.
@@ -31,93 +30,114 @@
parser = argparse.ArgumentParser(description=__doc__.strip(),
formatter_class=argparse.RawTextHelpFormatter)
+parser.add_argument('-i',
+ '--input-jar',
+ default=utils.R8_JAR,
+ help='Input JAR to use (default: build/libs/r8.jar)')
parser.add_argument(
- '-i', '--input-jar', default=utils.R8_JAR,
- help='Input JAR to use (default: build/libs/r8.jar)')
-parser.add_argument(
- '-o', '--output-jar',
+ '-o',
+ '--output-jar',
help='Path to output JAR (default: build/libs/<MainClass>-min.jar)')
+parser.add_argument('-l',
+ '--lib',
+ default=utils.RT_JAR,
+ help='Path to rt.jar to use instead of OpenJDK 1.8')
parser.add_argument(
- '-l', '--lib', default=utils.RT_JAR,
- help='Path to rt.jar to use instead of OpenJDK 1.8')
-parser.add_argument(
- '-m', '--mainclass',
+ '-m',
+ '--mainclass',
help='Create/overwrite MANIFEST.MF with the given Main-Class')
-parser.add_argument(
- '-O', '--no-debug', dest='debug', action='store_false',
- help='Disable assertions when running R8')
-parser.add_argument(
- '--benchmark-name',
- help='Print benchmarks with the given name')
+parser.add_argument('-O',
+ '--no-debug',
+ dest='debug',
+ action='store_false',
+ help='Disable assertions when running R8')
+parser.add_argument('--benchmark-name',
+ help='Print benchmarks with the given name')
+
def generate_output_name(input_jar, mainclass):
- if not mainclass:
- input_base, input_ext = os.path.splitext(input_jar)
- return '%s-min%s' % (input_base, input_ext)
- base = mainclass[mainclass.rindex('.')+1:] if '.' in mainclass else mainclass
- return os.path.join(utils.LIBS, '%s-min.jar' % base)
+ if not mainclass:
+ input_base, input_ext = os.path.splitext(input_jar)
+ return '%s-min%s' % (input_base, input_ext)
+ base = mainclass[mainclass.rindex('.') +
+ 1:] if '.' in mainclass else mainclass
+ return os.path.join(utils.LIBS, '%s-min.jar' % base)
+
def repackage(input_jar, output_jar, mainclass):
- print("Repackaging %s to %s with Main-Class: %s..." %
- (input_jar, output_jar, mainclass))
- manifest = MANIFEST % mainclass
- with zipfile.ZipFile(input_jar, 'r') as input_zf:
- with zipfile.ZipFile(output_jar, 'w') as output_zf:
- for zipinfo in input_zf.infolist():
- if zipinfo.filename.upper() == MANIFEST_PATH:
- assert manifest is not None
- output_zf.writestr(MANIFEST_PATH, manifest)
- manifest = None
- else:
- output_zf.writestr(zipinfo, input_zf.read(zipinfo))
- if manifest is not None:
- output_zf.writestr(MANIFEST_PATH, manifest)
+ print("Repackaging %s to %s with Main-Class: %s..." %
+ (input_jar, output_jar, mainclass))
+ manifest = MANIFEST % mainclass
+ with zipfile.ZipFile(input_jar, 'r') as input_zf:
+ with zipfile.ZipFile(output_jar, 'w') as output_zf:
+ for zipinfo in input_zf.infolist():
+ if zipinfo.filename.upper() == MANIFEST_PATH:
+ assert manifest is not None
+ output_zf.writestr(MANIFEST_PATH, manifest)
+ manifest = None
+ else:
+ output_zf.writestr(zipinfo, input_zf.read(zipinfo))
+ if manifest is not None:
+ output_zf.writestr(MANIFEST_PATH, manifest)
+
def extract_mainclass(input_jar):
- with zipfile.ZipFile(input_jar, 'r') as input_zf:
- try:
- manifest = input_zf.getinfo(MANIFEST_PATH)
- except KeyError:
- raise SystemExit('No --mainclass specified and no manifest in input JAR.')
- mo = re.search(MANIFEST_PATTERN, input_zf.read(manifest))
- if not mo:
- raise SystemExit(
- 'No --mainclass specified and no Main-Class in input JAR manifest.')
- return mo.group(1)
+ with zipfile.ZipFile(input_jar, 'r') as input_zf:
+ try:
+ manifest = input_zf.getinfo(MANIFEST_PATH)
+ except KeyError:
+ raise SystemExit(
+ 'No --mainclass specified and no manifest in input JAR.')
+ mo = re.search(MANIFEST_PATTERN, input_zf.read(manifest))
+ if not mo:
+ raise SystemExit(
+ 'No --mainclass specified and no Main-Class in input JAR manifest.'
+ )
+ return mo.group(1)
-def minify_tool(mainclass=None, input_jar=utils.R8_JAR, output_jar=None,
- lib=utils.RT_JAR, debug=True, build=True, benchmark_name=None,
- track_memory_file=None, additional_args=[], java_args=[]):
- if output_jar is None:
- output_jar = generate_output_name(input_jar, mainclass)
- with utils.TempDir() as path:
- if mainclass:
- tmp_input_path = os.path.join(path, 'input.jar')
- repackage(input_jar, tmp_input_path, mainclass)
- else:
- tmp_input_path = input_jar
- mainclass = extract_mainclass(input_jar)
- keep_path = os.path.join(path, 'keep.txt')
- with open(keep_path, 'w') as fp:
- fp.write(KEEP % mainclass)
- args = ['--lib', lib,
- '--classfile',
- '--output', output_jar,
- '--pg-conf', keep_path,
- '--release',
- tmp_input_path] + additional_args
- start_time = time.time()
- return_code = toolhelper.run('r8', args, debug=debug, build=build,
- track_memory_file=track_memory_file,
- extra_args=java_args)
- if benchmark_name:
- elapsed_ms = 1000 * (time.time() - start_time)
- print('%s(RunTimeRaw): %s ms' % (benchmark_name, elapsed_ms))
- if track_memory_file:
- print('%s(MemoryUse): %s' %
- (benchmark_name, utils.grep_memoryuse(track_memory_file)))
- return return_code
+def minify_tool(mainclass=None,
+ input_jar=utils.R8_JAR,
+ output_jar=None,
+ lib=utils.RT_JAR,
+ debug=True,
+ build=True,
+ benchmark_name=None,
+ track_memory_file=None,
+ additional_args=[],
+ java_args=[]):
+ if output_jar is None:
+ output_jar = generate_output_name(input_jar, mainclass)
+ with utils.TempDir() as path:
+ if mainclass:
+ tmp_input_path = os.path.join(path, 'input.jar')
+ repackage(input_jar, tmp_input_path, mainclass)
+ else:
+ tmp_input_path = input_jar
+ mainclass = extract_mainclass(input_jar)
+ keep_path = os.path.join(path, 'keep.txt')
+ with open(keep_path, 'w') as fp:
+ fp.write(KEEP % mainclass)
+ args = [
+ '--lib', lib, '--classfile', '--output', output_jar, '--pg-conf',
+ keep_path, '--release', tmp_input_path
+ ] + additional_args
+ start_time = time.time()
+ return_code = toolhelper.run('r8',
+ args,
+ debug=debug,
+ build=build,
+ track_memory_file=track_memory_file,
+ extra_args=java_args)
+ if benchmark_name:
+ elapsed_ms = 1000 * (time.time() - start_time)
+ print('%s(RunTimeRaw): %s ms' % (benchmark_name, elapsed_ms))
+ if track_memory_file:
+ print('%s(MemoryUse): %s' %
+ (benchmark_name, utils.grep_memoryuse(track_memory_file)))
+
+ return return_code
+
if __name__ == '__main__':
- sys.exit(minify_tool(**vars(parser.parse_args())))
+ sys.exit(minify_tool(**vars(parser.parse_args())))
diff --git a/tools/nest_data.py b/tools/nest_data.py
index 5724e43..bb9d709 100644
--- a/tools/nest_data.py
+++ b/tools/nest_data.py
@@ -15,23 +15,30 @@
ANDROID_JAR = utils.get_android_jar(25)
VERSIONS = {
- '20180926': {
- 'dex' : {
- 'inputs': [os.path.join(V20180926_BASE, 'obsidian-development-debug.apk')],
- 'libraries' : [ANDROID_JAR],
- 'min-api' : ANDROID_L_API,
+ '20180926': {
+ 'dex': {
+ 'inputs': [
+ os.path.join(V20180926_BASE, 'obsidian-development-debug.apk')
+ ],
+ 'libraries': [ANDROID_JAR],
+ 'min-api': ANDROID_L_API,
+ },
+ 'deploy': {
+ 'inputs': [
+ os.path.join(V20180926_BASE, 'obsidian-development-debug.jar')
+ ],
+ 'libraries': [ANDROID_JAR],
+ 'allow-type-errors': 1,
+ 'pgconf': [
+ os.path.join(V20180926_BASE, 'proguard', 'proguard.cfg'),
+ os.path.join(V20180926_BASE, 'proguard',
+ 'proguard-no-optimizations.cfg'),
+ os.path.join(V20180926_BASE, 'proguard',
+ 'proguard-ignore-warnings.cfg'),
+ utils.IGNORE_WARNINGS_RULES
+ ],
+ # Build for native multi dex
+ 'min-api': ANDROID_L_API,
+ }
},
- 'deploy' : {
- 'inputs': [os.path.join(V20180926_BASE, 'obsidian-development-debug.jar')],
- 'libraries' : [ANDROID_JAR],
- 'allow-type-errors' : 1,
- 'pgconf': [
- os.path.join(V20180926_BASE, 'proguard', 'proguard.cfg'),
- os.path.join(V20180926_BASE, 'proguard', 'proguard-no-optimizations.cfg'),
- os.path.join(V20180926_BASE, 'proguard', 'proguard-ignore-warnings.cfg'),
- utils.IGNORE_WARNINGS_RULES],
- # Build for native multi dex
- 'min-api' : ANDROID_L_API,
- }
- },
}
diff --git a/tools/notify.py b/tools/notify.py
index 34676ba..4dfceb3 100644
--- a/tools/notify.py
+++ b/tools/notify.py
@@ -4,17 +4,18 @@
# BSD-style license that can be found in the LICENSE file.
try:
- import gi
- gi.require_version('Notify', '0.7')
- from gi.repository import Notify
- Notify.init('R8 build tools')
+ import gi
+ gi.require_version('Notify', '0.7')
+ from gi.repository import Notify
+ Notify.init('R8 build tools')
- def notify(message):
- try:
- Notify.Notification.new('R8 build tools', message).show()
- except:
- return
+ def notify(message):
+ try:
+ Notify.Notification.new('R8 build tools', message).show()
+ except:
+ return
except (ImportError, ValueError):
- def notify(message):
- return
+
+ def notify(message):
+ return
diff --git a/tools/performance_try.py b/tools/performance_try.py
index 0c09e70..5f1b4b3 100755
--- a/tools/performance_try.py
+++ b/tools/performance_try.py
@@ -11,12 +11,16 @@
SCRIPT = '/google/data/ro/teams/dart/golem/bin/golem4.dart'
DART = os.path.join(utils.THIRD_PARTY, 'dart-sdk', 'bin', 'dart')
+
def Main():
- args = sys.argv[1:]
- if len(args) != 1 or '--help' in args:
- print('Performance tracking takes exactly one argument, the name for display')
- return 1
- subprocess.check_call([DART, SCRIPT, args[0]])
+ args = sys.argv[1:]
+ if len(args) != 1 or '--help' in args:
+ print(
+ 'Performance tracking takes exactly one argument, the name for display'
+ )
+ return 1
+ subprocess.check_call([DART, SCRIPT, args[0]])
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/printseeds.py b/tools/printseeds.py
index 6c0ee8c..4ec2e9e 100755
--- a/tools/printseeds.py
+++ b/tools/printseeds.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('printseeds', sys.argv[1:]))
+ sys.exit(toolhelper.run('printseeds', sys.argv[1:]))
diff --git a/tools/printuses.py b/tools/printuses.py
index 5253796..c18432f 100755
--- a/tools/printuses.py
+++ b/tools/printuses.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('printuses', sys.argv[1:]))
+ sys.exit(toolhelper.run('printuses', sys.argv[1:]))
diff --git a/tools/proguard.py b/tools/proguard.py
index 4a3ae7d..312533e 100755
--- a/tools/proguard.py
+++ b/tools/proguard.py
@@ -18,55 +18,72 @@
DEFAULT = 'pg6'
DEFAULT_ALIAS = 'pg'
VERSIONS = {
- 'pg5': os.path.join(PG_DIR, 'proguard5.2.1', 'lib', 'proguard.jar'),
- 'pg6': os.path.join(PG_DIR, 'proguard6.0.1', 'lib', 'proguard.jar'),
- 'pg7': os.path.join(PG_DIR, 'proguard-7.0.0', 'lib', 'proguard.jar'),
- 'pg_internal': os.path.join(
- PG_DIR, 'proguard_internal_159423826', 'ProGuard_deploy.jar'),
+ 'pg5':
+ os.path.join(PG_DIR, 'proguard5.2.1', 'lib', 'proguard.jar'),
+ 'pg6':
+ os.path.join(PG_DIR, 'proguard6.0.1', 'lib', 'proguard.jar'),
+ 'pg7':
+ os.path.join(PG_DIR, 'proguard-7.0.0', 'lib', 'proguard.jar'),
+ 'pg_internal':
+ os.path.join(PG_DIR, 'proguard_internal_159423826',
+ 'ProGuard_deploy.jar'),
}
# Add alias for the default version.
VERSIONS[DEFAULT_ALIAS] = VERSIONS[DEFAULT]
+
# Get versions sorted (nice for argument lists)
def getVersions():
- versions = list(VERSIONS.keys())
- versions.sort()
- return versions
+ versions = list(VERSIONS.keys())
+ versions.sort()
+ return versions
+
def isValidVersion(version):
- return version in VERSIONS
+ return version in VERSIONS
+
def getValidatedVersion(version):
- if not isValidVersion(version):
- raise ValueError("Invalid PG version: '%s'" % version)
- return version
+ if not isValidVersion(version):
+ raise ValueError("Invalid PG version: '%s'" % version)
+ return version
+
def getJar(version=DEFAULT):
- return VERSIONS[getValidatedVersion(version)]
+ return VERSIONS[getValidatedVersion(version)]
+
def getRetraceJar(version=DEFAULT):
- if version == 'pg_internal':
- raise ValueError("No retrace in internal distribution")
- return getJar().replace('proguard.jar', 'retrace.jar')
+ if version == 'pg_internal':
+ raise ValueError("No retrace in internal distribution")
+ return getJar().replace('proguard.jar', 'retrace.jar')
+
def getCmd(args, version=DEFAULT, jvmArgs=None):
- cmd = []
- if jvmArgs:
- cmd.extend(jvmArgs)
- cmd.extend([jdk.GetJavaExecutable(), '-jar', getJar(version)])
- cmd.extend(args)
- return cmd
+ cmd = []
+ if jvmArgs:
+ cmd.extend(jvmArgs)
+ cmd.extend([jdk.GetJavaExecutable(), '-jar', getJar(version)])
+ cmd.extend(args)
+ return cmd
-def run(args, version=DEFAULT, track_memory_file=None, stdout=None, stderr=None):
- cmd = []
- if track_memory_file:
- cmd.extend(['tools/track_memory.sh', track_memory_file])
- cmd.extend(getCmd(args, version))
- utils.PrintCmd(cmd)
- subprocess.call(cmd, stdout=stdout, stderr=stderr)
+
+def run(args,
+ version=DEFAULT,
+ track_memory_file=None,
+ stdout=None,
+ stderr=None):
+ cmd = []
+ if track_memory_file:
+ cmd.extend(['tools/track_memory.sh', track_memory_file])
+ cmd.extend(getCmd(args, version))
+ utils.PrintCmd(cmd)
+ subprocess.call(cmd, stdout=stdout, stderr=stderr)
+
def Main():
- run(sys.argv[1:])
+ run(sys.argv[1:])
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/r8.py b/tools/r8.py
index 0a9e09a..5a2f5e3 100755
--- a/tools/r8.py
+++ b/tools/r8.py
@@ -9,77 +9,73 @@
import toolhelper
import utils
+
def ParseOptions(argv):
- parser = optparse.OptionParser(usage='%prog [options] -- [R8 options]')
- parser.add_option(
- '-c',
- '--commit-hash',
- '--commit_hash',
- help='Commit hash of R8 to use.',
- default=None)
- parser.add_option(
- '--debug-agent',
- help='Enable Java debug agent and suspend compilation (default disabled)',
- default=False,
- action='store_true')
- parser.add_option(
- '--ea',
- help='Enable Java assertions when running the compiler (default disabled)',
- default=False,
- action='store_true')
- parser.add_option(
- '--lib-android',
- help='Add the android.jar for the given API level',
- default=None,
- type=int)
- parser.add_option(
- '--lib-rt',
- help='Add rt.jar from openjdk-1.8',
- default=False,
- action='store_true')
- parser.add_option(
- '--no-build', '--no_build',
- help='Do not build R8',
- default=False,
- action='store_true')
- parser.add_option(
- '--print-runtimeraw', '--print_runtimeraw',
- metavar='BENCHMARKNAME',
- help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
- ' <elapsed> ms\' at the end where <elapsed> is' +
- ' the elapsed time in milliseconds.')
- parser.add_option(
- '--tag',
- help='Tag of R8 to use.',
- default=None)
- parser.add_option(
- '--version',
- help='Version of R8 to use.',
- default=None)
- return parser.parse_args(argv)
+ parser = optparse.OptionParser(usage='%prog [options] -- [R8 options]')
+ parser.add_option('-c',
+ '--commit-hash',
+ '--commit_hash',
+ help='Commit hash of R8 to use.',
+ default=None)
+ parser.add_option(
+ '--debug-agent',
+ help=
+ 'Enable Java debug agent and suspend compilation (default disabled)',
+ default=False,
+ action='store_true')
+ parser.add_option(
+ '--ea',
+ help=
+ 'Enable Java assertions when running the compiler (default disabled)',
+ default=False,
+ action='store_true')
+ parser.add_option('--lib-android',
+ help='Add the android.jar for the given API level',
+ default=None,
+ type=int)
+ parser.add_option('--lib-rt',
+ help='Add rt.jar from openjdk-1.8',
+ default=False,
+ action='store_true')
+ parser.add_option('--no-build',
+ '--no_build',
+ help='Do not build R8',
+ default=False,
+ action='store_true')
+ parser.add_option('--print-runtimeraw',
+ '--print_runtimeraw',
+ metavar='BENCHMARKNAME',
+ help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
+ ' <elapsed> ms\' at the end where <elapsed> is' +
+ ' the elapsed time in milliseconds.')
+ parser.add_option('--tag', help='Tag of R8 to use.', default=None)
+ parser.add_option('--version', help='Version of R8 to use.', default=None)
+ return parser.parse_args(argv)
+
def main(argv):
- (options, args) = ParseOptions(sys.argv)
- r8_args = args[1:]
- if options.lib_android:
- r8_args.extend(['--lib', utils.get_android_jar(options.lib_android)])
- if options.lib_rt:
- r8_args.extend(['--lib', utils.RT_JAR])
- time_consumer = lambda duration : print_duration(duration, options)
- return toolhelper.run(
- 'r8',
- r8_args,
- build=not options.no_build,
- debug=options.ea,
- debug_agent=options.debug_agent,
- jar=utils.find_r8_jar_from_options(options),
- main='com.android.tools.r8.R8',
- time_consumer=time_consumer)
+ (options, args) = ParseOptions(sys.argv)
+ r8_args = args[1:]
+ if options.lib_android:
+ r8_args.extend(['--lib', utils.get_android_jar(options.lib_android)])
+ if options.lib_rt:
+ r8_args.extend(['--lib', utils.RT_JAR])
+ time_consumer = lambda duration: print_duration(duration, options)
+ return toolhelper.run('r8',
+ r8_args,
+ build=not options.no_build,
+ debug=options.ea,
+ debug_agent=options.debug_agent,
+ jar=utils.find_r8_jar_from_options(options),
+ main='com.android.tools.r8.R8',
+ time_consumer=time_consumer)
+
def print_duration(duration, options):
- benchmark_name = options.print_runtimeraw
- if benchmark_name:
- print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration))
+ benchmark_name = options.print_runtimeraw
+ if benchmark_name:
+ print('%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration))
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/r8_get.py b/tools/r8_get.py
index cc11e90..44cc9f1 100755
--- a/tools/r8_get.py
+++ b/tools/r8_get.py
@@ -7,26 +7,29 @@
import argparse
import compiledump
+
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Helper to fetch r8.jar from cloudstorage.')
- parser.add_argument(
- '-v',
- '--version',
- help='Version or commit-hash to download '
- '(e.g., 3.3.50 or 33ae86d80351efc4d632452331d06cb97e42f2a7).',
- required=True)
- parser.add_argument(
- '--outdir',
- help='Output directory to place the r8.jar in (default cwd).',
- default=None)
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='Helper to fetch r8.jar from cloudstorage.')
+ parser.add_argument(
+ '-v',
+ '--version',
+ help='Version or commit-hash to download '
+ '(e.g., 3.3.50 or 33ae86d80351efc4d632452331d06cb97e42f2a7).',
+ required=True)
+ parser.add_argument(
+ '--outdir',
+ help='Output directory to place the r8.jar in (default cwd).',
+ default=None)
+ return parser.parse_args()
+
def main():
- args = parse_arguments()
- outdir = args.outdir if args.outdir else ''
- print(compiledump.download_distribution(args.version, True, outdir))
- return 0
+ args = parse_arguments()
+ outdir = args.outdir if args.outdir else ''
+ print(compiledump.download_distribution(args.version, True, outdir))
+ return 0
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/r8_release.py b/tools/r8_release.py
index 0491a0d..8aaca29 100755
--- a/tools/r8_release.py
+++ b/tools/r8_release.py
@@ -17,8 +17,8 @@
import utils
R8_DEV_BRANCH = '8.3'
-R8_VERSION_FILE = os.path.join(
- 'src', 'main', 'java', 'com', 'android', 'tools', 'r8', 'Version.java')
+R8_VERSION_FILE = os.path.join('src', 'main', 'java', 'com', 'android', 'tools',
+ 'r8', 'Version.java')
THIS_FILE_RELATIVE = os.path.join('tools', 'r8_release.py')
GMAVEN_PUBLISHER = '/google/bin/releases/android-devtools/gmaven/publisher/gmaven-publisher'
@@ -28,255 +28,265 @@
GITHUB_DESUGAR_JDK_LIBS = 'https://github.com/google/desugar_jdk_libs'
+
def install_gerrit_change_id_hook(checkout_dir):
- with utils.ChangedWorkingDirectory(checkout_dir):
- # Fancy way of getting the string ".git".
- git_dir = subprocess.check_output(
- ['git', 'rev-parse', '--git-dir']).decode('utf-8').strip()
- commit_msg_hooks = '%s/hooks/commit-msg' % git_dir
- if not os.path.exists(os.path.dirname(commit_msg_hooks)):
- os.mkdir(os.path.dirname(commit_msg_hooks))
- # Install commit hook to generate Gerrit 'Change-Id:'.
- urllib.request.urlretrieve(
- 'https://gerrit-review.googlesource.com/tools/hooks/commit-msg',
- commit_msg_hooks)
- st = os.stat(commit_msg_hooks)
- os.chmod(
- commit_msg_hooks,
- st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+ with utils.ChangedWorkingDirectory(checkout_dir):
+ # Fancy way of getting the string ".git".
+ git_dir = subprocess.check_output(['git', 'rev-parse', '--git-dir'
+ ]).decode('utf-8').strip()
+ commit_msg_hooks = '%s/hooks/commit-msg' % git_dir
+ if not os.path.exists(os.path.dirname(commit_msg_hooks)):
+ os.mkdir(os.path.dirname(commit_msg_hooks))
+ # Install commit hook to generate Gerrit 'Change-Id:'.
+ urllib.request.urlretrieve(
+ 'https://gerrit-review.googlesource.com/tools/hooks/commit-msg',
+ commit_msg_hooks)
+ st = os.stat(commit_msg_hooks)
+ os.chmod(commit_msg_hooks,
+ st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
def checkout_r8(temp, branch):
- subprocess.check_call(['git', 'clone', utils.REPO_SOURCE, temp])
- with utils.ChangedWorkingDirectory(temp):
- subprocess.check_call([
- 'git',
- 'new-branch',
- '--upstream',
- 'origin/%s' % branch,
- 'dev-release'])
- install_gerrit_change_id_hook(temp)
- return temp
+ subprocess.check_call(['git', 'clone', utils.REPO_SOURCE, temp])
+ with utils.ChangedWorkingDirectory(temp):
+ subprocess.check_call([
+ 'git', 'new-branch', '--upstream',
+ 'origin/%s' % branch, 'dev-release'
+ ])
+ install_gerrit_change_id_hook(temp)
+ return temp
def prepare_release(args):
- if args.version:
- print("Cannot manually specify version when making a dev release.")
- sys.exit(1)
+ if args.version:
+ print("Cannot manually specify version when making a dev release.")
+ sys.exit(1)
- def make_release(args):
- commithash = args.dev_release
+ def make_release(args):
+ commithash = args.dev_release
- with utils.TempDir() as temp:
- with utils.ChangedWorkingDirectory(checkout_r8(temp, R8_DEV_BRANCH)):
- # Compute the current and new version on the branch.
- result = None
- for line in open(R8_VERSION_FILE, 'r'):
- result = re.match(
- r'.*LABEL = "%s\.(\d+)\-dev";' % R8_DEV_BRANCH, line)
- if result:
- break
- if not result or not result.group(1):
- print('Failed to find version label matching %s(\d+)-dev'\
- % R8_DEV_BRANCH)
- sys.exit(1)
- try:
- patch_version = int(result.group(1))
- except ValueError:
- print('Failed to convert version to integer: %s' % result.group(1))
+ with utils.TempDir() as temp:
+ with utils.ChangedWorkingDirectory(checkout_r8(temp,
+ R8_DEV_BRANCH)):
+ # Compute the current and new version on the branch.
+ result = None
+ for line in open(R8_VERSION_FILE, 'r'):
+ result = re.match(
+ r'.*LABEL = "%s\.(\d+)\-dev";' % R8_DEV_BRANCH, line)
+ if result:
+ break
+ if not result or not result.group(1):
+ print('Failed to find version label matching %s(\d+)-dev'\
+ % R8_DEV_BRANCH)
+ sys.exit(1)
+ try:
+ patch_version = int(result.group(1))
+ except ValueError:
+ print('Failed to convert version to integer: %s' %
+ result.group(1))
- old_version = '%s.%s-dev' % (R8_DEV_BRANCH, patch_version)
- version = '%s.%s-dev' % (R8_DEV_BRANCH, patch_version + 1)
+ old_version = '%s.%s-dev' % (R8_DEV_BRANCH, patch_version)
+ version = '%s.%s-dev' % (R8_DEV_BRANCH, patch_version + 1)
- # Verify that the merge point from main is not empty.
- merge_diff_output = subprocess.check_output([
- 'git', 'diff', 'HEAD..%s' % commithash]).decode('utf-8')
- other_diff = version_change_diff(
- merge_diff_output, old_version, "main")
- if not other_diff:
- print('Merge point from main (%s)' % commithash, \
- 'is the same as exiting release (%s).' % old_version)
- sys.exit(1)
+ # Verify that the merge point from main is not empty.
+ merge_diff_output = subprocess.check_output(
+ ['git', 'diff', 'HEAD..%s' % commithash]).decode('utf-8')
+ other_diff = version_change_diff(merge_diff_output, old_version,
+ "main")
+ if not other_diff:
+ print('Merge point from main (%s)' % commithash, \
+ 'is the same as exiting release (%s).' % old_version)
+ sys.exit(1)
- subprocess.check_call([
- 'git', 'cl', 'new-branch', 'release-%s' % version])
+ subprocess.check_call(
+ ['git', 'cl', 'new-branch',
+ 'release-%s' % version])
- if args.dev_pre_cherry_pick:
- for pre_commit in args.dev_pre_cherry_pick:
- subprocess.check_call([
- 'git', 'cherry-pick', '--no-edit', pre_commit])
+ if args.dev_pre_cherry_pick:
+ for pre_commit in args.dev_pre_cherry_pick:
+ subprocess.check_call(
+ ['git', 'cherry-pick', '--no-edit', pre_commit])
- # Merge the desired commit from main on to the branch.
- subprocess.check_call(['git', 'merge', '--no-ff', '--no-edit', commithash])
+ # Merge the desired commit from main on to the branch.
+ subprocess.check_call(
+ ['git', 'merge', '--no-ff', '--no-edit', commithash])
- # Rewrite the version, commit and validate.
- sed(old_version, version, R8_VERSION_FILE)
+ # Rewrite the version, commit and validate.
+ sed(old_version, version, R8_VERSION_FILE)
- subprocess.check_call([
- 'git', 'commit', '-a', '-m', 'Version %s' % version])
+ subprocess.check_call(
+ ['git', 'commit', '-a', '-m',
+ 'Version %s' % version])
- version_diff_output = subprocess.check_output([
- 'git', 'diff', '%s..HEAD' % commithash]).decode('utf-8')
+ version_diff_output = subprocess.check_output(
+ ['git', 'diff', '%s..HEAD' % commithash]).decode('utf-8')
- validate_version_change_diff(version_diff_output, "main", version)
+ validate_version_change_diff(version_diff_output, "main",
+ version)
- cmd = ['git', 'cl', 'upload', '--no-squash']
- if args.bypass_hooks:
- cmd.append('--bypass-hooks')
- maybe_check_call(args, cmd)
+ cmd = ['git', 'cl', 'upload', '--no-squash']
+ if args.bypass_hooks:
+ cmd.append('--bypass-hooks')
+ maybe_check_call(args, cmd)
- if args.dry_run:
- input(
- 'DryRun: check %s for content of version %s [enter to continue]:'
- % (temp, version))
+ if args.dry_run:
+ input(
+ 'DryRun: check %s for content of version %s [enter to continue]:'
+ % (temp, version))
- return "%s dev version %s from hash %s for review" % (
- 'DryRun: omitted upload of' if args.dry_run else 'Uploaded',
- version,
- commithash)
+ return "%s dev version %s from hash %s for review" % (
+ 'DryRun: omitted upload of' if args.dry_run else 'Uploaded',
+ version, commithash)
- return make_release
+ return make_release
def maybe_tag(args, version):
- maybe_check_call(args, [
- 'git', 'tag', '-a', version, '-m', '"%s"' % version])
- maybe_check_call(args, [
- 'git', 'push', 'origin', 'refs/tags/%s' % version])
+ maybe_check_call(args,
+ ['git', 'tag', '-a', version, '-m',
+ '"%s"' % version])
+ maybe_check_call(args, ['git', 'push', 'origin', 'refs/tags/%s' % version])
def version_change_diff(diff, old_version, new_version):
- invalid_line = None
- for line in str(diff).splitlines():
- if line.startswith('- ') and \
- line != '- public static final String LABEL = "%s";' % old_version:
- invalid_line = line
- elif line.startswith('+ ') and \
- line != '+ public static final String LABEL = "%s";' % new_version:
- invalid_line = line
- return invalid_line
+ invalid_line = None
+ for line in str(diff).splitlines():
+ if line.startswith('- ') and \
+ line != '- public static final String LABEL = "%s";' % old_version:
+ invalid_line = line
+ elif line.startswith('+ ') and \
+ line != '+ public static final String LABEL = "%s";' % new_version:
+ invalid_line = line
+ return invalid_line
def validate_version_change_diff(version_diff_output, old_version, new_version):
- invalid = version_change_diff(version_diff_output, old_version, new_version)
- if invalid:
- print("Unexpected diff:")
- print("=" * 80)
- print(version_diff_output)
- print("=" * 80)
- accept_string = 'THE DIFF IS OK!'
- answer = input(
- "Accept the additonal diff as part of the release? "
- "Type '%s' to accept: " % accept_string)
- if answer != accept_string:
- print("You did not type '%s'" % accept_string)
- print('Aborting dev release for %s' % version)
- sys.exit(1)
+ invalid = version_change_diff(version_diff_output, old_version, new_version)
+ if invalid:
+ print("Unexpected diff:")
+ print("=" * 80)
+ print(version_diff_output)
+ print("=" * 80)
+ accept_string = 'THE DIFF IS OK!'
+ answer = input("Accept the additonal diff as part of the release? "
+ "Type '%s' to accept: " % accept_string)
+ if answer != accept_string:
+ print("You did not type '%s'" % accept_string)
+ print('Aborting dev release for %s' % version)
+ sys.exit(1)
def maybe_check_call(args, cmd):
- if args.dry_run:
- print('DryRun:', ' '.join(cmd))
- else:
- print(' '.join(cmd))
- return subprocess.check_call(cmd)
+ if args.dry_run:
+ print('DryRun:', ' '.join(cmd))
+ else:
+ print(' '.join(cmd))
+ return subprocess.check_call(cmd)
def update_prebuilds(r8_checkout, version, checkout, keepanno=False):
- path = os.path.join(r8_checkout, 'tools', 'update_prebuilds_in_android.py')
- commit_arg = '--commit_hash=' if len(version) == 40 else '--version='
- cmd = [path, '--targets=lib', '--maps', commit_arg + version, checkout]
- if keepanno:
- cmd.append("--keepanno")
- subprocess.check_call(cmd)
+ path = os.path.join(r8_checkout, 'tools', 'update_prebuilds_in_android.py')
+ commit_arg = '--commit_hash=' if len(version) == 40 else '--version='
+ cmd = [path, '--targets=lib', '--maps', commit_arg + version, checkout]
+ if keepanno:
+ cmd.append("--keepanno")
+ subprocess.check_call(cmd)
-def release_studio_or_aosp(r8_checkout, path, options, git_message, keepanno=False):
- with utils.ChangedWorkingDirectory(path):
- if not options.use_existing_work_branch:
- subprocess.call(['repo', 'abandon', 'update-r8'])
- if not options.no_sync:
- subprocess.check_call(['repo', 'sync', '-cq', '-j', '16'])
+def release_studio_or_aosp(r8_checkout,
+ path,
+ options,
+ git_message,
+ keepanno=False):
+ with utils.ChangedWorkingDirectory(path):
+ if not options.use_existing_work_branch:
+ subprocess.call(['repo', 'abandon', 'update-r8'])
+ if not options.no_sync:
+ subprocess.check_call(['repo', 'sync', '-cq', '-j', '16'])
- prebuilts_r8 = os.path.join(path, 'prebuilts', 'r8')
+ prebuilts_r8 = os.path.join(path, 'prebuilts', 'r8')
- if not options.use_existing_work_branch:
- with utils.ChangedWorkingDirectory(prebuilts_r8):
- subprocess.check_call(['repo', 'start', 'update-r8'])
+ if not options.use_existing_work_branch:
+ with utils.ChangedWorkingDirectory(prebuilts_r8):
+ subprocess.check_call(['repo', 'start', 'update-r8'])
- update_prebuilds(r8_checkout, options.version, path, keepanno)
+ update_prebuilds(r8_checkout, options.version, path, keepanno)
- with utils.ChangedWorkingDirectory(prebuilts_r8):
- if not options.use_existing_work_branch:
- subprocess.check_call(['git', 'commit', '-a', '-m', git_message])
- else:
- print('Not committing when --use-existing-work-branch. '
- + 'Commit message should be:\n\n'
- + git_message
- + '\n')
- # Don't upload if requested not to, or if changes are not committed due
- # to --use-existing-work-branch
- if not options.no_upload and not options.use_existing_work_branch:
- process = subprocess.Popen(['repo', 'upload', '.', '--verify',
- '--current-branch'],
- stdin=subprocess.PIPE)
- return process.communicate(input=b'y\n')[0]
+ with utils.ChangedWorkingDirectory(prebuilts_r8):
+ if not options.use_existing_work_branch:
+ subprocess.check_call(
+ ['git', 'commit', '-a', '-m', git_message])
+ else:
+ print('Not committing when --use-existing-work-branch. ' +
+ 'Commit message should be:\n\n' + git_message + '\n')
+ # Don't upload if requested not to, or if changes are not committed due
+ # to --use-existing-work-branch
+ if not options.no_upload and not options.use_existing_work_branch:
+ process = subprocess.Popen(
+ ['repo', 'upload', '.', '--verify', '--current-branch'],
+ stdin=subprocess.PIPE)
+ return process.communicate(input=b'y\n')[0]
def prepare_aosp(args):
- assert args.version
- assert os.path.exists(args.aosp), "Could not find AOSP path %s" % args.aosp
+ assert args.version
+ assert os.path.exists(args.aosp), "Could not find AOSP path %s" % args.aosp
- def release_aosp(options):
- print("Releasing for AOSP")
- if options.dry_run:
- return 'DryRun: omitting AOSP release for %s' % options.version
+ def release_aosp(options):
+ print("Releasing for AOSP")
+ if options.dry_run:
+ return 'DryRun: omitting AOSP release for %s' % options.version
- git_message = ("""Update D8 and R8 to %s
+ git_message = ("""Update D8 and R8 to %s
Version: %s
This build IS NOT suitable for preview or public release.
Built here: go/r8-releases/raw/%s
-Test: TARGET_PRODUCT=aosp_arm64 m -j core-oj"""
- % (args.version, args.version, args.version))
- # Fixes to Android U branch is based of 8.2.2-dev where the keepanno library
- # is not built.
- keepanno = not args.version.startswith('8.2.2-udc')
- return release_studio_or_aosp(
- utils.REPO_ROOT, args.aosp, options, git_message, keepanno=keepanno)
+Test: TARGET_PRODUCT=aosp_arm64 m -j core-oj""" %
+ (args.version, args.version, args.version))
+ # Fixes to Android U branch is based of 8.2.2-dev where the keepanno library
+ # is not built.
+ keepanno = not args.version.startswith('8.2.2-udc')
+ return release_studio_or_aosp(utils.REPO_ROOT,
+ args.aosp,
+ options,
+ git_message,
+ keepanno=keepanno)
- return release_aosp
+ return release_aosp
def prepare_maven(args):
- assert args.version
+ assert args.version
- def release_maven(options):
- gfile = '/bigstore/r8-releases/raw/%s/r8lib.zip' % args.version
- release_id = gmaven_publisher_stage(options, [gfile])
+ def release_maven(options):
+ gfile = '/bigstore/r8-releases/raw/%s/r8lib.zip' % args.version
+ release_id = gmaven_publisher_stage(options, [gfile])
- print("Staged Release ID " + release_id + ".\n")
- gmaven_publisher_stage_redir_test_info(
- release_id, "com.android.tools:r8:%s" % args.version, "r8lib.jar")
+ print("Staged Release ID " + release_id + ".\n")
+ gmaven_publisher_stage_redir_test_info(
+ release_id, "com.android.tools:r8:%s" % args.version, "r8lib.jar")
- print
- answer = input("Continue with publishing [y/N]:")
+ print
+ answer = input("Continue with publishing [y/N]:")
- if answer != 'y':
- print('Aborting release to Google maven')
- sys.exit(1)
+ if answer != 'y':
+ print('Aborting release to Google maven')
+ sys.exit(1)
- gmaven_publisher_publish(args, release_id)
+ gmaven_publisher_publish(args, release_id)
- print("")
- print("Published. Use the email workflow for approval.")
+ print("")
+ print("Published. Use the email workflow for approval.")
- return release_maven
+ return release_maven
+
# ------------------------------------------------------ column 70 --v
def git_message_dev(version, bugs):
- return """Update D8 R8 to %s
+ return """Update D8 R8 to %s
This is a development snapshot, it's fine to use for studio canary
build, but not for BETA or release, for those we would need a release
@@ -289,400 +299,422 @@
def git_message_release(version, bugs):
- return """D8 R8 version %s
+ return """D8 R8 version %s
Built here: go/r8-releases/raw/%s/
Test: ./gradlew check
Bug: %s""" % (version, version, '\nBug: '.join(map(bug_fmt, bugs)))
+
def bug_fmt(bug):
- return "b/%s" % bug
+ return "b/%s" % bug
+
def prepare_studio(args):
- assert args.version
- assert os.path.exists(args.studio), ("Could not find STUDIO path %s"
- % args.studio)
+ assert args.version
+ assert os.path.exists(args.studio), ("Could not find STUDIO path %s" %
+ args.studio)
- def release_studio(options):
- print("Releasing for STUDIO")
- if options.dry_run:
- return 'DryRun: omitting studio release for %s' % options.version
+ def release_studio(options):
+ print("Releasing for STUDIO")
+ if options.dry_run:
+ return 'DryRun: omitting studio release for %s' % options.version
- if 'dev' in options.version:
- git_message = git_message_dev(options.version, options.bug)
- r8_checkout = utils.REPO_ROOT
- return release_studio_or_aosp(
- r8_checkout, args.studio, options, git_message)
- else:
- with utils.TempDir() as temp:
- checkout_r8(temp, options.version[0:options.version.rindex('.')])
- git_message = git_message_release(options.version, options.bug)
- return release_studio_or_aosp(temp, args.studio, options, git_message)
+ if 'dev' in options.version:
+ git_message = git_message_dev(options.version, options.bug)
+ r8_checkout = utils.REPO_ROOT
+ return release_studio_or_aosp(r8_checkout, args.studio, options,
+ git_message)
+ else:
+ with utils.TempDir() as temp:
+ checkout_r8(temp,
+ options.version[0:options.version.rindex('.')])
+ git_message = git_message_release(options.version, options.bug)
+ return release_studio_or_aosp(temp, args.studio, options,
+ git_message)
- return release_studio
+ return release_studio
def g4_cp(old, new, file):
- subprocess.check_call('g4 cp {%s,%s}/%s' % (old, new, file), shell=True)
+ subprocess.check_call('g4 cp {%s,%s}/%s' % (old, new, file), shell=True)
def g4_open(file):
- if not os.access(file, os.W_OK):
- subprocess.check_call('g4 open %s' % file, shell=True)
+ if not os.access(file, os.W_OK):
+ subprocess.check_call('g4 open %s' % file, shell=True)
def g4_change(version):
- return subprocess.check_output(
- 'g4 change --desc "Update R8 to version %s\n"' % (version),
- shell=True).decode('utf-8')
+ return subprocess.check_output(
+ 'g4 change --desc "Update R8 to version %s\n"' % (version),
+ shell=True).decode('utf-8')
+
def get_cl_id(c4_change_output):
- startIndex = c4_change_output.find('Change ') + len('Change ')
- endIndex = c4_change_output.find(' ', startIndex)
- cl = c4_change_output[startIndex:endIndex]
- assert cl.isdigit()
- return cl
+ startIndex = c4_change_output.find('Change ') + len('Change ')
+ endIndex = c4_change_output.find(' ', startIndex)
+ cl = c4_change_output[startIndex:endIndex]
+ assert cl.isdigit()
+ return cl
+
def sed(pattern, replace, path):
- with open(path, "r") as sources:
- lines = sources.readlines()
- with open(path, "w") as sources:
- for line in lines:
- sources.write(re.sub(pattern, replace, line))
+ with open(path, "r") as sources:
+ lines = sources.readlines()
+ with open(path, "w") as sources:
+ for line in lines:
+ sources.write(re.sub(pattern, replace, line))
def download_file(version, file, dst):
- dir = 'raw' if len(version) != 40 else 'raw/main'
- urllib.request.urlretrieve(
- ('https://storage.googleapis.com/r8-releases/%s/%s/%s' % (dir, version, file)),
- dst)
+ dir = 'raw' if len(version) != 40 else 'raw/main'
+ urllib.request.urlretrieve(
+ ('https://storage.googleapis.com/r8-releases/%s/%s/%s' %
+ (dir, version, file)), dst)
+
def download_gfile(gfile, dst):
- if not gfile.startswith('/bigstore/r8-releases'):
- print('Unexpected gfile prefix for %s' % gfile)
- sys.exit(1)
+ if not gfile.startswith('/bigstore/r8-releases'):
+ print('Unexpected gfile prefix for %s' % gfile)
+ sys.exit(1)
- urllib.request.urlretrieve(
- 'https://storage.googleapis.com/%s' % gfile[len('/bigstore/'):],
- dst)
+ urllib.request.urlretrieve(
+ 'https://storage.googleapis.com/%s' % gfile[len('/bigstore/'):], dst)
+
def blaze_run(target):
- return subprocess.check_output(
- 'blaze run %s' % target, shell=True, stderr=subprocess.STDOUT).decode('utf-8')
+ return subprocess.check_output('blaze run %s' % target,
+ shell=True,
+ stderr=subprocess.STDOUT).decode('utf-8')
def prepare_google3(args):
- assert args.version
- # Check if an existing client exists.
- if not args.use_existing_work_branch:
- check_no_google3_client(args, args.p4_client)
+ assert args.version
+ # Check if an existing client exists.
+ if not args.use_existing_work_branch:
+ check_no_google3_client(args, args.p4_client)
- def release_google3(options):
- print("Releasing for Google 3")
- if options.dry_run:
- return 'DryRun: omitting g3 release for %s' % options.version
+ def release_google3(options):
+ print("Releasing for Google 3")
+ if options.dry_run:
+ return 'DryRun: omitting g3 release for %s' % options.version
- google3_base = subprocess.check_output(
- ['p4', 'g4d', '-f', args.p4_client]).decode('utf-8').rstrip()
- third_party_r8 = os.path.join(google3_base, 'third_party', 'java', 'r8')
- today = datetime.date.today()
- with utils.ChangedWorkingDirectory(third_party_r8):
- # download files
- g4_open('full.jar')
- g4_open('src.jar')
- g4_open('lib.jar')
- g4_open('lib.jar.map')
- g4_open('retrace_full.jar')
- g4_open('retrace_lib.jar')
- g4_open('retrace_lib.jar.map')
- g4_open('desugar_jdk_libs_configuration.jar')
- download_file(options.version, 'r8-full-exclude-deps.jar', 'full.jar')
- download_file(options.version, 'r8-full-exclude-deps.jar', 'retrace_full.jar')
- download_file(options.version, 'r8-src.jar', 'src.jar')
- download_file(options.version, 'r8lib-exclude-deps.jar', 'lib.jar')
- download_file(
- options.version, 'r8lib-exclude-deps.jar.map', 'lib.jar.map')
- download_file(options.version, 'desugar_jdk_libs_configuration.jar',
- 'desugar_jdk_libs_configuration.jar')
- download_file(options.version, 'r8retrace-exclude-deps.jar', 'retrace_lib.jar')
- download_file(options.version, 'r8retrace-exclude-deps.jar.map', 'retrace_lib.jar.map')
- g4_open('METADATA')
- metadata_path = os.path.join(third_party_r8, 'METADATA')
- match_count = 0
- version_match_regexp = r'[1-9]\.[0-9]{1,2}\.[0-9]{1,3}-dev'
- for line in open(metadata_path, 'r'):
- result = re.search(version_match_regexp, line)
- if result:
- match_count = match_count + 1
- if match_count != 7:
- print((
- "Could not find the previous -dev release string to replace in "
- + "METADATA. Expected to find is mentioned 7 times, but "
- + "found %s occurrences. Please update %s manually and run "
- + "again with options --google3 "
- + "--use-existing-work-branch.")
- % (match_count, metadata_path))
- sys.exit(1)
- sed(version_match_regexp, options.version, metadata_path)
- sed(r'\{ year.*\}',
- ('{ year: %i month: %i day: %i }'
- % (today.year, today.month, today.day)),
- metadata_path)
- subprocess.check_output('chmod u+w *', shell=True)
+ google3_base = subprocess.check_output(
+ ['p4', 'g4d', '-f', args.p4_client]).decode('utf-8').rstrip()
+ third_party_r8 = os.path.join(google3_base, 'third_party', 'java', 'r8')
+ today = datetime.date.today()
+ with utils.ChangedWorkingDirectory(third_party_r8):
+ # download files
+ g4_open('full.jar')
+ g4_open('src.jar')
+ g4_open('lib.jar')
+ g4_open('lib.jar.map')
+ g4_open('retrace_full.jar')
+ g4_open('retrace_lib.jar')
+ g4_open('retrace_lib.jar.map')
+ g4_open('desugar_jdk_libs_configuration.jar')
+ download_file(options.version, 'r8-full-exclude-deps.jar',
+ 'full.jar')
+ download_file(options.version, 'r8-full-exclude-deps.jar',
+ 'retrace_full.jar')
+ download_file(options.version, 'r8-src.jar', 'src.jar')
+ download_file(options.version, 'r8lib-exclude-deps.jar', 'lib.jar')
+ download_file(options.version, 'r8lib-exclude-deps.jar.map',
+ 'lib.jar.map')
+ download_file(options.version, 'desugar_jdk_libs_configuration.jar',
+ 'desugar_jdk_libs_configuration.jar')
+ download_file(options.version, 'r8retrace-exclude-deps.jar',
+ 'retrace_lib.jar')
+ download_file(options.version, 'r8retrace-exclude-deps.jar.map',
+ 'retrace_lib.jar.map')
+ g4_open('METADATA')
+ metadata_path = os.path.join(third_party_r8, 'METADATA')
+ match_count = 0
+ version_match_regexp = r'[1-9]\.[0-9]{1,2}\.[0-9]{1,3}-dev'
+ for line in open(metadata_path, 'r'):
+ result = re.search(version_match_regexp, line)
+ if result:
+ match_count = match_count + 1
+ if match_count != 7:
+ print((
+ "Could not find the previous -dev release string to replace in "
+ + "METADATA. Expected to find is mentioned 7 times, but " +
+ "found %s occurrences. Please update %s manually and run " +
+ "again with options --google3 " +
+ "--use-existing-work-branch.") %
+ (match_count, metadata_path))
+ sys.exit(1)
+ sed(version_match_regexp, options.version, metadata_path)
+ sed(r'\{ year.*\}', ('{ year: %i month: %i day: %i }' %
+ (today.year, today.month, today.day)),
+ metadata_path)
+ subprocess.check_output('chmod u+w *', shell=True)
- with utils.ChangedWorkingDirectory(google3_base):
- blaze_result = blaze_run('//third_party/java/r8:d8 -- --version')
+ with utils.ChangedWorkingDirectory(google3_base):
+ blaze_result = blaze_run('//third_party/java/r8:d8 -- --version')
- assert options.version in blaze_result
+ assert options.version in blaze_result
- if not options.no_upload:
- change_result = g4_change(options.version)
- change_result += 'Run \'(g4d ' + args.p4_client \
- + ' && tap_presubmit -p all --train -c ' \
- + get_cl_id(change_result) + ')\' for running TAP global' \
- + ' presubmit using the train.\n' \
- + 'Run \'(g4d ' + args.p4_client \
- + ' && tap_presubmit -p all --notrain --detach --email' \
- + ' --skip_flaky_targets --skip_already_failing -c ' \
- + get_cl_id(change_result) + ')\' for running an isolated' \
- + ' TAP global presubmit.'
- return change_result
+ if not options.no_upload:
+ change_result = g4_change(options.version)
+ change_result += 'Run \'(g4d ' + args.p4_client \
+ + ' && tap_presubmit -p all --train -c ' \
+ + get_cl_id(change_result) + ')\' for running TAP global' \
+ + ' presubmit using the train.\n' \
+ + 'Run \'(g4d ' + args.p4_client \
+ + ' && tap_presubmit -p all --notrain --detach --email' \
+ + ' --skip_flaky_targets --skip_already_failing -c ' \
+ + get_cl_id(change_result) + ')\' for running an isolated' \
+ + ' TAP global presubmit.'
+ return change_result
- return release_google3
+ return release_google3
def prepare_google3_retrace(args):
- assert args.version
- # Check if an existing client exists.
- if not args.use_existing_work_branch:
- check_no_google3_client(args, args.p4_client)
+ assert args.version
+ # Check if an existing client exists.
+ if not args.use_existing_work_branch:
+ check_no_google3_client(args, args.p4_client)
- def release_google3_retrace(options):
- print("Releasing Retrace for Google 3")
- if options.dry_run:
- return 'DryRun: omitting g3 release for %s' % options.version
+ def release_google3_retrace(options):
+ print("Releasing Retrace for Google 3")
+ if options.dry_run:
+ return 'DryRun: omitting g3 release for %s' % options.version
- google3_base = subprocess.check_output(
- ['p4', 'g4d', '-f', args.p4_client]).decode('utf-8').rstrip()
- third_party_r8 = os.path.join(google3_base, 'third_party', 'java', 'r8')
- with utils.ChangedWorkingDirectory(third_party_r8):
- # download files
- g4_open('retrace_full.jar')
- g4_open('retrace_lib.jar')
- g4_open('retrace_lib.jar.map')
- download_file(options.version, 'r8-full-exclude-deps.jar', 'retrace_full.jar')
- download_file(options.version, 'r8retrace-exclude-deps.jar', 'retrace_lib.jar')
- download_file(
- options.version, 'r8lib-exclude-deps.jar.map', 'retrace_lib.jar.map')
- g4_open('METADATA')
- metadata_path = os.path.join(third_party_r8, 'METADATA')
- match_count = 0
- version_match_regexp = r'[1-9]\.[0-9]{1,2}\.[0-9]{1,3}-dev/r8retrace-exclude-deps.jar'
- for line in open(metadata_path, 'r'):
- result = re.search(version_match_regexp, line)
- if result:
- match_count = match_count + 1
- if match_count != 1:
- print(("Could not find the previous retrace release string to replace in " +
- "METADATA. Expected to find is mentioned 1 times. Please update %s " +
- "manually and run again with options --google3retrace " +
- "--use-existing-work-branch.") % metadata_path)
- sys.exit(1)
- sed(version_match_regexp, options.version + "/r8retrace-exclude-deps.jar", metadata_path)
- subprocess.check_output('chmod u+w *', shell=True)
+ google3_base = subprocess.check_output(
+ ['p4', 'g4d', '-f', args.p4_client]).decode('utf-8').rstrip()
+ third_party_r8 = os.path.join(google3_base, 'third_party', 'java', 'r8')
+ with utils.ChangedWorkingDirectory(third_party_r8):
+ # download files
+ g4_open('retrace_full.jar')
+ g4_open('retrace_lib.jar')
+ g4_open('retrace_lib.jar.map')
+ download_file(options.version, 'r8-full-exclude-deps.jar',
+ 'retrace_full.jar')
+ download_file(options.version, 'r8retrace-exclude-deps.jar',
+ 'retrace_lib.jar')
+ download_file(options.version, 'r8lib-exclude-deps.jar.map',
+ 'retrace_lib.jar.map')
+ g4_open('METADATA')
+ metadata_path = os.path.join(third_party_r8, 'METADATA')
+ match_count = 0
+ version_match_regexp = r'[1-9]\.[0-9]{1,2}\.[0-9]{1,3}-dev/r8retrace-exclude-deps.jar'
+ for line in open(metadata_path, 'r'):
+ result = re.search(version_match_regexp, line)
+ if result:
+ match_count = match_count + 1
+ if match_count != 1:
+ print((
+ "Could not find the previous retrace release string to replace in "
+ +
+ "METADATA. Expected to find is mentioned 1 times. Please update %s "
+ + "manually and run again with options --google3retrace " +
+ "--use-existing-work-branch.") % metadata_path)
+ sys.exit(1)
+ sed(version_match_regexp,
+ options.version + "/r8retrace-exclude-deps.jar", metadata_path)
+ subprocess.check_output('chmod u+w *', shell=True)
- with utils.ChangedWorkingDirectory(google3_base):
- blaze_result = blaze_run('//third_party/java/r8:retrace -- --version')
+ with utils.ChangedWorkingDirectory(google3_base):
+ blaze_result = blaze_run(
+ '//third_party/java/r8:retrace -- --version')
- print(blaze_result)
- assert options.version in blaze_result
+ print(blaze_result)
+ assert options.version in blaze_result
- if not options.no_upload:
- change_result = g4_change(options.version)
- change_result += 'Run \'(g4d ' + args.p4_client \
- + ' && tap_presubmit -p all --train -c ' \
- + get_cl_id(change_result) + ')\' for running TAP global' \
- + ' presubmit using the train.\n' \
- + 'Run \'(g4d ' + args.p4_client \
- + ' && tap_presubmit -p all --notrain --detach --email' \
- + ' --skip_flaky_targets --skip_already_failing -c ' \
- + get_cl_id(change_result) + ')\' for running an isolated' \
- + ' TAP global presubmit.'
- return change_result
+ if not options.no_upload:
+ change_result = g4_change(options.version)
+ change_result += 'Run \'(g4d ' + args.p4_client \
+ + ' && tap_presubmit -p all --train -c ' \
+ + get_cl_id(change_result) + ')\' for running TAP global' \
+ + ' presubmit using the train.\n' \
+ + 'Run \'(g4d ' + args.p4_client \
+ + ' && tap_presubmit -p all --notrain --detach --email' \
+ + ' --skip_flaky_targets --skip_already_failing -c ' \
+ + get_cl_id(change_result) + ')\' for running an isolated' \
+ + ' TAP global presubmit.'
+ return change_result
- return release_google3_retrace
+ return release_google3_retrace
+
def update_desugar_library_in_studio(args):
- assert os.path.exists(args.studio), ("Could not find STUDIO path %s"
- % args.studio)
+ assert os.path.exists(args.studio), ("Could not find STUDIO path %s" %
+ args.studio)
- def make_release(args):
- library_version = args.update_desugar_library_in_studio[0]
- configuration_version = args.update_desugar_library_in_studio[1]
- change_name = 'update-desugar-library-dependencies'
+ def make_release(args):
+ library_version = args.update_desugar_library_in_studio[0]
+ configuration_version = args.update_desugar_library_in_studio[1]
+ change_name = 'update-desugar-library-dependencies'
- with utils.ChangedWorkingDirectory(args.studio):
- if not args.use_existing_work_branch:
- subprocess.call(['repo', 'abandon', change_name])
- if not args.no_sync:
- subprocess.check_call(['repo', 'sync', '-cq', '-j', '16'])
+ with utils.ChangedWorkingDirectory(args.studio):
+ if not args.use_existing_work_branch:
+ subprocess.call(['repo', 'abandon', change_name])
+ if not args.no_sync:
+ subprocess.check_call(['repo', 'sync', '-cq', '-j', '16'])
- cmd = ['tools/base/bazel/bazel',
- 'run',
- '//tools/base/bazel:add_dependency',
- '--',
- '--repo=https://maven.google.com com.android.tools:desugar_jdk_libs:%s' % library_version]
- utils.PrintCmd(cmd)
- subprocess.check_call(" ".join(cmd), shell=True)
- cmd = ['tools/base/bazel/bazel', 'shutdown']
- utils.PrintCmd(cmd)
- subprocess.check_call(cmd)
+ cmd = [
+ 'tools/base/bazel/bazel', 'run',
+ '//tools/base/bazel:add_dependency', '--',
+ '--repo=https://maven.google.com com.android.tools:desugar_jdk_libs:%s'
+ % library_version
+ ]
+ utils.PrintCmd(cmd)
+ subprocess.check_call(" ".join(cmd), shell=True)
+ cmd = ['tools/base/bazel/bazel', 'shutdown']
+ utils.PrintCmd(cmd)
+ subprocess.check_call(cmd)
- prebuilts_tools = os.path.join(args.studio, 'prebuilts', 'tools')
- with utils.ChangedWorkingDirectory(prebuilts_tools):
- if not args.use_existing_work_branch:
+ prebuilts_tools = os.path.join(args.studio, 'prebuilts', 'tools')
with utils.ChangedWorkingDirectory(prebuilts_tools):
- subprocess.check_call(['repo', 'start', change_name])
- m2_dir = os.path.join(
- 'common', 'm2', 'repository', 'com', 'android', 'tools')
- subprocess.check_call(
- ['git',
- 'add',
- os.path.join(m2_dir, DESUGAR_JDK_LIBS, library_version)])
- subprocess.check_call(
- ['git',
- 'add',
- os.path.join(
- m2_dir, DESUGAR_JDK_LIBS_CONFIGURATION, configuration_version)])
+ if not args.use_existing_work_branch:
+ with utils.ChangedWorkingDirectory(prebuilts_tools):
+ subprocess.check_call(['repo', 'start', change_name])
+ m2_dir = os.path.join('common', 'm2', 'repository', 'com',
+ 'android', 'tools')
+ subprocess.check_call([
+ 'git', 'add',
+ os.path.join(m2_dir, DESUGAR_JDK_LIBS, library_version)
+ ])
+ subprocess.check_call([
+ 'git', 'add',
+ os.path.join(m2_dir, DESUGAR_JDK_LIBS_CONFIGURATION,
+ configuration_version)
+ ])
- git_message = ("""Update library desugaring dependencies
+ git_message = ("""Update library desugaring dependencies
com.android.tools:desugar_jdk_libs:%s
com.android.tools:desugar_jdk_libs_configuration:%s
Bug: %s
-Test: L8ToolTest, L8DexDesugarTest"""
- % (library_version,
- configuration_version,
- '\nBug: '.join(map(bug_fmt, args.bug))))
+Test: L8ToolTest, L8DexDesugarTest""" %
+ (library_version, configuration_version,
+ '\nBug: '.join(map(bug_fmt, args.bug))))
- if not args.use_existing_work_branch:
- subprocess.check_call(['git', 'commit', '-a', '-m', git_message])
- else:
- print('Not committing when --use-existing-work-branch. '
- + 'Commit message should be:\n\n'
- + git_message
- + '\n')
- # Don't upload if requested not to, or if changes are not committed due
- # to --use-existing-work-branch
- if not args.no_upload and not args.use_existing_work_branch:
- process = subprocess.Popen(['repo', 'upload', '.', '--verify'],
- stdin=subprocess.PIPE)
- return process.communicate(input='y\n')[0]
+ if not args.use_existing_work_branch:
+ subprocess.check_call(
+ ['git', 'commit', '-a', '-m', git_message])
+ else:
+ print('Not committing when --use-existing-work-branch. ' +
+ 'Commit message should be:\n\n' + git_message + '\n')
+ # Don't upload if requested not to, or if changes are not committed due
+ # to --use-existing-work-branch
+ if not args.no_upload and not args.use_existing_work_branch:
+ process = subprocess.Popen(['repo', 'upload', '.', '--verify'],
+ stdin=subprocess.PIPE)
+ return process.communicate(input='y\n')[0]
- return make_release
+ return make_release
def prepare_desugar_library(args):
- def make_release(args):
- library_version = args.desugar_library[0]
- configuration_version = args.desugar_library[1]
+ def make_release(args):
+ library_version = args.desugar_library[0]
+ configuration_version = args.desugar_library[1]
- # TODO(b/237636871): Cleanup and generalize.
- if (not (library_version.startswith('1.1')
- or library_version.startswith('1.2')
- or library_version.startswith('2.0'))):
- print("Release script does not support desugared library version %s"
- % library_version)
- sys.exit(1)
+ # TODO(b/237636871): Cleanup and generalize.
+ if (not (library_version.startswith('1.1') or
+ library_version.startswith('1.2') or
+ library_version.startswith('2.0'))):
+ print(
+ "Release script does not support desugared library version %s" %
+ library_version)
+ sys.exit(1)
- postfixes = ['']
- if library_version.startswith('1.2'):
- postfixes = ['_legacy']
- if library_version.startswith('2.0'):
- postfixes = ['_minimal', '', '_nio']
+ postfixes = ['']
+ if library_version.startswith('1.2'):
+ postfixes = ['_legacy']
+ if library_version.startswith('2.0'):
+ postfixes = ['_minimal', '', '_nio']
- with utils.TempDir() as temp:
- with utils.ChangedWorkingDirectory(temp):
- artifacts = []
- for postfix in postfixes:
- group_postfix = ('' if postfix == '_legacy' else postfix)
- archive_postfix = (postfix if library_version.startswith('1.1') else '_jdk11' + postfix)
- library_jar = DESUGAR_JDK_LIBS + postfix + '.jar'
- library_archive = DESUGAR_JDK_LIBS + archive_postfix + '.zip'
- configuration_archive = DESUGAR_JDK_LIBS_CONFIGURATION + archive_postfix + '.zip'
- library_gfile = ('/bigstore/r8-releases/raw/%s/%s/%s'
- % (DESUGAR_JDK_LIBS + group_postfix, library_version, library_archive))
- configuration_gfile = ('/bigstore/r8-releases/raw/main/%s/%s'
- % (configuration_version, configuration_archive))
+ with utils.TempDir() as temp:
+ with utils.ChangedWorkingDirectory(temp):
+ artifacts = []
+ for postfix in postfixes:
+ group_postfix = ('' if postfix == '_legacy' else postfix)
+ archive_postfix = (postfix
+ if library_version.startswith('1.1') else
+ '_jdk11' + postfix)
+ library_jar = DESUGAR_JDK_LIBS + postfix + '.jar'
+ library_archive = DESUGAR_JDK_LIBS + archive_postfix + '.zip'
+ configuration_archive = DESUGAR_JDK_LIBS_CONFIGURATION + archive_postfix + '.zip'
+ library_gfile = ('/bigstore/r8-releases/raw/%s/%s/%s' %
+ (DESUGAR_JDK_LIBS + group_postfix,
+ library_version, library_archive))
+ configuration_gfile = (
+ '/bigstore/r8-releases/raw/main/%s/%s' %
+ (configuration_version, configuration_archive))
- download_gfile(library_gfile, library_archive)
- download_gfile(configuration_gfile, configuration_archive)
- check_configuration(configuration_archive, group_postfix)
- artifacts.append(library_gfile)
- artifacts.append(configuration_gfile)
+ download_gfile(library_gfile, library_archive)
+ download_gfile(configuration_gfile, configuration_archive)
+ check_configuration(configuration_archive, group_postfix)
+ artifacts.append(library_gfile)
+ artifacts.append(configuration_gfile)
- release_id = gmaven_publisher_stage(args, artifacts)
+ release_id = gmaven_publisher_stage(args, artifacts)
- print("Staged Release ID " + release_id + ".\n")
- library_artifact_id = \
- '%s:%s:%s' % (ANDROID_TOOLS_PACKAGE, DESUGAR_JDK_LIBS, library_version)
- gmaven_publisher_stage_redir_test_info(
- release_id,
- library_artifact_id,
- library_jar)
+ print("Staged Release ID " + release_id + ".\n")
+ library_artifact_id = \
+ '%s:%s:%s' % (ANDROID_TOOLS_PACKAGE, DESUGAR_JDK_LIBS, library_version)
+ gmaven_publisher_stage_redir_test_info(release_id,
+ library_artifact_id,
+ library_jar)
- print("")
- answer = input("Continue with publishing [y/N]:")
+ print("")
+ answer = input("Continue with publishing [y/N]:")
- if answer != 'y':
- print('Aborting release to Google maven')
- sys.exit(1)
+ if answer != 'y':
+ print('Aborting release to Google maven')
+ sys.exit(1)
- gmaven_publisher_publish(args, release_id)
+ gmaven_publisher_publish(args, release_id)
- print("")
- print("Published. Use the email workflow for approval.")
+ print("")
+ print("Published. Use the email workflow for approval.")
- return make_release
+ return make_release
def check_configuration(configuration_archive, postfix):
- zip = zipfile.ZipFile(configuration_archive)
- zip.extractall()
- dirs = os.listdir(
- os.path.join('com', 'android', 'tools', DESUGAR_JDK_LIBS_CONFIGURATION + postfix))
- if len(dirs) != 1:
- print('Unexpected archive content, %s' + dirs)
- sys.exit(1)
+ zip = zipfile.ZipFile(configuration_archive)
+ zip.extractall()
+ dirs = os.listdir(
+ os.path.join('com', 'android', 'tools',
+ DESUGAR_JDK_LIBS_CONFIGURATION + postfix))
+ if len(dirs) != 1:
+ print('Unexpected archive content, %s' + dirs)
+ sys.exit(1)
- version = dirs[0]
- pom_file = os.path.join(
- 'com',
- 'android',
- 'tools',
- DESUGAR_JDK_LIBS_CONFIGURATION + postfix,
- version,
- '%s-%s.pom' % (DESUGAR_JDK_LIBS_CONFIGURATION + postfix, version))
- version_from_pom = extract_version_from_pom(pom_file)
- if version != version_from_pom:
- print('Version mismatch, %s != %s' % (version, version_from_pom))
- sys.exit(1)
+ version = dirs[0]
+ pom_file = os.path.join(
+ 'com', 'android', 'tools', DESUGAR_JDK_LIBS_CONFIGURATION + postfix,
+ version,
+ '%s-%s.pom' % (DESUGAR_JDK_LIBS_CONFIGURATION + postfix, version))
+ version_from_pom = extract_version_from_pom(pom_file)
+ if version != version_from_pom:
+ print('Version mismatch, %s != %s' % (version, version_from_pom))
+ sys.exit(1)
+
def check_no_google3_client(args, client_name):
- if not args.use_existing_work_branch:
- clients = subprocess.check_output('g4 myclients', shell=True).decode('utf-8')
- if ':%s:' % client_name in clients:
- if args.delete_work_branch:
- subprocess.check_call('g4 citc -d -f %s' % client_name, shell=True)
- else:
- print(("Remove the existing '%s' client before continuing " +
- "(force delete: 'g4 citc -d -f %s'), " +
- "or use either --use-existing-work-branch or " +
- "--delete-work-branch.") % (client_name, client_name))
- sys.exit(1)
+ if not args.use_existing_work_branch:
+ clients = subprocess.check_output('g4 myclients',
+ shell=True).decode('utf-8')
+ if ':%s:' % client_name in clients:
+ if args.delete_work_branch:
+ subprocess.check_call('g4 citc -d -f %s' % client_name,
+ shell=True)
+ else:
+ print(("Remove the existing '%s' client before continuing " +
+ "(force delete: 'g4 citc -d -f %s'), " +
+ "or use either --use-existing-work-branch or " +
+ "--delete-work-branch.") % (client_name, client_name))
+ sys.exit(1)
def extract_version_from_pom(pom_file):
@@ -693,53 +725,55 @@
return tree.getroot().find("{%s}version" % ns).text
-GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN = re.compile('Release ID = ([0-9a-f\-]+)')
+GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN = re.compile(
+ 'Release ID = ([0-9a-f\-]+)')
def gmaven_publisher_stage(args, gfiles):
- if args.dry_run:
- print('Dry-run, would have staged %s' % gfiles)
- return 'dry-run-release-id'
+ if args.dry_run:
+ print('Dry-run, would have staged %s' % gfiles)
+ return 'dry-run-release-id'
- print("Staging: %s" % ', '.join(gfiles))
- print("")
+ print("Staging: %s" % ', '.join(gfiles))
+ print("")
- cmd = [GMAVEN_PUBLISHER, 'stage', '--gfile', ','.join(gfiles)]
- output = subprocess.check_output(cmd)
+ cmd = [GMAVEN_PUBLISHER, 'stage', '--gfile', ','.join(gfiles)]
+ output = subprocess.check_output(cmd)
- # Expect output to contain:
- # [INFO] 06/19/2020 09:35:12 CEST: >>>>>>>>>> Staged
- # [INFO] 06/19/2020 09:35:12 CEST: Release ID = 9171d015-18f6-4a90-9984-1c362589dc1b
- # [INFO] 06/19/2020 09:35:12 CEST: Stage Path = /bigstore/studio_staging/maven2/sgjesse/9171d015-18f6-4a90-9984-1c362589dc1b
+ # Expect output to contain:
+ # [INFO] 06/19/2020 09:35:12 CEST: >>>>>>>>>> Staged
+ # [INFO] 06/19/2020 09:35:12 CEST: Release ID = 9171d015-18f6-4a90-9984-1c362589dc1b
+ # [INFO] 06/19/2020 09:35:12 CEST: Stage Path = /bigstore/studio_staging/maven2/sgjesse/9171d015-18f6-4a90-9984-1c362589dc1b
- matches = GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN.findall(output.decode("utf-8"))
- if matches == None or len(matches) > 1:
- print("Could not determine the release ID from the gmaven_publisher " +
- "output. Expected a line with 'Release ID = <release id>'.")
- print("Output was:")
+ matches = GMAVEN_PUBLISH_STAGE_RELEASE_ID_PATTERN.findall(
+ output.decode("utf-8"))
+ if matches == None or len(matches) > 1:
+ print("Could not determine the release ID from the gmaven_publisher " +
+ "output. Expected a line with 'Release ID = <release id>'.")
+ print("Output was:")
+ print(output)
+ sys.exit(1)
+
print(output)
- sys.exit(1)
- print(output)
-
- release_id = matches[0]
- return release_id
+ release_id = matches[0]
+ return release_id
def gmaven_publisher_stage_redir_test_info(release_id, artifact, dst):
- redir_command = ("/google/data/ro/teams/android-devtools-infra/tools/redir "
- + "--alsologtostderr "
- + "--gcs_bucket_path=/bigstore/gmaven-staging/${USER}/%s "
- + "--port=1480") % release_id
+ redir_command = ("/google/data/ro/teams/android-devtools-infra/tools/redir "
+ + "--alsologtostderr " +
+ "--gcs_bucket_path=/bigstore/gmaven-staging/${USER}/%s " +
+ "--port=1480") % release_id
- get_command = ("mvn org.apache.maven.plugins:maven-dependency-plugin:2.4:get "
- + "-Dmaven.repo.local=/tmp/maven_repo_local "
- + "-DremoteRepositories=http://localhost:1480 "
- + "-Dartifact=%s "
- + "-Ddest=%s") % (artifact, dst)
+ get_command = (
+ "mvn org.apache.maven.plugins:maven-dependency-plugin:2.4:get " +
+ "-Dmaven.repo.local=/tmp/maven_repo_local " +
+ "-DremoteRepositories=http://localhost:1480 " + "-Dartifact=%s " +
+ "-Ddest=%s") % (artifact, dst)
- print("""To test the staged content with 'redir' run:
+ print("""To test the staged content with 'redir' run:
%s
@@ -771,297 +805,319 @@
def gmaven_publisher_publish(args, release_id):
- if args.dry_run:
- print('Dry-run, would have published %s' % release_id)
- return
+ if args.dry_run:
+ print('Dry-run, would have published %s' % release_id)
+ return
- cmd = [GMAVEN_PUBLISHER, 'publish', release_id]
- output = subprocess.check_output(cmd)
+ cmd = [GMAVEN_PUBLISHER, 'publish', release_id]
+ output = subprocess.check_output(cmd)
+
def branch_change_diff(diff, old_version, new_version):
- invalid_line = None
- for line in str(diff).splitlines():
- if line.startswith('-R8') and \
- line != "-R8_DEV_BRANCH = '%s'" % old_version:
- print(line)
- invalid_line = line
- elif line.startswith('+R8') and \
- line != "+R8_DEV_BRANCH = '%s'" % new_version:
- print(line)
- invalid_line = line
- return invalid_line
+ invalid_line = None
+ for line in str(diff).splitlines():
+ if line.startswith('-R8') and \
+ line != "-R8_DEV_BRANCH = '%s'" % old_version:
+ print(line)
+ invalid_line = line
+ elif line.startswith('+R8') and \
+ line != "+R8_DEV_BRANCH = '%s'" % new_version:
+ print(line)
+ invalid_line = line
+ return invalid_line
def validate_branch_change_diff(version_diff_output, old_version, new_version):
- invalid = branch_change_diff(version_diff_output, old_version, new_version)
- if invalid:
- print("")
- print("The diff for the branch change in tools/release.py is not as expected:")
- print("")
- print("=" * 80)
- print(version_diff_output)
- print("=" * 80)
- print("")
- print("Validate the uploaded CL before landing.")
- print("")
+ invalid = branch_change_diff(version_diff_output, old_version, new_version)
+ if invalid:
+ print("")
+ print(
+ "The diff for the branch change in tools/release.py is not as expected:"
+ )
+ print("")
+ print("=" * 80)
+ print(version_diff_output)
+ print("=" * 80)
+ print("")
+ print("Validate the uploaded CL before landing.")
+ print("")
def prepare_branch(args):
- branch_version = args.new_dev_branch[0]
- commithash = args.new_dev_branch[1]
+ branch_version = args.new_dev_branch[0]
+ commithash = args.new_dev_branch[1]
- current_semver = utils.check_basic_semver_version(
- R8_DEV_BRANCH, ", current release branch version should be x.y", 2)
- semver = utils.check_basic_semver_version(
- branch_version, ", release branch version should be x.y", 2)
- if not semver.larger_than(current_semver):
- print('New branch version "'
- + branch_version
- + '" must be strictly larger than the current "'
- + R8_DEV_BRANCH
- + '"')
- sys.exit(1)
+ current_semver = utils.check_basic_semver_version(
+ R8_DEV_BRANCH, ", current release branch version should be x.y", 2)
+ semver = utils.check_basic_semver_version(
+ branch_version, ", release branch version should be x.y", 2)
+ if not semver.larger_than(current_semver):
+ print('New branch version "' + branch_version +
+ '" must be strictly larger than the current "' + R8_DEV_BRANCH +
+ '"')
+ sys.exit(1)
- def make_branch(options):
- with utils.TempDir() as temp:
- subprocess.check_call(['git', 'clone', utils.REPO_SOURCE, temp])
- with utils.ChangedWorkingDirectory(temp):
- subprocess.check_call(['git', 'branch', branch_version, commithash])
+ def make_branch(options):
+ with utils.TempDir() as temp:
+ subprocess.check_call(['git', 'clone', utils.REPO_SOURCE, temp])
+ with utils.ChangedWorkingDirectory(temp):
+ subprocess.check_call(
+ ['git', 'branch', branch_version, commithash])
- subprocess.check_call(['git', 'checkout', branch_version])
+ subprocess.check_call(['git', 'checkout', branch_version])
- # Rewrite the version, commit and validate.
- old_version = 'main'
- full_version = branch_version + '.0-dev'
- version_prefix = 'public static final String LABEL = "'
- sed(version_prefix + old_version,
- version_prefix + full_version,
- R8_VERSION_FILE)
+ # Rewrite the version, commit and validate.
+ old_version = 'main'
+ full_version = branch_version + '.0-dev'
+ version_prefix = 'public static final String LABEL = "'
+ sed(version_prefix + old_version, version_prefix + full_version,
+ R8_VERSION_FILE)
- subprocess.check_call([
- 'git', 'commit', '-a', '-m', 'Version %s' % full_version])
+ subprocess.check_call(
+ ['git', 'commit', '-a', '-m',
+ 'Version %s' % full_version])
- version_diff_output = subprocess.check_output([
- 'git', 'diff', '%s..HEAD' % commithash])
+ version_diff_output = subprocess.check_output(
+ ['git', 'diff', '%s..HEAD' % commithash])
- validate_version_change_diff(version_diff_output, old_version, full_version)
+ validate_version_change_diff(version_diff_output, old_version,
+ full_version)
- # Double check that we want to create a new release branch.
- if not options.dry_run:
- answer = input('Create new branch for %s [y/N]:' % branch_version)
- if answer != 'y':
- print('Aborting new branch for %s' % branch_version)
- sys.exit(1)
+ # Double check that we want to create a new release branch.
+ if not options.dry_run:
+ answer = input('Create new branch for %s [y/N]:' %
+ branch_version)
+ if answer != 'y':
+ print('Aborting new branch for %s' % branch_version)
+ sys.exit(1)
- maybe_check_call(options, [
- 'git', 'push', 'origin', 'HEAD:%s' % branch_version])
- maybe_tag(options, full_version)
+ maybe_check_call(
+ options,
+ ['git', 'push', 'origin',
+ 'HEAD:%s' % branch_version])
+ maybe_tag(options, full_version)
- print('Updating tools/r8_release.py to make new dev releases on %s'
- % branch_version)
+ print(
+ 'Updating tools/r8_release.py to make new dev releases on %s'
+ % branch_version)
- subprocess.check_call(['git', 'new-branch', 'update-release-script'])
+ subprocess.check_call(
+ ['git', 'new-branch', 'update-release-script'])
- # Check this file for the setting of the current dev branch.
- result = None
- for line in open(THIS_FILE_RELATIVE, 'r'):
- result = re.match(
- r"^R8_DEV_BRANCH = '(\d+).(\d+)'", line)
- if result:
- break
- if not result or not result.group(1):
- print('Failed to find version label in %s' % THIS_FILE_RELATIVE)
- sys.exit(1)
+ # Check this file for the setting of the current dev branch.
+ result = None
+ for line in open(THIS_FILE_RELATIVE, 'r'):
+ result = re.match(r"^R8_DEV_BRANCH = '(\d+).(\d+)'", line)
+ if result:
+ break
+ if not result or not result.group(1):
+ print('Failed to find version label in %s' %
+ THIS_FILE_RELATIVE)
+ sys.exit(1)
- # Update this file with the new dev branch.
- sed("R8_DEV_BRANCH = '%s.%s" % (result.group(1), result.group(2)),
- "R8_DEV_BRANCH = '%s.%s" % (str(semver.major), str(semver.minor)),
- THIS_FILE_RELATIVE)
+ # Update this file with the new dev branch.
+ sed(
+ "R8_DEV_BRANCH = '%s.%s" %
+ (result.group(1), result.group(2)),
+ "R8_DEV_BRANCH = '%s.%s" %
+ (str(semver.major), str(semver.minor)), THIS_FILE_RELATIVE)
- message = \
- 'Prepare %s for branch %s' % (THIS_FILE_RELATIVE, branch_version)
- subprocess.check_call(['git', 'commit', '-a', '-m', message])
+ message = \
+ 'Prepare %s for branch %s' % (THIS_FILE_RELATIVE, branch_version)
+ subprocess.check_call(['git', 'commit', '-a', '-m', message])
- branch_diff_output = subprocess.check_output(['git', 'diff', 'HEAD~'])
+ branch_diff_output = subprocess.check_output(
+ ['git', 'diff', 'HEAD~'])
- validate_branch_change_diff(
- branch_diff_output, R8_DEV_BRANCH, branch_version)
+ validate_branch_change_diff(branch_diff_output, R8_DEV_BRANCH,
+ branch_version)
- maybe_check_call(options, ['git', 'cl', 'upload', '-f', '-m', message])
+ maybe_check_call(options,
+ ['git', 'cl', 'upload', '-f', '-m', message])
- print('')
- print('Make sure to send out the branch change CL for review.')
- print('')
+ print('')
+ print('Make sure to send out the branch change CL for review.')
+ print('')
- return make_branch
+ return make_branch
def parse_options():
- result = argparse.ArgumentParser(description='Release r8')
- group = result.add_mutually_exclusive_group()
- group.add_argument('--dev-release',
- metavar=('<main hash>'),
- help='The hash to use for the new dev version of R8')
- group.add_argument('--version',
- metavar=('<version>'),
- help='The new version of R8 (e.g., 1.4.51) to release to selected channels')
- group.add_argument('--desugar-library',
- nargs=2,
- metavar=('<version>', '<configuration hash>'),
- help='The new version of com.android.tools:desugar_jdk_libs')
- group.add_argument('--update-desugar-library-in-studio',
- nargs=2,
- metavar=('<version>', '<configuration version>'),
- help='Update studio mirror of com.android.tools:desugar_jdk_libs')
- group.add_argument('--new-dev-branch',
- nargs=2,
- metavar=('<version>', '<main hash>'),
- help='Create a new branch starting a version line (e.g. 2.0)')
- result.add_argument('--dev-pre-cherry-pick',
- metavar=('<main hash(s)>'),
- default=[],
- action='append',
- help='List of commits to cherry pick before doing full '
- 'merge, mostly used for reverting cherry picks')
- result.add_argument('--no-sync', '--no_sync',
- default=False,
- action='store_true',
- help='Do not sync repos before uploading')
- result.add_argument('--bug',
- metavar=('<bug(s)>'),
- default=[],
- action='append',
- help='List of bugs for release version')
- result.add_argument('--no-bugs',
- default=False,
- action='store_true',
- help='Allow Studio release without specifying any bugs')
- result.add_argument('--studio',
- metavar=('<path>'),
- help='Release for studio by setting the path to a studio '
- 'checkout')
- result.add_argument('--aosp',
- metavar=('<path>'),
- help='Release for aosp by setting the path to the '
- 'checkout')
- result.add_argument('--maven',
- default=False,
- action='store_true',
- help='Release to Google Maven')
- result.add_argument('--google3',
- default=False,
- action='store_true',
- help='Release for google 3')
- result.add_argument('--google3retrace',
- default=False,
- action='store_true',
- help='Release retrace for google 3')
- result.add_argument('--p4-client',
- default='update-r8',
- metavar=('<client name>'),
- help='P4 client name for google 3')
- result.add_argument('--use-existing-work-branch', '--use_existing_work_branch',
- default=False,
- action='store_true',
- help='Use existing work branch/CL in aosp/studio/google3')
- result.add_argument('--delete-work-branch', '--delete_work_branch',
- default=False,
- action='store_true',
- help='Delete CL in google3')
- result.add_argument('--bypass-hooks', '--bypass_hooks',
- default=False,
- action='store_true',
- help="Bypass hooks when uploading")
- result.add_argument('--no-upload', '--no_upload',
- default=False,
- action='store_true',
- help="Don't upload for code review")
- result.add_argument('--dry-run',
- default=False,
- action='store_true',
- help='Only perform non-commiting tasks and print others.')
- result.add_argument('--dry-run-output', '--dry_run_output',
- default=os.getcwd(),
- metavar=('<path>'),
- help='Location for dry run output.')
- args = result.parse_args()
- if (len(args.bug) > 0 and args.no_bugs):
- print("Use of '--bug' and '--no-bugs' are mutually exclusive")
- sys.exit(1)
+ result = argparse.ArgumentParser(description='Release r8')
+ group = result.add_mutually_exclusive_group()
+ group.add_argument('--dev-release',
+ metavar=('<main hash>'),
+ help='The hash to use for the new dev version of R8')
+ group.add_argument(
+ '--version',
+ metavar=('<version>'),
+ help=
+ 'The new version of R8 (e.g., 1.4.51) to release to selected channels')
+ group.add_argument(
+ '--desugar-library',
+ nargs=2,
+ metavar=('<version>', '<configuration hash>'),
+ help='The new version of com.android.tools:desugar_jdk_libs')
+ group.add_argument(
+ '--update-desugar-library-in-studio',
+ nargs=2,
+ metavar=('<version>', '<configuration version>'),
+ help='Update studio mirror of com.android.tools:desugar_jdk_libs')
+ group.add_argument(
+ '--new-dev-branch',
+ nargs=2,
+ metavar=('<version>', '<main hash>'),
+ help='Create a new branch starting a version line (e.g. 2.0)')
+ result.add_argument('--dev-pre-cherry-pick',
+ metavar=('<main hash(s)>'),
+ default=[],
+ action='append',
+ help='List of commits to cherry pick before doing full '
+ 'merge, mostly used for reverting cherry picks')
+ result.add_argument('--no-sync',
+ '--no_sync',
+ default=False,
+ action='store_true',
+ help='Do not sync repos before uploading')
+ result.add_argument('--bug',
+ metavar=('<bug(s)>'),
+ default=[],
+ action='append',
+ help='List of bugs for release version')
+ result.add_argument('--no-bugs',
+ default=False,
+ action='store_true',
+ help='Allow Studio release without specifying any bugs')
+ result.add_argument(
+ '--studio',
+ metavar=('<path>'),
+ help='Release for studio by setting the path to a studio '
+ 'checkout')
+ result.add_argument('--aosp',
+ metavar=('<path>'),
+ help='Release for aosp by setting the path to the '
+ 'checkout')
+ result.add_argument('--maven',
+ default=False,
+ action='store_true',
+ help='Release to Google Maven')
+ result.add_argument('--google3',
+ default=False,
+ action='store_true',
+ help='Release for google 3')
+ result.add_argument('--google3retrace',
+ default=False,
+ action='store_true',
+ help='Release retrace for google 3')
+ result.add_argument('--p4-client',
+ default='update-r8',
+ metavar=('<client name>'),
+ help='P4 client name for google 3')
+ result.add_argument(
+ '--use-existing-work-branch',
+ '--use_existing_work_branch',
+ default=False,
+ action='store_true',
+ help='Use existing work branch/CL in aosp/studio/google3')
+ result.add_argument('--delete-work-branch',
+ '--delete_work_branch',
+ default=False,
+ action='store_true',
+ help='Delete CL in google3')
+ result.add_argument('--bypass-hooks',
+ '--bypass_hooks',
+ default=False,
+ action='store_true',
+ help="Bypass hooks when uploading")
+ result.add_argument('--no-upload',
+ '--no_upload',
+ default=False,
+ action='store_true',
+ help="Don't upload for code review")
+ result.add_argument(
+ '--dry-run',
+ default=False,
+ action='store_true',
+ help='Only perform non-commiting tasks and print others.')
+ result.add_argument('--dry-run-output',
+ '--dry_run_output',
+ default=os.getcwd(),
+ metavar=('<path>'),
+ help='Location for dry run output.')
+ args = result.parse_args()
+ if (len(args.bug) > 0 and args.no_bugs):
+ print("Use of '--bug' and '--no-bugs' are mutually exclusive")
+ sys.exit(1)
- if (args.studio
- and args.version
- and not 'dev' in args.version
- and args.bug == []
- and not args.no_bugs):
- print("When releasing a release version to Android Studio add the "
- + "list of bugs by using '--bug'")
- sys.exit(1)
+ if (args.studio and args.version and not 'dev' in args.version and
+ args.bug == [] and not args.no_bugs):
+ print("When releasing a release version to Android Studio add the " +
+ "list of bugs by using '--bug'")
+ sys.exit(1)
- if args.version and not 'dev' in args.version and args.google3:
- print("WARNING: You should not roll a release version into google 3")
+ if args.version and not 'dev' in args.version and args.google3:
+ print("WARNING: You should not roll a release version into google 3")
- return args
+ return args
def main():
- args = parse_options()
- targets_to_run = []
+ args = parse_options()
+ targets_to_run = []
- if args.new_dev_branch:
- if args.google3 or args.studio or args.aosp:
- print('Cannot create a branch and roll at the same time.')
- sys.exit(1)
- targets_to_run.append(prepare_branch(args))
+ if args.new_dev_branch:
+ if args.google3 or args.studio or args.aosp:
+ print('Cannot create a branch and roll at the same time.')
+ sys.exit(1)
+ targets_to_run.append(prepare_branch(args))
- if args.dev_release:
- if args.google3 or args.studio or args.aosp:
- print('Cannot create a dev release and roll at the same time.')
- sys.exit(1)
- targets_to_run.append(prepare_release(args))
+ if args.dev_release:
+ if args.google3 or args.studio or args.aosp:
+ print('Cannot create a dev release and roll at the same time.')
+ sys.exit(1)
+ targets_to_run.append(prepare_release(args))
- if (args.google3
- or args.maven
- or (args.studio and not args.no_sync)
- or (args.desugar_library and not args.dry_run)):
- utils.check_gcert()
+ if (args.google3 or args.maven or (args.studio and not args.no_sync) or
+ (args.desugar_library and not args.dry_run)):
+ utils.check_gcert()
- if args.google3:
- targets_to_run.append(prepare_google3(args))
- if args.google3retrace:
- targets_to_run.append(prepare_google3_retrace(args))
- if args.studio and not args.update_desugar_library_in_studio:
- targets_to_run.append(prepare_studio(args))
- if args.aosp:
- targets_to_run.append(prepare_aosp(args))
- if args.maven:
- targets_to_run.append(prepare_maven(args))
+ if args.google3:
+ targets_to_run.append(prepare_google3(args))
+ if args.google3retrace:
+ targets_to_run.append(prepare_google3_retrace(args))
+ if args.studio and not args.update_desugar_library_in_studio:
+ targets_to_run.append(prepare_studio(args))
+ if args.aosp:
+ targets_to_run.append(prepare_aosp(args))
+ if args.maven:
+ targets_to_run.append(prepare_maven(args))
- if args.desugar_library:
- targets_to_run.append(prepare_desugar_library(args))
+ if args.desugar_library:
+ targets_to_run.append(prepare_desugar_library(args))
- if args.update_desugar_library_in_studio:
- if not args.studio:
- print("--studio required")
- sys.exit(1)
- if args.bug == []:
- print("Update studio mirror of com.android.tools:desugar_jdk_libs "
- + "requires at least one bug by using '--bug'")
- sys.exit(1)
- targets_to_run.append(update_desugar_library_in_studio(args))
+ if args.update_desugar_library_in_studio:
+ if not args.studio:
+ print("--studio required")
+ sys.exit(1)
+ if args.bug == []:
+ print(
+ "Update studio mirror of com.android.tools:desugar_jdk_libs " +
+ "requires at least one bug by using '--bug'")
+ sys.exit(1)
+ targets_to_run.append(update_desugar_library_in_studio(args))
- final_results = []
- for target_closure in targets_to_run:
- final_results.append(target_closure(args))
+ final_results = []
+ for target_closure in targets_to_run:
+ final_results.append(target_closure(args))
- print('\n\n**************************************************************')
- print('PRINTING SUMMARY')
- print('**************************************************************\n\n')
+ print('\n\n**************************************************************')
+ print('PRINTING SUMMARY')
+ print('**************************************************************\n\n')
- for result in final_results:
- if result is not None:
- print(result)
+ for result in final_results:
+ if result is not None:
+ print(result)
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/r8bisect.py b/tools/r8bisect.py
index 1b35eb3..cc829c3 100755
--- a/tools/r8bisect.py
+++ b/tools/r8bisect.py
@@ -7,4 +7,4 @@
import toolhelper
if __name__ == '__main__':
- sys.exit(toolhelper.run('bisect', sys.argv[1:]))
+ sys.exit(toolhelper.run('bisect', sys.argv[1:]))
diff --git a/tools/release_smali.py b/tools/release_smali.py
index 902e517..51f261a 100755
--- a/tools/release_smali.py
+++ b/tools/release_smali.py
@@ -12,41 +12,45 @@
ARCHIVE_BUCKET = 'r8-releases'
REPO = 'https://github.com/google/smali'
+
def parse_options():
- result = argparse.ArgumentParser(description='Release Smali')
- result.add_argument('--version',
- required=True,
- metavar=('<version>'),
- help='The version of smali to release.')
- result.add_argument('--dry-run',
- default=False,
- action='store_true',
- help='Only perform non-commiting tasks and print others.')
- return result.parse_args()
+ result = argparse.ArgumentParser(description='Release Smali')
+ result.add_argument('--version',
+ required=True,
+ metavar=('<version>'),
+ help='The version of smali to release.')
+ result.add_argument(
+ '--dry-run',
+ default=False,
+ action='store_true',
+ help='Only perform non-commiting tasks and print others.')
+ return result.parse_args()
def Main():
- options = parse_options()
- utils.check_gcert()
- gfile = ('/bigstore/r8-releases/smali/%s/smali-maven-release-%s.zip'
- % (options.version, options.version))
- release_id = gmaven.publisher_stage([gfile], options.dry_run)
+ options = parse_options()
+ utils.check_gcert()
+ gfile = ('/bigstore/r8-releases/smali/%s/smali-maven-release-%s.zip' %
+ (options.version, options.version))
+ release_id = gmaven.publisher_stage([gfile], options.dry_run)
- print('Staged Release ID %s.\n' % release_id)
- gmaven.publisher_stage_redir_test_info(
- release_id, 'com.android.tools.smali:smali:%s' % options.version, 'smali.jar')
+ print('Staged Release ID %s.\n' % release_id)
+ gmaven.publisher_stage_redir_test_info(
+ release_id, 'com.android.tools.smali:smali:%s' % options.version,
+ 'smali.jar')
- print()
- answer = input('Continue with publishing [y/N]:')
+ print()
+ answer = input('Continue with publishing [y/N]:')
- if answer != 'y':
- print('Aborting release to Google maven')
- sys.exit(1)
+ if answer != 'y':
+ print('Aborting release to Google maven')
+ sys.exit(1)
- gmaven.publisher_publish(release_id, options.dry_run)
+ gmaven.publisher_publish(release_id, options.dry_run)
- print()
- print('Published. Use the email workflow for approval.')
+ print()
+ print('Published. Use the email workflow for approval.')
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/retrace.py b/tools/retrace.py
index 1a9deb5..6642224 100755
--- a/tools/retrace.py
+++ b/tools/retrace.py
@@ -15,204 +15,201 @@
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'R8lib wrapper for retrace tool.')
- parser.add_argument(
- '-c',
- '--commit-hash',
- '--commit_hash',
- help='Commit hash to download r8lib map file for.',
- default=None)
- parser.add_argument(
- '--version',
- help='Version to download r8lib map file for.',
- default=None)
- parser.add_argument(
- '--tag',
- help='Tag to download r8lib map file for.',
- default=None)
- parser.add_argument(
- '--exclude-deps', '--exclude_deps',
- default=None,
- action='store_true',
- help='Use the exclude-deps version of the mapping file.')
- parser.add_argument(
- '--map',
- help='Path to r8lib map.',
- default=None)
- parser.add_argument(
- '--r8jar',
- help='Path to r8 jar.',
- default=None)
- parser.add_argument(
- '--no-r8lib',
- '--no_r8lib',
- default=False,
- action='store_true',
- help='Use r8.jar and not r8lib.jar.')
- parser.add_argument(
- '--stacktrace',
- help='Path to stacktrace file (read from stdin if not passed).',
- default=None)
- parser.add_argument(
- '--quiet',
- default=None,
- action='store_true',
- help='Disables diagnostics printing to stdout.')
- parser.add_argument(
- '--debug-agent',
- '--debug_agent',
- default=None,
- action='store_true',
- help='Attach a debug-agent to the retracer java process.')
- parser.add_argument(
- '--regex',
- default=None,
- help='Sets a custom regular expression used for parsing'
- )
- parser.add_argument(
- '--verbose',
- default=None,
- action='store_true',
- help='Enables verbose retracing.')
- parser.add_argument(
- '--disable-map-validation',
- default=None,
- action='store_true',
- help='Disable validation of map hash.')
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='R8lib wrapper for retrace tool.')
+ parser.add_argument('-c',
+ '--commit-hash',
+ '--commit_hash',
+ help='Commit hash to download r8lib map file for.',
+ default=None)
+ parser.add_argument('--version',
+ help='Version to download r8lib map file for.',
+ default=None)
+ parser.add_argument('--tag',
+ help='Tag to download r8lib map file for.',
+ default=None)
+ parser.add_argument(
+ '--exclude-deps',
+ '--exclude_deps',
+ default=None,
+ action='store_true',
+ help='Use the exclude-deps version of the mapping file.')
+ parser.add_argument('--map', help='Path to r8lib map.', default=None)
+ parser.add_argument('--r8jar', help='Path to r8 jar.', default=None)
+ parser.add_argument('--no-r8lib',
+ '--no_r8lib',
+ default=False,
+ action='store_true',
+ help='Use r8.jar and not r8lib.jar.')
+ parser.add_argument(
+ '--stacktrace',
+ help='Path to stacktrace file (read from stdin if not passed).',
+ default=None)
+ parser.add_argument('--quiet',
+ default=None,
+ action='store_true',
+ help='Disables diagnostics printing to stdout.')
+ parser.add_argument(
+ '--debug-agent',
+ '--debug_agent',
+ default=None,
+ action='store_true',
+ help='Attach a debug-agent to the retracer java process.')
+ parser.add_argument(
+ '--regex',
+ default=None,
+ help='Sets a custom regular expression used for parsing')
+ parser.add_argument('--verbose',
+ default=None,
+ action='store_true',
+ help='Enables verbose retracing.')
+ parser.add_argument('--disable-map-validation',
+ default=None,
+ action='store_true',
+ help='Disable validation of map hash.')
+ return parser.parse_args()
def get_map_file(args, temp):
- # default to using the specified map file.
- if args.map:
- return args.map
+ # default to using the specified map file.
+ if args.map:
+ return args.map
- # next try to extract it from the tag/version options.
- map_path = utils.find_cloud_storage_file_from_options('r8lib.jar.map', args)
- if map_path:
- return map_path
+ # next try to extract it from the tag/version options.
+ map_path = utils.find_cloud_storage_file_from_options('r8lib.jar.map', args)
+ if map_path:
+ return map_path
- # next try to extract it from the stack-trace source-file content.
- if not args.stacktrace:
- if not args.quiet:
- print('Waiting for stack-trace input...')
- args.stacktrace = os.path.join(temp, 'stacktrace.txt')
- open(args.stacktrace, 'w').writelines(sys.stdin.readlines())
+ # next try to extract it from the stack-trace source-file content.
+ if not args.stacktrace:
+ if not args.quiet:
+ print('Waiting for stack-trace input...')
+ args.stacktrace = os.path.join(temp, 'stacktrace.txt')
+ open(args.stacktrace, 'w').writelines(sys.stdin.readlines())
- r8_source_file = None
- for line in open(args.stacktrace, 'r'):
- start = line.rfind("(R8_")
- if start > 0:
- end = line.find(":", start)
- content = line[start + 1: end]
- if r8_source_file:
- if content != r8_source_file:
- print('WARNING: there are multiple distinct R8 source files:')
- print(' ' + r8_source_file)
- print(' ' + content)
- else:
- r8_source_file = content
+ r8_source_file = None
+ for line in open(args.stacktrace, 'r'):
+ start = line.rfind("(R8_")
+ if start > 0:
+ end = line.find(":", start)
+ content = line[start + 1:end]
+ if r8_source_file:
+ if content != r8_source_file:
+ print(
+ 'WARNING: there are multiple distinct R8 source files:')
+ print(' ' + r8_source_file)
+ print(' ' + content)
+ else:
+ r8_source_file = content
- if r8_source_file:
- (header, r8_version_or_hash, maphash) = r8_source_file.split('_')
- # If the command-line specified --exclude-deps then assume it is as previous
- # versions will not be marked as such in the source-file line.
- is_excldeps = args.exclude_deps
- excldeps_start = r8_version_or_hash.find('+excldeps')
- if (excldeps_start > 0):
- is_excldeps = True
- r8_version_or_hash = r8_version_or_hash[0:excldeps_start]
- if len(r8_version_or_hash) < 40:
- args.version = r8_version_or_hash
- else:
- args.commit_hash = r8_version_or_hash
- map_path = None
- if path.exists(utils.R8LIB_MAP) and get_hash_from_map_file(utils.R8LIB_MAP) == maphash:
- return utils.R8LIB_MAP
+ if r8_source_file:
+ (header, r8_version_or_hash, maphash) = r8_source_file.split('_')
+ # If the command-line specified --exclude-deps then assume it is as previous
+ # versions will not be marked as such in the source-file line.
+ is_excldeps = args.exclude_deps
+ excldeps_start = r8_version_or_hash.find('+excldeps')
+ if (excldeps_start > 0):
+ is_excldeps = True
+ r8_version_or_hash = r8_version_or_hash[0:excldeps_start]
+ if len(r8_version_or_hash) < 40:
+ args.version = r8_version_or_hash
+ else:
+ args.commit_hash = r8_version_or_hash
+ map_path = None
+ if path.exists(utils.R8LIB_MAP) and get_hash_from_map_file(
+ utils.R8LIB_MAP) == maphash:
+ return utils.R8LIB_MAP
- try:
- map_path = utils.find_cloud_storage_file_from_options(
- 'r8lib' + ('-exclude-deps' if is_excldeps else '') + '.jar.map', args)
- except Exception as e:
- print(e)
- print('WARNING: Falling back to using local mapping file.')
+ try:
+ map_path = utils.find_cloud_storage_file_from_options(
+ 'r8lib' + ('-exclude-deps' if is_excldeps else '') + '.jar.map',
+ args)
+ except Exception as e:
+ print(e)
+ print('WARNING: Falling back to using local mapping file.')
- if map_path and not args.disable_map_validation:
- check_maphash(map_path, maphash, args)
- return map_path
+ if map_path and not args.disable_map_validation:
+ check_maphash(map_path, maphash, args)
+ return map_path
- # If no other map file was found, use the local mapping file.
- if args.r8jar:
- return args.r8jar + ".map"
- return utils.R8LIB_MAP
+ # If no other map file was found, use the local mapping file.
+ if args.r8jar:
+ return args.r8jar + ".map"
+ return utils.R8LIB_MAP
def check_maphash(mapping_path, maphash, args):
- infile_maphash = get_hash_from_map_file(mapping_path)
- if infile_maphash != maphash:
- print('ERROR: The mapping file hash does not match the R8 line')
- print(' In mapping file: ' + infile_maphash)
- print(' In source file: ' + maphash)
- if (not args.exclude_deps):
- print('If this could be a version without internalized dependencies '
- + 'try passing --exclude-deps')
- sys.exit(1)
+ infile_maphash = get_hash_from_map_file(mapping_path)
+ if infile_maphash != maphash:
+ print('ERROR: The mapping file hash does not match the R8 line')
+ print(' In mapping file: ' + infile_maphash)
+ print(' In source file: ' + maphash)
+ if (not args.exclude_deps):
+ print(
+ 'If this could be a version without internalized dependencies '
+ + 'try passing --exclude-deps')
+ sys.exit(1)
+
def get_hash_from_map_file(mapping_path):
- map_hash_header = "# pg_map_hash: SHA-256 "
- for line in open(mapping_path, 'r'):
- if line.startswith(map_hash_header):
- return line[len(map_hash_header):].strip()
+ map_hash_header = "# pg_map_hash: SHA-256 "
+ for line in open(mapping_path, 'r'):
+ if line.startswith(map_hash_header):
+ return line[len(map_hash_header):].strip()
+
def main():
- args = parse_arguments()
- with utils.TempDir() as temp:
- map_path = get_map_file(args, temp)
- return run(
- map_path,
- args.stacktrace,
- args.r8jar,
- args.no_r8lib,
- quiet=args.quiet,
- debug=args.debug_agent,
- regex=args.regex,
- verbose=args.verbose)
+ args = parse_arguments()
+ with utils.TempDir() as temp:
+ map_path = get_map_file(args, temp)
+ return run(map_path,
+ args.stacktrace,
+ args.r8jar,
+ args.no_r8lib,
+ quiet=args.quiet,
+ debug=args.debug_agent,
+ regex=args.regex,
+ verbose=args.verbose)
-def run(map_path, stacktrace, r8jar, no_r8lib, quiet=False, debug=False, regex=None, verbose=False):
- retrace_args = [jdk.GetJavaExecutable()]
+def run(map_path,
+ stacktrace,
+ r8jar,
+ no_r8lib,
+ quiet=False,
+ debug=False,
+ regex=None,
+ verbose=False):
+ retrace_args = [jdk.GetJavaExecutable()]
- if debug:
- retrace_args.append(
- '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005')
+ if debug:
+ retrace_args.append(
+ '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005'
+ )
- if not r8jar:
- r8jar = utils.R8_JAR if no_r8lib else utils.R8RETRACE_JAR
+ if not r8jar:
+ r8jar = utils.R8_JAR if no_r8lib else utils.R8RETRACE_JAR
- retrace_args += [
- '-cp',
- r8jar,
- 'com.android.tools.r8.retrace.Retrace',
- map_path
- ]
+ retrace_args += [
+ '-cp', r8jar, 'com.android.tools.r8.retrace.Retrace', map_path
+ ]
- if regex:
- retrace_args.append('--regex')
- retrace_args.append(regex)
+ if regex:
+ retrace_args.append('--regex')
+ retrace_args.append(regex)
- if quiet:
- retrace_args.append('--quiet')
+ if quiet:
+ retrace_args.append('--quiet')
- if stacktrace:
- retrace_args.append(stacktrace)
+ if stacktrace:
+ retrace_args.append(stacktrace)
- if verbose:
- retrace_args.append('--verbose')
+ if verbose:
+ retrace_args.append('--verbose')
- utils.PrintCmd(retrace_args, quiet=quiet)
- return subprocess.call(retrace_args)
+ utils.PrintCmd(retrace_args, quiet=quiet)
+ return subprocess.call(retrace_args)
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/run-jdwp-tests.py b/tools/run-jdwp-tests.py
index e241fb4..591a608 100755
--- a/tools/run-jdwp-tests.py
+++ b/tools/run-jdwp-tests.py
@@ -14,166 +14,171 @@
TEST_PACKAGE = 'org.apache.harmony.jpda.tests.jdwp'
VERSIONS = [
- 'default',
- '10.0.0',
- '9.0.0',
- '8.1.0',
- '7.0.0',
- '6.0.1',
- '5.1.1',
- '4.4.4'
+ 'default', '10.0.0', '9.0.0', '8.1.0', '7.0.0', '6.0.1', '5.1.1', '4.4.4'
]
-JUNIT_HOSTDEX = os.path.join(
- utils.REPO_ROOT,
- 'third_party', 'jdwp-tests', 'junit-hostdex.jar')
+JUNIT_HOSTDEX = os.path.join(utils.REPO_ROOT, 'third_party', 'jdwp-tests',
+ 'junit-hostdex.jar')
-JDWP_TESTS_HOSTDEX = os.path.join(
- utils.REPO_ROOT,
- 'third_party', 'jdwp-tests', 'apache-harmony-jdwp-tests-hostdex.jar')
+JDWP_TESTS_HOSTDEX = os.path.join(utils.REPO_ROOT, 'third_party', 'jdwp-tests',
+ 'apache-harmony-jdwp-tests-hostdex.jar')
-IMAGE='/system/non/existent/jdwp/image.art'
+IMAGE = '/system/non/existent/jdwp/image.art'
# Timeout in ms
-TIMEOUT=10000
+TIMEOUT = 10000
DEBUGGER_EXTRA_FLAGS = [
- '-Xjnigreflimit:2000',
- '-Duser.language=en',
- '-Duser.region=US',
- '-Djpda.settings.verbose=true',
- '-Djpda.settings.transportAddress=127.0.0.1:55107',
- '-Djpda.settings.timeout=%d' % TIMEOUT,
- '-Djpda.settings.waitingTime=%d' % TIMEOUT
+ '-Xjnigreflimit:2000', '-Duser.language=en', '-Duser.region=US',
+ '-Djpda.settings.verbose=true',
+ '-Djpda.settings.transportAddress=127.0.0.1:55107',
+ '-Djpda.settings.timeout=%d' % TIMEOUT,
+ '-Djpda.settings.waitingTime=%d' % TIMEOUT
]
-DEBUGGEE_EXTRA_FLAGS = [
-]
+DEBUGGEE_EXTRA_FLAGS = []
+
def get_art_dir(version):
- if version == '4.4.4':
- art_dir = 'dalvik'
- else:
- art_dir = version == 'default' and 'art' or 'art-%s' % version
- return os.path.join(utils.REPO_ROOT, 'tools', 'linux', art_dir)
+ if version == '4.4.4':
+ art_dir = 'dalvik'
+ else:
+ art_dir = version == 'default' and 'art' or 'art-%s' % version
+ return os.path.join(utils.REPO_ROOT, 'tools', 'linux', art_dir)
+
def get_lib_dir(version):
- return os.path.join(get_art_dir(version), 'lib')
+ return os.path.join(get_art_dir(version), 'lib')
+
def get_fw_dir(version):
- return os.path.join(get_art_dir(version), 'framework')
+ return os.path.join(get_art_dir(version), 'framework')
+
def get_vm(version):
- return os.path.join(get_art_dir(version), 'bin', 'dalvikvm')
+ return os.path.join(get_art_dir(version), 'bin', 'dalvikvm')
+
def setup_environment(version):
- art_dir = get_art_dir(version)
- lib_dir = get_lib_dir(version)
- android_data = os.path.join(utils.REPO_ROOT, 'build', 'tmp', version)
- if not os.path.isdir(android_data):
- os.makedirs(android_data)
- if version == '4.4.4':
- # Dalvik expects that the dalvik-cache dir already exists.
- dalvik_cache_dir = os.path.join(android_data, 'dalvik-cache')
- if not os.path.isdir(dalvik_cache_dir):
- os.makedirs(dalvik_cache_dir)
- os.environ['ANDROID_DATA'] = android_data
- os.environ['ANDROID_ROOT'] = art_dir
- os.environ['LD_LIBRARY_PATH'] = lib_dir
- os.environ['DYLD_LIBRARY_PATH'] = lib_dir
- os.environ['LD_USE_LOAD_BIAS'] = '1'
+ art_dir = get_art_dir(version)
+ lib_dir = get_lib_dir(version)
+ android_data = os.path.join(utils.REPO_ROOT, 'build', 'tmp', version)
+ if not os.path.isdir(android_data):
+ os.makedirs(android_data)
+ if version == '4.4.4':
+ # Dalvik expects that the dalvik-cache dir already exists.
+ dalvik_cache_dir = os.path.join(android_data, 'dalvik-cache')
+ if not os.path.isdir(dalvik_cache_dir):
+ os.makedirs(dalvik_cache_dir)
+ os.environ['ANDROID_DATA'] = android_data
+ os.environ['ANDROID_ROOT'] = art_dir
+ os.environ['LD_LIBRARY_PATH'] = lib_dir
+ os.environ['DYLD_LIBRARY_PATH'] = lib_dir
+ os.environ['LD_USE_LOAD_BIAS'] = '1'
+
def get_boot_libs(version):
- boot_libs = []
- if version == '4.4.4':
- # Dalvik
- boot_libs.extend(['core-hostdex.jar'])
- else:
- # ART
- boot_libs.extend(['core-libart-hostdex.jar'])
- if version != '5.1.1' and version != '6.0.1':
- boot_libs.extend(['core-oj-hostdex.jar'])
- boot_libs.extend(['apache-xml-hostdex.jar'])
- return [os.path.join(get_fw_dir(version), lib) for lib in boot_libs]
+ boot_libs = []
+ if version == '4.4.4':
+ # Dalvik
+ boot_libs.extend(['core-hostdex.jar'])
+ else:
+ # ART
+ boot_libs.extend(['core-libart-hostdex.jar'])
+ if version != '5.1.1' and version != '6.0.1':
+ boot_libs.extend(['core-oj-hostdex.jar'])
+ boot_libs.extend(['apache-xml-hostdex.jar'])
+ return [os.path.join(get_fw_dir(version), lib) for lib in boot_libs]
+
def get_common_flags(version):
- flags = []
- flags.extend(['-Xbootclasspath:%s' % ':'.join(get_boot_libs(version))])
- if version != '4.4.4':
- flags.extend(['-Ximage:%s' % IMAGE])
- if version != '5.1.1':
- flags.extend(['-Xcompiler-option', '--debuggable'])
- if version == '9.0.0' or version == '10.0.0':
- flags.extend(['-XjdwpProvider:internal'])
- return flags
+ flags = []
+ flags.extend(['-Xbootclasspath:%s' % ':'.join(get_boot_libs(version))])
+ if version != '4.4.4':
+ flags.extend(['-Ximage:%s' % IMAGE])
+ if version != '5.1.1':
+ flags.extend(['-Xcompiler-option', '--debuggable'])
+ if version == '9.0.0' or version == '10.0.0':
+ flags.extend(['-XjdwpProvider:internal'])
+ return flags
+
def get_debuggee_flags(version):
- return get_common_flags(version) + DEBUGGEE_EXTRA_FLAGS
+ return get_common_flags(version) + DEBUGGEE_EXTRA_FLAGS
+
def get_debugger_flags(version):
- return get_common_flags(version) + DEBUGGER_EXTRA_FLAGS
+ return get_common_flags(version) + DEBUGGER_EXTRA_FLAGS
+
def runDebuggee(version, args):
- art_dir = get_art_dir(version)
- lib_dir = get_lib_dir(version)
- fw_dir = get_fw_dir(version)
- cmd = [get_vm(version)]
- cmd.extend(get_debuggee_flags(version))
- cmd.extend(args)
- setup_environment(version)
- print("Running debuggee as: %s" % cmd)
- return subprocess.check_call(cmd)
+ art_dir = get_art_dir(version)
+ lib_dir = get_lib_dir(version)
+ fw_dir = get_fw_dir(version)
+ cmd = [get_vm(version)]
+ cmd.extend(get_debuggee_flags(version))
+ cmd.extend(args)
+ setup_environment(version)
+ print("Running debuggee as: %s" % cmd)
+ return subprocess.check_call(cmd)
+
def runDebugger(version, classpath, args):
- art_dir = get_art_dir(version)
- lib_dir = get_lib_dir(version)
- fw_dir = get_fw_dir(version)
- dalvikvm = os.path.join(art_dir, 'bin', 'dalvikvm')
- cmd = [dalvikvm]
- cmd.extend(['-classpath', '%s:%s' % (classpath, JUNIT_HOSTDEX)])
- cmd.extend(get_debugger_flags(version))
- cmd.append('-Djpda.settings.debuggeeJavaPath=%s %s' %\
- (dalvikvm, ' '.join(get_debuggee_flags(version))))
- cmd.extend(args)
- setup_environment(version)
- print("Running debugger as: " % cmd)
- return subprocess.check_call(cmd)
+ art_dir = get_art_dir(version)
+ lib_dir = get_lib_dir(version)
+ fw_dir = get_fw_dir(version)
+ dalvikvm = os.path.join(art_dir, 'bin', 'dalvikvm')
+ cmd = [dalvikvm]
+ cmd.extend(['-classpath', '%s:%s' % (classpath, JUNIT_HOSTDEX)])
+ cmd.extend(get_debugger_flags(version))
+ cmd.append('-Djpda.settings.debuggeeJavaPath=%s %s' %\
+ (dalvikvm, ' '.join(get_debuggee_flags(version))))
+ cmd.extend(args)
+ setup_environment(version)
+ print("Running debugger as: " % cmd)
+ return subprocess.check_call(cmd)
+
def usage():
- print("Usage: %s [--debuggee] [--version=<version>] [--classpath=<classpath>] <args>" % (sys.argv[0]))
- print("where <version> is one of:", ', '.join(VERSIONS))
- print(" and <classpath> is optional classpath (default: %s)" % JDWP_TESTS_HOSTDEX)
- print(" and <args> will be passed on as arguments to the art runtime.")
+ print(
+ "Usage: %s [--debuggee] [--version=<version>] [--classpath=<classpath>] <args>"
+ % (sys.argv[0]))
+ print("where <version> is one of:", ', '.join(VERSIONS))
+ print(" and <classpath> is optional classpath (default: %s)" %
+ JDWP_TESTS_HOSTDEX)
+ print(" and <args> will be passed on as arguments to the art runtime.")
+
def main():
- version = 'default'
- debuggee = False
- args = []
- classpath = JDWP_TESTS_HOSTDEX
- for arg in sys.argv[1:]:
- if arg == '--help':
- usage()
- return 0
- elif arg.startswith('--version='):
- version = arg[len('--version='):]
- elif arg.startswith('--classpath='):
- classpath = arg[len('--classpath='):]
+ version = 'default'
+ debuggee = False
+ args = []
+ classpath = JDWP_TESTS_HOSTDEX
+ for arg in sys.argv[1:]:
+ if arg == '--help':
+ usage()
+ return 0
+ elif arg.startswith('--version='):
+ version = arg[len('--version='):]
+ elif arg.startswith('--classpath='):
+ classpath = arg[len('--classpath='):]
+ else:
+ args.append(arg)
+ if version not in VERSIONS:
+ print("Invalid version %s" % version)
+ usage()
+ return 1
+ if not debuggee and len(args) == 0:
+ args.append(DEFAULT_TEST)
+ if debuggee:
+ return runDebuggee(version, args)
else:
- args.append(arg)
- if version not in VERSIONS:
- print("Invalid version %s" % version)
- usage()
- return 1
- if not debuggee and len(args) == 0:
- args.append(DEFAULT_TEST)
- if debuggee:
- return runDebuggee(version, args)
- else:
- if len(args) == 0:
- args.append(DEFAULT_TEST)
- elif len(args) == 1:
- args = [TEST_RUNNER, args[0]]
- return runDebugger(version, classpath, args)
+ if len(args) == 0:
+ args.append(DEFAULT_TEST)
+ elif len(args) == 1:
+ args = [TEST_RUNNER, args[0]]
+ return runDebugger(version, classpath, args)
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/run-r8-on-gmscore.py b/tools/run-r8-on-gmscore.py
index 280941e..0f7d3b2 100755
--- a/tools/run-r8-on-gmscore.py
+++ b/tools/run-r8-on-gmscore.py
@@ -7,6 +7,7 @@
import run_on_app
if __name__ == '__main__':
- # Default compiler is R8.
- sys.exit(run_on_app.main(sys.argv[1:]
- + ['--app', 'gmscore', '--compiler', 'r8']))
+ # Default compiler is R8.
+ sys.exit(
+ run_on_app.main(sys.argv[1:] +
+ ['--app', 'gmscore', '--compiler', 'r8']))
diff --git a/tools/run_benchmark.py b/tools/run_benchmark.py
index 0d1e4fd..f98c8b7 100755
--- a/tools/run_benchmark.py
+++ b/tools/run_benchmark.py
@@ -19,117 +19,134 @@
]
GOLEM_BUILD_TARGETS = [utils.GRADLE_TASK_R8LIB] + GOLEM_BUILD_TARGETS_TESTS
+
def get_golem_resource_path(benchmark):
- return os.path.join('benchmarks', benchmark)
+ return os.path.join('benchmarks', benchmark)
+
def get_jdk_home(options, benchmark):
- if options.golem:
- return os.path.join(get_golem_resource_path(benchmark), 'linux')
- return None
+ if options.golem:
+ return os.path.join(get_golem_resource_path(benchmark), 'linux')
+ return None
+
def parse_options(argv):
- result = argparse.ArgumentParser(description = 'Run test-based benchmarks.')
- result.add_argument('--golem',
- help='Indicate this as a run on golem',
- default=False,
- action='store_true')
- result.add_argument('--benchmark',
- help='The test benchmark to run',
- required=True)
- result.add_argument('--target',
- help='The test target to run',
- required=True,
- # These should 1:1 with benchmarks/BenchmarkTarget.java
- choices=['d8', 'r8-full', 'r8-force', 'r8-compat'])
- result.add_argument('--nolib', '--no-lib', '--no-r8lib',
- help='Run the non-lib R8 build (default false)',
- default=False,
- action='store_true')
- result.add_argument('--no-build', '--no_build',
- help='Run without building first (default false)',
- default=False,
- action='store_true')
- result.add_argument('--enable-assertions', '--enable_assertions', '-ea',
- help='Enable assertions when running',
- default=False,
- action='store_true')
- result.add_argument('--print-times',
- help='Print timing information from r8',
- default=False,
- action='store_true')
- result.add_argument('--version', '-v',
- help='Use R8 version/hash for the run (default local build)',
- default=None)
- result.add_argument('--temp',
- help='A directory to use for temporaries and outputs.',
- default=None)
- return result.parse_known_args(argv)
+ result = argparse.ArgumentParser(description='Run test-based benchmarks.')
+ result.add_argument('--golem',
+ help='Indicate this as a run on golem',
+ default=False,
+ action='store_true')
+ result.add_argument('--benchmark',
+ help='The test benchmark to run',
+ required=True)
+ result.add_argument(
+ '--target',
+ help='The test target to run',
+ required=True,
+ # These should 1:1 with benchmarks/BenchmarkTarget.java
+ choices=['d8', 'r8-full', 'r8-force', 'r8-compat'])
+ result.add_argument('--nolib',
+ '--no-lib',
+ '--no-r8lib',
+ help='Run the non-lib R8 build (default false)',
+ default=False,
+ action='store_true')
+ result.add_argument('--no-build',
+ '--no_build',
+ help='Run without building first (default false)',
+ default=False,
+ action='store_true')
+ result.add_argument('--enable-assertions',
+ '--enable_assertions',
+ '-ea',
+ help='Enable assertions when running',
+ default=False,
+ action='store_true')
+ result.add_argument('--print-times',
+ help='Print timing information from r8',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--version',
+ '-v',
+ help='Use R8 version/hash for the run (default local build)',
+ default=None)
+ result.add_argument('--temp',
+ help='A directory to use for temporaries and outputs.',
+ default=None)
+ return result.parse_known_args(argv)
+
def main(argv, temp):
- (options, args) = parse_options(argv)
+ (options, args) = parse_options(argv)
- if options.temp:
- temp = options.temp
+ if options.temp:
+ temp = options.temp
- if options.golem:
- options.no_build = True
+ if options.golem:
+ options.no_build = True
+ if options.nolib:
+ print("Error: golem should always run r8lib")
+ return 1
+
if options.nolib:
- print("Error: golem should always run r8lib")
- return 1
+ testBuildTargets = [
+ utils.GRADLE_TASK_TEST_JAR, utils.GRADLE_TASK_TEST_DEPS_JAR
+ ]
+ buildTargets = [utils.GRADLE_TASK_R8] + testBuildTargets
+ r8jar = utils.R8_JAR
+ testjars = [utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR]
+ else:
+ testBuildTargets = GOLEM_BUILD_TARGETS_TESTS
+ buildTargets = GOLEM_BUILD_TARGETS
+ r8jar = utils.R8LIB_JAR
+ testjars = [
+ os.path.join(utils.R8LIB_TESTS_JAR),
+ os.path.join(utils.R8LIB_TESTS_DEPS_JAR)
+ ]
- if options.nolib:
- testBuildTargets = [utils.GRADLE_TASK_TEST_JAR, utils.GRADLE_TASK_TEST_DEPS_JAR]
- buildTargets = [utils.GRADLE_TASK_R8] + testBuildTargets
- r8jar = utils.R8_JAR
- testjars = [utils.R8_TESTS_JAR, utils.R8_TESTS_DEPS_JAR]
- else:
- testBuildTargets = GOLEM_BUILD_TARGETS_TESTS
- buildTargets = GOLEM_BUILD_TARGETS
- r8jar = utils.R8LIB_JAR
- testjars = [
- os.path.join(utils.R8LIB_TESTS_JAR),
- os.path.join(utils.R8LIB_TESTS_DEPS_JAR)
- ]
+ if options.version:
+ # r8 is downloaded so only test jar needs to be built.
+ buildTargets = testBuildTargets
+ r8jar = compiledump.download_distribution(options.version,
+ options.nolib, temp)
- if options.version:
- # r8 is downloaded so only test jar needs to be built.
- buildTargets = testBuildTargets
- r8jar = compiledump.download_distribution(options.version, options.nolib, temp)
+ if not options.no_build:
+ gradle.RunGradle(buildTargets + ['-Pno_internal'])
- if not options.no_build:
- gradle.RunGradle(buildTargets + ['-Pno_internal'])
+ if not options.golem:
+ # When running locally, change the working directory to be in 'temp'.
+ # This is hard to do properly within the JVM so we do it here.
+ with utils.ChangedWorkingDirectory(temp):
+ return run(options, r8jar, testjars)
+ else:
+ return run(options, r8jar, testjars)
- if not options.golem:
- # When running locally, change the working directory to be in 'temp'.
- # This is hard to do properly within the JVM so we do it here.
- with utils.ChangedWorkingDirectory(temp):
- return run(options, r8jar, testjars)
- else:
- return run(options, r8jar, testjars)
def run(options, r8jar, testjars):
- jdkhome = get_jdk_home(options, options.benchmark)
- cmd = [jdk.GetJavaExecutable(jdkhome)]
- if options.enable_assertions:
- cmd.append('-ea')
- if options.print_times:
- cmd.append('-Dcom.android.tools.r8.printtimes=1')
- if not options.golem:
+ jdkhome = get_jdk_home(options, options.benchmark)
+ cmd = [jdk.GetJavaExecutable(jdkhome)]
+ if options.enable_assertions:
+ cmd.append('-ea')
+ if options.print_times:
+ cmd.append('-Dcom.android.tools.r8.printtimes=1')
+ if not options.golem:
+ cmd.extend([
+ '-DUSE_NEW_GRADLE_SETUP=true',
+ f'-DTEST_DATA_LOCATION={utils.REPO_ROOT}/d8_r8/test_modules/tests_java_8/build/classes/java/test'
+ ])
+ cmd.extend(['-cp', ':'.join([r8jar] + testjars)])
cmd.extend([
- '-DUSE_NEW_GRADLE_SETUP=true',
- f'-DTEST_DATA_LOCATION={utils.REPO_ROOT}/d8_r8/test_modules/tests_java_8/build/classes/java/test'
- ])
- cmd.extend(['-cp', ':'.join([r8jar] + testjars)])
- cmd.extend([
- 'com.android.tools.r8.benchmarks.BenchmarkMainEntryRunner',
- options.benchmark,
- options.target,
- # When running locally the working directory is moved and we pass the
- # repository root as an argument. The runner can then setup dependencies.
- 'golem' if options.golem else utils.REPO_ROOT,
+ 'com.android.tools.r8.benchmarks.BenchmarkMainEntryRunner',
+ options.benchmark,
+ options.target,
+ # When running locally the working directory is moved and we pass the
+ # repository root as an argument. The runner can then setup dependencies.
+ 'golem' if options.golem else utils.REPO_ROOT,
])
- return subprocess.check_call(cmd)
+ return subprocess.check_call(cmd)
+
if __name__ == '__main__':
- with utils.TempDir() as temp:
- sys.exit(main(sys.argv[1:], temp))
+ with utils.TempDir() as temp:
+ sys.exit(main(sys.argv[1:], temp))
diff --git a/tools/run_kotlin_benchmarks.py b/tools/run_kotlin_benchmarks.py
index 86adbe2..f7befb3 100755
--- a/tools/run_kotlin_benchmarks.py
+++ b/tools/run_kotlin_benchmarks.py
@@ -12,7 +12,6 @@
import toolhelper
import utils
-
BENCHMARK_ROOT = os.path.join(utils.REPO_ROOT, 'third_party', 'benchmarks',
'kotlin-benches')
@@ -31,75 +30,87 @@
-allowaccessmodification
"""
-DEVICE_TEMP='/data/local/temp/bench'
+DEVICE_TEMP = '/data/local/temp/bench'
def parse_options():
- result = optparse.OptionParser()
- result.add_option('--api',
- help='Android api level',
- default='26',
- choices=['21', '22', '23', '24', '25', '26'])
- result.add_option('--benchmark',
- help='The benchmark to run',
- default='rgx',
- choices=['rgx', 'deltablue', 'sta', 'empty'])
- result.add_option('--use-device',
- help='Run the benchmark on an attaced device',
- default=False, action='store_true')
- return result.parse_args()
+ result = optparse.OptionParser()
+ result.add_option('--api',
+ help='Android api level',
+ default='26',
+ choices=['21', '22', '23', '24', '25', '26'])
+ result.add_option('--benchmark',
+ help='The benchmark to run',
+ default='rgx',
+ choices=['rgx', 'deltablue', 'sta', 'empty'])
+ result.add_option('--use-device',
+ help='Run the benchmark on an attaced device',
+ default=False,
+ action='store_true')
+ return result.parse_args()
def get_jar_for_benchmark(benchmark):
- return os.path.join(BENCHMARK_ROOT,
- BENCHMARK_PATTERN.format(benchmark=benchmark))
+ return os.path.join(BENCHMARK_ROOT,
+ BENCHMARK_PATTERN.format(benchmark=benchmark))
+
def run_art(dex):
- command = ['bash', ART, '-cp', dex, BENCHMARK_MAIN_CLASS]
- utils.PrintCmd(command)
- benchmark_output = subprocess.check_output(command)
- return get_result(benchmark_output)
+ command = ['bash', ART, '-cp', dex, BENCHMARK_MAIN_CLASS]
+ utils.PrintCmd(command)
+ benchmark_output = subprocess.check_output(command)
+ return get_result(benchmark_output)
+
def adb(args):
- command = ['adb'] + args
- utils.PrintCmd(command)
- return subprocess.check_output(['adb'] + args)
+ command = ['adb'] + args
+ utils.PrintCmd(command)
+ return subprocess.check_output(['adb'] + args)
+
def get_result(output):
- # There is a lot of debug output, with the actual results being in the line with:
- # RESULTS,KtBench,KtBench,15719
- # structure.
- for result in [s for s in output.splitlines() if s.startswith('RESULTS')]:
- return s.split('RESULTS,KtBench,KtBench,')[1]
+ # There is a lot of debug output, with the actual results being in the line with:
+ # RESULTS,KtBench,KtBench,15719
+ # structure.
+ for result in [s for s in output.splitlines() if s.startswith('RESULTS')]:
+ return s.split('RESULTS,KtBench,KtBench,')[1]
+
def run_art_device(dex):
- adb(['wait-for-device', 'root'])
- device_dst = os.path.join(DEVICE_TEMP, os.path.basename(dex))
- adb(['push', dex, device_dst])
- benchmark_output = adb(['shell', 'dalvikvm', '-cp', device_dst, BENCHMARK_MAIN_CLASS])
- return get_result(benchmark_output)
+ adb(['wait-for-device', 'root'])
+ device_dst = os.path.join(DEVICE_TEMP, os.path.basename(dex))
+ adb(['push', dex, device_dst])
+ benchmark_output = adb(
+ ['shell', 'dalvikvm', '-cp', device_dst, BENCHMARK_MAIN_CLASS])
+ return get_result(benchmark_output)
+
def Main():
- (options, args) = parse_options()
- with utils.TempDir() as temp:
- dex_path = os.path.join(temp, "classes.jar")
- proguard_conf = os.path.join(temp, 'proguard.conf')
- with open(proguard_conf, 'w') as f:
- f.write(PROGUARD_CONF)
- benchmark_jar = get_jar_for_benchmark(options.benchmark)
- r8_args = [
- '--lib', utils.get_android_jar(26), # Only works with api 26
- '--output', dex_path,
- '--pg-conf', proguard_conf,
- '--min-api', str(options.api),
- benchmark_jar
- ]
- toolhelper.run('r8', r8_args, True)
- if options.use_device:
- result = run_art_device(dex_path)
- else:
- result = run_art(dex_path)
- print('Kotlin_{}(RunTimeRaw): {} ms'.format(options.benchmark, result))
+ (options, args) = parse_options()
+ with utils.TempDir() as temp:
+ dex_path = os.path.join(temp, "classes.jar")
+ proguard_conf = os.path.join(temp, 'proguard.conf')
+ with open(proguard_conf, 'w') as f:
+ f.write(PROGUARD_CONF)
+ benchmark_jar = get_jar_for_benchmark(options.benchmark)
+ r8_args = [
+ '--lib',
+ utils.get_android_jar(26), # Only works with api 26
+ '--output',
+ dex_path,
+ '--pg-conf',
+ proguard_conf,
+ '--min-api',
+ str(options.api),
+ benchmark_jar
+ ]
+ toolhelper.run('r8', r8_args, True)
+ if options.use_device:
+ result = run_art_device(dex_path)
+ else:
+ result = run_art(dex_path)
+ print('Kotlin_{}(RunTimeRaw): {} ms'.format(options.benchmark, result))
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/run_on_app.py b/tools/run_on_app.py
index 0196e3f..8684f89 100755
--- a/tools/run_on_app.py
+++ b/tools/run_on_app.py
@@ -40,164 +40,167 @@
FIND_MIN_XMX_FILE = 'find_min_xmx_results'
FIND_MIN_XMX_DIR = 'find_min_xmx'
+
def ParseOptions(argv):
- result = argparse.ArgumentParser()
- result.add_argument('--compiler',
- help='The compiler to use',
- choices=COMPILERS)
- result.add_argument('--compiler-build',
- help='Compiler build to use',
- choices=COMPILER_BUILDS,
- default='lib')
- result.add_argument('--no-fail-fast',
- help='Whether run_on_app.py should report all failures '
- 'and not just the first one',
- default=False,
- action='store_true')
- result.add_argument('--hash',
- help='The version of D8/R8 to use')
- result.add_argument('--app',
- help='What app to run on',
- choices=APPS)
- result.add_argument('--run-all',
- help='Compile all possible combinations',
- default=False,
- action='store_true')
- result.add_argument('--expect-oom',
- help='Expect that compilation will fail with an OOM',
- default=False,
- action='store_true')
- result.add_argument('--type',
- help='Default for R8: deploy, for D8: proguarded',
- choices=TYPES)
- result.add_argument('--out',
- help='Where to place the output',
- default=utils.BUILD)
- result.add_argument('--no-build',
- help='Run without building first',
- default=False,
- action='store_true')
- result.add_argument('--max-memory',
- help='The maximum memory in MB to run with',
- type=int)
- result.add_argument('--find-min-xmx',
- help='Find the minimum amount of memory we can run in',
- default=False,
- action='store_true')
- result.add_argument('--find-min-xmx-min-memory',
- help='Setting the minimum memory baseline to run in',
- type=int)
- result.add_argument('--find-min-xmx-max-memory',
- help='Setting the maximum memory baseline to run in',
- type=int)
- result.add_argument('--find-min-xmx-range-size',
- help='Setting the size of the acceptable memory range',
- type=int,
- default=32)
- result.add_argument('--find-min-xmx-archive',
- help='Archive find-min-xmx results on GCS',
- default=False,
- action='store_true')
- result.add_argument('--no-extra-pgconf', '--no_extra_pgconf',
- help='Build without the following extra rules: ' +
- '-printconfiguration, -printmapping, -printseeds, ' +
- '-printusage',
- default=False,
- action='store_true')
- result.add_argument('--timeout',
- type=int,
- default=0,
- help='Set timeout instead of waiting for OOM.')
- result.add_argument('--ignore-java-version',
- help='Do not check java version',
- default=False,
- action='store_true')
- result.add_argument('--no-libraries',
- help='Do not pass in libraries, even if they exist in conf',
- default=False,
- action='store_true')
- result.add_argument('--disable-assertions', '--disable_assertions', '-da',
- help='Disable Java assertions when running the compiler '
- '(default enabled)',
- default=False,
- action='store_true')
- result.add_argument('--debug-agent',
- help='Run with debug agent.',
- default=False,
- action='store_true')
- result.add_argument('--version',
- help='The version of the app to run')
- result.add_argument('-k',
- help='Override the default ProGuard keep rules')
- result.add_argument('--compiler-flags',
- help='Additional option(s) for the compiler. ' +
- 'If passing several options use a quoted string.')
- result.add_argument('--r8-flags',
- help='Additional option(s) for the compiler. ' +
- 'Same as --compiler-flags, keeping it for backward'
- ' compatibility. ' +
- 'If passing several options use a quoted string.')
- result.add_argument('--track-memory-to-file',
- help='Track how much memory the jvm is using while ' +
- ' compiling. Output to the specified file.')
- result.add_argument('--profile',
- help='Profile R8 run.',
- default=False,
- action='store_true')
- result.add_argument('--dump-args-file',
- help='Dump a file with the arguments for the specified ' +
- 'configuration. For use as a @<file> argument to perform ' +
- 'the run.')
- result.add_argument('--print-runtimeraw',
- metavar='BENCHMARKNAME',
- help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
+ result = argparse.ArgumentParser()
+ result.add_argument('--compiler',
+ help='The compiler to use',
+ choices=COMPILERS)
+ result.add_argument('--compiler-build',
+ help='Compiler build to use',
+ choices=COMPILER_BUILDS,
+ default='lib')
+ result.add_argument('--no-fail-fast',
+ help='Whether run_on_app.py should report all failures '
+ 'and not just the first one',
+ default=False,
+ action='store_true')
+ result.add_argument('--hash', help='The version of D8/R8 to use')
+ result.add_argument('--app', help='What app to run on', choices=APPS)
+ result.add_argument('--run-all',
+ help='Compile all possible combinations',
+ default=False,
+ action='store_true')
+ result.add_argument('--expect-oom',
+ help='Expect that compilation will fail with an OOM',
+ default=False,
+ action='store_true')
+ result.add_argument('--type',
+ help='Default for R8: deploy, for D8: proguarded',
+ choices=TYPES)
+ result.add_argument('--out',
+ help='Where to place the output',
+ default=utils.BUILD)
+ result.add_argument('--no-build',
+ help='Run without building first',
+ default=False,
+ action='store_true')
+ result.add_argument('--max-memory',
+ help='The maximum memory in MB to run with',
+ type=int)
+ result.add_argument('--find-min-xmx',
+ help='Find the minimum amount of memory we can run in',
+ default=False,
+ action='store_true')
+ result.add_argument('--find-min-xmx-min-memory',
+ help='Setting the minimum memory baseline to run in',
+ type=int)
+ result.add_argument('--find-min-xmx-max-memory',
+ help='Setting the maximum memory baseline to run in',
+ type=int)
+ result.add_argument('--find-min-xmx-range-size',
+ help='Setting the size of the acceptable memory range',
+ type=int,
+ default=32)
+ result.add_argument('--find-min-xmx-archive',
+ help='Archive find-min-xmx results on GCS',
+ default=False,
+ action='store_true')
+ result.add_argument('--no-extra-pgconf',
+ '--no_extra_pgconf',
+ help='Build without the following extra rules: ' +
+ '-printconfiguration, -printmapping, -printseeds, ' +
+ '-printusage',
+ default=False,
+ action='store_true')
+ result.add_argument('--timeout',
+ type=int,
+ default=0,
+ help='Set timeout instead of waiting for OOM.')
+ result.add_argument('--ignore-java-version',
+ help='Do not check java version',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--no-libraries',
+ help='Do not pass in libraries, even if they exist in conf',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--disable-assertions',
+ '--disable_assertions',
+ '-da',
+ help='Disable Java assertions when running the compiler '
+ '(default enabled)',
+ default=False,
+ action='store_true')
+ result.add_argument('--debug-agent',
+ help='Run with debug agent.',
+ default=False,
+ action='store_true')
+ result.add_argument('--version', help='The version of the app to run')
+ result.add_argument('-k', help='Override the default ProGuard keep rules')
+ result.add_argument('--compiler-flags',
+ help='Additional option(s) for the compiler. ' +
+ 'If passing several options use a quoted string.')
+ result.add_argument('--r8-flags',
+ help='Additional option(s) for the compiler. ' +
+ 'Same as --compiler-flags, keeping it for backward'
+ ' compatibility. ' +
+ 'If passing several options use a quoted string.')
+ result.add_argument('--track-memory-to-file',
+ help='Track how much memory the jvm is using while ' +
+ ' compiling. Output to the specified file.')
+ result.add_argument('--profile',
+ help='Profile R8 run.',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--dump-args-file',
+ help='Dump a file with the arguments for the specified ' +
+ 'configuration. For use as a @<file> argument to perform ' + 'the run.')
+ result.add_argument('--print-runtimeraw',
+ metavar='BENCHMARKNAME',
+ help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
' <elapsed> ms\' at the end where <elapsed> is' +
' the elapsed time in milliseconds.')
- result.add_argument('--print-memoryuse',
- metavar='BENCHMARKNAME',
- help='Print the line \'<BENCHMARKNAME>(MemoryUse):' +
+ result.add_argument('--print-memoryuse',
+ metavar='BENCHMARKNAME',
+ help='Print the line \'<BENCHMARKNAME>(MemoryUse):' +
' <mem>\' at the end where <mem> is the peak' +
' peak resident set size (VmHWM) in bytes.')
- result.add_argument('--print-dexsegments',
- metavar='BENCHMARKNAME',
- help='Print the sizes of individual dex segments as ' +
+ result.add_argument('--print-dexsegments',
+ metavar='BENCHMARKNAME',
+ help='Print the sizes of individual dex segments as ' +
'\'<BENCHMARKNAME>-<segment>(CodeSize): <bytes>\'')
- result.add_argument('--track-time-in-memory',
- help='Plot the times taken from memory starting point to '
- 'end-point with defined memory increment',
- default=False,
- action='store_true')
- result.add_argument('--track-time-in-memory-max',
- help='Setting the maximum memory baseline to run in',
- type=int)
- result.add_argument('--track-time-in-memory-min',
- help='Setting the minimum memory baseline to run in',
- type=int)
- result.add_argument('--track-time-in-memory-increment',
- help='Setting the increment',
- type=int,
- default=32)
- result.add_argument('--print-times',
- help='Include timing',
- default=False,
- action='store_true')
- result.add_argument('--cpu-list',
- help='Run under \'taskset\' with these CPUs. See '
- 'the \'taskset\' -c option for the format')
- result.add_argument('--quiet',
- help='Disable compiler logging',
- default=False,
- action='store_true')
- result.add_argument('--workers',
- help='Number of workers to use',
- default=1,
- type=int)
- (options, args) = result.parse_known_args(argv)
- assert not options.hash or options.no_build, (
- 'Argument --no-build is required when using --hash')
- assert not options.hash or options.compiler_build == 'full', (
- 'Compiler build lib not yet supported with --hash')
- return (options, args)
+ result.add_argument(
+ '--track-time-in-memory',
+ help='Plot the times taken from memory starting point to '
+ 'end-point with defined memory increment',
+ default=False,
+ action='store_true')
+ result.add_argument('--track-time-in-memory-max',
+ help='Setting the maximum memory baseline to run in',
+ type=int)
+ result.add_argument('--track-time-in-memory-min',
+ help='Setting the minimum memory baseline to run in',
+ type=int)
+ result.add_argument('--track-time-in-memory-increment',
+ help='Setting the increment',
+ type=int,
+ default=32)
+ result.add_argument('--print-times',
+ help='Include timing',
+ default=False,
+ action='store_true')
+ result.add_argument('--cpu-list',
+ help='Run under \'taskset\' with these CPUs. See '
+ 'the \'taskset\' -c option for the format')
+ result.add_argument('--quiet',
+ help='Disable compiler logging',
+ default=False,
+ action='store_true')
+ result.add_argument('--workers',
+ help='Number of workers to use',
+ default=1,
+ type=int)
+ (options, args) = result.parse_known_args(argv)
+ assert not options.hash or options.no_build, (
+ 'Argument --no-build is required when using --hash')
+ assert not options.hash or options.compiler_build == 'full', (
+ 'Compiler build lib not yet supported with --hash')
+ return (options, args)
+
# Most apps have -printmapping, -printseeds, -printusage and
# -printconfiguration in the Proguard configuration. However we don't
@@ -205,570 +208,597 @@
# Instead generate an auxiliary Proguard configuration placing these
# output files together with the dex output.
def GenerateAdditionalProguardConfiguration(temp, outdir):
- name = "output.config"
- with open(os.path.join(temp, name), 'w') as f:
- f.write('-printmapping ' + os.path.join(outdir, 'proguard.map') + "\n")
- f.write('-printseeds ' + os.path.join(outdir, 'proguard.seeds') + "\n")
- f.write('-printusage ' + os.path.join(outdir, 'proguard.usage') + "\n")
- f.write('-printconfiguration ' + os.path.join(outdir, 'proguard.config') + "\n")
- return os.path.abspath(f.name)
+ name = "output.config"
+ with open(os.path.join(temp, name), 'w') as f:
+ f.write('-printmapping ' + os.path.join(outdir, 'proguard.map') + "\n")
+ f.write('-printseeds ' + os.path.join(outdir, 'proguard.seeds') + "\n")
+ f.write('-printusage ' + os.path.join(outdir, 'proguard.usage') + "\n")
+ f.write('-printconfiguration ' +
+ os.path.join(outdir, 'proguard.config') + "\n")
+ return os.path.abspath(f.name)
+
# Please add bug number for disabled permutations and please explicitly
# do Bug: #BUG in the commit message of disabling to ensure re-enabling
DISABLED_PERMUTATIONS = [
- # (app, version, type), e.g., ('gmail', '180826.15', 'deploy')
+ # (app, version, type), e.g., ('gmail', '180826.15', 'deploy')
]
+
def get_permutations():
- data_providers = {
- 'nest': nest_data,
- 'youtube': youtube_data,
- 'chrome': chrome_data,
- 'gmail': gmail_data,
- }
- # Check to ensure that we add all variants here.
- assert len(APPS) == len(data_providers)
- for app, data in data_providers.items():
- for version in data.VERSIONS:
- for type in data.VERSIONS[version]:
- if (app, version, type) not in DISABLED_PERMUTATIONS:
- # Only run with R8 lib to reduce cycle times.
- for use_r8lib in [True]:
- yield app, version, type, use_r8lib
+ data_providers = {
+ 'nest': nest_data,
+ 'youtube': youtube_data,
+ 'chrome': chrome_data,
+ 'gmail': gmail_data,
+ }
+ # Check to ensure that we add all variants here.
+ assert len(APPS) == len(data_providers)
+ for app, data in data_providers.items():
+ for version in data.VERSIONS:
+ for type in data.VERSIONS[version]:
+ if (app, version, type) not in DISABLED_PERMUTATIONS:
+ # Only run with R8 lib to reduce cycle times.
+ for use_r8lib in [True]:
+ yield app, version, type, use_r8lib
+
def run_all(options, args):
- # Build first so that each job won't.
- if should_build(options):
- gradle.RunGradle([utils.GRADLE_TASK_R8LIB])
- options.no_build = True
- assert not should_build(options)
+ # Build first so that each job won't.
+ if should_build(options):
+ gradle.RunGradle([utils.GRADLE_TASK_R8LIB])
+ options.no_build = True
+ assert not should_build(options)
- # Args will be destroyed
- assert len(args) == 0
- jobs = []
- for name, version, type, use_r8lib in get_permutations():
- compiler = 'r8' if type == 'deploy' else 'd8'
- compiler_build = 'lib' if use_r8lib else 'full'
- fixed_options = copy.copy(options)
- fixed_options.app = name
- fixed_options.version = version
- fixed_options.compiler = compiler
- fixed_options.compiler_build = compiler_build
- fixed_options.type = type
- jobs.append(
- create_job(
- compiler, compiler_build, name, fixed_options, type, version))
- exit_code = thread_utils.run_in_parallel(
- jobs,
- number_of_workers=options.workers,
- stop_on_first_failure=not options.no_fail_fast)
- exit(exit_code)
+ # Args will be destroyed
+ assert len(args) == 0
+ jobs = []
+ for name, version, type, use_r8lib in get_permutations():
+ compiler = 'r8' if type == 'deploy' else 'd8'
+ compiler_build = 'lib' if use_r8lib else 'full'
+ fixed_options = copy.copy(options)
+ fixed_options.app = name
+ fixed_options.version = version
+ fixed_options.compiler = compiler
+ fixed_options.compiler_build = compiler_build
+ fixed_options.type = type
+ jobs.append(
+ create_job(compiler, compiler_build, name, fixed_options, type,
+ version))
+ exit_code = thread_utils.run_in_parallel(
+ jobs,
+ number_of_workers=options.workers,
+ stop_on_first_failure=not options.no_fail_fast)
+ exit(exit_code)
+
def create_job(compiler, compiler_build, name, options, type, version):
- return lambda worker_id: run_job(
- compiler, compiler_build, name, options, type, version, worker_id)
+ return lambda worker_id: run_job(compiler, compiler_build, name, options,
+ type, version, worker_id)
-def run_job(
- compiler, compiler_build, name, options, type, version, worker_id):
- print_thread(
- 'Executing %s/%s with %s %s %s'
- % (compiler, compiler_build, name, version, type),
- worker_id)
- if worker_id is not None:
- options.out = os.path.join(options.out, str(worker_id))
- os.makedirs(options.out, exist_ok=True)
- exit_code = run_with_options(options, [], worker_id=worker_id)
- if exit_code:
+
+def run_job(compiler, compiler_build, name, options, type, version, worker_id):
print_thread(
- 'Failed %s %s %s with %s/%s'
- % (name, version, type, compiler, compiler_build),
- worker_id)
- return exit_code
+ 'Executing %s/%s with %s %s %s' %
+ (compiler, compiler_build, name, version, type), worker_id)
+ if worker_id is not None:
+ options.out = os.path.join(options.out, str(worker_id))
+ os.makedirs(options.out, exist_ok=True)
+ exit_code = run_with_options(options, [], worker_id=worker_id)
+ if exit_code:
+ print_thread(
+ 'Failed %s %s %s with %s/%s' %
+ (name, version, type, compiler, compiler_build), worker_id)
+ return exit_code
+
def find_min_xmx(options, args):
- # Args will be destroyed
- assert len(args) == 0
- # If we can run in 128 MB then we are good (which we can for small examples
- # or D8 on medium sized examples)
- if options.find_min_xmx_min_memory:
- not_working = options.find_min_xmx_min_memory
- elif options.compiler == 'd8':
- not_working = 128
- else:
- not_working = 1024
- if options.find_min_xmx_max_memory:
- working = options.find_min_xmx_max_memory
- else:
- working = 1024 * 8
- exit_code = 0
- range = int(options.find_min_xmx_range_size)
- while working - not_working > range:
- next_candidate = int(working - ((working - not_working)/2))
- print('working: %s, non_working: %s, next_candidate: %s' %
- (working, not_working, next_candidate))
- extra_args = ['-Xmx%sM' % next_candidate]
- t0 = time.time()
- exit_code = run_with_options(options, [], extra_args)
- t1 = time.time()
- print('Running took: %s ms' % (1000.0 * (t1 - t0)))
- if exit_code != 0:
- if exit_code not in [OOM_EXIT_CODE, TIMEOUT_KILL_CODE]:
- print('Non OOM/Timeout error executing, exiting')
- return 2
- if exit_code == 0:
- working = next_candidate
- elif exit_code == TIMEOUT_KILL_CODE:
- print('Timeout. Continue to the next candidate.')
- not_working = next_candidate
+ # Args will be destroyed
+ assert len(args) == 0
+ # If we can run in 128 MB then we are good (which we can for small examples
+ # or D8 on medium sized examples)
+ if options.find_min_xmx_min_memory:
+ not_working = options.find_min_xmx_min_memory
+ elif options.compiler == 'd8':
+ not_working = 128
else:
- assert exit_code == OOM_EXIT_CODE
- not_working = next_candidate
+ not_working = 1024
+ if options.find_min_xmx_max_memory:
+ working = options.find_min_xmx_max_memory
+ else:
+ working = 1024 * 8
+ exit_code = 0
+ range = int(options.find_min_xmx_range_size)
+ while working - not_working > range:
+ next_candidate = int(working - ((working - not_working) / 2))
+ print('working: %s, non_working: %s, next_candidate: %s' %
+ (working, not_working, next_candidate))
+ extra_args = ['-Xmx%sM' % next_candidate]
+ t0 = time.time()
+ exit_code = run_with_options(options, [], extra_args)
+ t1 = time.time()
+ print('Running took: %s ms' % (1000.0 * (t1 - t0)))
+ if exit_code != 0:
+ if exit_code not in [OOM_EXIT_CODE, TIMEOUT_KILL_CODE]:
+ print('Non OOM/Timeout error executing, exiting')
+ return 2
+ if exit_code == 0:
+ working = next_candidate
+ elif exit_code == TIMEOUT_KILL_CODE:
+ print('Timeout. Continue to the next candidate.')
+ not_working = next_candidate
+ else:
+ assert exit_code == OOM_EXIT_CODE
+ not_working = next_candidate
- assert working - not_working <= range
- found_range = 'Found range: %s - %s' % (not_working, working)
- print(found_range)
+ assert working - not_working <= range
+ found_range = 'Found range: %s - %s' % (not_working, working)
+ print(found_range)
- if options.find_min_xmx_archive:
- sha = utils.get_HEAD_sha1()
- (version, _) = get_version_and_data(options)
- destination = os.path.join(
- utils.R8_TEST_RESULTS_BUCKET,
- FIND_MIN_XMX_DIR,
- sha,
- options.compiler,
- options.compiler_build,
- options.app,
- version,
- get_type(options))
- gs_destination = 'gs://%s' % destination
- utils.archive_value(FIND_MIN_XMX_FILE, gs_destination, found_range + '\n')
+ if options.find_min_xmx_archive:
+ sha = utils.get_HEAD_sha1()
+ (version, _) = get_version_and_data(options)
+ destination = os.path.join(utils.R8_TEST_RESULTS_BUCKET,
+ FIND_MIN_XMX_DIR, sha, options.compiler,
+ options.compiler_build, options.app, version,
+ get_type(options))
+ gs_destination = 'gs://%s' % destination
+ utils.archive_value(FIND_MIN_XMX_FILE, gs_destination,
+ found_range + '\n')
- return 0
+ return 0
+
def print_min_xmx_ranges_for_hash(hash, compiler, compiler_build):
- app_directory = os.path.join(
- utils.R8_TEST_RESULTS_BUCKET,
- FIND_MIN_XMX_DIR,
- hash,
- compiler,
- compiler_build)
- gs_base = 'gs://%s' % app_directory
- for app in utils.ls_files_on_cloud_storage(gs_base).strip().split('\n'):
- for version in utils.ls_files_on_cloud_storage(app).strip().split('\n'):
- for type in utils.ls_files_on_cloud_storage(version).strip().split('\n'):
- gs_location = '%s%s' % (type, FIND_MIN_XMX_FILE)
- value = utils.cat_file_on_cloud_storage(gs_location, ignore_errors=True)
- print('%s\n' % value)
+ app_directory = os.path.join(utils.R8_TEST_RESULTS_BUCKET, FIND_MIN_XMX_DIR,
+ hash, compiler, compiler_build)
+ gs_base = 'gs://%s' % app_directory
+ for app in utils.ls_files_on_cloud_storage(gs_base).strip().split('\n'):
+ for version in utils.ls_files_on_cloud_storage(app).strip().split('\n'):
+ for type in utils.ls_files_on_cloud_storage(version).strip().split(
+ '\n'):
+ gs_location = '%s%s' % (type, FIND_MIN_XMX_FILE)
+ value = utils.cat_file_on_cloud_storage(gs_location,
+ ignore_errors=True)
+ print('%s\n' % value)
+
def track_time_in_memory(options, args):
- # Args will be destroyed
- assert len(args) == 0
- if not options.track_time_in_memory_min:
- raise Exception(
- 'You have to specify --track_time_in_memory_min when running with '
- '--track-time-in-memory')
- if not options.track_time_in_memory_max:
- raise Exception(
- 'You have to specify --track_time_in_memory_max when running with '
- '--track-time-in-memory')
- if not options.track_time_in_memory_increment:
- raise Exception(
- 'You have to specify --track_time_in_memory_increment when running '
- 'with --track-time-in-memory')
- current = options.track_time_in_memory_min
- print('Memory (KB)\tTime (ms)')
- with utils.TempDir() as temp:
- stdout = os.path.join(temp, 'stdout')
- stdout_fd = open(stdout, 'w')
- while current <= options.track_time_in_memory_max:
- extra_args = ['-Xmx%sM' % current]
- t0 = time.time()
- exit_code = run_with_options(options, [], extra_args, stdout_fd, quiet=True)
- t1 = time.time()
- total = (1000.0 * (t1 - t0)) if exit_code == 0 else -1
- print('%s\t%s' % (current, total))
- current += options.track_time_in_memory_increment
+ # Args will be destroyed
+ assert len(args) == 0
+ if not options.track_time_in_memory_min:
+ raise Exception(
+ 'You have to specify --track_time_in_memory_min when running with '
+ '--track-time-in-memory')
+ if not options.track_time_in_memory_max:
+ raise Exception(
+ 'You have to specify --track_time_in_memory_max when running with '
+ '--track-time-in-memory')
+ if not options.track_time_in_memory_increment:
+ raise Exception(
+ 'You have to specify --track_time_in_memory_increment when running '
+ 'with --track-time-in-memory')
+ current = options.track_time_in_memory_min
+ print('Memory (KB)\tTime (ms)')
+ with utils.TempDir() as temp:
+ stdout = os.path.join(temp, 'stdout')
+ stdout_fd = open(stdout, 'w')
+ while current <= options.track_time_in_memory_max:
+ extra_args = ['-Xmx%sM' % current]
+ t0 = time.time()
+ exit_code = run_with_options(options, [],
+ extra_args,
+ stdout_fd,
+ quiet=True)
+ t1 = time.time()
+ total = (1000.0 * (t1 - t0)) if exit_code == 0 else -1
+ print('%s\t%s' % (current, total))
+ current += options.track_time_in_memory_increment
- return 0
+ return 0
+
def main(argv):
- (options, args) = ParseOptions(argv)
- if options.expect_oom and not options.max_memory:
- raise Exception(
- 'You should only use --expect-oom if also specifying --max-memory')
- if options.expect_oom and options.timeout:
- raise Exception(
- 'You should not use --timeout when also specifying --expect-oom')
- if options.find_min_xmx and options.track_time_in_memory:
- raise Exception(
- 'You cannot both find the min xmx and track time at the same time')
- if options.run_all:
- return run_all(options, args)
- if options.find_min_xmx:
- return find_min_xmx(options, args)
- if options.track_time_in_memory:
- return track_time_in_memory(options, args)
- exit_code = run_with_options(options, args, quiet=options.quiet)
- if options.expect_oom:
- exit_code = 0 if exit_code == OOM_EXIT_CODE else 1
- return exit_code
+ (options, args) = ParseOptions(argv)
+ if options.expect_oom and not options.max_memory:
+ raise Exception(
+ 'You should only use --expect-oom if also specifying --max-memory')
+ if options.expect_oom and options.timeout:
+ raise Exception(
+ 'You should not use --timeout when also specifying --expect-oom')
+ if options.find_min_xmx and options.track_time_in_memory:
+ raise Exception(
+ 'You cannot both find the min xmx and track time at the same time')
+ if options.run_all:
+ return run_all(options, args)
+ if options.find_min_xmx:
+ return find_min_xmx(options, args)
+ if options.track_time_in_memory:
+ return track_time_in_memory(options, args)
+ exit_code = run_with_options(options, args, quiet=options.quiet)
+ if options.expect_oom:
+ exit_code = 0 if exit_code == OOM_EXIT_CODE else 1
+ return exit_code
+
def get_version_and_data(options):
- if options.app == 'nest':
- version = options.version or '20180926'
- data = nest_data
- elif options.app == 'youtube':
- version = options.version or youtube_data.LATEST_VERSION
- data = youtube_data
- elif options.app == 'chrome':
- version = options.version or '180917'
- data = chrome_data
- elif options.app == 'gmail':
- version = options.version or '170604.16'
- data = gmail_data
- else:
- raise Exception("You need to specify '--app={}'".format('|'.join(APPS)))
- return version, data
+ if options.app == 'nest':
+ version = options.version or '20180926'
+ data = nest_data
+ elif options.app == 'youtube':
+ version = options.version or youtube_data.LATEST_VERSION
+ data = youtube_data
+ elif options.app == 'chrome':
+ version = options.version or '180917'
+ data = chrome_data
+ elif options.app == 'gmail':
+ version = options.version or '170604.16'
+ data = gmail_data
+ else:
+ raise Exception("You need to specify '--app={}'".format('|'.join(APPS)))
+ return version, data
+
def get_type(options):
- if not options.type:
- return 'deploy' if options.compiler == 'r8' else 'proguarded'
- return options.type
+ if not options.type:
+ return 'deploy' if options.compiler == 'r8' else 'proguarded'
+ return options.type
+
def has_injars_and_libraryjars(pgconfs):
- # Check if there are -injars and -libraryjars in the configuration.
- has_injars = False
- has_libraryjars = False
- for pgconf in pgconfs:
- pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
- with open(pgconf) as pgconf_file:
- for line in pgconf_file:
- trimmed = line.strip()
- if trimmed.startswith('-injars'):
- has_injars = True
- elif trimmed.startswith('-libraryjars'):
- has_libraryjars = True
- if has_injars and has_libraryjars:
- return True
- return False
+ # Check if there are -injars and -libraryjars in the configuration.
+ has_injars = False
+ has_libraryjars = False
+ for pgconf in pgconfs:
+ pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
+ with open(pgconf) as pgconf_file:
+ for line in pgconf_file:
+ trimmed = line.strip()
+ if trimmed.startswith('-injars'):
+ has_injars = True
+ elif trimmed.startswith('-libraryjars'):
+ has_libraryjars = True
+ if has_injars and has_libraryjars:
+ return True
+ return False
+
def check_no_injars_and_no_libraryjars(pgconfs):
- # Ensure that there are no -injars or -libraryjars in the configuration.
- for pgconf in pgconfs:
- pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
- with open(pgconf) as pgconf_file:
- for line in pgconf_file:
- trimmed = line.strip()
- if trimmed.startswith('-injars'):
- raise Exception("Unexpected -injars found in " + pgconf)
- elif trimmed.startswith('-libraryjars'):
- raise Exception("Unexpected -libraryjars found in " + pgconf)
+ # Ensure that there are no -injars or -libraryjars in the configuration.
+ for pgconf in pgconfs:
+ pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
+ with open(pgconf) as pgconf_file:
+ for line in pgconf_file:
+ trimmed = line.strip()
+ if trimmed.startswith('-injars'):
+ raise Exception("Unexpected -injars found in " + pgconf)
+ elif trimmed.startswith('-libraryjars'):
+ raise Exception("Unexpected -libraryjars found in " +
+ pgconf)
+
def should_build(options):
- return not options.no_build
+ return not options.no_build
-def build_desugared_library_dex(
- options,
- quiet,
- temp,
- android_java8_libs,
- desugared_lib_pg_conf,
- inputs,
- outdir):
- if not inputs:
- raise Exception(
- "If 'android_java8_libs' is specified the inputs must be explicit"
- + "(not defined using '-injars' in Proguard configuration files)")
- if outdir.endswith('.zip') or outdir.endswith('.jar'):
- raise Exception(
- "If 'android_java8_libs' is specified the output must be a directory")
- jar = None
- main = None
- if options.hash:
- jar = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
- main = 'com.android.tools.r8.R8'
+def build_desugared_library_dex(options, quiet, temp, android_java8_libs,
+ desugared_lib_pg_conf, inputs, outdir):
+ if not inputs:
+ raise Exception(
+ "If 'android_java8_libs' is specified the inputs must be explicit" +
+ "(not defined using '-injars' in Proguard configuration files)")
+ if outdir.endswith('.zip') or outdir.endswith('.jar'):
+ raise Exception(
+ "If 'android_java8_libs' is specified the output must be a directory"
+ )
- # Determine the l8 tool.
- assert(options.compiler_build in ['full', 'lib'])
- lib_prefix = 'r8lib-' if options.compiler_build == 'lib' else ''
- tool = lib_prefix + 'l8'
+ jar = None
+ main = None
+ if options.hash:
+ jar = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
+ main = 'com.android.tools.r8.R8'
- # Prepare out directory.
- android_java8_libs_output = os.path.join(temp, 'android_java8_libs')
- os.makedirs(android_java8_libs_output)
+ # Determine the l8 tool.
+ assert (options.compiler_build in ['full', 'lib'])
+ lib_prefix = 'r8lib-' if options.compiler_build == 'lib' else ''
+ tool = lib_prefix + 'l8'
- # Prepare arguments for L8.
- args = [
- '--desugared-lib', android_java8_libs['config'],
- '--lib', android_java8_libs['library'],
- '--output', android_java8_libs_output,
- '--pg-conf', desugared_lib_pg_conf,
- '--release',
- ]
- if 'pgconf' in android_java8_libs:
- for pgconf in android_java8_libs['pgconf']:
- args.extend(['--pg-conf', pgconf])
- args.extend(android_java8_libs['program'])
+ # Prepare out directory.
+ android_java8_libs_output = os.path.join(temp, 'android_java8_libs')
+ os.makedirs(android_java8_libs_output)
- # Run L8.
- exit_code = toolhelper.run(
- tool, args,
- build=should_build(options),
- debug=not options.disable_assertions,
- quiet=quiet,
- jar=jar,
- main=main)
+ # Prepare arguments for L8.
+ args = [
+ '--desugared-lib',
+ android_java8_libs['config'],
+ '--lib',
+ android_java8_libs['library'],
+ '--output',
+ android_java8_libs_output,
+ '--pg-conf',
+ desugared_lib_pg_conf,
+ '--release',
+ ]
+ if 'pgconf' in android_java8_libs:
+ for pgconf in android_java8_libs['pgconf']:
+ args.extend(['--pg-conf', pgconf])
+ args.extend(android_java8_libs['program'])
- # Copy the desugared library DEX to the output.
- dex_file_name = (
- 'classes' + str(len(glob(os.path.join(outdir, '*.dex'))) + 1) + '.dex')
- shutil.copyfile(
- os.path.join(android_java8_libs_output, 'classes.dex'),
- os.path.join(outdir, dex_file_name))
+ # Run L8.
+ exit_code = toolhelper.run(tool,
+ args,
+ build=should_build(options),
+ debug=not options.disable_assertions,
+ quiet=quiet,
+ jar=jar,
+ main=main)
-def run_with_options(
- options, args, extra_args=None, stdout=None, quiet=False, worker_id=None):
- if extra_args is None:
- extra_args = []
- app_provided_pg_conf = False;
- # todo(121018500): remove when memory is under control
- if not any('-Xmx' in arg for arg in extra_args):
- if options.max_memory:
- extra_args.append('-Xmx%sM' % options.max_memory)
+ # Copy the desugared library DEX to the output.
+ dex_file_name = ('classes' +
+ str(len(glob(os.path.join(outdir, '*.dex'))) + 1) + '.dex')
+ shutil.copyfile(os.path.join(android_java8_libs_output, 'classes.dex'),
+ os.path.join(outdir, dex_file_name))
+
+
+def run_with_options(options,
+ args,
+ extra_args=None,
+ stdout=None,
+ quiet=False,
+ worker_id=None):
+ if extra_args is None:
+ extra_args = []
+ app_provided_pg_conf = False
+ # todo(121018500): remove when memory is under control
+ if not any('-Xmx' in arg for arg in extra_args):
+ if options.max_memory:
+ extra_args.append('-Xmx%sM' % options.max_memory)
+ else:
+ extra_args.append('-Xmx8G')
+ if not options.ignore_java_version:
+ utils.check_java_version()
+
+ if options.print_times:
+ extra_args.append('-Dcom.android.tools.r8.printtimes=1')
+
+ if not options.disable_assertions:
+ extra_args.append('-Dcom.android.tools.r8.enableTestAssertions=1')
+
+ outdir = options.out
+ (version_id, data) = get_version_and_data(options)
+
+ if options.compiler not in COMPILERS:
+ raise Exception("You need to specify '--compiler={}'".format(
+ '|'.join(COMPILERS)))
+
+ if options.compiler_build not in COMPILER_BUILDS:
+ raise Exception("You need to specify '--compiler-build={}'".format(
+ '|'.join(COMPILER_BUILDS)))
+
+ if not version_id in data.VERSIONS.keys():
+ print('No version {} for application {}'.format(version_id,
+ options.app))
+ print('Valid versions are {}'.format(data.VERSIONS.keys()))
+ return 1
+
+ version = data.VERSIONS[version_id]
+
+ type = get_type(options)
+
+ if type not in version:
+ print('No type {} for version {}'.format(type, version))
+ print('Valid types are {}'.format(version.keys()))
+ return 1
+ values = version[type]
+
+ args.extend(['--output', outdir])
+ if 'min-api' in values:
+ args.extend(['--min-api', values['min-api']])
+
+ if 'main-dex-list' in values:
+ args.extend(['--main-dex-list', values['main-dex-list']])
+
+ inputs = values['inputs']
+ libraries = values['libraries'] if 'libraries' in values else []
+
+ if options.compiler == 'r8':
+ if 'pgconf' in values and not options.k:
+ sanitized_lib_path = os.path.join(os.path.abspath(outdir),
+ 'sanitized_lib.jar')
+ if has_injars_and_libraryjars(values['pgconf']):
+ sanitized_pgconf_path = os.path.join(os.path.abspath(outdir),
+ 'sanitized.config')
+ SanitizeLibrariesInPgconf(sanitized_lib_path,
+ sanitized_pgconf_path,
+ values['pgconf'])
+ libraries = [sanitized_lib_path]
+ args.extend(['--pg-conf', sanitized_pgconf_path])
+ inputs = []
+ else:
+ # -injars without -libraryjars or vice versa is not supported.
+ check_no_injars_and_no_libraryjars(values['pgconf'])
+ for pgconf in values['pgconf']:
+ args.extend(['--pg-conf', pgconf])
+ if 'sanitize_libraries' in values and values[
+ 'sanitize_libraries']:
+ SanitizeLibraries(sanitized_lib_path, values['libraries'],
+ values['inputs'])
+ libraries = [sanitized_lib_path]
+ app_provided_pg_conf = True
+ if 'pgconf_extra' in values:
+ extra_conf = os.path.join(os.path.abspath(outdir),
+ 'pgconf_extra')
+ with open(extra_conf, 'w') as extra_f:
+ extra_f.write(values['pgconf_extra'])
+ args.extend(['--pg-conf', extra_conf])
+ if options.k:
+ args.extend(['--pg-conf', options.k])
+ if 'maindexrules' in values:
+ for rules in values['maindexrules']:
+ args.extend(['--main-dex-rules', rules])
+ if 'allow-type-errors' in values:
+ extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
+ extra_args.append(
+ '-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1')
+ if 'system-properties' in values:
+ for system_property in values['system-properties']:
+ extra_args.append(system_property)
+
+ if options.debug_agent:
+ if not options.compiler_build == 'full':
+ print(
+ 'WARNING: Running debugging agent on r8lib is questionable...')
+ extra_args.append(
+ '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+ )
+
+ if not options.no_libraries:
+ for lib in libraries:
+ args.extend(['--lib', lib])
+
+ if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
+ and not os.path.exists(outdir):
+ os.makedirs(outdir)
+
+ if options.hash:
+ # Download r8-<hash>.jar from
+ # https://storage.googleapis.com/r8-releases/raw/<hash>/.
+ download_path = archive.GetUploadDestination(options.hash, 'r8.jar',
+ True)
+ assert utils.file_exists_on_cloud_storage(download_path), (
+ 'Could not find r8.jar file from provided hash: %s' % options.hash)
+ destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
+ utils.download_file_from_cloud_storage(download_path,
+ destination,
+ quiet=quiet)
+
+ # Additional flags for the compiler from the configuration file.
+ if 'flags' in values:
+ args.extend(values['flags'].split(' '))
+ if options.compiler == 'r8':
+ if 'r8-flags' in values:
+ args.extend(values['r8-flags'].split(' '))
+
+ # Additional flags for the compiler from the command line.
+ if options.compiler_flags:
+ args.extend(options.compiler_flags.split(' '))
+ if options.r8_flags:
+ args.extend(options.r8_flags.split(' '))
+
+ # Feature jars.
+ features = values['features'] if 'features' in values else []
+ for i, feature in enumerate(features, start=1):
+ feature_out = os.path.join(outdir, 'feature-%d.zip' % i)
+ for feature_jar in feature['inputs']:
+ args.extend(['--feature', feature_jar, feature_out])
+
+ args.extend(inputs)
+
+ t0 = None
+ if options.dump_args_file:
+ with open(options.dump_args_file, 'w') as args_file:
+ args_file.writelines([arg + os.linesep for arg in args])
else:
- extra_args.append('-Xmx8G')
- if not options.ignore_java_version:
- utils.check_java_version()
+ with utils.TempDir() as temp:
+ if options.print_memoryuse and not options.track_memory_to_file:
+ options.track_memory_to_file = os.path.join(
+ temp, utils.MEMORY_USE_TMP_FILE)
+ if options.compiler == 'r8' and app_provided_pg_conf:
+ # Ensure that output of -printmapping and -printseeds go to the output
+ # location and not where the app Proguard configuration places them.
+ if outdir.endswith('.zip') or outdir.endswith('.jar'):
+ pg_outdir = os.path.dirname(outdir)
+ else:
+ pg_outdir = outdir
+ if not options.no_extra_pgconf:
+ additional_pg_conf = GenerateAdditionalProguardConfiguration(
+ temp, os.path.abspath(pg_outdir))
+ args.extend(['--pg-conf', additional_pg_conf])
- if options.print_times:
- extra_args.append('-Dcom.android.tools.r8.printtimes=1')
+ android_java8_libs = values.get('android_java8_libs')
+ if android_java8_libs:
+ desugared_lib_pg_conf = os.path.join(
+ temp, 'desugared-lib-pg-conf.txt')
+ args.extend(['--desugared-lib', android_java8_libs['config']])
+ args.extend(
+ ['--desugared-lib-pg-conf-output', desugared_lib_pg_conf])
- if not options.disable_assertions:
- extra_args.append('-Dcom.android.tools.r8.enableTestAssertions=1')
+ stderr_path = os.path.join(temp, 'stderr')
+ with open(stderr_path, 'w') as stderr:
+ jar = None
+ main = None
+ if options.compiler_build == 'full':
+ tool = options.compiler
+ else:
+ assert (options.compiler_build == 'lib')
+ tool = 'r8lib-' + options.compiler
+ if options.hash:
+ jar = os.path.join(utils.LIBS,
+ 'r8-' + options.hash + '.jar')
+ main = 'com.android.tools.r8.' + options.compiler.upper()
+ if should_build(options):
+ gradle.RunGradle([
+ utils.GRADLE_TASK_R8LIB
+ if tool.startswith('r8lib') else UTILS.GRADLE_TASK_R8
+ ])
+ t0 = time.time()
+ exit_code = toolhelper.run(
+ tool,
+ args,
+ build=False,
+ debug=not options.disable_assertions,
+ profile=options.profile,
+ track_memory_file=options.track_memory_to_file,
+ extra_args=extra_args,
+ stdout=stdout,
+ stderr=stderr,
+ timeout=options.timeout,
+ quiet=quiet,
+ cmd_prefix=['taskset', '-c', options.cpu_list]
+ if options.cpu_list else [],
+ jar=jar,
+ main=main,
+ worker_id=worker_id)
+ if exit_code != 0:
+ with open(stderr_path) as stderr:
+ stderr_text = stderr.read()
+ if not quiet:
+ print(stderr_text)
+ if 'java.lang.OutOfMemoryError' in stderr_text:
+ if not quiet:
+ print('Failure was OOM')
+ return OOM_EXIT_CODE
+ return exit_code
- outdir = options.out
- (version_id, data) = get_version_and_data(options)
+ if options.print_memoryuse:
+ print('{}(MemoryUse): {}'.format(
+ options.print_memoryuse,
+ utils.grep_memoryuse(options.track_memory_to_file)))
- if options.compiler not in COMPILERS:
- raise Exception("You need to specify '--compiler={}'"
- .format('|'.join(COMPILERS)))
+ if android_java8_libs:
+ build_desugared_library_dex(options, quiet, temp,
+ android_java8_libs,
+ desugared_lib_pg_conf, inputs,
+ outdir)
- if options.compiler_build not in COMPILER_BUILDS:
- raise Exception("You need to specify '--compiler-build={}'"
- .format('|'.join(COMPILER_BUILDS)))
+ if options.print_runtimeraw:
+ print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
+ 1000.0 * (time.time() - t0)))
- if not version_id in data.VERSIONS.keys():
- print('No version {} for application {}'
- .format(version_id, options.app))
- print('Valid versions are {}'.format(data.VERSIONS.keys()))
- return 1
-
- version = data.VERSIONS[version_id]
-
- type = get_type(options)
-
- if type not in version:
- print('No type {} for version {}'.format(type, version))
- print('Valid types are {}'.format(version.keys()))
- return 1
- values = version[type]
-
- args.extend(['--output', outdir])
- if 'min-api' in values:
- args.extend(['--min-api', values['min-api']])
-
- if 'main-dex-list' in values:
- args.extend(['--main-dex-list', values['main-dex-list']])
-
- inputs = values['inputs']
- libraries = values['libraries'] if 'libraries' in values else []
-
- if options.compiler == 'r8':
- if 'pgconf' in values and not options.k:
- sanitized_lib_path = os.path.join(
- os.path.abspath(outdir), 'sanitized_lib.jar')
- if has_injars_and_libraryjars(values['pgconf']):
- sanitized_pgconf_path = os.path.join(
- os.path.abspath(outdir), 'sanitized.config')
- SanitizeLibrariesInPgconf(
- sanitized_lib_path, sanitized_pgconf_path, values['pgconf'])
- libraries = [sanitized_lib_path]
- args.extend(['--pg-conf', sanitized_pgconf_path])
- inputs = []
- else:
- # -injars without -libraryjars or vice versa is not supported.
- check_no_injars_and_no_libraryjars(values['pgconf'])
- for pgconf in values['pgconf']:
- args.extend(['--pg-conf', pgconf])
- if 'sanitize_libraries' in values and values['sanitize_libraries']:
- SanitizeLibraries(
- sanitized_lib_path, values['libraries'], values['inputs'])
- libraries = [sanitized_lib_path]
- app_provided_pg_conf = True
- if 'pgconf_extra' in values:
- extra_conf = os.path.join(os.path.abspath(outdir), 'pgconf_extra')
- with open(extra_conf, 'w') as extra_f:
- extra_f.write(values['pgconf_extra'])
- args.extend(['--pg-conf', extra_conf])
- if options.k:
- args.extend(['--pg-conf', options.k])
- if 'maindexrules' in values:
- for rules in values['maindexrules']:
- args.extend(['--main-dex-rules', rules])
- if 'allow-type-errors' in values:
- extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
- extra_args.append(
- '-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1')
- if 'system-properties' in values:
- for system_property in values['system-properties']:
- extra_args.append(system_property)
-
- if options.debug_agent:
- if not options.compiler_build == 'full':
- print('WARNING: Running debugging agent on r8lib is questionable...')
- extra_args.append(
- '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005')
-
- if not options.no_libraries:
- for lib in libraries:
- args.extend(['--lib', lib])
-
- if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
- and not os.path.exists(outdir):
- os.makedirs(outdir)
-
- if options.hash:
- # Download r8-<hash>.jar from
- # https://storage.googleapis.com/r8-releases/raw/<hash>/.
- download_path = archive.GetUploadDestination(options.hash, 'r8.jar', True)
- assert utils.file_exists_on_cloud_storage(download_path), (
- 'Could not find r8.jar file from provided hash: %s' % options.hash)
- destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
- utils.download_file_from_cloud_storage(
- download_path, destination, quiet=quiet)
-
- # Additional flags for the compiler from the configuration file.
- if 'flags' in values:
- args.extend(values['flags'].split(' '))
- if options.compiler == 'r8':
- if 'r8-flags' in values:
- args.extend(values['r8-flags'].split(' '))
-
- # Additional flags for the compiler from the command line.
- if options.compiler_flags:
- args.extend(options.compiler_flags.split(' '))
- if options.r8_flags:
- args.extend(options.r8_flags.split(' '))
-
- # Feature jars.
- features = values['features'] if 'features' in values else []
- for i, feature in enumerate(features, start=1):
- feature_out = os.path.join(outdir, 'feature-%d.zip' % i)
- for feature_jar in feature['inputs']:
- args.extend(['--feature', feature_jar, feature_out])
-
- args.extend(inputs)
-
- t0 = None
- if options.dump_args_file:
- with open(options.dump_args_file, 'w') as args_file:
- args_file.writelines([arg + os.linesep for arg in args])
- else:
- with utils.TempDir() as temp:
- if options.print_memoryuse and not options.track_memory_to_file:
- options.track_memory_to_file = os.path.join(temp,
- utils.MEMORY_USE_TMP_FILE)
- if options.compiler == 'r8' and app_provided_pg_conf:
- # Ensure that output of -printmapping and -printseeds go to the output
- # location and not where the app Proguard configuration places them.
- if outdir.endswith('.zip') or outdir.endswith('.jar'):
- pg_outdir = os.path.dirname(outdir)
- else:
- pg_outdir = outdir
- if not options.no_extra_pgconf:
- additional_pg_conf = GenerateAdditionalProguardConfiguration(
- temp, os.path.abspath(pg_outdir))
- args.extend(['--pg-conf', additional_pg_conf])
-
- android_java8_libs = values.get('android_java8_libs')
- if android_java8_libs:
- desugared_lib_pg_conf = os.path.join(
- temp, 'desugared-lib-pg-conf.txt')
- args.extend(['--desugared-lib', android_java8_libs['config']])
- args.extend(
- ['--desugared-lib-pg-conf-output', desugared_lib_pg_conf])
-
- stderr_path = os.path.join(temp, 'stderr')
- with open(stderr_path, 'w') as stderr:
- jar = None
- main = None
- if options.compiler_build == 'full':
- tool = options.compiler
- else:
- assert(options.compiler_build == 'lib')
- tool = 'r8lib-' + options.compiler
- if options.hash:
- jar = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
- main = 'com.android.tools.r8.' + options.compiler.upper()
- if should_build(options):
- gradle.RunGradle([
- utils.GRADLE_TASK_R8LIB if tool.startswith('r8lib')
- else UTILS.GRADLE_TASK_R8])
- t0 = time.time()
- exit_code = toolhelper.run(tool, args,
- build=False,
- debug=not options.disable_assertions,
- profile=options.profile,
- track_memory_file=options.track_memory_to_file,
- extra_args=extra_args,
- stdout=stdout,
- stderr=stderr,
- timeout=options.timeout,
- quiet=quiet,
- cmd_prefix=[
- 'taskset', '-c', options.cpu_list] if options.cpu_list else [],
- jar=jar,
- main=main,
- worker_id=worker_id)
- if exit_code != 0:
- with open(stderr_path) as stderr:
- stderr_text = stderr.read()
- if not quiet:
- print(stderr_text)
- if 'java.lang.OutOfMemoryError' in stderr_text:
- if not quiet:
- print('Failure was OOM')
- return OOM_EXIT_CODE
- return exit_code
-
- if options.print_memoryuse:
- print('{}(MemoryUse): {}'
- .format(options.print_memoryuse,
- utils.grep_memoryuse(options.track_memory_to_file)))
-
- if android_java8_libs:
- build_desugared_library_dex(
- options, quiet, temp, android_java8_libs,
- desugared_lib_pg_conf, inputs, outdir)
-
-
- if options.print_runtimeraw:
- print('{}(RunTimeRaw): {} ms'
- .format(options.print_runtimeraw, 1000.0 * (time.time() - t0)))
-
- if options.print_dexsegments:
- dex_files = glob(os.path.join(outdir, '*.dex'))
- utils.print_dexsegments(options.print_dexsegments, dex_files)
- print('{}-Total(CodeSize): {}'.format(
+ if options.print_dexsegments:
+ dex_files = glob(os.path.join(outdir, '*.dex'))
+ utils.print_dexsegments(options.print_dexsegments, dex_files)
+ print('{}-Total(CodeSize): {}'.format(
options.print_dexsegments, compute_size_of_dex_files(dex_files)))
- return 0
+ return 0
+
def compute_size_of_dex_files(dex_files):
- dex_size = 0
- for dex_file in dex_files:
- dex_size += os.path.getsize(dex_file)
- return dex_size
+ dex_size = 0
+ for dex_file in dex_files:
+ dex_size += os.path.getsize(dex_file)
+ return dex_size
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/run_on_app_dump.py b/tools/run_on_app_dump.py
index 172f8b7..902c675 100755
--- a/tools/run_on_app_dump.py
+++ b/tools/run_on_app_dump.py
@@ -23,13 +23,15 @@
import update_prebuilds_in_android
import utils
-GOLEM_BUILD_TARGETS = [utils.GRADLE_TASK_R8LIB,
- utils.GRADLE_TASK_RETRACE]
+GOLEM_BUILD_TARGETS = [utils.GRADLE_TASK_R8LIB, utils.GRADLE_TASK_RETRACE]
SHRINKERS = ['r8', 'r8-full', 'r8-nolib', 'r8-nolib-full']
+
class AttrDict(dict):
- def __getattr__(self, name):
- return self.get(name, None)
+
+ def __getattr__(self, name):
+ return self.get(name, None)
+
# To generate the files for a new app, navigate to the app source folder and
# run:
@@ -40,1230 +42,1314 @@
# ./gradlew assembleAndroidTest -Dcom.android.tools.r8.dumpinputtodirectory=<path>
# will also generate dumps and apk for tests.
+
class App(object):
- def __init__(self, fields):
- defaults = {
- 'id': None,
- 'name': None,
- 'collections': [],
- 'dump_app': None,
- 'apk_app': None,
- 'dump_test': None,
- 'apk_test': None,
- 'skip': False,
- 'url': None, # url is not used but nice to have for updating apps
- 'revision': None,
- 'folder': None,
- 'skip_recompilation': False,
- 'compiler_properties': [],
- 'internal': False,
- 'golem_duration': None,
- }
- # This below does not work in python3
- defaults.update(fields.items())
- self.__dict__ = defaults
+
+ def __init__(self, fields):
+ defaults = {
+ 'id': None,
+ 'name': None,
+ 'collections': [],
+ 'dump_app': None,
+ 'apk_app': None,
+ 'dump_test': None,
+ 'apk_test': None,
+ 'skip': False,
+ 'url': None, # url is not used but nice to have for updating apps
+ 'revision': None,
+ 'folder': None,
+ 'skip_recompilation': False,
+ 'compiler_properties': [],
+ 'internal': False,
+ 'golem_duration': None,
+ }
+ # This below does not work in python3
+ defaults.update(fields.items())
+ self.__dict__ = defaults
class AppCollection(object):
- def __init__(self, fields):
- defaults = {
- 'name': None
- }
- # This below does not work in python3
- defaults.update(fields.items())
- self.__dict__ = defaults
+
+ def __init__(self, fields):
+ defaults = {'name': None}
+ # This below does not work in python3
+ defaults.update(fields.items())
+ self.__dict__ = defaults
APPS = [
- App({
- 'id': 'com.numix.calculator',
- 'name': 'Calculator',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- # Compiling tests fail: Library class android.content.res.XmlResourceParser
- # implements program class org.xmlpull.v1.XmlPullParser. Nothing to really
- # do about that.
- 'id_test': 'com.numix.calculator.test',
- 'dump_test': 'dump_test.zip',
- 'apk_test': 'app-release-androidTest.apk',
- 'url': 'https://github.com/numixproject/android-suite/tree/master/Calculator',
- 'revision': 'f58e1b53f7278c9b675d5855842c6d8a44cccb1f',
- 'folder': 'android-suite-calculator',
- }),
- App({
- 'id': 'dev.dworks.apps.anexplorer.pro',
- 'name': 'AnExplorer',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'AnExplorer-googleMobileProRelease-4.0.3.apk',
- 'url': 'https://github.com/christofferqa/AnExplorer',
- 'revision': '365927477b8eab4052a1882d5e358057ae3dee4d',
- 'folder': 'anexplorer',
- }),
- App({
- 'id': 'de.danoeh.antennapod',
- 'name': 'AntennaPod',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-free-release.apk',
- # TODO(b/172452102): Tests and monkey do not work
- 'id_test': 'de.danoeh.antennapod.test',
- 'dump_test': 'dump_test.zip',
- 'apk_test': 'app-free-release-androidTest.apk',
- 'url': 'https://github.com/christofferqa/AntennaPod.git',
- 'revision': '77e94f4783a16abe9cc5b78dc2d2b2b1867d8c06',
- 'folder': 'antennapod',
- }),
- App({
- 'id': 'com.example.applymapping',
- 'name': 'applymapping',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'id_test': 'com.example.applymapping.test',
- 'dump_test': 'dump_test.zip',
- 'apk_test': 'app-release-androidTest.apk',
- 'url': 'https://github.com/mkj-gram/applymapping',
- 'revision': 'e3ae14b8c16fa4718e5dea8f7ad00937701b3c48',
- 'folder': 'applymapping',
- }),
- App({
- 'id': 'com.chanapps.four.activity',
- 'name': 'chanu',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'url': 'https://github.com/mkj-gram/chanu.git',
- 'revision': '6e53458f167b6d78398da60c20fd0da01a232617',
- 'folder': 'chanu',
- # The app depends on a class file that has access flags interface but
- # not abstract
- 'compiler_properties': ['-Dcom.android.tools.r8.allowInvalidCfAccessFlags=true']
- }),
- App({
- 'id': 'com.example.myapplication',
- 'name': 'empty-activity',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'url': 'https://github.com/christofferqa/empty_android_activity.git',
- 'revision': '2d297ec3373dadb03cbae916b9feba4792563156',
- 'folder': 'empty-activity',
- }),
- App({
- 'id': 'com.example.emptycomposeactivity',
- 'name': 'empty-compose-activity',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'url': 'https://github.com/christofferqa/empty_android_compose_activity.git',
- 'revision': '3c8111b8b7d6e9184049a07e2b96702d7b33d03e',
- 'folder': 'empty-compose-activity',
- }),
- # TODO(b/172539375): Monkey runner fails on recompilation.
- App({
- 'id': 'com.google.firebase.example.fireeats',
- 'name': 'FriendlyEats',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/firebase/friendlyeats-android',
- 'revision': '7c6dd016fc31ea5ecb948d5166b8479efc3775cc',
- 'folder': 'friendlyeats',
- }),
- App({
- 'id': 'com.google.samples.apps.sunflower',
- 'name': 'Sunflower',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-debug.apk',
- # TODO(b/172549283): Compiling tests fails
- 'id_test': 'com.google.samples.apps.sunflower.test',
- 'dump_test': 'dump_test.zip',
- 'apk_test': 'app-debug-androidTest.apk',
- 'url': 'https://github.com/android/sunflower',
- 'revision': '0c4c88fdad2a74791199dffd1a6559559b1dbd4a',
- 'folder': 'sunflower',
- }),
- # TODO(b/172565385): Monkey runner fails on recompilation
- App({
- 'id': 'com.google.samples.apps.iosched',
- 'name': 'iosched',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'mobile-release.apk',
- 'url': 'https://github.com/christofferqa/iosched.git',
- 'revision': '581cbbe2253711775dbccb753cdb53e7e506cb02',
- 'folder': 'iosched',
- }),
- App({
- 'id': 'fr.neamar.kiss',
- 'name': 'KISS',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- # TODO(b/172569220): Running tests fails due to missing keep rules
- 'id_test': 'fr.neamar.kiss.test',
- 'dump_test': 'dump_test.zip',
- 'apk_test': 'app-release-androidTest.apk',
- 'url': 'https://github.com/Neamar/KISS',
- 'revision': '8ccffaadaf0d0b8fc4418ed2b4281a0935d3d971',
- 'folder': 'kiss',
- }),
- # TODO(b/172577344): Monkey runner not working.
- App({
- 'id': 'io.github.hidroh.materialistic',
- 'name': 'materialistic',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'url': 'https://github.com/christofferqa/materialistic.git',
- 'revision': '2b2b2ee25ce9e672d5aab1dc90a354af1522b1d9',
- 'folder': 'materialistic',
- }),
- App({
- 'id': 'com.avjindersinghsekhon.minimaltodo',
- 'name': 'MinimalTodo',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'url': 'https://github.com/christofferqa/Minimal-Todo',
- 'revision': '9d8c73746762cd376b718858ec1e8783ca07ba7c',
- 'folder': 'minimal-todo',
- }),
- App({
- 'id': 'net.nurik.roman.muzei',
- 'name': 'muzei',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'muzei-release.apk',
- 'url': 'https://github.com/romannurik/muzei',
- 'revision': '9eac6e98aebeaf0ae40bdcd85f16dd2886551138',
- 'folder': 'muzei',
- }),
- # TODO(b/172806281): Monkey runner does not work.
- App({
- 'id': 'org.schabi.newpipe',
- 'name': 'NewPipe',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/TeamNewPipe/NewPipe',
- 'revision': 'f4435f90313281beece70c544032f784418d85fa',
- 'folder': 'newpipe',
- }),
- # TODO(b/172806808): Monkey runner does not work.
- App({
- 'id': 'io.rover.app.debug',
- 'name': 'Rover',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'example-app-release-unsigned.apk',
- 'url': 'https://github.com/RoverPlatform/rover-android',
- 'revision': '94342117097770ea3ca2c6df6ab496a1a55c3ce7',
- 'folder': 'rover-android',
- }),
- # TODO(b/172808159): Monkey runner does not work
- App({
- 'id': 'com.google.android.apps.santatracker',
- 'name': 'SantaTracker',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'santa-tracker-release.apk',
- 'url': 'https://github.com/christofferqa/santa-tracker-android',
- 'revision': '8dee74be7d9ee33c69465a07088c53087d24a6dd',
- 'folder': 'santa-tracker',
- }),
- App({
- 'id': 'org.thoughtcrime.securesms',
- 'name': 'Signal',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'Signal-Android-play-prod-universal-release-4.76.2.apk',
- # TODO(b/172812839): Instrumentation test fails.
- 'id_test': 'org.thoughtcrime.securesms.test',
- 'dump_test': 'dump_test.zip',
- 'apk_test': 'Signal-Android-play-prod-release-androidTest.apk',
- 'url': 'https://github.com/signalapp/Signal-Android',
- 'revision': '91ca19f294362ccee2c2b43c247eba228e2b30a1',
- 'folder': 'signal-android',
- }),
- # TODO(b/172815827): Monkey runner does not work
- App({
- 'id': 'com.simplemobiletools.calendar.pro',
- 'name': 'Simple-Calendar',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'calendar-release.apk',
- 'url': 'https://github.com/SimpleMobileTools/Simple-Calendar',
- 'revision': '906209874d0a091c7fce5a57972472f272d6b068',
- 'folder': 'simple-calendar',
- }),
- # TODO(b/172815534): Monkey runner does not work
- App({
- 'id': 'com.simplemobiletools.camera.pro',
- 'name': 'Simple-Camera',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'camera-release.apk',
- 'url': 'https://github.com/SimpleMobileTools/Simple-Camera',
- 'revision': 'ebf9820c51e960912b3238287e30a131244fdee6',
- 'folder': 'simple-camera',
- }),
- App({
- 'id': 'com.simplemobiletools.filemanager.pro',
- 'name': 'Simple-File-Manager',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'file-manager-release.apk',
- 'url': 'https://github.com/SimpleMobileTools/Simple-File-Manager',
- 'revision': '2b7fa68ea251222cc40cf6d62ad1de260a6f54d9',
- 'folder': 'simple-file-manager',
- }),
- App({
- 'id': 'com.simplemobiletools.gallery.pro',
- 'name': 'Simple-Gallery',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'gallery-326-foss-release.apk',
- 'url': 'https://github.com/SimpleMobileTools/Simple-Gallery',
- 'revision': '564e56b20d33b28d0018c8087ec705beeb60785e',
- 'folder': 'simple-gallery',
- }),
- App({
- 'id': 'com.example.sqldelight.hockey',
- 'name': 'SQLDelight',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'android-release.apk',
- 'url': 'https://github.com/christofferqa/sqldelight',
- 'revision': '2e67a1126b6df05e4119d1e3a432fde51d76cdc8',
- 'folder': 'sqldelight',
- }),
- # TODO(b/172824096): Monkey runner does not work.
- App({
- 'id': 'eu.kanade.tachiyomi',
- 'name': 'Tachiyomi',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-dev-release.apk',
- 'url': 'https://github.com/inorichi/tachiyomi',
- 'revision': '8aa6486bf76ab9a61a5494bee284b1a5e9180bf3',
- 'folder': 'tachiyomi',
- }),
- # TODO(b/172862042): Monkey runner does not work.
- App({
- 'id': 'app.tivi',
- 'name': 'Tivi',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release.apk',
- 'url': 'https://github.com/chrisbanes/tivi',
- 'revision': '5c6d9ed338885c59b1fc64050d92d056417bb4de',
- 'folder': 'tivi',
- 'golem_duration': 300
- }),
- App({
- 'id': 'com.keylesspalace.tusky',
- 'name': 'Tusky',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-blue-release.apk',
- 'url': 'https://github.com/tuskyapp/Tusky',
- 'revision': '814a9b8f9bacf8d26f712b06a0313a3534a2be95',
- 'folder': 'tusky',
- }),
- App({
- 'id': 'org.wikipedia',
- 'name': 'Wikipedia',
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-prod-release.apk',
- 'url': 'https://github.com/wikimedia/apps-android-wikipedia',
- 'revision': '0fa7cad843c66313be8e25790ef084cf1a1fa67e',
- 'folder': 'wikipedia',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'androidx.compose.samples.crane',
- 'name': 'compose-crane',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/crane',
- 'golem_duration': 240
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.jetcaster',
- 'name': 'compose-jetcaster',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/jetcaster',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.compose.jetchat',
- 'name': 'compose-jetchat',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/jetchat',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.jetnews',
- 'name': 'compose-jetnews',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/jetnews',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.jetsnack',
- 'name': 'compose-jetsnack',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/jetsnack',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.compose.jetsurvey',
- 'name': 'compose-jetsurvey',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/jetsurvey',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.owl',
- 'name': 'compose-owl',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/owl',
- }),
- # TODO(b/173167253): Check if monkey testing works.
- App({
- 'id': 'com.example.compose.rally',
- 'name': 'compose-rally',
- 'collections': ['compose-samples'],
- 'dump_app': 'dump_app.zip',
- 'apk_app': 'app-release-unsigned.apk',
- 'url': 'https://github.com/android/compose-samples',
- 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
- 'folder': 'android/compose-samples/rally',
- }),
+ App({
+ 'id':
+ 'com.numix.calculator',
+ 'name':
+ 'Calculator',
+ 'dump_app':
+ 'dump_app.zip',
+ 'apk_app':
+ 'app-release.apk',
+ # Compiling tests fail: Library class android.content.res.XmlResourceParser
+ # implements program class org.xmlpull.v1.XmlPullParser. Nothing to really
+ # do about that.
+ 'id_test':
+ 'com.numix.calculator.test',
+ 'dump_test':
+ 'dump_test.zip',
+ 'apk_test':
+ 'app-release-androidTest.apk',
+ 'url':
+ 'https://github.com/numixproject/android-suite/tree/master/Calculator',
+ 'revision':
+ 'f58e1b53f7278c9b675d5855842c6d8a44cccb1f',
+ 'folder':
+ 'android-suite-calculator',
+ }),
+ App({
+ 'id': 'dev.dworks.apps.anexplorer.pro',
+ 'name': 'AnExplorer',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'AnExplorer-googleMobileProRelease-4.0.3.apk',
+ 'url': 'https://github.com/christofferqa/AnExplorer',
+ 'revision': '365927477b8eab4052a1882d5e358057ae3dee4d',
+ 'folder': 'anexplorer',
+ }),
+ App({
+ 'id': 'de.danoeh.antennapod',
+ 'name': 'AntennaPod',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-free-release.apk',
+ # TODO(b/172452102): Tests and monkey do not work
+ 'id_test': 'de.danoeh.antennapod.test',
+ 'dump_test': 'dump_test.zip',
+ 'apk_test': 'app-free-release-androidTest.apk',
+ 'url': 'https://github.com/christofferqa/AntennaPod.git',
+ 'revision': '77e94f4783a16abe9cc5b78dc2d2b2b1867d8c06',
+ 'folder': 'antennapod',
+ }),
+ App({
+ 'id': 'com.example.applymapping',
+ 'name': 'applymapping',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release.apk',
+ 'id_test': 'com.example.applymapping.test',
+ 'dump_test': 'dump_test.zip',
+ 'apk_test': 'app-release-androidTest.apk',
+ 'url': 'https://github.com/mkj-gram/applymapping',
+ 'revision': 'e3ae14b8c16fa4718e5dea8f7ad00937701b3c48',
+ 'folder': 'applymapping',
+ }),
+ App({
+ 'id':
+ 'com.chanapps.four.activity',
+ 'name':
+ 'chanu',
+ 'dump_app':
+ 'dump_app.zip',
+ 'apk_app':
+ 'app-release.apk',
+ 'url':
+ 'https://github.com/mkj-gram/chanu.git',
+ 'revision':
+ '6e53458f167b6d78398da60c20fd0da01a232617',
+ 'folder':
+ 'chanu',
+ # The app depends on a class file that has access flags interface but
+ # not abstract
+ 'compiler_properties': [
+ '-Dcom.android.tools.r8.allowInvalidCfAccessFlags=true'
+ ]
+ }),
+ App({
+ 'id': 'com.example.myapplication',
+ 'name': 'empty-activity',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release.apk',
+ 'url': 'https://github.com/christofferqa/empty_android_activity.git',
+ 'revision': '2d297ec3373dadb03cbae916b9feba4792563156',
+ 'folder': 'empty-activity',
+ }),
+ App({
+ 'id':
+ 'com.example.emptycomposeactivity',
+ 'name':
+ 'empty-compose-activity',
+ 'dump_app':
+ 'dump_app.zip',
+ 'apk_app':
+ 'app-release.apk',
+ 'url':
+ 'https://github.com/christofferqa/empty_android_compose_activity.git',
+ 'revision':
+ '3c8111b8b7d6e9184049a07e2b96702d7b33d03e',
+ 'folder':
+ 'empty-compose-activity',
+ }),
+ # TODO(b/172539375): Monkey runner fails on recompilation.
+ App({
+ 'id': 'com.google.firebase.example.fireeats',
+ 'name': 'FriendlyEats',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/firebase/friendlyeats-android',
+ 'revision': '7c6dd016fc31ea5ecb948d5166b8479efc3775cc',
+ 'folder': 'friendlyeats',
+ }),
+ App({
+ 'id': 'com.google.samples.apps.sunflower',
+ 'name': 'Sunflower',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-debug.apk',
+ # TODO(b/172549283): Compiling tests fails
+ 'id_test': 'com.google.samples.apps.sunflower.test',
+ 'dump_test': 'dump_test.zip',
+ 'apk_test': 'app-debug-androidTest.apk',
+ 'url': 'https://github.com/android/sunflower',
+ 'revision': '0c4c88fdad2a74791199dffd1a6559559b1dbd4a',
+ 'folder': 'sunflower',
+ }),
+ # TODO(b/172565385): Monkey runner fails on recompilation
+ App({
+ 'id': 'com.google.samples.apps.iosched',
+ 'name': 'iosched',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'mobile-release.apk',
+ 'url': 'https://github.com/christofferqa/iosched.git',
+ 'revision': '581cbbe2253711775dbccb753cdb53e7e506cb02',
+ 'folder': 'iosched',
+ }),
+ App({
+ 'id': 'fr.neamar.kiss',
+ 'name': 'KISS',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release.apk',
+ # TODO(b/172569220): Running tests fails due to missing keep rules
+ 'id_test': 'fr.neamar.kiss.test',
+ 'dump_test': 'dump_test.zip',
+ 'apk_test': 'app-release-androidTest.apk',
+ 'url': 'https://github.com/Neamar/KISS',
+ 'revision': '8ccffaadaf0d0b8fc4418ed2b4281a0935d3d971',
+ 'folder': 'kiss',
+ }),
+ # TODO(b/172577344): Monkey runner not working.
+ App({
+ 'id': 'io.github.hidroh.materialistic',
+ 'name': 'materialistic',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release.apk',
+ 'url': 'https://github.com/christofferqa/materialistic.git',
+ 'revision': '2b2b2ee25ce9e672d5aab1dc90a354af1522b1d9',
+ 'folder': 'materialistic',
+ }),
+ App({
+ 'id': 'com.avjindersinghsekhon.minimaltodo',
+ 'name': 'MinimalTodo',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release.apk',
+ 'url': 'https://github.com/christofferqa/Minimal-Todo',
+ 'revision': '9d8c73746762cd376b718858ec1e8783ca07ba7c',
+ 'folder': 'minimal-todo',
+ }),
+ App({
+ 'id': 'net.nurik.roman.muzei',
+ 'name': 'muzei',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'muzei-release.apk',
+ 'url': 'https://github.com/romannurik/muzei',
+ 'revision': '9eac6e98aebeaf0ae40bdcd85f16dd2886551138',
+ 'folder': 'muzei',
+ }),
+ # TODO(b/172806281): Monkey runner does not work.
+ App({
+ 'id': 'org.schabi.newpipe',
+ 'name': 'NewPipe',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/TeamNewPipe/NewPipe',
+ 'revision': 'f4435f90313281beece70c544032f784418d85fa',
+ 'folder': 'newpipe',
+ }),
+ # TODO(b/172806808): Monkey runner does not work.
+ App({
+ 'id': 'io.rover.app.debug',
+ 'name': 'Rover',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'example-app-release-unsigned.apk',
+ 'url': 'https://github.com/RoverPlatform/rover-android',
+ 'revision': '94342117097770ea3ca2c6df6ab496a1a55c3ce7',
+ 'folder': 'rover-android',
+ }),
+ # TODO(b/172808159): Monkey runner does not work
+ App({
+ 'id': 'com.google.android.apps.santatracker',
+ 'name': 'SantaTracker',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'santa-tracker-release.apk',
+ 'url': 'https://github.com/christofferqa/santa-tracker-android',
+ 'revision': '8dee74be7d9ee33c69465a07088c53087d24a6dd',
+ 'folder': 'santa-tracker',
+ }),
+ App({
+ 'id': 'org.thoughtcrime.securesms',
+ 'name': 'Signal',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'Signal-Android-play-prod-universal-release-4.76.2.apk',
+ # TODO(b/172812839): Instrumentation test fails.
+ 'id_test': 'org.thoughtcrime.securesms.test',
+ 'dump_test': 'dump_test.zip',
+ 'apk_test': 'Signal-Android-play-prod-release-androidTest.apk',
+ 'url': 'https://github.com/signalapp/Signal-Android',
+ 'revision': '91ca19f294362ccee2c2b43c247eba228e2b30a1',
+ 'folder': 'signal-android',
+ }),
+ # TODO(b/172815827): Monkey runner does not work
+ App({
+ 'id': 'com.simplemobiletools.calendar.pro',
+ 'name': 'Simple-Calendar',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'calendar-release.apk',
+ 'url': 'https://github.com/SimpleMobileTools/Simple-Calendar',
+ 'revision': '906209874d0a091c7fce5a57972472f272d6b068',
+ 'folder': 'simple-calendar',
+ }),
+ # TODO(b/172815534): Monkey runner does not work
+ App({
+ 'id': 'com.simplemobiletools.camera.pro',
+ 'name': 'Simple-Camera',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'camera-release.apk',
+ 'url': 'https://github.com/SimpleMobileTools/Simple-Camera',
+ 'revision': 'ebf9820c51e960912b3238287e30a131244fdee6',
+ 'folder': 'simple-camera',
+ }),
+ App({
+ 'id': 'com.simplemobiletools.filemanager.pro',
+ 'name': 'Simple-File-Manager',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'file-manager-release.apk',
+ 'url': 'https://github.com/SimpleMobileTools/Simple-File-Manager',
+ 'revision': '2b7fa68ea251222cc40cf6d62ad1de260a6f54d9',
+ 'folder': 'simple-file-manager',
+ }),
+ App({
+ 'id': 'com.simplemobiletools.gallery.pro',
+ 'name': 'Simple-Gallery',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'gallery-326-foss-release.apk',
+ 'url': 'https://github.com/SimpleMobileTools/Simple-Gallery',
+ 'revision': '564e56b20d33b28d0018c8087ec705beeb60785e',
+ 'folder': 'simple-gallery',
+ }),
+ App({
+ 'id': 'com.example.sqldelight.hockey',
+ 'name': 'SQLDelight',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'android-release.apk',
+ 'url': 'https://github.com/christofferqa/sqldelight',
+ 'revision': '2e67a1126b6df05e4119d1e3a432fde51d76cdc8',
+ 'folder': 'sqldelight',
+ }),
+ # TODO(b/172824096): Monkey runner does not work.
+ App({
+ 'id': 'eu.kanade.tachiyomi',
+ 'name': 'Tachiyomi',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-dev-release.apk',
+ 'url': 'https://github.com/inorichi/tachiyomi',
+ 'revision': '8aa6486bf76ab9a61a5494bee284b1a5e9180bf3',
+ 'folder': 'tachiyomi',
+ }),
+ # TODO(b/172862042): Monkey runner does not work.
+ App({
+ 'id': 'app.tivi',
+ 'name': 'Tivi',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release.apk',
+ 'url': 'https://github.com/chrisbanes/tivi',
+ 'revision': '5c6d9ed338885c59b1fc64050d92d056417bb4de',
+ 'folder': 'tivi',
+ 'golem_duration': 300
+ }),
+ App({
+ 'id': 'com.keylesspalace.tusky',
+ 'name': 'Tusky',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-blue-release.apk',
+ 'url': 'https://github.com/tuskyapp/Tusky',
+ 'revision': '814a9b8f9bacf8d26f712b06a0313a3534a2be95',
+ 'folder': 'tusky',
+ }),
+ App({
+ 'id': 'org.wikipedia',
+ 'name': 'Wikipedia',
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-prod-release.apk',
+ 'url': 'https://github.com/wikimedia/apps-android-wikipedia',
+ 'revision': '0fa7cad843c66313be8e25790ef084cf1a1fa67e',
+ 'folder': 'wikipedia',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'androidx.compose.samples.crane',
+ 'name': 'compose-crane',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/crane',
+ 'golem_duration': 240
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.jetcaster',
+ 'name': 'compose-jetcaster',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/jetcaster',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.compose.jetchat',
+ 'name': 'compose-jetchat',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/jetchat',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.jetnews',
+ 'name': 'compose-jetnews',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/jetnews',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.jetsnack',
+ 'name': 'compose-jetsnack',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/jetsnack',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.compose.jetsurvey',
+ 'name': 'compose-jetsurvey',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/jetsurvey',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.owl',
+ 'name': 'compose-owl',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/owl',
+ }),
+ # TODO(b/173167253): Check if monkey testing works.
+ App({
+ 'id': 'com.example.compose.rally',
+ 'name': 'compose-rally',
+ 'collections': ['compose-samples'],
+ 'dump_app': 'dump_app.zip',
+ 'apk_app': 'app-release-unsigned.apk',
+ 'url': 'https://github.com/android/compose-samples',
+ 'revision': '779cf9e187b8ee2c6b620b2abb4524719b3f10f8',
+ 'folder': 'android/compose-samples/rally',
+ }),
]
-
-APP_COLLECTIONS = [
- AppCollection({
+APP_COLLECTIONS = [AppCollection({
'name': 'compose-samples',
- })
-]
+})]
def remove_print_lines(file):
- with open(file) as f:
- lines = f.readlines()
- with open(file, 'w') as f:
- for line in lines:
- if '-printconfiguration' not in line:
- f.write(line)
+ with open(file) as f:
+ lines = f.readlines()
+ with open(file, 'w') as f:
+ for line in lines:
+ if '-printconfiguration' not in line:
+ f.write(line)
def download_sha(app_sha, internal, quiet=False):
- if internal:
- utils.DownloadFromX20(app_sha)
- else:
- utils.DownloadFromGoogleCloudStorage(app_sha, quiet=quiet)
+ if internal:
+ utils.DownloadFromX20(app_sha)
+ else:
+ utils.DownloadFromGoogleCloudStorage(app_sha, quiet=quiet)
def is_logging_enabled_for(app, options):
- if options.no_logging:
- return False
- if options.app_logging_filter and app.name not in options.app_logging_filter:
- return False
- return True
+ if options.no_logging:
+ return False
+ if options.app_logging_filter and app.name not in options.app_logging_filter:
+ return False
+ return True
def is_minified_r8(shrinker):
- return '-nolib' not in shrinker
+ return '-nolib' not in shrinker
def is_full_r8(shrinker):
- return '-full' in shrinker
+ return '-full' in shrinker
def version_is_built_jar(version):
- return version != 'main' and version != 'source'
+ return version != 'main' and version != 'source'
def compute_size_of_dex_files_in_package(path):
- dex_size = 0
- z = zipfile.ZipFile(path, 'r')
- for filename in z.namelist():
- if filename.endswith('.dex'):
- dex_size += z.getinfo(filename).file_size
- return dex_size
+ dex_size = 0
+ z = zipfile.ZipFile(path, 'r')
+ for filename in z.namelist():
+ if filename.endswith('.dex'):
+ dex_size += z.getinfo(filename).file_size
+ return dex_size
def dump_for_app(app_dir, app):
- return os.path.join(app_dir, app.dump_app)
+ return os.path.join(app_dir, app.dump_app)
def dump_test_for_app(app_dir, app):
- return os.path.join(app_dir, app.dump_test)
+ return os.path.join(app_dir, app.dump_test)
def get_r8_jar(options, temp_dir, shrinker):
- if (options.version == 'source'):
- return None
- jar = os.path.abspath(
- os.path.join(
- temp_dir,
- '..',
- 'r8lib.jar' if is_minified_r8(shrinker) else 'r8.jar'))
- return jar
+ if (options.version == 'source'):
+ return None
+ jar = os.path.abspath(
+ os.path.join(temp_dir, '..',
+ 'r8lib.jar' if is_minified_r8(shrinker) else 'r8.jar'))
+ return jar
def get_results_for_app(app, options, temp_dir, worker_id):
- app_folder = app.folder if app.folder else app.name + "_" + app.revision
- # Golem extraction will extract to the basename under the benchmarks dir.
- app_location = os.path.basename(app_folder) if options.golem else app_folder
- opensource_basedir = (os.path.join('benchmarks', app.name) if options.golem
- else utils.OPENSOURCE_DUMPS_DIR)
- app_dir = (os.path.join(utils.INTERNAL_DUMPS_DIR, app_location) if app.internal
- else os.path.join(opensource_basedir, app_location))
- if not os.path.exists(app_dir) and not options.golem:
- # Download the app from google storage.
- download_sha(app_dir + ".tar.gz.sha1", app.internal)
+ app_folder = app.folder if app.folder else app.name + "_" + app.revision
+ # Golem extraction will extract to the basename under the benchmarks dir.
+ app_location = os.path.basename(app_folder) if options.golem else app_folder
+ opensource_basedir = (os.path.join('benchmarks', app.name)
+ if options.golem else utils.OPENSOURCE_DUMPS_DIR)
+ app_dir = (os.path.join(utils.INTERNAL_DUMPS_DIR, app_location) if
+ app.internal else os.path.join(opensource_basedir, app_location))
+ if not os.path.exists(app_dir) and not options.golem:
+ # Download the app from google storage.
+ download_sha(app_dir + ".tar.gz.sha1", app.internal)
- # Ensure that the dumps are in place
- assert os.path.isfile(dump_for_app(app_dir, app)), "Could not find dump " \
- "for app " + app.name
+ # Ensure that the dumps are in place
+ assert os.path.isfile(dump_for_app(app_dir, app)), "Could not find dump " \
+ "for app " + app.name
- result = {}
- result['status'] = 'success'
- result_per_shrinker = build_app_with_shrinkers(
- app, options, temp_dir, app_dir, worker_id=worker_id)
- for shrinker, shrinker_result in result_per_shrinker.items():
- result[shrinker] = shrinker_result
- return result
+ result = {}
+ result['status'] = 'success'
+ result_per_shrinker = build_app_with_shrinkers(app,
+ options,
+ temp_dir,
+ app_dir,
+ worker_id=worker_id)
+ for shrinker, shrinker_result in result_per_shrinker.items():
+ result[shrinker] = shrinker_result
+ return result
def build_app_with_shrinkers(app, options, temp_dir, app_dir, worker_id):
- result_per_shrinker = {}
- for shrinker in options.shrinker:
- results = []
- build_app_and_run_with_shrinker(
- app, options, temp_dir, app_dir, shrinker, results, worker_id=worker_id)
- result_per_shrinker[shrinker] = results
- if len(options.apps) > 1:
- print_thread('', worker_id)
- log_results_for_app(app, result_per_shrinker, options, worker_id=worker_id)
- print_thread('', worker_id)
+ result_per_shrinker = {}
+ for shrinker in options.shrinker:
+ results = []
+ build_app_and_run_with_shrinker(app,
+ options,
+ temp_dir,
+ app_dir,
+ shrinker,
+ results,
+ worker_id=worker_id)
+ result_per_shrinker[shrinker] = results
+ if len(options.apps) > 1:
+ print_thread('', worker_id)
+ log_results_for_app(app,
+ result_per_shrinker,
+ options,
+ worker_id=worker_id)
+ print_thread('', worker_id)
- return result_per_shrinker
+ return result_per_shrinker
def is_last_build(index, compilation_steps):
- return index == compilation_steps - 1
+ return index == compilation_steps - 1
def build_app_and_run_with_shrinker(app, options, temp_dir, app_dir, shrinker,
results, worker_id):
- print_thread(
- '[{}] Building {} with {}'.format(
- datetime.now().strftime("%H:%M:%S"),
- app.name,
- shrinker),
- worker_id)
- print_thread(
- 'To compile locally: '
- 'tools/run_on_app_dump.py --shrinker {} --r8-compilation-steps {} '
- '--app {} --minify {} --optimize {} --shrink {}'.format(
- shrinker,
- options.r8_compilation_steps,
- app.name,
- options.minify,
- options.optimize,
- options.shrink),
- worker_id)
- print_thread(
- 'HINT: use --shrinker r8-nolib --no-build if you have a local R8.jar',
- worker_id)
- recomp_jar = None
- status = 'success'
- if options.r8_compilation_steps < 1:
- return
- compilation_steps = 1 if app.skip_recompilation else options.r8_compilation_steps
- for compilation_step in range(0, compilation_steps):
- if status != 'success':
- break
print_thread(
- 'Compiling {} of {}'.format(compilation_step + 1, compilation_steps),
+ '[{}] Building {} with {}'.format(datetime.now().strftime("%H:%M:%S"),
+ app.name, shrinker), worker_id)
+ print_thread(
+ 'To compile locally: '
+ 'tools/run_on_app_dump.py --shrinker {} --r8-compilation-steps {} '
+ '--app {} --minify {} --optimize {} --shrink {}'.format(
+ shrinker, options.r8_compilation_steps, app.name, options.minify,
+ options.optimize, options.shrink), worker_id)
+ print_thread(
+ 'HINT: use --shrinker r8-nolib --no-build if you have a local R8.jar',
worker_id)
- result = {}
- try:
- start = time.time()
- (app_jar, mapping, new_recomp_jar) = \
- build_app_with_shrinker(
- app, options, temp_dir, app_dir, shrinker, compilation_step,
- compilation_steps, recomp_jar, worker_id=worker_id)
- end = time.time()
- dex_size = compute_size_of_dex_files_in_package(app_jar)
- result['build_status'] = 'success'
- result['recompilation_status'] = 'success'
- result['output_jar'] = app_jar
- result['output_mapping'] = mapping
- result['dex_size'] = dex_size
- result['duration'] = int((end - start) * 1000) # Wall time
- if (new_recomp_jar is None
- and not is_last_build(compilation_step, compilation_steps)):
- result['recompilation_status'] = 'failed'
- warn('Failed to build {} with {}'.format(app.name, shrinker))
- recomp_jar = new_recomp_jar
- except Exception as e:
- warn('Failed to build {} with {}'.format(app.name, shrinker))
- if e:
- print_thread('Error: ' + str(e), worker_id)
- result['build_status'] = 'failed'
- status = 'failed'
+ recomp_jar = None
+ status = 'success'
+ if options.r8_compilation_steps < 1:
+ return
+ compilation_steps = 1 if app.skip_recompilation else options.r8_compilation_steps
+ for compilation_step in range(0, compilation_steps):
+ if status != 'success':
+ break
+ print_thread(
+ 'Compiling {} of {}'.format(compilation_step + 1,
+ compilation_steps), worker_id)
+ result = {}
+ try:
+ start = time.time()
+ (app_jar, mapping, new_recomp_jar) = \
+ build_app_with_shrinker(
+ app, options, temp_dir, app_dir, shrinker, compilation_step,
+ compilation_steps, recomp_jar, worker_id=worker_id)
+ end = time.time()
+ dex_size = compute_size_of_dex_files_in_package(app_jar)
+ result['build_status'] = 'success'
+ result['recompilation_status'] = 'success'
+ result['output_jar'] = app_jar
+ result['output_mapping'] = mapping
+ result['dex_size'] = dex_size
+ result['duration'] = int((end - start) * 1000) # Wall time
+ if (new_recomp_jar is None and
+ not is_last_build(compilation_step, compilation_steps)):
+ result['recompilation_status'] = 'failed'
+ warn('Failed to build {} with {}'.format(app.name, shrinker))
+ recomp_jar = new_recomp_jar
+ except Exception as e:
+ warn('Failed to build {} with {}'.format(app.name, shrinker))
+ if e:
+ print_thread('Error: ' + str(e), worker_id)
+ result['build_status'] = 'failed'
+ status = 'failed'
- original_app_apk = os.path.join(app_dir, app.apk_app)
- app_apk_destination = os.path.join(
- temp_dir,"{}_{}.apk".format(app.id, compilation_step))
+ original_app_apk = os.path.join(app_dir, app.apk_app)
+ app_apk_destination = os.path.join(
+ temp_dir, "{}_{}.apk".format(app.id, compilation_step))
- if result.get('build_status') == 'success' and options.monkey:
- # Make a copy of the given APK, move the newly generated dex files into the
- # copied APK, and then sign the APK.
- apk_masseur.masseur(
- original_app_apk, dex=app_jar, resources='META-INF/services/*',
- out=app_apk_destination,
- quiet=options.quiet, logging=is_logging_enabled_for(app, options),
- keystore=options.keystore)
+ if result.get('build_status') == 'success' and options.monkey:
+ # Make a copy of the given APK, move the newly generated dex files into the
+ # copied APK, and then sign the APK.
+ apk_masseur.masseur(original_app_apk,
+ dex=app_jar,
+ resources='META-INF/services/*',
+ out=app_apk_destination,
+ quiet=options.quiet,
+ logging=is_logging_enabled_for(app, options),
+ keystore=options.keystore)
- result['monkey_status'] = 'success' if adb.run_monkey(
- app.id, options.emulator_id, app_apk_destination, options.monkey_events,
- options.quiet, is_logging_enabled_for(app, options)) else 'failed'
+ result['monkey_status'] = 'success' if adb.run_monkey(
+ app.id, options.emulator_id, app_apk_destination,
+ options.monkey_events, options.quiet,
+ is_logging_enabled_for(app, options)) else 'failed'
- if (result.get('build_status') == 'success'
- and options.run_tests and app.dump_test):
- if not os.path.isfile(app_apk_destination):
- apk_masseur.masseur(
- original_app_apk, dex=app_jar, resources='META-INF/services/*',
- out=app_apk_destination,
- quiet=options.quiet, logging=is_logging_enabled_for(app, options),
- keystore=options.keystore)
+ if (result.get('build_status') == 'success' and options.run_tests and
+ app.dump_test):
+ if not os.path.isfile(app_apk_destination):
+ apk_masseur.masseur(original_app_apk,
+ dex=app_jar,
+ resources='META-INF/services/*',
+ out=app_apk_destination,
+ quiet=options.quiet,
+ logging=is_logging_enabled_for(
+ app, options),
+ keystore=options.keystore)
- # Compile the tests with the mapping file.
- test_jar = build_test_with_shrinker(
- app, options, temp_dir, app_dir,shrinker, compilation_step,
- result['output_mapping'])
- if not test_jar:
- result['instrumentation_test_status'] = 'compilation_failed'
- else:
- original_test_apk = os.path.join(app_dir, app.apk_test)
- test_apk_destination = os.path.join(
- temp_dir,"{}_{}.test.apk".format(app.id_test, compilation_step))
- apk_masseur.masseur(
- original_test_apk, dex=test_jar, resources='META-INF/services/*',
- out=test_apk_destination,
- quiet=options.quiet, logging=is_logging_enabled_for(app, options),
- keystore=options.keystore)
- result['instrumentation_test_status'] = 'success' if adb.run_instrumented(
- app.id, app.id_test, options.emulator_id, app_apk_destination,
- test_apk_destination, options.quiet,
- is_logging_enabled_for(app, options)) else 'failed'
+ # Compile the tests with the mapping file.
+ test_jar = build_test_with_shrinker(app, options, temp_dir, app_dir,
+ shrinker, compilation_step,
+ result['output_mapping'])
+ if not test_jar:
+ result['instrumentation_test_status'] = 'compilation_failed'
+ else:
+ original_test_apk = os.path.join(app_dir, app.apk_test)
+ test_apk_destination = os.path.join(
+ temp_dir, "{}_{}.test.apk".format(app.id_test,
+ compilation_step))
+ apk_masseur.masseur(original_test_apk,
+ dex=test_jar,
+ resources='META-INF/services/*',
+ out=test_apk_destination,
+ quiet=options.quiet,
+ logging=is_logging_enabled_for(
+ app, options),
+ keystore=options.keystore)
+ result[
+ 'instrumentation_test_status'] = 'success' if adb.run_instrumented(
+ app.id, app.id_test, options.emulator_id,
+ app_apk_destination,
+ test_apk_destination, options.quiet,
+ is_logging_enabled_for(app, options)) else 'failed'
- results.append(result)
- if result.get('recompilation_status') != 'success':
- break
+ results.append(result)
+ if result.get('recompilation_status') != 'success':
+ break
+
def get_jdk_home(options, app):
- if options.golem:
- return os.path.join('benchmarks', app.name, 'linux')
- return None
+ if options.golem:
+ return os.path.join('benchmarks', app.name, 'linux')
+ return None
+
def build_app_with_shrinker(app, options, temp_dir, app_dir, shrinker,
compilation_step_index, compilation_steps,
prev_recomp_jar, worker_id):
- def config_files_consumer(files):
- for file in files:
- compiledump.clean_config(file, options)
- remove_print_lines(file)
- args = AttrDict({
- 'dump': dump_for_app(app_dir, app),
- 'r8_jar': get_r8_jar(options, temp_dir, shrinker),
- 'r8_flags': options.r8_flags,
- 'disable_assertions': options.disable_assertions,
- 'version': options.version,
- 'compiler': 'r8full' if is_full_r8(shrinker) else 'r8',
- 'debug_agent': options.debug_agent,
- 'program_jar': prev_recomp_jar,
- 'nolib': not is_minified_r8(shrinker),
- 'config_files_consumer': config_files_consumer,
- 'properties': app.compiler_properties,
- 'disable_desugared_lib': False,
- 'print_times': options.print_times,
- })
- app_jar = os.path.join(
- temp_dir, '{}_{}_{}_dex_out.jar'.format(
- app.name, shrinker, compilation_step_index))
- app_mapping = os.path.join(
- temp_dir, '{}_{}_{}_dex_out.jar.map'.format(
- app.name, shrinker, compilation_step_index))
- recomp_jar = None
- jdkhome = get_jdk_home(options, app)
- with utils.TempDir() as compile_temp_dir:
- compile_result = compiledump.run1(
- compile_temp_dir, args, [], jdkhome, worker_id=worker_id)
- out_jar = os.path.join(compile_temp_dir, "out.jar")
- out_mapping = os.path.join(compile_temp_dir, "out.jar.map")
+ def config_files_consumer(files):
+ for file in files:
+ compiledump.clean_config(file, options)
+ remove_print_lines(file)
- if compile_result != 0 or not os.path.isfile(out_jar):
- assert False, 'Compilation of {} failed'.format(dump_for_app(app_dir, app))
- shutil.move(out_jar, app_jar)
- shutil.move(out_mapping, app_mapping)
+ args = AttrDict({
+ 'dump': dump_for_app(app_dir, app),
+ 'r8_jar': get_r8_jar(options, temp_dir, shrinker),
+ 'r8_flags': options.r8_flags,
+ 'disable_assertions': options.disable_assertions,
+ 'version': options.version,
+ 'compiler': 'r8full' if is_full_r8(shrinker) else 'r8',
+ 'debug_agent': options.debug_agent,
+ 'program_jar': prev_recomp_jar,
+ 'nolib': not is_minified_r8(shrinker),
+ 'config_files_consumer': config_files_consumer,
+ 'properties': app.compiler_properties,
+ 'disable_desugared_lib': False,
+ 'print_times': options.print_times,
+ })
- if compilation_step_index < compilation_steps - 1:
- args['classfile'] = True
- args['min_api'] = "10000"
- args['disable_desugared_lib'] = True
- compile_result = compiledump.run1(compile_temp_dir, args, [], jdkhome)
- if compile_result == 0:
- recomp_jar = os.path.join(
- temp_dir, '{}_{}_{}_cf_out.jar'.format(
- app.name, shrinker, compilation_step_index))
- shutil.move(out_jar, recomp_jar)
+ app_jar = os.path.join(
+ temp_dir, '{}_{}_{}_dex_out.jar'.format(app.name, shrinker,
+ compilation_step_index))
+ app_mapping = os.path.join(
+ temp_dir, '{}_{}_{}_dex_out.jar.map'.format(app.name, shrinker,
+ compilation_step_index))
+ recomp_jar = None
+ jdkhome = get_jdk_home(options, app)
+ with utils.TempDir() as compile_temp_dir:
+ compile_result = compiledump.run1(compile_temp_dir,
+ args, [],
+ jdkhome,
+ worker_id=worker_id)
+ out_jar = os.path.join(compile_temp_dir, "out.jar")
+ out_mapping = os.path.join(compile_temp_dir, "out.jar.map")
- return (app_jar, app_mapping, recomp_jar)
+ if compile_result != 0 or not os.path.isfile(out_jar):
+ assert False, 'Compilation of {} failed'.format(
+ dump_for_app(app_dir, app))
+ shutil.move(out_jar, app_jar)
+ shutil.move(out_mapping, app_mapping)
+
+ if compilation_step_index < compilation_steps - 1:
+ args['classfile'] = True
+ args['min_api'] = "10000"
+ args['disable_desugared_lib'] = True
+ compile_result = compiledump.run1(compile_temp_dir, args, [],
+ jdkhome)
+ if compile_result == 0:
+ recomp_jar = os.path.join(
+ temp_dir,
+ '{}_{}_{}_cf_out.jar'.format(app.name, shrinker,
+ compilation_step_index))
+ shutil.move(out_jar, recomp_jar)
+
+ return (app_jar, app_mapping, recomp_jar)
def build_test_with_shrinker(app, options, temp_dir, app_dir, shrinker,
compilation_step_index, mapping):
- def rewrite_files(files):
- add_applymapping = True
- for file in files:
- compiledump.clean_config(file, options)
- remove_print_lines(file)
- with open(file) as f:
- lines = f.readlines()
- with open(file, 'w') as f:
- for line in lines:
- if '-applymapping' not in line:
- f.write(line + '\n')
- if add_applymapping:
- f.write("-applymapping " + mapping + '\n')
- add_applymapping = False
+ def rewrite_files(files):
+ add_applymapping = True
+ for file in files:
+ compiledump.clean_config(file, options)
+ remove_print_lines(file)
+ with open(file) as f:
+ lines = f.readlines()
+ with open(file, 'w') as f:
+ for line in lines:
+ if '-applymapping' not in line:
+ f.write(line + '\n')
+ if add_applymapping:
+ f.write("-applymapping " + mapping + '\n')
+ add_applymapping = False
- args = AttrDict({
- 'dump': dump_test_for_app(app_dir, app),
- 'r8_jar': get_r8_jar(options, temp_dir, shrinker),
- 'disable_assertions': options.disable_assertions,
- 'version': options.version,
- 'compiler': 'r8full' if is_full_r8(shrinker) else 'r8',
- 'debug_agent': options.debug_agent,
- 'nolib': not is_minified_r8(shrinker),
- # The config file will have an -applymapping reference to an old map.
- # Update it to point to mapping file build in the compilation of the app.
- 'config_files_consumer': rewrite_files,
- })
+ args = AttrDict({
+ 'dump': dump_test_for_app(app_dir, app),
+ 'r8_jar': get_r8_jar(options, temp_dir, shrinker),
+ 'disable_assertions': options.disable_assertions,
+ 'version': options.version,
+ 'compiler': 'r8full' if is_full_r8(shrinker) else 'r8',
+ 'debug_agent': options.debug_agent,
+ 'nolib': not is_minified_r8(shrinker),
+ # The config file will have an -applymapping reference to an old map.
+ # Update it to point to mapping file build in the compilation of the app.
+ 'config_files_consumer': rewrite_files,
+ })
- test_jar = os.path.join(
- temp_dir, '{}_{}_{}_test_out.jar'.format(
- app.name, shrinker, compilation_step_index))
+ test_jar = os.path.join(
+ temp_dir, '{}_{}_{}_test_out.jar'.format(app.name, shrinker,
+ compilation_step_index))
- with utils.TempDir() as compile_temp_dir:
- jdkhome = get_jdk_home(options, app)
- compile_result = compiledump.run1(compile_temp_dir, args, [], jdkhome)
- out_jar = os.path.join(compile_temp_dir, "out.jar")
- if compile_result != 0 or not os.path.isfile(out_jar):
- return None
- shutil.move(out_jar, test_jar)
+ with utils.TempDir() as compile_temp_dir:
+ jdkhome = get_jdk_home(options, app)
+ compile_result = compiledump.run1(compile_temp_dir, args, [], jdkhome)
+ out_jar = os.path.join(compile_temp_dir, "out.jar")
+ if compile_result != 0 or not os.path.isfile(out_jar):
+ return None
+ shutil.move(out_jar, test_jar)
- return test_jar
+ return test_jar
def log_results_for_apps(result_per_shrinker_per_app, options):
- print('')
- app_errors = 0
- for (app, result_per_shrinker) in result_per_shrinker_per_app:
- app_errors += (1 if log_results_for_app(app, result_per_shrinker, options)
- else 0)
- return app_errors
+ print('')
+ app_errors = 0
+ for (app, result_per_shrinker) in result_per_shrinker_per_app:
+ app_errors += (1 if log_results_for_app(app, result_per_shrinker,
+ options) else 0)
+ return app_errors
def log_results_for_app(app, result_per_shrinker, options, worker_id=None):
- if options.print_dexsegments:
- log_segments_for_app(app, result_per_shrinker, options, worker_id=worker_id)
- return False
- else:
- return log_comparison_results_for_app(app, result_per_shrinker, options, worker_id=worker_id)
+ if options.print_dexsegments:
+ log_segments_for_app(app,
+ result_per_shrinker,
+ options,
+ worker_id=worker_id)
+ return False
+ else:
+ return log_comparison_results_for_app(app,
+ result_per_shrinker,
+ options,
+ worker_id=worker_id)
def log_segments_for_app(app, result_per_shrinker, options, worker_id):
- for shrinker in SHRINKERS:
- if shrinker not in result_per_shrinker:
- continue
- for result in result_per_shrinker.get(shrinker):
- benchmark_name = '{}-{}'.format(options.print_dexsegments, app.name)
- utils.print_dexsegments(
- benchmark_name, [result.get('output_jar')], worker_id=worker_id)
- duration = result.get('duration')
- print_thread(
- '%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration),
- worker_id)
- print_thread(
- '%s-Total(CodeSize): %s' % (benchmark_name, result.get('dex_size')),
- worker_id)
+ for shrinker in SHRINKERS:
+ if shrinker not in result_per_shrinker:
+ continue
+ for result in result_per_shrinker.get(shrinker):
+ benchmark_name = '{}-{}'.format(options.print_dexsegments, app.name)
+ utils.print_dexsegments(benchmark_name, [result.get('output_jar')],
+ worker_id=worker_id)
+ duration = result.get('duration')
+ print_thread(
+ '%s-Total(RunTimeRaw): %s ms' % (benchmark_name, duration),
+ worker_id)
+ print_thread(
+ '%s-Total(CodeSize): %s' %
+ (benchmark_name, result.get('dex_size')), worker_id)
def percentage_diff_as_string(before, after):
- if after < before:
- return '-' + str(round((1.0 - after / before) * 100)) + '%'
- else:
- return '+' + str(round((after - before) / before * 100)) + '%'
+ if after < before:
+ return '-' + str(round((1.0 - after / before) * 100)) + '%'
+ else:
+ return '+' + str(round((after - before) / before * 100)) + '%'
-def log_comparison_results_for_app(app, result_per_shrinker, options, worker_id):
- print_thread(app.name + ':', worker_id)
- app_error = False
- if result_per_shrinker.get('status', 'success') != 'success':
- error_message = result_per_shrinker.get('error_message')
- print_thread(' skipped ({})'.format(error_message), worker_id)
- return
+def log_comparison_results_for_app(app, result_per_shrinker, options,
+ worker_id):
+ print_thread(app.name + ':', worker_id)
+ app_error = False
+ if result_per_shrinker.get('status', 'success') != 'success':
+ error_message = result_per_shrinker.get('error_message')
+ print_thread(' skipped ({})'.format(error_message), worker_id)
+ return
- proguard_result = result_per_shrinker.get('pg', {})
- proguard_dex_size = float(proguard_result.get('dex_size', -1))
+ proguard_result = result_per_shrinker.get('pg', {})
+ proguard_dex_size = float(proguard_result.get('dex_size', -1))
- for shrinker in SHRINKERS:
- if shrinker not in result_per_shrinker:
- continue
- compilation_index = 1
- for result in result_per_shrinker.get(shrinker):
- build_status = result.get('build_status')
- if build_status != 'success' and build_status is not None:
- app_error = True
- warn(' {}-#{}: {}'.format(shrinker, compilation_index, build_status))
- continue
+ for shrinker in SHRINKERS:
+ if shrinker not in result_per_shrinker:
+ continue
+ compilation_index = 1
+ for result in result_per_shrinker.get(shrinker):
+ build_status = result.get('build_status')
+ if build_status != 'success' and build_status is not None:
+ app_error = True
+ warn(' {}-#{}: {}'.format(shrinker, compilation_index,
+ build_status))
+ continue
- if options.golem:
- print_thread(
- '%s(RunTimeRaw): %s ms' % (app.name, result.get('duration')),
- worker_id)
- print_thread(
- '%s(CodeSize): %s' % (app.name, result.get('dex_size')), worker_id)
- continue
+ if options.golem:
+ print_thread(
+ '%s(RunTimeRaw): %s ms' %
+ (app.name, result.get('duration')), worker_id)
+ print_thread(
+ '%s(CodeSize): %s' % (app.name, result.get('dex_size')),
+ worker_id)
+ continue
- print_thread(' {}-#{}:'.format(shrinker, compilation_index), worker_id)
- dex_size = result.get('dex_size')
- msg = ' dex size: {}'.format(dex_size)
- if options.print_runtimeraw:
- print_thread(
- ' run time raw: {} ms'.format(result.get('duration')), worker_id)
- if dex_size != proguard_dex_size and proguard_dex_size >= 0:
- msg = '{} ({}, {})'.format(
- msg, dex_size - proguard_dex_size,
- percentage_diff_as_string(proguard_dex_size, dex_size))
- success(msg) if dex_size < proguard_dex_size else warn(msg)
- else:
- print_thread(msg, worker_id)
+ print_thread(' {}-#{}:'.format(shrinker, compilation_index),
+ worker_id)
+ dex_size = result.get('dex_size')
+ msg = ' dex size: {}'.format(dex_size)
+ if options.print_runtimeraw:
+ print_thread(
+ ' run time raw: {} ms'.format(result.get('duration')),
+ worker_id)
+ if dex_size != proguard_dex_size and proguard_dex_size >= 0:
+ msg = '{} ({}, {})'.format(
+ msg, dex_size - proguard_dex_size,
+ percentage_diff_as_string(proguard_dex_size, dex_size))
+ success(msg) if dex_size < proguard_dex_size else warn(msg)
+ else:
+ print_thread(msg, worker_id)
- if options.monkey:
- monkey_status = result.get('monkey_status')
- if monkey_status != 'success':
- app_error = True
- warn(' monkey: {}'.format(monkey_status))
- else:
- success(' monkey: {}'.format(monkey_status))
+ if options.monkey:
+ monkey_status = result.get('monkey_status')
+ if monkey_status != 'success':
+ app_error = True
+ warn(' monkey: {}'.format(monkey_status))
+ else:
+ success(' monkey: {}'.format(monkey_status))
- if options.run_tests and 'instrumentation_test_status' in result:
- test_status = result.get('instrumentation_test_status')
- if test_status != 'success':
- warn(' instrumentation_tests: {}'.format(test_status))
- else:
- success(' instrumentation_tests: {}'.format(test_status))
+ if options.run_tests and 'instrumentation_test_status' in result:
+ test_status = result.get('instrumentation_test_status')
+ if test_status != 'success':
+ warn(' instrumentation_tests: {}'.format(test_status))
+ else:
+ success(' instrumentation_tests: {}'.format(test_status))
- recompilation_status = result.get('recompilation_status', '')
- if recompilation_status == 'failed':
- app_error = True
- warn(' recompilation {}-#{}: failed'.format(shrinker,
- compilation_index))
- continue
+ recompilation_status = result.get('recompilation_status', '')
+ if recompilation_status == 'failed':
+ app_error = True
+ warn(' recompilation {}-#{}: failed'.format(
+ shrinker, compilation_index))
+ continue
- compilation_index += 1
+ compilation_index += 1
- return app_error
+ return app_error
def parse_options(argv):
- result = argparse.ArgumentParser(description = 'Run/compile dump artifacts.')
- result.add_argument('--app',
- help='What app to run on',
- choices=[app.name for app in APPS],
- action='append')
- result.add_argument('--app-collection', '--app_collection',
- help='What app collection to run',
- choices=[collection.name for collection in
- APP_COLLECTIONS],
- action='append')
- result.add_argument('--app-logging-filter', '--app_logging_filter',
- help='The apps for which to turn on logging',
- action='append')
- result.add_argument('--bot',
- help='Running on bot, use third_party dependency.',
- default=False,
- action='store_true')
- result.add_argument('--generate-golem-config', '--generate_golem_config',
- help='Generate a new config for golem.',
- default=False,
- action='store_true')
- result.add_argument('--debug-agent',
- help='Enable Java debug agent and suspend compilation '
- '(default disabled)',
- default=False,
- action='store_true')
- result.add_argument('--disable-assertions', '--disable_assertions', '-da',
- help='Disable Java assertions when running the compiler '
- '(default enabled)',
- default=False,
- action='store_true')
- result.add_argument('--emulator-id', '--emulator_id',
- help='Id of the emulator to use',
- default='emulator-5554')
- result.add_argument('--golem',
- help='Running on golem, do not download',
- default=False,
- action='store_true')
- result.add_argument('--hash',
- help='The commit of R8 to use')
- result.add_argument('--internal',
- help='Run internal apps if set, otherwise run opensource',
- default=False,
- action='store_true')
- result.add_argument('--keystore',
- help='Path to app.keystore',
- default=os.path.join(utils.TOOLS_DIR, 'debug.keystore'))
- result.add_argument('--keystore-password', '--keystore_password',
- help='Password for app.keystore',
- default='android')
- result.add_argument('--minify',
- help='Force enable/disable minification' +
- ' (defaults to app proguard config)',
- choices=['default', 'force-enable', 'force-disable'],
- default='default')
- result.add_argument('--monkey',
- help='Whether to install and run app(s) with monkey',
- default=False,
- action='store_true')
- result.add_argument('--monkey-events', '--monkey_events',
- help='Number of events that the monkey should trigger',
- default=250,
- type=int)
- result.add_argument('--no-build', '--no_build',
- help='Run without building first (only when using ToT)',
- default=False,
- action='store_true')
- result.add_argument('--no-logging', '--no_logging',
- help='Disable logging except for errors',
- default=False,
- action='store_true')
- result.add_argument('--optimize',
- help='Force enable/disable optimizations' +
- ' (defaults to app proguard config)',
- choices=['default', 'force-enable', 'force-disable'],
- default='default')
- result.add_argument('--print-times',
- help='Print timing information from r8',
- default=False,
- action='store_true')
- result.add_argument('--print-dexsegments',
- metavar='BENCHMARKNAME',
- help='Print the sizes of individual dex segments as ' +
- '\'<BENCHMARKNAME>-<APP>-<segment>(CodeSize): '
- '<bytes>\'')
- result.add_argument('--print-runtimeraw',
- metavar='BENCHMARKNAME',
- help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
- ' <elapsed> ms\' at the end where <elapsed> is' +
- ' the elapsed time in milliseconds.')
- result.add_argument('--quiet',
- help='Disable verbose logging',
- default=False,
- action='store_true')
- result.add_argument('--r8-compilation-steps', '--r8_compilation_steps',
- help='Number of times R8 should be run on each app',
- default=2,
- type=int)
- result.add_argument('--r8-flags', '--r8_flags',
- help='Additional option(s) for the compiler.')
- result.add_argument('--run-tests', '--run_tests',
- help='Whether to run instrumentation tests',
- default=False,
- action='store_true')
- result.add_argument('--shrink',
- help='Force enable/disable shrinking' +
- ' (defaults to app proguard config)',
- choices=['default', 'force-enable', 'force-disable'],
- default='default')
- result.add_argument('--sign-apks', '--sign_apks',
- help='Whether the APKs should be signed',
- default=False,
- action='store_true')
- result.add_argument('--shrinker',
- help='The shrinkers to use (by default, all are run)',
- action='append')
- result.add_argument('--temp',
- help='A directory to use for temporaries and outputs.',
- default=None)
- result.add_argument('--version',
- default='main',
- help='The version of R8 to use (e.g., 1.4.51)')
- result.add_argument('--workers',
- help='Number of workers to use',
- default=1,
- type=int)
- (options, args) = result.parse_known_args(argv)
+ result = argparse.ArgumentParser(description='Run/compile dump artifacts.')
+ result.add_argument('--app',
+ help='What app to run on',
+ choices=[app.name for app in APPS],
+ action='append')
+ result.add_argument(
+ '--app-collection',
+ '--app_collection',
+ help='What app collection to run',
+ choices=[collection.name for collection in APP_COLLECTIONS],
+ action='append')
+ result.add_argument('--app-logging-filter',
+ '--app_logging_filter',
+ help='The apps for which to turn on logging',
+ action='append')
+ result.add_argument('--bot',
+ help='Running on bot, use third_party dependency.',
+ default=False,
+ action='store_true')
+ result.add_argument('--generate-golem-config',
+ '--generate_golem_config',
+ help='Generate a new config for golem.',
+ default=False,
+ action='store_true')
+ result.add_argument('--debug-agent',
+ help='Enable Java debug agent and suspend compilation '
+ '(default disabled)',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--disable-assertions',
+ '--disable_assertions',
+ '-da',
+ help='Disable Java assertions when running the compiler '
+ '(default enabled)',
+ default=False,
+ action='store_true')
+ result.add_argument('--emulator-id',
+ '--emulator_id',
+ help='Id of the emulator to use',
+ default='emulator-5554')
+ result.add_argument('--golem',
+ help='Running on golem, do not download',
+ default=False,
+ action='store_true')
+ result.add_argument('--hash', help='The commit of R8 to use')
+ result.add_argument(
+ '--internal',
+ help='Run internal apps if set, otherwise run opensource',
+ default=False,
+ action='store_true')
+ result.add_argument('--keystore',
+ help='Path to app.keystore',
+ default=os.path.join(utils.TOOLS_DIR, 'debug.keystore'))
+ result.add_argument('--keystore-password',
+ '--keystore_password',
+ help='Password for app.keystore',
+ default='android')
+ result.add_argument('--minify',
+ help='Force enable/disable minification' +
+ ' (defaults to app proguard config)',
+ choices=['default', 'force-enable', 'force-disable'],
+ default='default')
+ result.add_argument('--monkey',
+ help='Whether to install and run app(s) with monkey',
+ default=False,
+ action='store_true')
+ result.add_argument('--monkey-events',
+ '--monkey_events',
+ help='Number of events that the monkey should trigger',
+ default=250,
+ type=int)
+ result.add_argument('--no-build',
+ '--no_build',
+ help='Run without building first (only when using ToT)',
+ default=False,
+ action='store_true')
+ result.add_argument('--no-logging',
+ '--no_logging',
+ help='Disable logging except for errors',
+ default=False,
+ action='store_true')
+ result.add_argument('--optimize',
+ help='Force enable/disable optimizations' +
+ ' (defaults to app proguard config)',
+ choices=['default', 'force-enable', 'force-disable'],
+ default='default')
+ result.add_argument('--print-times',
+ help='Print timing information from r8',
+ default=False,
+ action='store_true')
+ result.add_argument('--print-dexsegments',
+ metavar='BENCHMARKNAME',
+ help='Print the sizes of individual dex segments as ' +
+ '\'<BENCHMARKNAME>-<APP>-<segment>(CodeSize): '
+ '<bytes>\'')
+ result.add_argument('--print-runtimeraw',
+ metavar='BENCHMARKNAME',
+ help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
+ ' <elapsed> ms\' at the end where <elapsed> is' +
+ ' the elapsed time in milliseconds.')
+ result.add_argument('--quiet',
+ help='Disable verbose logging',
+ default=False,
+ action='store_true')
+ result.add_argument('--r8-compilation-steps',
+ '--r8_compilation_steps',
+ help='Number of times R8 should be run on each app',
+ default=2,
+ type=int)
+ result.add_argument('--r8-flags',
+ '--r8_flags',
+ help='Additional option(s) for the compiler.')
+ result.add_argument('--run-tests',
+ '--run_tests',
+ help='Whether to run instrumentation tests',
+ default=False,
+ action='store_true')
+ result.add_argument('--shrink',
+ help='Force enable/disable shrinking' +
+ ' (defaults to app proguard config)',
+ choices=['default', 'force-enable', 'force-disable'],
+ default='default')
+ result.add_argument('--sign-apks',
+ '--sign_apks',
+ help='Whether the APKs should be signed',
+ default=False,
+ action='store_true')
+ result.add_argument('--shrinker',
+ help='The shrinkers to use (by default, all are run)',
+ action='append')
+ result.add_argument('--temp',
+ help='A directory to use for temporaries and outputs.',
+ default=None)
+ result.add_argument('--version',
+ default='main',
+ help='The version of R8 to use (e.g., 1.4.51)')
+ result.add_argument('--workers',
+ help='Number of workers to use',
+ default=1,
+ type=int)
+ (options, args) = result.parse_known_args(argv)
- if options.app or options.app_collection:
- if not options.app:
- options.app = []
- if not options.app_collection:
- options.app_collection = []
- options.apps = [
- app
- for app in APPS
- if app.name in options.app
- or any(collection in options.app_collection
- for collection in app.collections)]
- del options.app
- del options.app_collection
- else:
- options.apps = [app for app in APPS if app.internal == options.internal]
+ if options.app or options.app_collection:
+ if not options.app:
+ options.app = []
+ if not options.app_collection:
+ options.app_collection = []
+ options.apps = [
+ app for app in APPS if app.name in options.app or any(
+ collection in options.app_collection
+ for collection in app.collections)
+ ]
+ del options.app
+ del options.app_collection
+ else:
+ options.apps = [app for app in APPS if app.internal == options.internal]
- if options.app_logging_filter:
- for app_name in options.app_logging_filter:
- assert any(app.name == app_name for app in options.apps)
- if options.shrinker:
- for shrinker in options.shrinker:
- assert shrinker in SHRINKERS, (
- 'Shrinker must be one of %s' % ', '.join(SHRINKERS))
- else:
- options.shrinker = [shrinker for shrinker in SHRINKERS]
+ if options.app_logging_filter:
+ for app_name in options.app_logging_filter:
+ assert any(app.name == app_name for app in options.apps)
+ if options.shrinker:
+ for shrinker in options.shrinker:
+ assert shrinker in SHRINKERS, ('Shrinker must be one of %s' %
+ ', '.join(SHRINKERS))
+ else:
+ options.shrinker = [shrinker for shrinker in SHRINKERS]
- if options.hash or version_is_built_jar(options.version):
- # No need to build R8 if a specific version should be used.
- options.no_build = True
- if 'r8-nolib' in options.shrinker:
- warn('Skipping shrinker r8-nolib because a specific version '
- + 'of r8 was specified')
- options.shrinker.remove('r8-nolib')
- if 'r8-nolib-full' in options.shrinker:
- warn('Skipping shrinker r8-nolib-full because a specific version '
- + 'of r8 was specified')
- options.shrinker.remove('r8-nolib-full')
- return (options, args)
+ if options.hash or version_is_built_jar(options.version):
+ # No need to build R8 if a specific version should be used.
+ options.no_build = True
+ if 'r8-nolib' in options.shrinker:
+ warn('Skipping shrinker r8-nolib because a specific version ' +
+ 'of r8 was specified')
+ options.shrinker.remove('r8-nolib')
+ if 'r8-nolib-full' in options.shrinker:
+ warn('Skipping shrinker r8-nolib-full because a specific version ' +
+ 'of r8 was specified')
+ options.shrinker.remove('r8-nolib-full')
+ return (options, args)
def print_indented(s, indent):
- print(' ' * indent + s)
+ print(' ' * indent + s)
def get_sha256(gz_file):
- with open(gz_file, 'rb') as f:
- bytes = f.read() # read entire file as bytes
- return hashlib.sha256(bytes).hexdigest();
+ with open(gz_file, 'rb') as f:
+ bytes = f.read() # read entire file as bytes
+ return hashlib.sha256(bytes).hexdigest()
def get_sha_from_file(sha_file):
- with open(sha_file, 'r') as f:
- return f.readlines()[0]
+ with open(sha_file, 'r') as f:
+ return f.readlines()[0]
def print_golem_config(options):
- print('// AUTOGENERATED FILE from tools/run_on_app_dump.py in R8 repo')
- print('part of r8_config;')
- print('')
- print('final Suite dumpsSuite = Suite("OpenSourceAppDumps");')
- print('')
- print('createOpenSourceAppBenchmarks() {')
- print_indented('final cpus = ["Lenovo M90"];', 2)
- print_indented('final targetsCompat = ["R8"];', 2)
- print_indented('final targetsFull = ["R8-full-minify-optimize-shrink"];', 2)
- # Avoid calculating this for every app
- jdk_gz = jdk.GetJdkHome() + '.tar.gz'
- add_golem_resource(2, jdk_gz, 'openjdk')
- for app in options.apps:
- if app.folder and not app.internal:
- indentation = 2;
- print_indented('{', indentation)
- indentation = 4
- print_indented('final name = "%s";' % app.name, indentation)
- print_indented('final benchmark =', indentation)
- print_indented(
- 'StandardBenchmark(name, [Metric.RunTimeRaw, Metric.CodeSize]);',
- indentation + 4)
- if app.golem_duration != None:
- print_indented(
- 'final timeout = const Duration(seconds: %s);' % app.golem_duration,
- indentation)
- print_indented(
- 'ExecutionManagement.addTimeoutConstraint'
- '(timeout, benchmark: benchmark);', indentation)
- app_gz = os.path.join(utils.OPENSOURCE_DUMPS_DIR, app.folder + '.tar.gz')
- name = 'appResource'
- add_golem_resource(indentation, app_gz, name)
- print_golem_config_target('Compat', 'r8', app, indentation)
- print_golem_config_target(
- 'Full',
- 'r8-full',
- app,
- indentation,
- minify='force-enable',
- optimize='force-enable',
- shrink='force-enable')
- print_indented('dumpsSuite.addBenchmark(name);', indentation)
- indentation = 2
- print_indented('}', indentation)
- print('}')
+ print('// AUTOGENERATED FILE from tools/run_on_app_dump.py in R8 repo')
+ print('part of r8_config;')
+ print('')
+ print('final Suite dumpsSuite = Suite("OpenSourceAppDumps");')
+ print('')
+ print('createOpenSourceAppBenchmarks() {')
+ print_indented('final cpus = ["Lenovo M90"];', 2)
+ print_indented('final targetsCompat = ["R8"];', 2)
+ print_indented('final targetsFull = ["R8-full-minify-optimize-shrink"];', 2)
+ # Avoid calculating this for every app
+ jdk_gz = jdk.GetJdkHome() + '.tar.gz'
+ add_golem_resource(2, jdk_gz, 'openjdk')
+ for app in options.apps:
+ if app.folder and not app.internal:
+ indentation = 2
+ print_indented('{', indentation)
+ indentation = 4
+ print_indented('final name = "%s";' % app.name, indentation)
+ print_indented('final benchmark =', indentation)
+ print_indented(
+ 'StandardBenchmark(name, [Metric.RunTimeRaw, Metric.CodeSize]);',
+ indentation + 4)
+ if app.golem_duration != None:
+ print_indented(
+ 'final timeout = const Duration(seconds: %s);' %
+ app.golem_duration, indentation)
+ print_indented(
+ 'ExecutionManagement.addTimeoutConstraint'
+ '(timeout, benchmark: benchmark);', indentation)
+ app_gz = os.path.join(utils.OPENSOURCE_DUMPS_DIR,
+ app.folder + '.tar.gz')
+ name = 'appResource'
+ add_golem_resource(indentation, app_gz, name)
+ print_golem_config_target('Compat', 'r8', app, indentation)
+ print_golem_config_target('Full',
+ 'r8-full',
+ app,
+ indentation,
+ minify='force-enable',
+ optimize='force-enable',
+ shrink='force-enable')
+ print_indented('dumpsSuite.addBenchmark(name);', indentation)
+ indentation = 2
+ print_indented('}', indentation)
+ print('}')
-def print_golem_config_target(
- target, shrinker, app, indentation,
- minify='default', optimize='default', shrink='default'):
- options="options" + target
- print_indented(
- 'final %s = benchmark.addTargets(noImplementation, targets%s);'
- % (options, target),
- indentation)
- print_indented('%s.cpus = cpus;' % options, indentation)
- print_indented('%s.isScript = true;' % options, indentation)
- print_indented('%s.fromRevision = 9700;' % options, indentation);
- print_indented('%s.mainFile = "tools/run_on_app_dump.py "' % options,
- indentation)
- print_indented('"--golem --disable-assertions --quiet --shrinker %s --app %s "'
- % (shrinker, app.name),
- indentation + 4)
- print_indented('"--minify %s --optimize %s --shrink %s";'
- % (minify, optimize, shrink),
- indentation + 4)
- print_indented('%s.resources.add(appResource);' % options, indentation)
- print_indented('%s.resources.add(openjdk);' % options, indentation)
+
+def print_golem_config_target(target,
+ shrinker,
+ app,
+ indentation,
+ minify='default',
+ optimize='default',
+ shrink='default'):
+ options = "options" + target
+ print_indented(
+ 'final %s = benchmark.addTargets(noImplementation, targets%s);' %
+ (options, target), indentation)
+ print_indented('%s.cpus = cpus;' % options, indentation)
+ print_indented('%s.isScript = true;' % options, indentation)
+ print_indented('%s.fromRevision = 9700;' % options, indentation)
+ print_indented('%s.mainFile = "tools/run_on_app_dump.py "' % options,
+ indentation)
+ print_indented(
+ '"--golem --disable-assertions --quiet --shrinker %s --app %s "' %
+ (shrinker, app.name), indentation + 4)
+ print_indented(
+ '"--minify %s --optimize %s --shrink %s";' % (minify, optimize, shrink),
+ indentation + 4)
+ print_indented('%s.resources.add(appResource);' % options, indentation)
+ print_indented('%s.resources.add(openjdk);' % options, indentation)
+
def add_golem_resource(indentation, gz, name, sha256=None):
- sha = gz + '.sha1'
- if not sha256:
- # Golem uses a sha256 of the file in the cache, and you need to specify that.
- download_sha(sha, False, quiet=True)
- sha256 = get_sha256(gz)
- sha = get_sha_from_file(sha)
- print_indented('final %s = BenchmarkResource("",' % name, indentation)
- print_indented('type: BenchmarkResourceType.storage,', indentation + 4)
- print_indented('uri: "gs://r8-deps/%s",' % sha, indentation + 4)
- # Make dart formatter happy.
- if indentation > 2:
- print_indented('hash:', indentation + 4)
- print_indented('"%s",' % sha256, indentation + 8)
- else:
- print_indented('hash: "%s",' % sha256, indentation + 4)
- print_indented('extract: "gz");', indentation + 4);
+ sha = gz + '.sha1'
+ if not sha256:
+ # Golem uses a sha256 of the file in the cache, and you need to specify that.
+ download_sha(sha, False, quiet=True)
+ sha256 = get_sha256(gz)
+ sha = get_sha_from_file(sha)
+ print_indented('final %s = BenchmarkResource("",' % name, indentation)
+ print_indented('type: BenchmarkResourceType.storage,', indentation + 4)
+ print_indented('uri: "gs://r8-deps/%s",' % sha, indentation + 4)
+ # Make dart formatter happy.
+ if indentation > 2:
+ print_indented('hash:', indentation + 4)
+ print_indented('"%s",' % sha256, indentation + 8)
+ else:
+ print_indented('hash: "%s",' % sha256, indentation + 4)
+ print_indented('extract: "gz");', indentation + 4)
+
def main(argv):
- (options, args) = parse_options(argv)
+ (options, args) = parse_options(argv)
- if options.bot:
- options.no_logging = True
- options.shrinker = ['r8', 'r8-full']
- print(options.shrinker)
+ if options.bot:
+ options.no_logging = True
+ options.shrinker = ['r8', 'r8-full']
+ print(options.shrinker)
- if options.golem:
- options.disable_assertions = True
- options.no_build = True
- options.r8_compilation_steps = 1
- options.quiet = True
- options.no_logging = True
+ if options.golem:
+ options.disable_assertions = True
+ options.no_build = True
+ options.r8_compilation_steps = 1
+ options.quiet = True
+ options.no_logging = True
- if options.generate_golem_config:
- print_golem_config(options)
- return 0
+ if options.generate_golem_config:
+ print_golem_config(options)
+ return 0
- with utils.TempDir() as temp_dir:
- if options.temp:
- temp_dir = options.temp
- os.makedirs(temp_dir, exist_ok=True)
- if options.hash:
- # Download r8-<hash>.jar from
- # https://storage.googleapis.com/r8-releases/raw/.
- target = 'r8-{}.jar'.format(options.hash)
- update_prebuilds_in_android.download_hash(
- temp_dir, 'com/android/tools/r8/' + options.hash, target)
- as_utils.MoveFile(
- os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
- quiet=options.quiet)
- elif version_is_built_jar(options.version):
- # Download r8-<version>.jar from
- # https://storage.googleapis.com/r8-releases/raw/.
- target = 'r8-{}.jar'.format(options.version)
- update_prebuilds_in_android.download_version(
- temp_dir, 'com/android/tools/r8/' + options.version, target)
- as_utils.MoveFile(
- os.path.join(temp_dir, target), os.path.join(temp_dir, 'r8lib.jar'),
- quiet=options.quiet)
- elif options.version == 'main':
- if not options.no_build:
- gradle.RunGradle([utils.GRADLE_TASK_RETRACE, utils.GRADLE_TASK_R8,
- '-Pno_internal'])
- build_r8lib = False
- for shrinker in options.shrinker:
- if is_minified_r8(shrinker):
- build_r8lib = True
- if build_r8lib:
- gradle.RunGradle([utils.GRADLE_TASK_R8LIB, '-Pno_internal'])
- # Make a copy of r8.jar and r8lib.jar such that they stay the same for
- # the entire execution of this script.
- if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
- assert os.path.isfile(utils.R8_JAR), 'Cannot build without r8.jar'
- shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
- if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
- assert os.path.isfile(utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
- shutil.copyfile(utils.R8LIB_JAR, os.path.join(temp_dir, 'r8lib.jar'))
+ with utils.TempDir() as temp_dir:
+ if options.temp:
+ temp_dir = options.temp
+ os.makedirs(temp_dir, exist_ok=True)
+ if options.hash:
+ # Download r8-<hash>.jar from
+ # https://storage.googleapis.com/r8-releases/raw/.
+ target = 'r8-{}.jar'.format(options.hash)
+ update_prebuilds_in_android.download_hash(
+ temp_dir, 'com/android/tools/r8/' + options.hash, target)
+ as_utils.MoveFile(os.path.join(temp_dir, target),
+ os.path.join(temp_dir, 'r8lib.jar'),
+ quiet=options.quiet)
+ elif version_is_built_jar(options.version):
+ # Download r8-<version>.jar from
+ # https://storage.googleapis.com/r8-releases/raw/.
+ target = 'r8-{}.jar'.format(options.version)
+ update_prebuilds_in_android.download_version(
+ temp_dir, 'com/android/tools/r8/' + options.version, target)
+ as_utils.MoveFile(os.path.join(temp_dir, target),
+ os.path.join(temp_dir, 'r8lib.jar'),
+ quiet=options.quiet)
+ elif options.version == 'main':
+ if not options.no_build:
+ gradle.RunGradle([
+ utils.GRADLE_TASK_RETRACE, utils.GRADLE_TASK_R8,
+ '-Pno_internal'
+ ])
+ build_r8lib = False
+ for shrinker in options.shrinker:
+ if is_minified_r8(shrinker):
+ build_r8lib = True
+ if build_r8lib:
+ gradle.RunGradle([utils.GRADLE_TASK_R8LIB, '-Pno_internal'])
+ # Make a copy of r8.jar and r8lib.jar such that they stay the same for
+ # the entire execution of this script.
+ if 'r8-nolib' in options.shrinker or 'r8-nolib-full' in options.shrinker:
+ assert os.path.isfile(
+ utils.R8_JAR), 'Cannot build without r8.jar'
+ shutil.copyfile(utils.R8_JAR, os.path.join(temp_dir, 'r8.jar'))
+ if 'r8' in options.shrinker or 'r8-full' in options.shrinker:
+ assert os.path.isfile(
+ utils.R8LIB_JAR), 'Cannot build without r8lib.jar'
+ shutil.copyfile(utils.R8LIB_JAR,
+ os.path.join(temp_dir, 'r8lib.jar'))
- jobs = []
- result_per_shrinker_per_app = []
- for app in options.apps:
- if app.skip:
- continue
- result = {}
- result_per_shrinker_per_app.append((app, result))
- jobs.append(create_job(app, options, result, temp_dir))
- thread_utils.run_in_parallel(
- jobs,
- number_of_workers=options.workers,
- stop_on_first_failure=False)
- errors = log_results_for_apps(result_per_shrinker_per_app, options)
- if errors > 0:
- dest = 'gs://r8-test-results/r8-libs/' + str(int(time.time()))
- utils.upload_file_to_cloud_storage(os.path.join(temp_dir, 'r8lib.jar'), dest)
- print('R8lib saved to %s' % dest)
- return errors
+ jobs = []
+ result_per_shrinker_per_app = []
+ for app in options.apps:
+ if app.skip:
+ continue
+ result = {}
+ result_per_shrinker_per_app.append((app, result))
+ jobs.append(create_job(app, options, result, temp_dir))
+ thread_utils.run_in_parallel(jobs,
+ number_of_workers=options.workers,
+ stop_on_first_failure=False)
+ errors = log_results_for_apps(result_per_shrinker_per_app, options)
+ if errors > 0:
+ dest = 'gs://r8-test-results/r8-libs/' + str(int(time.time()))
+ utils.upload_file_to_cloud_storage(
+ os.path.join(temp_dir, 'r8lib.jar'), dest)
+ print('R8lib saved to %s' % dest)
+ return errors
+
def create_job(app, options, result, temp_dir):
- return lambda worker_id: run_job(
- app, options, result, temp_dir, worker_id)
+ return lambda worker_id: run_job(app, options, result, temp_dir, worker_id)
+
def run_job(app, options, result, temp_dir, worker_id):
- job_temp_dir = os.path.join(temp_dir, str(worker_id or 0))
- os.makedirs(job_temp_dir, exist_ok=True)
- result.update(get_results_for_app(app, options, job_temp_dir, worker_id))
- return 0
+ job_temp_dir = os.path.join(temp_dir, str(worker_id or 0))
+ os.makedirs(job_temp_dir, exist_ok=True)
+ result.update(get_results_for_app(app, options, job_temp_dir, worker_id))
+ return 0
+
def success(message):
- CGREEN = '\033[32m'
- CEND = '\033[0m'
- print(CGREEN + message + CEND)
+ CGREEN = '\033[32m'
+ CEND = '\033[0m'
+ print(CGREEN + message + CEND)
def warn(message):
- CRED = '\033[91m'
- CEND = '\033[0m'
- print(CRED + message + CEND)
+ CRED = '\033[91m'
+ CEND = '\033[0m'
+ print(CRED + message + CEND)
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/sanitize_libraries.py b/tools/sanitize_libraries.py
index bc61d50..671d3a9 100755
--- a/tools/sanitize_libraries.py
+++ b/tools/sanitize_libraries.py
@@ -7,102 +7,103 @@
import sys
import zipfile
+
# Proguard lookup program classes before library classes. In R8 this is
# not the behaviour (it used to be) R8 will check library classes before
# program classes. Some apps have duplicate classes in the library and program.
# To make these apps work with R8 simulate program classes before library
# classes by creating a new library jar which have all the provided library
# classes which are not also in program classes.
-def SanitizeLibrariesInPgconf(
- sanitized_lib_path,
- sanitized_pgconf_path,
- pgconfs,
- injars = None,
- libraryjars = None):
- with open(sanitized_pgconf_path, 'w') as sanitized_pgconf:
- injars = [] if injars is None else injars
- libraryjars = [] if libraryjars is None else libraryjars
- for pgconf in pgconfs:
- pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
- first_library_jar = True
- with open(pgconf) as pgconf_file:
- for line in pgconf_file:
- trimmed = line.strip()
- if trimmed.startswith('-injars'):
- # Collect -injars and leave them in the configuration.
- injar = os.path.join(
- pgconf_dirname, trimmed[len('-injars'):].strip())
- injars.append(injar)
- sanitized_pgconf.write('-injars {}\n'.format(injar))
- elif trimmed.startswith('-libraryjars'):
- # Collect -libraryjars and replace them with the sanitized library.
- libraryjar = os.path.join(
- pgconf_dirname, trimmed[len('-libraryjars'):].strip())
- libraryjars.append(libraryjar)
- if first_library_jar:
- sanitized_pgconf.write(
- '-libraryjars {}\n'.format(sanitized_lib_path))
- first_library_jar = False
- sanitized_pgconf.write('# {}'.format(line))
- else:
- sanitized_pgconf.write(line)
+def SanitizeLibrariesInPgconf(sanitized_lib_path,
+ sanitized_pgconf_path,
+ pgconfs,
+ injars=None,
+ libraryjars=None):
+ with open(sanitized_pgconf_path, 'w') as sanitized_pgconf:
+ injars = [] if injars is None else injars
+ libraryjars = [] if libraryjars is None else libraryjars
+ for pgconf in pgconfs:
+ pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
+ first_library_jar = True
+ with open(pgconf) as pgconf_file:
+ for line in pgconf_file:
+ trimmed = line.strip()
+ if trimmed.startswith('-injars'):
+ # Collect -injars and leave them in the configuration.
+ injar = os.path.join(pgconf_dirname,
+ trimmed[len('-injars'):].strip())
+ injars.append(injar)
+ sanitized_pgconf.write('-injars {}\n'.format(injar))
+ elif trimmed.startswith('-libraryjars'):
+ # Collect -libraryjars and replace them with the sanitized library.
+ libraryjar = os.path.join(
+ pgconf_dirname,
+ trimmed[len('-libraryjars'):].strip())
+ libraryjars.append(libraryjar)
+ if first_library_jar:
+ sanitized_pgconf.write(
+ '-libraryjars {}\n'.format(sanitized_lib_path))
+ first_library_jar = False
+ sanitized_pgconf.write('# {}'.format(line))
+ else:
+ sanitized_pgconf.write(line)
- SanitizeLibraries(sanitized_lib_path, libraryjars, injars)
+ SanitizeLibraries(sanitized_lib_path, libraryjars, injars)
def SanitizeLibraries(sanitized_lib_path, libraryjars, injars):
- program_entries = set()
- library_entries = set()
+ program_entries = set()
+ library_entries = set()
- for injar in injars:
- with zipfile.ZipFile(injar, 'r') as injar_zf:
- for zipinfo in injar_zf.infolist():
- program_entries.add(zipinfo.filename)
+ for injar in injars:
+ with zipfile.ZipFile(injar, 'r') as injar_zf:
+ for zipinfo in injar_zf.infolist():
+ program_entries.add(zipinfo.filename)
- with zipfile.ZipFile(sanitized_lib_path, 'w') as output_zf:
- for libraryjar in libraryjars:
- with zipfile.ZipFile(libraryjar, 'r') as input_zf:
- for zipinfo in input_zf.infolist():
- if (not zipinfo.filename in program_entries
- and not zipinfo.filename in library_entries):
- library_entries.add(zipinfo.filename)
- output_zf.writestr(zipinfo, input_zf.read(zipinfo))
+ with zipfile.ZipFile(sanitized_lib_path, 'w') as output_zf:
+ for libraryjar in libraryjars:
+ with zipfile.ZipFile(libraryjar, 'r') as input_zf:
+ for zipinfo in input_zf.infolist():
+ if (not zipinfo.filename in program_entries and
+ not zipinfo.filename in library_entries):
+ library_entries.add(zipinfo.filename)
+ output_zf.writestr(zipinfo, input_zf.read(zipinfo))
def usage(argv, error):
- print(error)
- print("Usage: sanitize_libraries.py <sanitized_lib> <sanitized_pgconf> ("
- + "--injar <existing_injar>"
- + "|--libraryjar <existing_library_jar>"
- + "|--pgconf <existing_pgconf>)+")
- return 1
+ print(error)
+ print("Usage: sanitize_libraries.py <sanitized_lib> <sanitized_pgconf> (" +
+ "--injar <existing_injar>" + "|--libraryjar <existing_library_jar>" +
+ "|--pgconf <existing_pgconf>)+")
+ return 1
def main(argv):
- if (len(argv) < 4):
- return usage(argv, "Wrong number of arguments!")
- pgconfs = []
- injars = []
- libraryjars = []
- i = 2
- while i < len(argv):
- directive = argv[i]
- if directive not in ['--pgconf', '--injar', '--libraryjar']:
- return usage(
- argv,
- 'Unexpected argument, expected one of --pgconf, --injar, and '
- + '--libraryjar.')
- if i + 1 >= len(argv):
- return usage(argv, 'Expected argument after ' + directive + '.')
- file = argv[i + 1]
- if directive == '--pgconf':
- pgconfs.append(file)
- elif directive == '--injar':
- injars.append(file)
- elif directive == '--libraryjar':
- libraryjars.append(file)
- i = i + 2
- SanitizeLibrariesInPgconf(argv[0], argv[1], pgconfs, injars, libraryjars)
+ if (len(argv) < 4):
+ return usage(argv, "Wrong number of arguments!")
+ pgconfs = []
+ injars = []
+ libraryjars = []
+ i = 2
+ while i < len(argv):
+ directive = argv[i]
+ if directive not in ['--pgconf', '--injar', '--libraryjar']:
+ return usage(
+ argv,
+ 'Unexpected argument, expected one of --pgconf, --injar, and ' +
+ '--libraryjar.')
+ if i + 1 >= len(argv):
+ return usage(argv, 'Expected argument after ' + directive + '.')
+ file = argv[i + 1]
+ if directive == '--pgconf':
+ pgconfs.append(file)
+ elif directive == '--injar':
+ injars.append(file)
+ elif directive == '--libraryjar':
+ libraryjars.append(file)
+ i = i + 2
+ SanitizeLibrariesInPgconf(argv[0], argv[1], pgconfs, injars, libraryjars)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/adb_utils.py b/tools/startup/adb_utils.py
index d4f70a1..6ce19ab 100755
--- a/tools/startup/adb_utils.py
+++ b/tools/startup/adb_utils.py
@@ -17,458 +17,511 @@
import profile_utils
import utils
-DEVNULL=subprocess.DEVNULL
+DEVNULL = subprocess.DEVNULL
+
class ProcessReader(threading.Thread):
- def __init__(self, process):
- threading.Thread.__init__(self)
- self.lines = []
- self.process = process
+ def __init__(self, process):
+ threading.Thread.__init__(self)
+ self.lines = []
+ self.process = process
- def run(self):
- for line in self.process.stdout:
- line = line.decode('utf-8').strip()
- self.lines.append(line)
+ def run(self):
+ for line in self.process.stdout:
+ line = line.decode('utf-8').strip()
+ self.lines.append(line)
- def stop(self):
- self.process.kill()
+ def stop(self):
+ self.process.kill()
+
class ScreenState(Enum):
- OFF_LOCKED = 1,
- OFF_UNLOCKED = 2
- ON_LOCKED = 3
- ON_UNLOCKED = 4
+ OFF_LOCKED = 1,
+ OFF_UNLOCKED = 2
+ ON_LOCKED = 3
+ ON_UNLOCKED = 4
- def is_off(self):
- return self == ScreenState.OFF_LOCKED or self == ScreenState.OFF_UNLOCKED
+ def is_off(self):
+ return self == ScreenState.OFF_LOCKED or self == ScreenState.OFF_UNLOCKED
- def is_on(self):
- return self == ScreenState.ON_LOCKED or self == ScreenState.ON_UNLOCKED
+ def is_on(self):
+ return self == ScreenState.ON_LOCKED or self == ScreenState.ON_UNLOCKED
- def is_on_and_locked(self):
- return self == ScreenState.ON_LOCKED
+ def is_on_and_locked(self):
+ return self == ScreenState.ON_LOCKED
- def is_on_and_unlocked(self):
- return self == ScreenState.ON_UNLOCKED
+ def is_on_and_unlocked(self):
+ return self == ScreenState.ON_UNLOCKED
+
def broadcast(action, component, device_id=None):
- print('Sending broadcast %s' % action)
- cmd = create_adb_cmd('shell am broadcast -a %s %s' % (action, component), device_id)
- return subprocess.check_output(cmd).decode('utf-8').strip().splitlines()
+ print('Sending broadcast %s' % action)
+ cmd = create_adb_cmd('shell am broadcast -a %s %s' % (action, component),
+ device_id)
+ return subprocess.check_output(cmd).decode('utf-8').strip().splitlines()
+
def build_apks_from_bundle(bundle, output, overwrite=False):
- print('Building %s' % bundle)
- cmd = [
- 'java', '-jar', utils.BUNDLETOOL_JAR,
- 'build-apks',
- '--bundle=%s' % bundle,
- '--output=%s' % output]
- if overwrite:
- cmd.append('--overwrite')
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ print('Building %s' % bundle)
+ cmd = [
+ 'java', '-jar', utils.BUNDLETOOL_JAR, 'build-apks',
+ '--bundle=%s' % bundle,
+ '--output=%s' % output
+ ]
+ if overwrite:
+ cmd.append('--overwrite')
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
def capture_screen(target, device_id=None):
- print('Taking screenshot to %s' % target)
- tmp = '/sdcard/screencap.png'
- cmd = create_adb_cmd('shell screencap -p %s' % tmp, device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
- pull(tmp, target, device_id)
+ print('Taking screenshot to %s' % target)
+ tmp = '/sdcard/screencap.png'
+ cmd = create_adb_cmd('shell screencap -p %s' % tmp, device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ pull(tmp, target, device_id)
+
def create_adb_cmd(arguments, device_id=None):
- assert isinstance(arguments, list) or isinstance(arguments, str)
- cmd = ['adb']
- if device_id is not None:
- cmd.append('-s')
- cmd.append(device_id)
- cmd.extend(arguments if isinstance(arguments, list) else arguments.split(' '))
- return cmd
+ assert isinstance(arguments, list) or isinstance(arguments, str)
+ cmd = ['adb']
+ if device_id is not None:
+ cmd.append('-s')
+ cmd.append(device_id)
+ cmd.extend(
+ arguments if isinstance(arguments, list) else arguments.split(' '))
+ return cmd
+
def capture_app_profile_data(app_id, device_id=None):
- ps_cmd = create_adb_cmd('shell ps -o NAME', device_id)
- stdout = subprocess.check_output(ps_cmd).decode('utf-8').strip()
- killed_any = False
- for process_name in stdout.splitlines():
- if process_name.startswith(app_id):
- print('Flushing profile for process %s' % process_name)
- killall_cmd = create_adb_cmd(
- 'shell killall -s SIGUSR1 %s' % process_name, device_id)
- killall_result = subprocess.run(killall_cmd, capture_output=True)
- stdout = killall_result.stdout.decode('utf-8')
- stderr = killall_result.stderr.decode('utf-8')
- if killall_result.returncode == 0:
- killed_any = True
- else:
- print('Error: stdout: %s, stderr: %s' % (stdout, stderr))
- time.sleep(5)
- assert killed_any, 'Expected to find at least one process'
+ ps_cmd = create_adb_cmd('shell ps -o NAME', device_id)
+ stdout = subprocess.check_output(ps_cmd).decode('utf-8').strip()
+ killed_any = False
+ for process_name in stdout.splitlines():
+ if process_name.startswith(app_id):
+ print('Flushing profile for process %s' % process_name)
+ killall_cmd = create_adb_cmd(
+ 'shell killall -s SIGUSR1 %s' % process_name, device_id)
+ killall_result = subprocess.run(killall_cmd, capture_output=True)
+ stdout = killall_result.stdout.decode('utf-8')
+ stderr = killall_result.stderr.decode('utf-8')
+ if killall_result.returncode == 0:
+ killed_any = True
+ else:
+ print('Error: stdout: %s, stderr: %s' % (stdout, stderr))
+ time.sleep(5)
+ assert killed_any, 'Expected to find at least one process'
+
def check_app_has_profile_data(app_id, device_id=None):
- profile_path = get_profile_path(app_id)
- cmd = create_adb_cmd(
- 'shell du /data/misc/profiles/cur/0/%s/primary.prof' % app_id,
- device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- size_str = stdout[:stdout.index('\t')]
- assert size_str.isdigit()
- size = int(size_str)
- if size == 4:
- raise ValueError('Expected size of profile at %s to be > 4K' % profile_path)
+ profile_path = get_profile_path(app_id)
+ cmd = create_adb_cmd(
+ 'shell du /data/misc/profiles/cur/0/%s/primary.prof' % app_id,
+ device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ size_str = stdout[:stdout.index('\t')]
+ assert size_str.isdigit()
+ size = int(size_str)
+ if size == 4:
+ raise ValueError('Expected size of profile at %s to be > 4K' %
+ profile_path)
+
def clear_logcat(device_id=None):
- cmd = create_adb_cmd('logcat -c', device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ cmd = create_adb_cmd('logcat -c', device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
def clear_profile_data(app_id, device_id=None):
- cmd = create_adb_cmd(
- 'shell cmd package compile --reset %s' % app_id, device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ cmd = create_adb_cmd('shell cmd package compile --reset %s' % app_id,
+ device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
def drop_caches(device_id=None):
- cmd = create_adb_cmd(
- ['shell', 'echo 3 > /proc/sys/vm/drop_caches'], device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ cmd = create_adb_cmd(['shell', 'echo 3 > /proc/sys/vm/drop_caches'],
+ device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
def ensure_screen_on(device_id=None):
- if get_screen_state(device_id).is_off():
- toggle_screen(device_id)
- assert get_screen_state(device_id).is_on()
+ if get_screen_state(device_id).is_off():
+ toggle_screen(device_id)
+ assert get_screen_state(device_id).is_on()
+
def ensure_screen_off(device_id=None):
- if get_screen_state(device_id).is_on():
- toggle_screen(device_id)
- assert get_screen_state(device_id).is_off()
+ if get_screen_state(device_id).is_on():
+ toggle_screen(device_id)
+ assert get_screen_state(device_id).is_off()
+
def force_compilation(app_id, device_id=None):
- print('Applying AOT (full)')
- cmd = create_adb_cmd(
- 'shell cmd package compile -m speed -f %s' % app_id, device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ print('Applying AOT (full)')
+ cmd = create_adb_cmd('shell cmd package compile -m speed -f %s' % app_id,
+ device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
def force_profile_compilation(app_id, device_id=None):
- print('Applying AOT (profile)')
- cmd = create_adb_cmd(
- 'shell cmd package compile -m speed-profile -f %s' % app_id, device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+ print('Applying AOT (profile)')
+ cmd = create_adb_cmd(
+ 'shell cmd package compile -m speed-profile -f %s' % app_id, device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
def get_apk_path(app_id, device_id=None):
- cmd = create_adb_cmd('shell pm path %s' % app_id, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- if not stdout.startswith('package:'):
- raise ValueError(
- 'Expected stdout to start with "package:", was: %s' % stdout)
- apk_path = stdout[len('package:'):]
- if not apk_path.endswith('.apk'):
- raise ValueError(
- 'Expected stdout to end with ".apk", was: %s' % stdout)
- return apk_path
+ cmd = create_adb_cmd('shell pm path %s' % app_id, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ if not stdout.startswith('package:'):
+ raise ValueError('Expected stdout to start with "package:", was: %s' %
+ stdout)
+ apk_path = stdout[len('package:'):]
+ if not apk_path.endswith('.apk'):
+ raise ValueError('Expected stdout to end with ".apk", was: %s' % stdout)
+ return apk_path
+
def get_component_name(app_id, activity):
- if activity.startswith(app_id):
- return '%s/.%s' % (app_id, activity[len(app_id)+1:])
- else:
- return '%s/%s' % (app_id, activity)
+ if activity.startswith(app_id):
+ return '%s/.%s' % (app_id, activity[len(app_id) + 1:])
+ else:
+ return '%s/%s' % (app_id, activity)
+
def get_meminfo(app_id, device_id=None):
- cmd = create_adb_cmd('shell dumpsys meminfo -s %s' % app_id, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- for line in stdout.splitlines():
- if 'TOTAL PSS: ' in line:
- elements = [s for s in line.replace('TOTAL ', 'TOTAL_').split()]
- assert elements[0] == 'TOTAL_PSS:', elements[0]
- assert elements[1].isdigit()
- assert elements[2] == 'TOTAL_RSS:'
- assert elements[3].isdigit()
- return { 'total_pss': int(elements[1]), 'total_rss': int(elements[3]) }
- raise ValueError('Unexpected stdout: %s' % stdout)
+ cmd = create_adb_cmd('shell dumpsys meminfo -s %s' % app_id, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ for line in stdout.splitlines():
+ if 'TOTAL PSS: ' in line:
+ elements = [s for s in line.replace('TOTAL ', 'TOTAL_').split()]
+ assert elements[0] == 'TOTAL_PSS:', elements[0]
+ assert elements[1].isdigit()
+ assert elements[2] == 'TOTAL_RSS:'
+ assert elements[3].isdigit()
+ return {
+ 'total_pss': int(elements[1]),
+ 'total_rss': int(elements[3])
+ }
+ raise ValueError('Unexpected stdout: %s' % stdout)
+
def get_profile_data(app_id, device_id=None):
- with utils.TempDir() as temp:
- source = get_profile_path(app_id)
- target = os.path.join(temp, 'primary.prof')
- pull(source, target, device_id)
- with open(target, 'rb') as f:
- return f.read()
+ with utils.TempDir() as temp:
+ source = get_profile_path(app_id)
+ target = os.path.join(temp, 'primary.prof')
+ pull(source, target, device_id)
+ with open(target, 'rb') as f:
+ return f.read()
+
def get_profile_path(app_id):
- return '/data/misc/profiles/cur/0/%s/primary.prof' % app_id
+ return '/data/misc/profiles/cur/0/%s/primary.prof' % app_id
+
def get_minor_major_page_faults(app_id, device_id=None):
- pid = get_pid(app_id, device_id)
- cmd = create_adb_cmd('shell ps -p %i -o MINFL,MAJFL' % pid, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8')
- lines_it = iter(stdout.splitlines())
- first_line = next(lines_it)
- assert first_line == ' MINFL MAJFL'
- second_line = next(lines_it)
- minfl, majfl = second_line.split()
- assert minfl.isdigit()
- assert majfl.isdigit()
- return (int(minfl), int(majfl))
+ pid = get_pid(app_id, device_id)
+ cmd = create_adb_cmd('shell ps -p %i -o MINFL,MAJFL' % pid, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8')
+ lines_it = iter(stdout.splitlines())
+ first_line = next(lines_it)
+ assert first_line == ' MINFL MAJFL'
+ second_line = next(lines_it)
+ minfl, majfl = second_line.split()
+ assert minfl.isdigit()
+ assert majfl.isdigit()
+ return (int(minfl), int(majfl))
+
def get_pid(app_id, device_id=None):
- cmd = create_adb_cmd('shell pidof %s' % app_id, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- assert stdout.isdigit()
- pid = int(stdout)
- return pid
+ cmd = create_adb_cmd('shell pidof %s' % app_id, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ assert stdout.isdigit()
+ pid = int(stdout)
+ return pid
+
def get_screen_state(device_id=None):
- cmd = create_adb_cmd('shell dumpsys nfc', device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- screen_state_value = None
- for line in stdout.splitlines():
- if line.startswith('mScreenState='):
- value_start_index = len('mScreenState=')
- screen_state_value=line[value_start_index:]
- if screen_state_value is None:
- raise ValueError('Expected to find mScreenState in: adb shell dumpsys nfc')
- if not hasattr(ScreenState, screen_state_value):
- raise ValueError(
- 'Expected mScreenState to be a value of ScreenState, was: %s'
- % screen_state_value)
- return ScreenState[screen_state_value]
+ cmd = create_adb_cmd('shell dumpsys nfc', device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ screen_state_value = None
+ for line in stdout.splitlines():
+ if line.startswith('mScreenState='):
+ value_start_index = len('mScreenState=')
+ screen_state_value = line[value_start_index:]
+ if screen_state_value is None:
+ raise ValueError(
+ 'Expected to find mScreenState in: adb shell dumpsys nfc')
+ if not hasattr(ScreenState, screen_state_value):
+ raise ValueError(
+ 'Expected mScreenState to be a value of ScreenState, was: %s' %
+ screen_state_value)
+ return ScreenState[screen_state_value]
+
def get_classes_and_methods_from_app_profile(app_id, device_id=None):
- apk_path = get_apk_path(app_id, device_id)
- profile_path = get_profile_path(app_id)
- cmd = create_adb_cmd(
- 'shell profman --dump-classes-and-methods'
- ' --profile-file=%s --apk=%s --dex-location=%s'
- % (profile_path, apk_path, apk_path), device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- lines = stdout.splitlines()
- return profile_utils.parse_art_profile(lines)
+ apk_path = get_apk_path(app_id, device_id)
+ profile_path = get_profile_path(app_id)
+ cmd = create_adb_cmd(
+ 'shell profman --dump-classes-and-methods'
+ ' --profile-file=%s --apk=%s --dex-location=%s' %
+ (profile_path, apk_path, apk_path), device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ lines = stdout.splitlines()
+ return profile_utils.parse_art_profile(lines)
+
def get_screen_off_timeout(device_id=None):
- cmd = create_adb_cmd(
- 'shell settings get system screen_off_timeout', device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- assert stdout.isdigit()
- screen_off_timeout = int(stdout)
- return screen_off_timeout
+ cmd = create_adb_cmd('shell settings get system screen_off_timeout',
+ device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ assert stdout.isdigit()
+ screen_off_timeout = int(stdout)
+ return screen_off_timeout
+
def grant(app_id, permission, device_id=None):
- cmd = create_adb_cmd('shell pm grant %s %s' % (app_id, permission), device_id)
- subprocess.check_call(cmd)
+ cmd = create_adb_cmd('shell pm grant %s %s' % (app_id, permission),
+ device_id)
+ subprocess.check_call(cmd)
+
def install(apk, device_id=None):
- print('Installing %s' % apk)
- cmd = create_adb_cmd('install %s' % apk, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8')
- assert 'Success' in stdout
+ print('Installing %s' % apk)
+ cmd = create_adb_cmd('install %s' % apk, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8')
+ assert 'Success' in stdout
+
def install_apks(apks, device_id=None, max_attempts=3):
- print('Installing %s' % apks)
- cmd = [
- 'java', '-jar', utils.BUNDLETOOL_JAR,
- 'install-apks',
- '--apks=%s' % apks]
- if device_id is not None:
- cmd.append('--device-id=%s' % device_id)
- for i in range(max_attempts):
+ print('Installing %s' % apks)
+ cmd = [
+ 'java', '-jar', utils.BUNDLETOOL_JAR, 'install-apks',
+ '--apks=%s' % apks
+ ]
+ if device_id is not None:
+ cmd.append('--device-id=%s' % device_id)
+ for i in range(max_attempts):
+ process_result = subprocess.run(cmd, capture_output=True)
+ stdout = process_result.stdout.decode('utf-8')
+ stderr = process_result.stderr.decode('utf-8')
+ if process_result.returncode == 0:
+ return
+ print('Failed to install %s' % apks)
+ print('Stdout: %s' % stdout)
+ print('Stderr: %s' % stderr)
+ print('Retrying...')
+ raise Exception('Unable to install apks in %s attempts' % max_attempts)
+
+
+def install_bundle(bundle, device_id=None):
+ print('Installing %s' % bundle)
+ with utils.TempDir() as temp:
+ apks = os.path.join(temp, 'Bundle.apks')
+ build_apks_from_bundle(bundle, apks)
+ install_apks(apks, device_id)
+
+
+def install_profile_using_adb(app_id, host_profile_path, device_id=None):
+ device_profile_path = get_profile_path(app_id)
+ cmd = create_adb_cmd('push %s %s' %
+ (host_profile_path, device_profile_path))
+ subprocess.check_call(cmd)
+ stop_app(app_id, device_id)
+ force_profile_compilation(app_id, device_id)
+
+
+def install_profile_using_profileinstaller(app_id, device_id=None):
+ # This assumes that the profileinstaller library has been added to the app,
+ # https://developer.android.com/jetpack/androidx/releases/profileinstaller.
+ action = 'androidx.profileinstaller.action.INSTALL_PROFILE'
+ component = '%s/androidx.profileinstaller.ProfileInstallReceiver' % app_id
+ stdout = broadcast(action, component, device_id)
+ assert len(stdout) == 2
+ assert stdout[0] == ('Broadcasting: Intent { act=%s flg=0x400000 cmp=%s }' %
+ (action, component))
+ assert stdout[1] == 'Broadcast completed: result=1', stdout[1]
+ stop_app(app_id, device_id)
+ force_profile_compilation(app_id, device_id)
+
+
+def issue_key_event(key_event, device_id=None, sleep_in_seconds=1):
+ cmd = create_adb_cmd('shell input keyevent %s' % key_event, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ assert len(stdout) == 0
+ time.sleep(sleep_in_seconds)
+
+
+def launch_activity(app_id,
+ activity,
+ device_id=None,
+ intent_data_uri=None,
+ wait_for_activity_to_launch=False):
+ args = ['shell', 'am', 'start', '-n', '%s/%s' % (app_id, activity)]
+ if intent_data_uri:
+ args.extend(['-d', intent_data_uri])
+ if wait_for_activity_to_launch:
+ args.append('-W')
+ cmd = create_adb_cmd(args, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ assert stdout.startswith('Starting: Intent {')
+ expected_component = 'cmp=%s' % get_component_name(app_id, activity)
+ assert expected_component in stdout, \
+ 'was %s, expected %s' % (stdout, expected_component)
+ lines = stdout.splitlines()
+ result = {}
+ for line in lines:
+ if line.startswith('TotalTime: '):
+ total_time_str = line.removeprefix('TotalTime: ')
+ assert total_time_str.isdigit()
+ result['total_time'] = int(total_time_str)
+ assert not wait_for_activity_to_launch or 'total_time' in result, lines
+ return result
+
+
+def navigate_to_home_screen(device_id=None):
+ cmd = create_adb_cmd('shell input keyevent KEYCODE_HOME', device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
+
+def prepare_for_interaction_with_device(device_id=None, device_pin=None):
+ # Increase screen off timeout to avoid device screen turns off.
+ twenty_four_hours_in_millis = 24 * 60 * 60 * 1000
+ previous_screen_off_timeout = get_screen_off_timeout(device_id)
+ set_screen_off_timeout(twenty_four_hours_in_millis, device_id)
+
+ # Unlock device.
+ unlock(device_id, device_pin)
+
+ teardown_options = {
+ 'previous_screen_off_timeout': previous_screen_off_timeout
+ }
+ return teardown_options
+
+
+def pull(source, target, device_id=None):
+ cmd = create_adb_cmd('pull %s %s' % (source, target), device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
+
+def root(device_id=None):
+ cmd = create_adb_cmd('root', device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
+
+def set_screen_off_timeout(screen_off_timeout_in_millis, device_id=None):
+ cmd = create_adb_cmd(
+ 'shell settings put system screen_off_timeout %i' %
+ screen_off_timeout_in_millis, device_id)
+ stdout = subprocess.check_output(cmd).decode('utf-8').strip()
+ assert len(stdout) == 0
+
+
+def start_logcat(device_id=None, format=None, filter=None, silent=False):
+ args = ['logcat']
+ if format:
+ args.extend(['--format', format])
+ if silent:
+ args.append('-s')
+ if filter:
+ args.append(filter)
+ cmd = create_adb_cmd(args, device_id)
+ logcat_process = subprocess.Popen(cmd,
+ bufsize=1024 * 1024,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ reader = ProcessReader(logcat_process)
+ reader.start()
+ return reader
+
+
+def stop_logcat(logcat_reader):
+ logcat_reader.stop()
+ logcat_reader.join()
+ return logcat_reader.lines
+
+
+def stop_app(app_id, device_id=None):
+ print('Shutting down %s' % app_id)
+ cmd = create_adb_cmd('shell am force-stop %s' % app_id, device_id)
+ subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
+
+
+def teardown_after_interaction_with_device(teardown_options, device_id=None):
+ # Reset screen off timeout.
+ set_screen_off_timeout(teardown_options['previous_screen_off_timeout'],
+ device_id)
+
+
+def toggle_screen(device_id=None):
+ issue_key_event('KEYCODE_POWER', device_id)
+
+
+def uninstall(app_id, device_id=None):
+ print('Uninstalling %s' % app_id)
+ cmd = create_adb_cmd('uninstall %s' % app_id, device_id)
process_result = subprocess.run(cmd, capture_output=True)
stdout = process_result.stdout.decode('utf-8')
stderr = process_result.stderr.decode('utf-8')
if process_result.returncode == 0:
- return
- print('Failed to install %s' % apks)
- print('Stdout: %s' % stdout)
- print('Stderr: %s' % stderr)
- print('Retrying...')
- raise Exception('Unable to install apks in %s attempts' % max_attempts)
+ assert 'Success' in stdout
+ elif stdout.startswith('cmd: Failure calling service package: Broken pipe'):
+ assert app_id == 'com.google.android.youtube'
+ print('Waiting after broken pipe')
+ time.sleep(15)
+ else:
+ expected_error = (
+ 'java.lang.IllegalArgumentException: Unknown package: %s' % app_id)
+ assert 'Failure [DELETE_FAILED_INTERNAL_ERROR]' in stdout \
+ or expected_error in stderr, \
+ 'stdout: %s, stderr: %s' % (stdout, stderr)
-def install_bundle(bundle, device_id=None):
- print('Installing %s' % bundle)
- with utils.TempDir() as temp:
- apks = os.path.join(temp, 'Bundle.apks')
- build_apks_from_bundle(bundle, apks)
- install_apks(apks, device_id)
-
-def install_profile_using_adb(app_id, host_profile_path, device_id=None):
- device_profile_path = get_profile_path(app_id)
- cmd = create_adb_cmd('push %s %s' % (host_profile_path, device_profile_path))
- subprocess.check_call(cmd)
- stop_app(app_id, device_id)
- force_profile_compilation(app_id, device_id)
-
-def install_profile_using_profileinstaller(app_id, device_id=None):
- # This assumes that the profileinstaller library has been added to the app,
- # https://developer.android.com/jetpack/androidx/releases/profileinstaller.
- action = 'androidx.profileinstaller.action.INSTALL_PROFILE'
- component = '%s/androidx.profileinstaller.ProfileInstallReceiver' % app_id
- stdout = broadcast(action, component, device_id)
- assert len(stdout) == 2
- assert stdout[0] == ('Broadcasting: Intent { act=%s flg=0x400000 cmp=%s }' % (action, component))
- assert stdout[1] == 'Broadcast completed: result=1', stdout[1]
- stop_app(app_id, device_id)
- force_profile_compilation(app_id, device_id)
-
-def issue_key_event(key_event, device_id=None, sleep_in_seconds=1):
- cmd = create_adb_cmd('shell input keyevent %s' % key_event, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- assert len(stdout) == 0
- time.sleep(sleep_in_seconds)
-
-def launch_activity(
- app_id,
- activity,
- device_id=None,
- intent_data_uri=None,
- wait_for_activity_to_launch=False):
- args = ['shell', 'am', 'start', '-n', '%s/%s' % (app_id, activity)]
- if intent_data_uri:
- args.extend(['-d', intent_data_uri])
- if wait_for_activity_to_launch:
- args.append('-W')
- cmd = create_adb_cmd(args, device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- assert stdout.startswith('Starting: Intent {')
- expected_component = 'cmp=%s' % get_component_name(app_id, activity)
- assert expected_component in stdout, \
- 'was %s, expected %s' % (stdout, expected_component)
- lines = stdout.splitlines()
- result = {}
- for line in lines:
- if line.startswith('TotalTime: '):
- total_time_str = line.removeprefix('TotalTime: ')
- assert total_time_str.isdigit()
- result['total_time'] = int(total_time_str)
- assert not wait_for_activity_to_launch or 'total_time' in result, lines
- return result
-
-def navigate_to_home_screen(device_id=None):
- cmd = create_adb_cmd('shell input keyevent KEYCODE_HOME', device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
-
-def prepare_for_interaction_with_device(device_id=None, device_pin=None):
- # Increase screen off timeout to avoid device screen turns off.
- twenty_four_hours_in_millis = 24 * 60 * 60 * 1000
- previous_screen_off_timeout = get_screen_off_timeout(device_id)
- set_screen_off_timeout(twenty_four_hours_in_millis, device_id)
-
- # Unlock device.
- unlock(device_id, device_pin)
-
- teardown_options = {
- 'previous_screen_off_timeout': previous_screen_off_timeout
- }
- return teardown_options
-
-def pull(source, target, device_id=None):
- cmd = create_adb_cmd('pull %s %s' % (source, target), device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
-
-def root(device_id=None):
- cmd = create_adb_cmd('root', device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
-
-def set_screen_off_timeout(screen_off_timeout_in_millis, device_id=None):
- cmd = create_adb_cmd(
- 'shell settings put system screen_off_timeout %i'
- % screen_off_timeout_in_millis,
- device_id)
- stdout = subprocess.check_output(cmd).decode('utf-8').strip()
- assert len(stdout) == 0
-
-def start_logcat(device_id=None, format=None, filter=None, silent=False):
- args = ['logcat']
- if format:
- args.extend(['--format', format])
- if silent:
- args.append('-s')
- if filter:
- args.append(filter)
- cmd = create_adb_cmd(args, device_id)
- logcat_process = subprocess.Popen(
- cmd, bufsize=1024*1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- reader = ProcessReader(logcat_process)
- reader.start()
- return reader
-
-def stop_logcat(logcat_reader):
- logcat_reader.stop()
- logcat_reader.join()
- return logcat_reader.lines
-
-def stop_app(app_id, device_id=None):
- print('Shutting down %s' % app_id)
- cmd = create_adb_cmd('shell am force-stop %s' % app_id, device_id)
- subprocess.check_call(cmd, stdout=DEVNULL, stderr=DEVNULL)
-
-def teardown_after_interaction_with_device(teardown_options, device_id=None):
- # Reset screen off timeout.
- set_screen_off_timeout(
- teardown_options['previous_screen_off_timeout'],
- device_id)
-
-def toggle_screen(device_id=None):
- issue_key_event('KEYCODE_POWER', device_id)
-
-def uninstall(app_id, device_id=None):
- print('Uninstalling %s' % app_id)
- cmd = create_adb_cmd('uninstall %s' % app_id, device_id)
- process_result = subprocess.run(cmd, capture_output=True)
- stdout = process_result.stdout.decode('utf-8')
- stderr = process_result.stderr.decode('utf-8')
- if process_result.returncode == 0:
- assert 'Success' in stdout
- elif stdout.startswith('cmd: Failure calling service package: Broken pipe'):
- assert app_id == 'com.google.android.youtube'
- print('Waiting after broken pipe')
- time.sleep(15)
- else:
- expected_error = (
- 'java.lang.IllegalArgumentException: Unknown package: %s' % app_id)
- assert 'Failure [DELETE_FAILED_INTERNAL_ERROR]' in stdout \
- or expected_error in stderr, \
- 'stdout: %s, stderr: %s' % (stdout, stderr)
def unlock(device_id=None, device_pin=None):
- ensure_screen_on(device_id)
- screen_state = get_screen_state(device_id)
- assert screen_state.is_on(), 'was %s' % screen_state
- if screen_state.is_on_and_locked():
- if device_pin is not None:
- raise NotImplementedError('Device unlocking with pin not implemented')
- issue_key_event('KEYCODE_MENU', device_id)
+ ensure_screen_on(device_id)
screen_state = get_screen_state(device_id)
- assert screen_state.is_on_and_unlocked(), 'was %s' % screen_state
+ assert screen_state.is_on(), 'was %s' % screen_state
+ if screen_state.is_on_and_locked():
+ if device_pin is not None:
+ raise NotImplementedError(
+ 'Device unlocking with pin not implemented')
+ issue_key_event('KEYCODE_MENU', device_id)
+ screen_state = get_screen_state(device_id)
+ assert screen_state.is_on_and_unlocked(), 'was %s' % screen_state
+
def parse_options(argv):
- result = argparse.ArgumentParser(description='Run adb utils.')
- result.add_argument('--capture-screen',
- help='Capture screen to given file')
- result.add_argument('--device-id',
- help='Device id (e.g., emulator-5554).')
- result.add_argument('--device-pin',
- help='Device pin code (e.g., 1234)')
- result.add_argument('--ensure-screen-off',
- help='Ensure screen off',
- action='store_true',
- default=False)
- result.add_argument('--get-screen-state',
- help='Get screen state',
- action='store_true',
- default=False)
- result.add_argument('--unlock',
- help='Unlock device',
- action='store_true',
- default=False)
- options, args = result.parse_known_args(argv)
- return options, args
+ result = argparse.ArgumentParser(description='Run adb utils.')
+ result.add_argument('--capture-screen', help='Capture screen to given file')
+ result.add_argument('--device-id', help='Device id (e.g., emulator-5554).')
+ result.add_argument('--device-pin', help='Device pin code (e.g., 1234)')
+ result.add_argument('--ensure-screen-off',
+ help='Ensure screen off',
+ action='store_true',
+ default=False)
+ result.add_argument('--get-screen-state',
+ help='Get screen state',
+ action='store_true',
+ default=False)
+ result.add_argument('--unlock',
+ help='Unlock device',
+ action='store_true',
+ default=False)
+ options, args = result.parse_known_args(argv)
+ return options, args
+
def main(argv):
- (options, args) = parse_options(argv)
- if options.capture_screen:
- capture_screen(options.capture_screen, options.device_id)
- if options.ensure_screen_off:
- ensure_screen_off(options.device_id)
- elif options.get_screen_state:
- print(get_screen_state(options.device_id))
- elif options.unlock:
- unlock(options.device_id, options.device_pin)
+ (options, args) = parse_options(argv)
+ if options.capture_screen:
+ capture_screen(options.capture_screen, options.device_id)
+ if options.ensure_screen_off:
+ ensure_screen_off(options.device_id)
+ elif options.get_screen_state:
+ print(get_screen_state(options.device_id))
+ elif options.unlock:
+ unlock(options.device_id, options.device_pin)
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/generate_startup_descriptors.py b/tools/startup/generate_startup_descriptors.py
index adae239..211be89 100755
--- a/tools/startup/generate_startup_descriptors.py
+++ b/tools/startup/generate_startup_descriptors.py
@@ -11,445 +11,474 @@
import sys
import time
+
class Device:
- def __init__(self, device_id, device_pin):
- self.device_id = device_id
- self.device_pin = device_pin
+ def __init__(self, device_id, device_pin):
+ self.device_id = device_id
+ self.device_pin = device_pin
+
def extend_startup_descriptors(startup_descriptors, iteration, device, options):
- (logcat, profile, profile_classes_and_methods) = \
- generate_startup_profile(device, options)
- if options.logcat:
- write_tmp_logcat(logcat, iteration, options)
- current_startup_descriptors = get_r8_startup_descriptors_from_logcat(
- logcat, options)
- else:
- write_tmp_profile(profile, iteration, options)
- write_tmp_profile_classes_and_methods(
- profile_classes_and_methods, iteration, options)
- current_startup_descriptors = \
- profile_utils.transform_art_profile_to_r8_startup_list(
- profile_classes_and_methods, options.generalize_synthetics)
- write_tmp_startup_descriptors(current_startup_descriptors, iteration, options)
- new_startup_descriptors = add_r8_startup_descriptors(
- startup_descriptors, current_startup_descriptors)
- number_of_new_startup_descriptors = \
- len(new_startup_descriptors) - len(startup_descriptors)
- if options.out is not None:
- print(
- 'Found %i new startup descriptors in iteration %i'
- % (number_of_new_startup_descriptors, iteration + 1))
- return new_startup_descriptors
+ (logcat, profile, profile_classes_and_methods) = \
+ generate_startup_profile(device, options)
+ if options.logcat:
+ write_tmp_logcat(logcat, iteration, options)
+ current_startup_descriptors = get_r8_startup_descriptors_from_logcat(
+ logcat, options)
+ else:
+ write_tmp_profile(profile, iteration, options)
+ write_tmp_profile_classes_and_methods(profile_classes_and_methods,
+ iteration, options)
+ current_startup_descriptors = \
+ profile_utils.transform_art_profile_to_r8_startup_list(
+ profile_classes_and_methods, options.generalize_synthetics)
+ write_tmp_startup_descriptors(current_startup_descriptors, iteration,
+ options)
+ new_startup_descriptors = add_r8_startup_descriptors(
+ startup_descriptors, current_startup_descriptors)
+ number_of_new_startup_descriptors = \
+ len(new_startup_descriptors) - len(startup_descriptors)
+ if options.out is not None:
+ print('Found %i new startup descriptors in iteration %i' %
+ (number_of_new_startup_descriptors, iteration + 1))
+ return new_startup_descriptors
+
def generate_startup_profile(device, options):
- logcat = None
- profile = None
- profile_classes_and_methods = None
- if options.use_existing_profile:
- # Verify presence of profile.
- adb_utils.check_app_has_profile_data(options.app_id, device.device_id)
- profile = adb_utils.get_profile_data(options.app_id, device.device_id)
- profile_classes_and_methods = \
- adb_utils.get_classes_and_methods_from_app_profile(
- options.app_id, device.device_id)
- else:
- # Unlock device.
- tear_down_options = adb_utils.prepare_for_interaction_with_device(
- device.device_id, device.device_pin)
-
- logcat_process = None
- if options.logcat:
- # Clear logcat and start capturing logcat.
- adb_utils.clear_logcat(device.device_id)
- logcat_process = adb_utils.start_logcat(
- device.device_id, format='tag', filter='R8:I ActivityTaskManager:I *:S')
+ logcat = None
+ profile = None
+ profile_classes_and_methods = None
+ if options.use_existing_profile:
+ # Verify presence of profile.
+ adb_utils.check_app_has_profile_data(options.app_id, device.device_id)
+ profile = adb_utils.get_profile_data(options.app_id, device.device_id)
+ profile_classes_and_methods = \
+ adb_utils.get_classes_and_methods_from_app_profile(
+ options.app_id, device.device_id)
else:
- # Clear existing profile data.
- adb_utils.clear_profile_data(options.app_id, device.device_id)
+ # Unlock device.
+ tear_down_options = adb_utils.prepare_for_interaction_with_device(
+ device.device_id, device.device_pin)
- # Launch activity to generate startup profile on device.
- adb_utils.launch_activity(
- options.app_id, options.main_activity, device.device_id)
+ logcat_process = None
+ if options.logcat:
+ # Clear logcat and start capturing logcat.
+ adb_utils.clear_logcat(device.device_id)
+ logcat_process = adb_utils.start_logcat(
+ device.device_id,
+ format='tag',
+ filter='R8:I ActivityTaskManager:I *:S')
+ else:
+ # Clear existing profile data.
+ adb_utils.clear_profile_data(options.app_id, device.device_id)
- # Wait for activity startup.
- time.sleep(options.startup_duration)
+ # Launch activity to generate startup profile on device.
+ adb_utils.launch_activity(options.app_id, options.main_activity,
+ device.device_id)
- if options.logcat:
- # Get startup descriptors from logcat.
- logcat = adb_utils.stop_logcat(logcat_process)
- else:
- # Capture startup profile.
- adb_utils.capture_app_profile_data(options.app_id, device.device_id)
- profile = adb_utils.get_profile_data(options.app_id, device.device_id)
- profile_classes_and_methods = \
- adb_utils.get_classes_and_methods_from_app_profile(
- options.app_id, device.device_id)
+ # Wait for activity startup.
+ time.sleep(options.startup_duration)
- # Shutdown app.
- adb_utils.stop_app(options.app_id, device.device_id)
- adb_utils.teardown_after_interaction_with_device(
- tear_down_options, device.device_id)
+ if options.logcat:
+ # Get startup descriptors from logcat.
+ logcat = adb_utils.stop_logcat(logcat_process)
+ else:
+ # Capture startup profile.
+ adb_utils.capture_app_profile_data(options.app_id, device.device_id)
+ profile = adb_utils.get_profile_data(options.app_id,
+ device.device_id)
+ profile_classes_and_methods = \
+ adb_utils.get_classes_and_methods_from_app_profile(
+ options.app_id, device.device_id)
- return (logcat, profile, profile_classes_and_methods)
+ # Shutdown app.
+ adb_utils.stop_app(options.app_id, device.device_id)
+ adb_utils.teardown_after_interaction_with_device(
+ tear_down_options, device.device_id)
+
+ return (logcat, profile, profile_classes_and_methods)
+
def get_r8_startup_descriptors_from_logcat(logcat, options):
- post_startup = False
- startup_descriptors = {}
- for line in logcat:
- line_elements = parse_logcat_line(line)
- if line_elements is None:
- continue
- (priority, tag, message) = line_elements
- if tag == 'ActivityTaskManager':
- if message.startswith('START') \
- or message.startswith('Activity pause timeout for') \
- or message.startswith('Activity top resumed state loss timeout for') \
- or message.startswith('Force removing') \
- or message.startswith(
- 'Launch timeout has expired, giving up wake lock!'):
- continue
- elif message.startswith('Displayed %s/' % options.app_id):
- print('Entering post startup: %s' % message)
- post_startup = True
- continue
- elif tag == 'R8':
- if is_startup_descriptor(message):
- startup_descriptors[message] = {
- 'conditional_startup': False,
- 'hot': False,
- 'post_startup': post_startup,
- 'startup': True
- }
- continue
- # Reaching here means we didn't expect this line.
- report_unrecognized_logcat_line(line)
- return startup_descriptors
+ post_startup = False
+ startup_descriptors = {}
+ for line in logcat:
+ line_elements = parse_logcat_line(line)
+ if line_elements is None:
+ continue
+ (priority, tag, message) = line_elements
+ if tag == 'ActivityTaskManager':
+ if message.startswith('START') \
+ or message.startswith('Activity pause timeout for') \
+ or message.startswith('Activity top resumed state loss timeout for') \
+ or message.startswith('Force removing') \
+ or message.startswith(
+ 'Launch timeout has expired, giving up wake lock!'):
+ continue
+ elif message.startswith('Displayed %s/' % options.app_id):
+ print('Entering post startup: %s' % message)
+ post_startup = True
+ continue
+ elif tag == 'R8':
+ if is_startup_descriptor(message):
+ startup_descriptors[message] = {
+ 'conditional_startup': False,
+ 'hot': False,
+ 'post_startup': post_startup,
+ 'startup': True
+ }
+ continue
+ # Reaching here means we didn't expect this line.
+ report_unrecognized_logcat_line(line)
+ return startup_descriptors
+
def is_startup_descriptor(string):
- # The descriptor should start with the holder (possibly prefixed with 'S').
- if not any(string.startswith('%sL' % flags) for flags in ['', 'S']):
- return False
- # The descriptor should end with ';', a primitive type, or void.
- if not string.endswith(';') \
- and not any(string.endswith(c) for c in get_primitive_descriptors()) \
- and not string.endswith('V'):
- return False
- return True
+ # The descriptor should start with the holder (possibly prefixed with 'S').
+ if not any(string.startswith('%sL' % flags) for flags in ['', 'S']):
+ return False
+ # The descriptor should end with ';', a primitive type, or void.
+ if not string.endswith(';') \
+ and not any(string.endswith(c) for c in get_primitive_descriptors()) \
+ and not string.endswith('V'):
+ return False
+ return True
+
def get_primitive_descriptors():
- return ['Z', 'B', 'S', 'C', 'I', 'F', 'J', 'D']
+ return ['Z', 'B', 'S', 'C', 'I', 'F', 'J', 'D']
+
def parse_logcat_line(line):
- if line == '--------- beginning of kernel':
- return None
- if line == '--------- beginning of main':
- return None
- if line == '--------- beginning of system':
- return None
+ if line == '--------- beginning of kernel':
+ return None
+ if line == '--------- beginning of main':
+ return None
+ if line == '--------- beginning of system':
+ return None
- priority = None
- tag = None
+ priority = None
+ tag = None
- try:
- priority_end = line.index('/')
- priority = line[0:priority_end]
- line = line[priority_end + 1:]
- except ValueError:
- return report_unrecognized_logcat_line(line)
+ try:
+ priority_end = line.index('/')
+ priority = line[0:priority_end]
+ line = line[priority_end + 1:]
+ except ValueError:
+ return report_unrecognized_logcat_line(line)
- try:
- tag_end = line.index(':')
- tag = line[0:tag_end].strip()
- line = line[tag_end + 1 :]
- except ValueError:
- return report_unrecognized_logcat_line(line)
+ try:
+ tag_end = line.index(':')
+ tag = line[0:tag_end].strip()
+ line = line[tag_end + 1:]
+ except ValueError:
+ return report_unrecognized_logcat_line(line)
- message = line.strip()
- return (priority, tag, message)
+ message = line.strip()
+ return (priority, tag, message)
+
def report_unrecognized_logcat_line(line):
- print('Unrecognized line in logcat: %s' % line)
+ print('Unrecognized line in logcat: %s' % line)
-def add_r8_startup_descriptors(old_startup_descriptors, startup_descriptors_to_add):
- new_startup_descriptors = {}
- if len(old_startup_descriptors) == 0:
- for startup_descriptor, flags in startup_descriptors_to_add.items():
- new_startup_descriptors[startup_descriptor] = flags.copy()
- else:
- # Merge the new startup descriptors with the old descriptors in a way so
- # that new startup descriptors are added next to the startup descriptors
- # they are close to in the newly generated list of startup descriptors.
- startup_descriptors_to_add_after_key = {}
- startup_descriptors_to_add_in_the_end = {}
- closest_seen_startup_descriptor = None
- for startup_descriptor, flags in startup_descriptors_to_add.items():
- if startup_descriptor in old_startup_descriptors:
- closest_seen_startup_descriptor = startup_descriptor
- else:
- if closest_seen_startup_descriptor is None:
- # Insert this new startup descriptor in the end of the result.
- startup_descriptors_to_add_in_the_end[startup_descriptor] = flags
- else:
- # Record that this should be inserted after
- # closest_seen_startup_descriptor.
- pending_startup_descriptors = \
- startup_descriptors_to_add_after_key.setdefault(
- closest_seen_startup_descriptor, {})
- pending_startup_descriptors[startup_descriptor] = flags
- for startup_descriptor, flags in old_startup_descriptors.items():
- # Merge flags if this also exists in startup_descriptors_to_add.
- if startup_descriptor in startup_descriptors_to_add:
- merged_flags = flags.copy()
- other_flags = startup_descriptors_to_add[startup_descriptor]
- assert not other_flags['conditional_startup']
- merged_flags['hot'] = \
- merged_flags['hot'] or other_flags['hot']
- merged_flags['startup'] = \
- merged_flags['startup'] or other_flags['startup']
- merged_flags['post_startup'] = \
- merged_flags['post_startup'] or other_flags['post_startup']
- new_startup_descriptors[startup_descriptor] = merged_flags
- else:
- new_startup_descriptors[startup_descriptor] = flags.copy()
- # Flush startup descriptors that followed this item in the new trace.
- if startup_descriptor in startup_descriptors_to_add_after_key:
- pending_startup_descriptors = \
- startup_descriptors_to_add_after_key[startup_descriptor]
- for pending_startup_descriptor, pending_flags \
- in pending_startup_descriptors.items():
- new_startup_descriptors[pending_startup_descriptor] = \
- pending_flags.copy()
- # Insert remaining new startup descriptors in the end.
- for startup_descriptor, flags \
- in startup_descriptors_to_add_in_the_end.items():
- assert startup_descriptor not in new_startup_descriptors
- new_startup_descriptors[startup_descriptor] = flags.copy()
- return new_startup_descriptors
+
+def add_r8_startup_descriptors(old_startup_descriptors,
+ startup_descriptors_to_add):
+ new_startup_descriptors = {}
+ if len(old_startup_descriptors) == 0:
+ for startup_descriptor, flags in startup_descriptors_to_add.items():
+ new_startup_descriptors[startup_descriptor] = flags.copy()
+ else:
+ # Merge the new startup descriptors with the old descriptors in a way so
+ # that new startup descriptors are added next to the startup descriptors
+ # they are close to in the newly generated list of startup descriptors.
+ startup_descriptors_to_add_after_key = {}
+ startup_descriptors_to_add_in_the_end = {}
+ closest_seen_startup_descriptor = None
+ for startup_descriptor, flags in startup_descriptors_to_add.items():
+ if startup_descriptor in old_startup_descriptors:
+ closest_seen_startup_descriptor = startup_descriptor
+ else:
+ if closest_seen_startup_descriptor is None:
+ # Insert this new startup descriptor in the end of the result.
+ startup_descriptors_to_add_in_the_end[
+ startup_descriptor] = flags
+ else:
+ # Record that this should be inserted after
+ # closest_seen_startup_descriptor.
+ pending_startup_descriptors = \
+ startup_descriptors_to_add_after_key.setdefault(
+ closest_seen_startup_descriptor, {})
+ pending_startup_descriptors[startup_descriptor] = flags
+ for startup_descriptor, flags in old_startup_descriptors.items():
+ # Merge flags if this also exists in startup_descriptors_to_add.
+ if startup_descriptor in startup_descriptors_to_add:
+ merged_flags = flags.copy()
+ other_flags = startup_descriptors_to_add[startup_descriptor]
+ assert not other_flags['conditional_startup']
+ merged_flags['hot'] = \
+ merged_flags['hot'] or other_flags['hot']
+ merged_flags['startup'] = \
+ merged_flags['startup'] or other_flags['startup']
+ merged_flags['post_startup'] = \
+ merged_flags['post_startup'] or other_flags['post_startup']
+ new_startup_descriptors[startup_descriptor] = merged_flags
+ else:
+ new_startup_descriptors[startup_descriptor] = flags.copy()
+ # Flush startup descriptors that followed this item in the new trace.
+ if startup_descriptor in startup_descriptors_to_add_after_key:
+ pending_startup_descriptors = \
+ startup_descriptors_to_add_after_key[startup_descriptor]
+ for pending_startup_descriptor, pending_flags \
+ in pending_startup_descriptors.items():
+ new_startup_descriptors[pending_startup_descriptor] = \
+ pending_flags.copy()
+ # Insert remaining new startup descriptors in the end.
+ for startup_descriptor, flags \
+ in startup_descriptors_to_add_in_the_end.items():
+ assert startup_descriptor not in new_startup_descriptors
+ new_startup_descriptors[startup_descriptor] = flags.copy()
+ return new_startup_descriptors
+
def write_tmp_binary_artifact(artifact, iteration, options, name):
- if not options.tmp_dir:
- return
- out_dir = os.path.join(options.tmp_dir, str(iteration))
- os.makedirs(out_dir, exist_ok=True)
- path = os.path.join(out_dir, name)
- with open(path, 'wb') as f:
- f.write(artifact)
+ if not options.tmp_dir:
+ return
+ out_dir = os.path.join(options.tmp_dir, str(iteration))
+ os.makedirs(out_dir, exist_ok=True)
+ path = os.path.join(out_dir, name)
+ with open(path, 'wb') as f:
+ f.write(artifact)
-def write_tmp_textual_artifact(artifact, iteration, options, name, item_to_string=None):
- if not options.tmp_dir:
- return
- out_dir = os.path.join(options.tmp_dir, str(iteration))
- os.makedirs(out_dir, exist_ok=True)
- path = os.path.join(out_dir, name)
- with open(path, 'w') as f:
- for item in artifact:
- f.write(item if item_to_string is None else item_to_string(item))
- f.write('\n')
+
+def write_tmp_textual_artifact(artifact,
+ iteration,
+ options,
+ name,
+ item_to_string=None):
+ if not options.tmp_dir:
+ return
+ out_dir = os.path.join(options.tmp_dir, str(iteration))
+ os.makedirs(out_dir, exist_ok=True)
+ path = os.path.join(out_dir, name)
+ with open(path, 'w') as f:
+ for item in artifact:
+ f.write(item if item_to_string is None else item_to_string(item))
+ f.write('\n')
+
def write_tmp_logcat(logcat, iteration, options):
- write_tmp_textual_artifact(logcat, iteration, options, 'logcat.txt')
+ write_tmp_textual_artifact(logcat, iteration, options, 'logcat.txt')
+
def write_tmp_profile(profile, iteration, options):
- write_tmp_binary_artifact(profile, iteration, options, 'primary.prof')
+ write_tmp_binary_artifact(profile, iteration, options, 'primary.prof')
-def write_tmp_profile_classes_and_methods(
- profile_classes_and_methods, iteration, options):
- def item_to_string(item):
- (descriptor, flags) = item
- return '%s%s%s%s' % (
- 'H' if flags.get('hot') else '',
- 'S' if flags.get('startup') else '',
- 'P' if flags.get('post_startup') else '',
- descriptor)
- write_tmp_textual_artifact(
- profile_classes_and_methods.items(),
- iteration,
- options,
- 'profile.txt',
- item_to_string)
+
+def write_tmp_profile_classes_and_methods(profile_classes_and_methods,
+ iteration, options):
+
+ def item_to_string(item):
+ (descriptor, flags) = item
+ return '%s%s%s%s' % ('H' if flags.get('hot') else '',
+ 'S' if flags.get('startup') else '', 'P'
+ if flags.get('post_startup') else '', descriptor)
+
+ write_tmp_textual_artifact(profile_classes_and_methods.items(), iteration,
+ options, 'profile.txt', item_to_string)
+
def write_tmp_startup_descriptors(startup_descriptors, iteration, options):
- lines = [
- startup_descriptor_to_string(startup_descriptor, flags)
- for startup_descriptor, flags in startup_descriptors.items()]
- write_tmp_textual_artifact(
- lines, iteration, options, 'startup-descriptors.txt')
+ lines = [
+ startup_descriptor_to_string(startup_descriptor, flags)
+ for startup_descriptor, flags in startup_descriptors.items()
+ ]
+ write_tmp_textual_artifact(lines, iteration, options,
+ 'startup-descriptors.txt')
+
def startup_descriptor_to_string(startup_descriptor, flags):
- result = ''
- if flags['hot']:
- result += 'H'
- if flags['startup']:
- result += 'S'
- if flags['post_startup']:
- result += 'P'
- result += startup_descriptor
- return result
+ result = ''
+ if flags['hot']:
+ result += 'H'
+ if flags['startup']:
+ result += 'S'
+ if flags['post_startup']:
+ result += 'P'
+ result += startup_descriptor
+ return result
+
def should_include_startup_descriptor(descriptor, flags, options):
- if flags.get('conditional_startup') \
- and not options.include_conditional_startup:
- return False
- if flags.get('post_startup') \
- and not flags.get('startup') \
- and not options.include_post_startup:
- return False
- return True
+ if flags.get('conditional_startup') \
+ and not options.include_conditional_startup:
+ return False
+ if flags.get('post_startup') \
+ and not flags.get('startup') \
+ and not options.include_post_startup:
+ return False
+ return True
+
def parse_options(argv):
- result = argparse.ArgumentParser(
- description='Generate a perfetto trace file.')
- result.add_argument('--apk',
- help='Path to the .apk')
- result.add_argument('--apks',
- help='Path to the .apks')
- result.add_argument('--app-id',
- help='The application ID of interest',
- required=True)
- result.add_argument('--bundle',
- help='Path to the .aab')
- result.add_argument('--device-id',
- help='Device id (e.g., emulator-5554).',
- action='append')
- result.add_argument('--device-pin',
- help='Device pin code (e.g., 1234)',
- action='append')
- result.add_argument('--generalize-synthetics',
- help='Whether synthetics should be abstracted into their '
- 'synthetic contexts',
- action='store_true',
- default=False)
- result.add_argument('--grant-post-notification-permission',
- help='Grants the android.permission.POST_NOTIFICATIONS '
- 'permission before launching the app',
- default=False,
- action='store_true')
- result.add_argument('--logcat',
- action='store_true',
- default=False)
- result.add_argument('--include-conditional-startup',
- help='Include conditional startup classes and methods in '
- 'the R8 startup descriptors',
- action='store_true',
- default=False)
- result.add_argument('--include-post-startup',
- help='Include post startup classes and methods in the R8 '
- 'startup descriptors',
- action='store_true',
- default=False)
- result.add_argument('--iterations',
- help='Number of profiles to generate',
- default=1,
- type=int)
- result.add_argument('--main-activity',
- help='Main activity class name')
- result.add_argument('--out',
- help='File where to store startup descriptors (defaults '
- 'to stdout)')
- result.add_argument('--startup-duration',
- help='Duration in seconds before shutting down app',
- default=15,
- type=int)
- result.add_argument('--tmp-dir',
- help='Directory where to store intermediate artifacts'
- ' (by default these are not emitted)')
- result.add_argument('--until-stable',
- help='Repeat profile generation until no new startup '
- 'descriptors are found',
- action='store_true',
- default=False)
- result.add_argument('--until-stable-iterations',
- help='Number of times that profile generation must must '
- 'not find new startup descriptors before exiting',
- default=1,
- type=int)
- result.add_argument('--use-existing-profile',
- help='Do not launch app to generate startup profile',
- action='store_true',
- default=False)
- options, args = result.parse_known_args(argv)
+ result = argparse.ArgumentParser(
+ description='Generate a perfetto trace file.')
+ result.add_argument('--apk', help='Path to the .apk')
+ result.add_argument('--apks', help='Path to the .apks')
+ result.add_argument('--app-id',
+ help='The application ID of interest',
+ required=True)
+ result.add_argument('--bundle', help='Path to the .aab')
+ result.add_argument('--device-id',
+ help='Device id (e.g., emulator-5554).',
+ action='append')
+ result.add_argument('--device-pin',
+ help='Device pin code (e.g., 1234)',
+ action='append')
+ result.add_argument(
+ '--generalize-synthetics',
+ help='Whether synthetics should be abstracted into their '
+ 'synthetic contexts',
+ action='store_true',
+ default=False)
+ result.add_argument('--grant-post-notification-permission',
+ help='Grants the android.permission.POST_NOTIFICATIONS '
+ 'permission before launching the app',
+ default=False,
+ action='store_true')
+ result.add_argument('--logcat', action='store_true', default=False)
+ result.add_argument(
+ '--include-conditional-startup',
+ help='Include conditional startup classes and methods in '
+ 'the R8 startup descriptors',
+ action='store_true',
+ default=False)
+ result.add_argument(
+ '--include-post-startup',
+ help='Include post startup classes and methods in the R8 '
+ 'startup descriptors',
+ action='store_true',
+ default=False)
+ result.add_argument('--iterations',
+ help='Number of profiles to generate',
+ default=1,
+ type=int)
+ result.add_argument('--main-activity', help='Main activity class name')
+ result.add_argument(
+ '--out',
+ help='File where to store startup descriptors (defaults '
+ 'to stdout)')
+ result.add_argument('--startup-duration',
+ help='Duration in seconds before shutting down app',
+ default=15,
+ type=int)
+ result.add_argument('--tmp-dir',
+ help='Directory where to store intermediate artifacts'
+ ' (by default these are not emitted)')
+ result.add_argument('--until-stable',
+ help='Repeat profile generation until no new startup '
+ 'descriptors are found',
+ action='store_true',
+ default=False)
+ result.add_argument(
+ '--until-stable-iterations',
+ help='Number of times that profile generation must must '
+ 'not find new startup descriptors before exiting',
+ default=1,
+ type=int)
+ result.add_argument('--use-existing-profile',
+ help='Do not launch app to generate startup profile',
+ action='store_true',
+ default=False)
+ options, args = result.parse_known_args(argv)
- # Read the device pins.
- device_pins = options.device_pin or []
- del options.device_pin
+ # Read the device pins.
+ device_pins = options.device_pin or []
+ del options.device_pin
- # Convert the device ids and pins into a list of devices.
- options.devices = []
- if options.device_id is None:
- # Assume a single device is attached.
- options.devices.append(
- Device(None, device_pins[0] if len(device_pins) > 0 else None))
- else:
- for i in range(len(options.device_id)):
- device_id = options.device_id[i]
- device_pin = device_pins[i] if i < len(device_pins) else None
- options.devices.append(Device(device_id, device_pin))
- del options.device_id
+ # Convert the device ids and pins into a list of devices.
+ options.devices = []
+ if options.device_id is None:
+ # Assume a single device is attached.
+ options.devices.append(
+ Device(None, device_pins[0] if len(device_pins) > 0 else None))
+ else:
+ for i in range(len(options.device_id)):
+ device_id = options.device_id[i]
+ device_pin = device_pins[i] if i < len(device_pins) else None
+ options.devices.append(Device(device_id, device_pin))
+ del options.device_id
- paths = [
- path for path in [options.apk, options.apks, options.bundle]
- if path is not None]
- assert len(paths) <= 1, 'Expected at most one .apk, .apks, or .aab file.'
- assert options.main_activity is not None or options.use_existing_profile, \
- 'Argument --main-activity is required except when running with ' \
- '--use-existing-profile.'
+ paths = [
+ path for path in [options.apk, options.apks, options.bundle]
+ if path is not None
+ ]
+ assert len(paths) <= 1, 'Expected at most one .apk, .apks, or .aab file.'
+ assert options.main_activity is not None or options.use_existing_profile, \
+ 'Argument --main-activity is required except when running with ' \
+ '--use-existing-profile.'
- return options, args
+ return options, args
+
def run_on_device(device, options, startup_descriptors):
- adb_utils.root(device.device_id)
- if options.apk:
- adb_utils.uninstall(options.app_id, device.device_id)
- adb_utils.install(options.apk, device.device_id)
- elif options.apks:
- adb_utils.uninstall(options.app_id, device.device_id)
- adb_utils.install_apks(options.apks, device.device_id)
- elif options.bundle:
- adb_utils.uninstall(options.app_id, device.device_id)
- adb_utils.install_bundle(options.bundle, device.device_id)
- # Grant notifications.
- if options.grant_post_notification_permission:
- adb_utils.grant(
- options.app_id,
- 'android.permission.POST_NOTIFICATIONS',
- device.device_id)
- if options.until_stable:
- iteration = 0
- stable_iterations = 0
- while True:
- old_startup_descriptors = startup_descriptors
- startup_descriptors = extend_startup_descriptors(
- old_startup_descriptors, iteration, device, options)
- diff = len(startup_descriptors) - len(old_startup_descriptors)
- if diff == 0:
- stable_iterations = stable_iterations + 1
- if stable_iterations == options.until_stable_iterations:
- break
- else:
+ adb_utils.root(device.device_id)
+ if options.apk:
+ adb_utils.uninstall(options.app_id, device.device_id)
+ adb_utils.install(options.apk, device.device_id)
+ elif options.apks:
+ adb_utils.uninstall(options.app_id, device.device_id)
+ adb_utils.install_apks(options.apks, device.device_id)
+ elif options.bundle:
+ adb_utils.uninstall(options.app_id, device.device_id)
+ adb_utils.install_bundle(options.bundle, device.device_id)
+ # Grant notifications.
+ if options.grant_post_notification_permission:
+ adb_utils.grant(options.app_id, 'android.permission.POST_NOTIFICATIONS',
+ device.device_id)
+ if options.until_stable:
+ iteration = 0
stable_iterations = 0
- iteration = iteration + 1
- else:
- for iteration in range(options.iterations):
- startup_descriptors = extend_startup_descriptors(
- startup_descriptors, iteration, device, options)
- return startup_descriptors
+ while True:
+ old_startup_descriptors = startup_descriptors
+ startup_descriptors = extend_startup_descriptors(
+ old_startup_descriptors, iteration, device, options)
+ diff = len(startup_descriptors) - len(old_startup_descriptors)
+ if diff == 0:
+ stable_iterations = stable_iterations + 1
+ if stable_iterations == options.until_stable_iterations:
+ break
+ else:
+ stable_iterations = 0
+ iteration = iteration + 1
+ else:
+ for iteration in range(options.iterations):
+ startup_descriptors = extend_startup_descriptors(
+ startup_descriptors, iteration, device, options)
+ return startup_descriptors
+
def main(argv):
- (options, args) = parse_options(argv)
- startup_descriptors = {}
- for device in options.devices:
- startup_descriptors = run_on_device(device, options, startup_descriptors)
- if options.out is not None:
- with open(options.out, 'w') as f:
- for startup_descriptor, flags in startup_descriptors.items():
- if should_include_startup_descriptor(startup_descriptor, flags, options):
- f.write(startup_descriptor_to_string(startup_descriptor, flags))
- f.write('\n')
- else:
- for startup_descriptor, flags in startup_descriptors.items():
- if should_include_startup_descriptor(startup_descriptor, flags, options):
- print(startup_descriptor_to_string(startup_descriptor, flags))
+ (options, args) = parse_options(argv)
+ startup_descriptors = {}
+ for device in options.devices:
+ startup_descriptors = run_on_device(device, options,
+ startup_descriptors)
+ if options.out is not None:
+ with open(options.out, 'w') as f:
+ for startup_descriptor, flags in startup_descriptors.items():
+ if should_include_startup_descriptor(startup_descriptor, flags,
+ options):
+ f.write(
+ startup_descriptor_to_string(startup_descriptor, flags))
+ f.write('\n')
+ else:
+ for startup_descriptor, flags in startup_descriptors.items():
+ if should_include_startup_descriptor(startup_descriptor, flags,
+ options):
+ print(startup_descriptor_to_string(startup_descriptor, flags))
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/instrument.py b/tools/startup/instrument.py
index ecefe55..9752670 100755
--- a/tools/startup/instrument.py
+++ b/tools/startup/instrument.py
@@ -18,117 +18,126 @@
import utils
import zip_utils
+
def parse_options(argv):
- result = argparse.ArgumentParser(
- description='Instrument the dex files of a given apk to print what is '
- 'executed.')
- result.add_argument('--apk',
- help='Path to the .apk',
- required=True)
- result.add_argument('--dex-files',
- action='append',
- help='Name of dex files to instrument')
- result.add_argument('--discard',
- action='append',
- help='Name of dex files to discard')
- result.add_argument('--out',
- help='Destination of resulting apk',
- required=True)
- options, args = result.parse_known_args(argv)
- return options, args
+ result = argparse.ArgumentParser(
+ description='Instrument the dex files of a given apk to print what is '
+ 'executed.')
+ result.add_argument('--apk', help='Path to the .apk', required=True)
+ result.add_argument('--dex-files',
+ action='append',
+ help='Name of dex files to instrument')
+ result.add_argument('--discard',
+ action='append',
+ help='Name of dex files to discard')
+ result.add_argument('--out',
+ help='Destination of resulting apk',
+ required=True)
+ options, args = result.parse_known_args(argv)
+ return options, args
+
def add_instrumented_dex(dex_file, instrumented_dex_index, instrumented_dir):
- dex_name = get_dex_name(instrumented_dex_index)
- destination = os.path.join(instrumented_dir, dex_name)
- shutil.move(dex_file, destination)
+ dex_name = get_dex_name(instrumented_dex_index)
+ destination = os.path.join(instrumented_dir, dex_name)
+ shutil.move(dex_file, destination)
+
def get_dex_name(dex_index):
- assert dex_index > 0
- return 'classes.dex' if dex_index == 1 else ('classes%s.dex' % dex_index)
+ assert dex_index > 0
+ return 'classes.dex' if dex_index == 1 else ('classes%s.dex' % dex_index)
-def instrument_dex_file(dex_file, include_instrumentation_server, options, tmp_dir):
- d8_cmd = [
- 'java',
- '-cp', utils.R8_JAR,
- '-Dcom.android.tools.r8.startup.instrumentation.instrument=1',
- '-Dcom.android.tools.r8.startup.instrumentation.instrumentationtag=R8']
- if not include_instrumentation_server:
- # We avoid injecting the InstrumentationServer by specifying it should only
- # be added if foo.bar.Baz is in the program.
- d8_cmd.append(
- '-Dcom.android.tools.r8.startup.instrumentation.instrumentationserversyntheticcontext=foo.bar.Baz')
- d8_cmd.extend([
- 'com.android.tools.r8.D8',
- '--min-api', str(apk_utils.get_min_api(options.apk)),
- '--output', tmp_dir,
- '--release',
- dex_file])
- subprocess.check_call(d8_cmd)
- instrumented_dex_files = []
- instrumented_dex_index = 1
- while True:
- instrumented_dex_name = get_dex_name(instrumented_dex_index)
- instrumented_dex_file = os.path.join(tmp_dir, instrumented_dex_name)
- if not os.path.exists(instrumented_dex_file):
- break
- instrumented_dex_files.append(instrumented_dex_file)
- instrumented_dex_index = instrumented_dex_index + 1
- assert len(instrumented_dex_files) > 0
- return instrumented_dex_files
+
+def instrument_dex_file(dex_file, include_instrumentation_server, options,
+ tmp_dir):
+ d8_cmd = [
+ 'java', '-cp', utils.R8_JAR,
+ '-Dcom.android.tools.r8.startup.instrumentation.instrument=1',
+ '-Dcom.android.tools.r8.startup.instrumentation.instrumentationtag=R8'
+ ]
+ if not include_instrumentation_server:
+ # We avoid injecting the InstrumentationServer by specifying it should only
+ # be added if foo.bar.Baz is in the program.
+ d8_cmd.append(
+ '-Dcom.android.tools.r8.startup.instrumentation.instrumentationserversyntheticcontext=foo.bar.Baz'
+ )
+ d8_cmd.extend([
+ 'com.android.tools.r8.D8', '--min-api',
+ str(apk_utils.get_min_api(options.apk)), '--output', tmp_dir,
+ '--release', dex_file
+ ])
+ subprocess.check_call(d8_cmd)
+ instrumented_dex_files = []
+ instrumented_dex_index = 1
+ while True:
+ instrumented_dex_name = get_dex_name(instrumented_dex_index)
+ instrumented_dex_file = os.path.join(tmp_dir, instrumented_dex_name)
+ if not os.path.exists(instrumented_dex_file):
+ break
+ instrumented_dex_files.append(instrumented_dex_file)
+ instrumented_dex_index = instrumented_dex_index + 1
+ assert len(instrumented_dex_files) > 0
+ return instrumented_dex_files
+
def should_discard_dex_file(dex_name, options):
- return options.discard is not None and dex_name in options.discard
+ return options.discard is not None and dex_name in options.discard
+
def should_instrument_dex_file(dex_name, options):
- return options.dex_files is not None and dex_name in options.dex_files
+ return options.dex_files is not None and dex_name in options.dex_files
+
def main(argv):
- options, args = parse_options(argv)
- with utils.TempDir() as tmp_dir:
- # Extract the dex files of the apk.
- uninstrumented_dir = os.path.join(tmp_dir, 'uninstrumented')
- os.mkdir(uninstrumented_dir)
+ options, args = parse_options(argv)
+ with utils.TempDir() as tmp_dir:
+ # Extract the dex files of the apk.
+ uninstrumented_dir = os.path.join(tmp_dir, 'uninstrumented')
+ os.mkdir(uninstrumented_dir)
- dex_predicate = \
- lambda name : name.startswith('classes') and name.endswith('.dex')
- zip_utils.extract_all_that_matches(
- options.apk, uninstrumented_dir, dex_predicate)
+ dex_predicate = \
+ lambda name : name.startswith('classes') and name.endswith('.dex')
+ zip_utils.extract_all_that_matches(options.apk, uninstrumented_dir,
+ dex_predicate)
- # Instrument each dex one by one.
- instrumented_dir = os.path.join(tmp_dir, 'instrumented')
- os.mkdir(instrumented_dir)
+ # Instrument each dex one by one.
+ instrumented_dir = os.path.join(tmp_dir, 'instrumented')
+ os.mkdir(instrumented_dir)
- include_instrumentation_server = True
- instrumented_dex_index = 1
- uninstrumented_dex_index = 1
- while True:
- dex_name = get_dex_name(uninstrumented_dex_index)
- dex_file = os.path.join(uninstrumented_dir, dex_name)
- if not os.path.exists(dex_file):
- break
- if not should_discard_dex_file(dex_name, options):
- if should_instrument_dex_file(dex_name, options):
- with utils.TempDir() as tmp_instrumentation_dir:
- instrumented_dex_files = \
- instrument_dex_file(
- dex_file,
- include_instrumentation_server,
- options,
- tmp_instrumentation_dir)
- for instrumented_dex_file in instrumented_dex_files:
- add_instrumented_dex(
- instrumented_dex_file, instrumented_dex_index, instrumented_dir)
- instrumented_dex_index = instrumented_dex_index + 1
- include_instrumentation_server = False
- else:
- add_instrumented_dex(dex_file, instrumented_dex_index, instrumented_dir)
- instrumented_dex_index = instrumented_dex_index + 1
- uninstrumented_dex_index = uninstrumented_dex_index + 1
+ include_instrumentation_server = True
+ instrumented_dex_index = 1
+ uninstrumented_dex_index = 1
+ while True:
+ dex_name = get_dex_name(uninstrumented_dex_index)
+ dex_file = os.path.join(uninstrumented_dir, dex_name)
+ if not os.path.exists(dex_file):
+ break
+ if not should_discard_dex_file(dex_name, options):
+ if should_instrument_dex_file(dex_name, options):
+ with utils.TempDir() as tmp_instrumentation_dir:
+ instrumented_dex_files = \
+ instrument_dex_file(
+ dex_file,
+ include_instrumentation_server,
+ options,
+ tmp_instrumentation_dir)
+ for instrumented_dex_file in instrumented_dex_files:
+ add_instrumented_dex(instrumented_dex_file,
+ instrumented_dex_index,
+ instrumented_dir)
+ instrumented_dex_index = instrumented_dex_index + 1
+ include_instrumentation_server = False
+ else:
+ add_instrumented_dex(dex_file, instrumented_dex_index,
+ instrumented_dir)
+ instrumented_dex_index = instrumented_dex_index + 1
+ uninstrumented_dex_index = uninstrumented_dex_index + 1
- assert instrumented_dex_index > 1
+ assert instrumented_dex_index > 1
- # Masseur APK.
- apk_masseur.masseur(options.apk, dex=instrumented_dir, out=options.out)
+ # Masseur APK.
+ apk_masseur.masseur(options.apk, dex=instrumented_dir, out=options.out)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/measure_startup.py b/tools/startup/measure_startup.py
index 57f6cb4..7e97735 100755
--- a/tools/startup/measure_startup.py
+++ b/tools/startup/measure_startup.py
@@ -18,446 +18,461 @@
import perfetto_utils
import utils
+
def setup(options):
- # Increase screen off timeout to avoid device screen turns off.
- twenty_four_hours_in_millis = 24 * 60 * 60 * 1000
- previous_screen_off_timeout = adb_utils.get_screen_off_timeout(
- options.device_id)
- adb_utils.set_screen_off_timeout(
- twenty_four_hours_in_millis, options.device_id)
+ # Increase screen off timeout to avoid device screen turns off.
+ twenty_four_hours_in_millis = 24 * 60 * 60 * 1000
+ previous_screen_off_timeout = adb_utils.get_screen_off_timeout(
+ options.device_id)
+ adb_utils.set_screen_off_timeout(twenty_four_hours_in_millis,
+ options.device_id)
- # Unlock device.
- adb_utils.unlock(options.device_id, options.device_pin)
+ # Unlock device.
+ adb_utils.unlock(options.device_id, options.device_pin)
- teardown_options = {
- 'previous_screen_off_timeout': previous_screen_off_timeout
- }
- return teardown_options
+ teardown_options = {
+ 'previous_screen_off_timeout': previous_screen_off_timeout
+ }
+ return teardown_options
+
def teardown(options, teardown_options):
- # Reset screen off timeout.
- adb_utils.set_screen_off_timeout(
- teardown_options['previous_screen_off_timeout'],
- options.device_id)
+ # Reset screen off timeout.
+ adb_utils.set_screen_off_timeout(
+ teardown_options['previous_screen_off_timeout'], options.device_id)
+
def run_all(apk_or_apks, options, tmp_dir):
- # Launch app while collecting information.
- data_total = {}
- for iteration in range(1, options.iterations + 1):
- print('Starting iteration %i' % iteration)
- out_dir = os.path.join(options.out_dir, str(iteration))
- teardown_options = setup_for_run(apk_or_apks, out_dir, options)
- data = run(out_dir, options, tmp_dir)
- teardown_for_run(out_dir, options, teardown_options)
- add_data(data_total, data)
- print('Result:')
- print(data)
- print(compute_data_summary(data_total))
- print('Done')
- print('Average result:')
- data_summary = compute_data_summary(data_total)
- print(data_summary)
- write_data_to_dir(options.out_dir, data_summary)
- if options.out:
- write_data_to_file(options.out, data_summary)
+ # Launch app while collecting information.
+ data_total = {}
+ for iteration in range(1, options.iterations + 1):
+ print('Starting iteration %i' % iteration)
+ out_dir = os.path.join(options.out_dir, str(iteration))
+ teardown_options = setup_for_run(apk_or_apks, out_dir, options)
+ data = run(out_dir, options, tmp_dir)
+ teardown_for_run(out_dir, options, teardown_options)
+ add_data(data_total, data)
+ print('Result:')
+ print(data)
+ print(compute_data_summary(data_total))
+ print('Done')
+ print('Average result:')
+ data_summary = compute_data_summary(data_total)
+ print(data_summary)
+ write_data_to_dir(options.out_dir, data_summary)
+ if options.out:
+ write_data_to_file(options.out, data_summary)
+
def compute_data_summary(data_total):
- data_summary = {}
- for key, value in data_total.items():
- if not isinstance(value, list):
- data_summary[key] = value
- continue
- data_summary['%s_avg' % key] = round(statistics.mean(value), 1)
- data_summary['%s_med' % key] = statistics.median(value)
- data_summary['%s_min' % key] = min(value)
- data_summary['%s_max' % key] = max(value)
- return data_summary
+ data_summary = {}
+ for key, value in data_total.items():
+ if not isinstance(value, list):
+ data_summary[key] = value
+ continue
+ data_summary['%s_avg' % key] = round(statistics.mean(value), 1)
+ data_summary['%s_med' % key] = statistics.median(value)
+ data_summary['%s_min' % key] = min(value)
+ data_summary['%s_max' % key] = max(value)
+ return data_summary
+
def setup_for_run(apk_or_apks, out_dir, options):
- adb_utils.root(options.device_id)
+ adb_utils.root(options.device_id)
- print('Installing')
- adb_utils.uninstall(options.app_id, options.device_id)
- if apk_or_apks['apk']:
- adb_utils.install(apk_or_apks['apk'], options.device_id)
- else:
- assert apk_or_apks['apks']
- adb_utils.install_apks(apk_or_apks['apks'], options.device_id)
-
- os.makedirs(out_dir, exist_ok=True)
-
- # Grant notifications.
- if options.grant_post_notification_permission:
- adb_utils.grant(
- options.app_id,
- 'android.permission.POST_NOTIFICATIONS',
- options.device_id)
-
- # AOT compile.
- if options.aot:
- print('AOT compiling')
- if options.baseline_profile:
- adb_utils.clear_profile_data(options.app_id, options.device_id)
- if options.baseline_profile_install == 'adb':
- adb_utils.install_profile_using_adb(
- options.app_id, options.baseline_profile, options.device_id)
- else:
- assert options.baseline_profile_install == 'profileinstaller'
- adb_utils.install_profile_using_profileinstaller(
- options.app_id, options.device_id)
+ print('Installing')
+ adb_utils.uninstall(options.app_id, options.device_id)
+ if apk_or_apks['apk']:
+ adb_utils.install(apk_or_apks['apk'], options.device_id)
else:
- adb_utils.force_compilation(options.app_id, options.device_id)
+ assert apk_or_apks['apks']
+ adb_utils.install_apks(apk_or_apks['apks'], options.device_id)
- # Cooldown and then unlock device.
- if options.cooldown > 0:
- print('Cooling down for %i seconds' % options.cooldown)
- assert adb_utils.get_screen_state(options.device_id).is_off()
- time.sleep(options.cooldown)
- teardown_options = adb_utils.prepare_for_interaction_with_device(
- options.device_id, options.device_pin)
- else:
- teardown_options = None
+ os.makedirs(out_dir, exist_ok=True)
- # Prelaunch for hot startup.
- if options.hot_startup:
- print('Prelaunching')
- adb_utils.launch_activity(
+ # Grant notifications.
+ if options.grant_post_notification_permission:
+ adb_utils.grant(options.app_id, 'android.permission.POST_NOTIFICATIONS',
+ options.device_id)
+
+ # AOT compile.
+ if options.aot:
+ print('AOT compiling')
+ if options.baseline_profile:
+ adb_utils.clear_profile_data(options.app_id, options.device_id)
+ if options.baseline_profile_install == 'adb':
+ adb_utils.install_profile_using_adb(options.app_id,
+ options.baseline_profile,
+ options.device_id)
+ else:
+ assert options.baseline_profile_install == 'profileinstaller'
+ adb_utils.install_profile_using_profileinstaller(
+ options.app_id, options.device_id)
+ else:
+ adb_utils.force_compilation(options.app_id, options.device_id)
+
+ # Cooldown and then unlock device.
+ if options.cooldown > 0:
+ print('Cooling down for %i seconds' % options.cooldown)
+ assert adb_utils.get_screen_state(options.device_id).is_off()
+ time.sleep(options.cooldown)
+ teardown_options = adb_utils.prepare_for_interaction_with_device(
+ options.device_id, options.device_pin)
+ else:
+ teardown_options = None
+
+ # Prelaunch for hot startup.
+ if options.hot_startup:
+ print('Prelaunching')
+ adb_utils.launch_activity(options.app_id,
+ options.main_activity,
+ options.device_id,
+ wait_for_activity_to_launch=False)
+ time.sleep(options.startup_duration)
+ adb_utils.navigate_to_home_screen(options.device_id)
+ time.sleep(1)
+
+ # Drop caches before run.
+ adb_utils.drop_caches(options.device_id)
+ return teardown_options
+
+
+def teardown_for_run(out_dir, options, teardown_options):
+ assert adb_utils.get_screen_state(options.device_id).is_on_and_unlocked()
+
+ if options.capture_screen:
+ target = os.path.join(out_dir, 'screen.png')
+ adb_utils.capture_screen(target, options.device_id)
+
+ if options.cooldown > 0:
+ adb_utils.teardown_after_interaction_with_device(
+ teardown_options, options.device_id)
+ adb_utils.ensure_screen_off(options.device_id)
+ else:
+ assert teardown_options is None
+
+
+def run(out_dir, options, tmp_dir):
+ assert adb_utils.get_screen_state(options.device_id).is_on_and_unlocked()
+
+ # Start logcat for time to fully drawn.
+ logcat_process = None
+ if options.fully_drawn_logcat_message:
+ adb_utils.clear_logcat(options.device_id)
+ logcat_process = adb_utils.start_logcat(
+ options.device_id,
+ format='time',
+ filter='%s ActivityTaskManager:I' %
+ options.fully_drawn_logcat_filter,
+ silent=True)
+
+ # Start perfetto trace collector.
+ perfetto_process = None
+ perfetto_trace_path = None
+ if options.perfetto:
+ perfetto_process, perfetto_trace_path = perfetto_utils.record_android_trace(
+ out_dir, tmp_dir, options.device_id)
+
+ # Launch main activity.
+ launch_activity_result = adb_utils.launch_activity(
options.app_id,
options.main_activity,
options.device_id,
- wait_for_activity_to_launch=False)
- time.sleep(options.startup_duration)
- adb_utils.navigate_to_home_screen(options.device_id)
- time.sleep(1)
+ intent_data_uri=options.intent_data_uri,
+ wait_for_activity_to_launch=True)
- # Drop caches before run.
- adb_utils.drop_caches(options.device_id)
- return teardown_options
+ # Wait for app to be fully drawn.
+ logcat = None
+ if logcat_process is not None:
+ wait_until_fully_drawn(logcat_process, options)
+ logcat = adb_utils.stop_logcat(logcat_process)
-def teardown_for_run(out_dir, options, teardown_options):
- assert adb_utils.get_screen_state(options.device_id).is_on_and_unlocked()
+ # Wait for perfetto trace collector to stop.
+ if options.perfetto:
+ perfetto_utils.stop_record_android_trace(perfetto_process, out_dir)
- if options.capture_screen:
- target = os.path.join(out_dir, 'screen.png')
- adb_utils.capture_screen(target, options.device_id)
+ # Get minor and major page faults from app process.
+ data = compute_data(launch_activity_result, logcat, perfetto_trace_path,
+ options)
+ write_data_to_dir(out_dir, data)
+ return data
- if options.cooldown > 0:
- adb_utils.teardown_after_interaction_with_device(
- teardown_options, options.device_id)
- adb_utils.ensure_screen_off(options.device_id)
- else:
- assert teardown_options is None
-
-def run(out_dir, options, tmp_dir):
- assert adb_utils.get_screen_state(options.device_id).is_on_and_unlocked()
-
- # Start logcat for time to fully drawn.
- logcat_process = None
- if options.fully_drawn_logcat_message:
- adb_utils.clear_logcat(options.device_id)
- logcat_process = adb_utils.start_logcat(
- options.device_id,
- format='time',
- filter='%s ActivityTaskManager:I' % options.fully_drawn_logcat_filter,
- silent=True)
-
- # Start perfetto trace collector.
- perfetto_process = None
- perfetto_trace_path = None
- if options.perfetto:
- perfetto_process, perfetto_trace_path = perfetto_utils.record_android_trace(
- out_dir, tmp_dir, options.device_id)
-
- # Launch main activity.
- launch_activity_result = adb_utils.launch_activity(
- options.app_id,
- options.main_activity,
- options.device_id,
- intent_data_uri=options.intent_data_uri,
- wait_for_activity_to_launch=True)
-
- # Wait for app to be fully drawn.
- logcat = None
- if logcat_process is not None:
- wait_until_fully_drawn(logcat_process, options)
- logcat = adb_utils.stop_logcat(logcat_process)
-
- # Wait for perfetto trace collector to stop.
- if options.perfetto:
- perfetto_utils.stop_record_android_trace(perfetto_process, out_dir)
-
- # Get minor and major page faults from app process.
- data = compute_data(
- launch_activity_result, logcat, perfetto_trace_path, options)
- write_data_to_dir(out_dir, data)
- return data
def wait_until_fully_drawn(logcat_process, options):
- print('Waiting until app is fully drawn')
- while True:
- is_fully_drawn = any(
- is_app_fully_drawn_logcat_message(line, options) \
- for line in logcat_process.lines)
- if is_fully_drawn:
- break
- time.sleep(1)
- print('Done')
+ print('Waiting until app is fully drawn')
+ while True:
+ is_fully_drawn = any(
+ is_app_fully_drawn_logcat_message(line, options) \
+ for line in logcat_process.lines)
+ if is_fully_drawn:
+ break
+ time.sleep(1)
+ print('Done')
+
def compute_time_to_fully_drawn_from_time_to_first_frame(logcat, options):
- displayed_time = None
- fully_drawn_time = None
- for line in logcat:
- if is_app_displayed_logcat_message(line, options):
- displayed_time = get_timestamp_from_logcat_message(line)
- elif is_app_fully_drawn_logcat_message(line, options):
- fully_drawn_time = get_timestamp_from_logcat_message(line)
- assert displayed_time is not None
- assert fully_drawn_time is not None
- assert fully_drawn_time >= displayed_time
- return fully_drawn_time - displayed_time
+ displayed_time = None
+ fully_drawn_time = None
+ for line in logcat:
+ if is_app_displayed_logcat_message(line, options):
+ displayed_time = get_timestamp_from_logcat_message(line)
+ elif is_app_fully_drawn_logcat_message(line, options):
+ fully_drawn_time = get_timestamp_from_logcat_message(line)
+ assert displayed_time is not None
+ assert fully_drawn_time is not None
+ assert fully_drawn_time >= displayed_time
+ return fully_drawn_time - displayed_time
+
def get_timestamp_from_logcat_message(line):
- time_end_index = len('00-00 00:00:00.000')
- time_format = '%m-%d %H:%M:%S.%f'
- time_str = line[0:time_end_index] + '000'
- time_seconds = datetime.datetime.strptime(time_str, time_format).timestamp()
- return int(time_seconds * 1000)
+ time_end_index = len('00-00 00:00:00.000')
+ time_format = '%m-%d %H:%M:%S.%f'
+ time_str = line[0:time_end_index] + '000'
+ time_seconds = datetime.datetime.strptime(time_str, time_format).timestamp()
+ return int(time_seconds * 1000)
+
def is_app_displayed_logcat_message(line, options):
- substring = 'Displayed %s' % adb_utils.get_component_name(
- options.app_id, options.main_activity)
- return substring in line
+ substring = 'Displayed %s' % adb_utils.get_component_name(
+ options.app_id, options.main_activity)
+ return substring in line
+
def is_app_fully_drawn_logcat_message(line, options):
- return re.search(options.fully_drawn_logcat_message, line)
+ return re.search(options.fully_drawn_logcat_message, line)
+
def add_data(data_total, data):
- for key, value in data.items():
- if key == 'app_id':
- assert data_total.get(key, value) == value
- data_total[key] = value
- if key == 'time':
- continue
- if key in data_total:
- if key == 'app_id':
- assert data_total[key] == value
- else:
- existing_value = data_total[key]
- assert isinstance(value, int)
- assert isinstance(existing_value, list)
- existing_value.append(value)
- else:
- assert isinstance(value, int), key
- data_total[key] = [value]
+ for key, value in data.items():
+ if key == 'app_id':
+ assert data_total.get(key, value) == value
+ data_total[key] = value
+ if key == 'time':
+ continue
+ if key in data_total:
+ if key == 'app_id':
+ assert data_total[key] == value
+ else:
+ existing_value = data_total[key]
+ assert isinstance(value, int)
+ assert isinstance(existing_value, list)
+ existing_value.append(value)
+ else:
+ assert isinstance(value, int), key
+ data_total[key] = [value]
+
def compute_data(launch_activity_result, logcat, perfetto_trace_path, options):
- minfl, majfl = adb_utils.get_minor_major_page_faults(
- options.app_id, options.device_id)
- meminfo = adb_utils.get_meminfo(options.app_id, options.device_id)
- data = {
- 'app_id': options.app_id,
- 'time': time.ctime(time.time()),
- 'minfl': minfl,
- 'majfl': majfl
- }
- data.update(meminfo)
- startup_data = compute_startup_data(
- launch_activity_result, logcat, perfetto_trace_path, options)
- return data | startup_data
+ minfl, majfl = adb_utils.get_minor_major_page_faults(
+ options.app_id, options.device_id)
+ meminfo = adb_utils.get_meminfo(options.app_id, options.device_id)
+ data = {
+ 'app_id': options.app_id,
+ 'time': time.ctime(time.time()),
+ 'minfl': minfl,
+ 'majfl': majfl
+ }
+ data.update(meminfo)
+ startup_data = compute_startup_data(launch_activity_result, logcat,
+ perfetto_trace_path, options)
+ return data | startup_data
-def compute_startup_data(
- launch_activity_result, logcat, perfetto_trace_path, options):
- time_to_first_frame = launch_activity_result.get('total_time')
- startup_data = {
- 'adb_startup': time_to_first_frame
- }
- # Time to fully drawn.
- if options.fully_drawn_logcat_message:
- startup_data['time_to_fully_drawn'] = \
- compute_time_to_fully_drawn_from_time_to_first_frame(logcat, options) \
- + time_to_first_frame
+def compute_startup_data(launch_activity_result, logcat, perfetto_trace_path,
+ options):
+ time_to_first_frame = launch_activity_result.get('total_time')
+ startup_data = {'adb_startup': time_to_first_frame}
- # Perfetto stats.
- perfetto_startup_data = {}
- if options.perfetto:
- TraceProcessor = perfetto_utils.get_trace_processor()
- trace_processor = TraceProcessor(file_path=perfetto_trace_path)
+ # Time to fully drawn.
+ if options.fully_drawn_logcat_message:
+ startup_data['time_to_fully_drawn'] = \
+ compute_time_to_fully_drawn_from_time_to_first_frame(logcat, options) \
+ + time_to_first_frame
- # Compute time to first frame according to the builtin android_startup
- # metric.
- startup_metric = trace_processor.metric(['android_startup'])
- time_to_first_frame_ms = \
- startup_metric.android_startup.startup[0].to_first_frame.dur_ms
- perfetto_startup_data['perfetto_startup'] = round(time_to_first_frame_ms)
+ # Perfetto stats.
+ perfetto_startup_data = {}
+ if options.perfetto:
+ TraceProcessor = perfetto_utils.get_trace_processor()
+ trace_processor = TraceProcessor(file_path=perfetto_trace_path)
- if not options.hot_startup:
- # Compute time to first and last doFrame event.
- bind_application_slice = perfetto_utils.find_unique_slice_by_name(
- 'bindApplication', options, trace_processor)
- activity_start_slice = perfetto_utils.find_unique_slice_by_name(
- 'activityStart', options, trace_processor)
- do_frame_slices = perfetto_utils.find_slices_by_name(
- 'Choreographer#doFrame', options, trace_processor)
- first_do_frame_slice = next(do_frame_slices)
- *_, last_do_frame_slice = do_frame_slices
+ # Compute time to first frame according to the builtin android_startup
+ # metric.
+ startup_metric = trace_processor.metric(['android_startup'])
+ time_to_first_frame_ms = \
+ startup_metric.android_startup.startup[0].to_first_frame.dur_ms
+ perfetto_startup_data['perfetto_startup'] = round(
+ time_to_first_frame_ms)
- perfetto_startup_data.update({
- 'time_to_first_choreographer_do_frame_ms':
- round(perfetto_utils.get_slice_end_since_start(
- first_do_frame_slice, bind_application_slice)),
- 'time_to_last_choreographer_do_frame_ms':
- round(perfetto_utils.get_slice_end_since_start(
- last_do_frame_slice, bind_application_slice))
- })
+ if not options.hot_startup:
+ # Compute time to first and last doFrame event.
+ bind_application_slice = perfetto_utils.find_unique_slice_by_name(
+ 'bindApplication', options, trace_processor)
+ activity_start_slice = perfetto_utils.find_unique_slice_by_name(
+ 'activityStart', options, trace_processor)
+ do_frame_slices = perfetto_utils.find_slices_by_name(
+ 'Choreographer#doFrame', options, trace_processor)
+ first_do_frame_slice = next(do_frame_slices)
+ *_, last_do_frame_slice = do_frame_slices
- # Return combined startup data.
- return startup_data | perfetto_startup_data
+ perfetto_startup_data.update({
+ 'time_to_first_choreographer_do_frame_ms':
+ round(
+ perfetto_utils.get_slice_end_since_start(
+ first_do_frame_slice, bind_application_slice)),
+ 'time_to_last_choreographer_do_frame_ms':
+ round(
+ perfetto_utils.get_slice_end_since_start(
+ last_do_frame_slice, bind_application_slice))
+ })
+
+ # Return combined startup data.
+ return startup_data | perfetto_startup_data
+
def write_data_to_dir(out_dir, data):
- data_path = os.path.join(out_dir, 'data.txt')
- write_data_to_file(data_path, data)
+ data_path = os.path.join(out_dir, 'data.txt')
+ write_data_to_file(data_path, data)
+
def write_data_to_file(out_file, data):
- with open(out_file, 'w') as f:
- for key, value in data.items():
- f.write('%s=%s\n' % (key, str(value)))
+ with open(out_file, 'w') as f:
+ for key, value in data.items():
+ f.write('%s=%s\n' % (key, str(value)))
+
def parse_options(argv):
- result = argparse.ArgumentParser(
- description='Generate a perfetto trace file.')
- result.add_argument('--app-id',
- help='The application ID of interest',
- required=True)
- result.add_argument('--aot',
- help='Enable force compilation',
- default=False,
- action='store_true')
- result.add_argument('--apk',
- help='Path to the .apk')
- result.add_argument('--apks',
- help='Path to the .apks')
- result.add_argument('--bundle',
- help='Path to the .aab')
- result.add_argument('--capture-screen',
- help='Take a screenshot after each test',
- default=False,
- action='store_true')
- result.add_argument('--cooldown',
- help='Seconds to wait before running each iteration',
- default=0,
- type=int)
- result.add_argument('--device-id',
- help='Device id (e.g., emulator-5554).')
- result.add_argument('--device-pin',
- help='Device pin code (e.g., 1234)')
- result.add_argument('--fully-drawn-logcat-filter',
- help='Logcat filter for the fully drawn message '
- '(e.g., "tag:I")')
- result.add_argument('--fully-drawn-logcat-message',
- help='Logcat message that indicates that the app is '
- 'fully drawn (regexp)')
- result.add_argument('--grant-post-notification-permission',
- help='Grants the android.permission.POST_NOTIFICATIONS '
- 'permission before launching the app',
- default=False,
- action='store_true')
- result.add_argument('--hot-startup',
- help='Measure hot startup instead of cold startup',
- default=False,
- action='store_true')
- result.add_argument('--intent-data-uri',
- help='Value to use for the -d argument to the intent '
- 'that is used to launch the app')
- result.add_argument('--iterations',
- help='Number of traces to generate',
- default=1,
- type=int)
- result.add_argument('--main-activity',
- help='Main activity class name',
- required=True)
- result.add_argument('--no-perfetto',
- help='Disables perfetto trace generation',
- action='store_true',
- default=False)
- result.add_argument('--out',
- help='File to store result in')
- result.add_argument('--out-dir',
- help='Directory to store trace files in',
- required=True)
- result.add_argument('--baseline-profile',
- help='Baseline profile (.prof) in binary format')
- result.add_argument('--baseline-profile-metadata',
- help='Baseline profile metadata (.profm) in binary '
- 'format')
- result.add_argument('--baseline-profile-install',
- help='Whether to install profile using adb or '
- 'profileinstaller',
- choices=['adb', 'profileinstaller'],
- default='profileinstaller')
- result.add_argument('--startup-duration',
- help='Duration in seconds before shutting down app',
- default=15,
- type=int)
- options, args = result.parse_known_args(argv)
- setattr(options, 'perfetto', not options.no_perfetto)
+ result = argparse.ArgumentParser(
+ description='Generate a perfetto trace file.')
+ result.add_argument('--app-id',
+ help='The application ID of interest',
+ required=True)
+ result.add_argument('--aot',
+ help='Enable force compilation',
+ default=False,
+ action='store_true')
+ result.add_argument('--apk', help='Path to the .apk')
+ result.add_argument('--apks', help='Path to the .apks')
+ result.add_argument('--bundle', help='Path to the .aab')
+ result.add_argument('--capture-screen',
+ help='Take a screenshot after each test',
+ default=False,
+ action='store_true')
+ result.add_argument('--cooldown',
+ help='Seconds to wait before running each iteration',
+ default=0,
+ type=int)
+ result.add_argument('--device-id', help='Device id (e.g., emulator-5554).')
+ result.add_argument('--device-pin', help='Device pin code (e.g., 1234)')
+ result.add_argument('--fully-drawn-logcat-filter',
+ help='Logcat filter for the fully drawn message '
+ '(e.g., "tag:I")')
+ result.add_argument('--fully-drawn-logcat-message',
+ help='Logcat message that indicates that the app is '
+ 'fully drawn (regexp)')
+ result.add_argument('--grant-post-notification-permission',
+ help='Grants the android.permission.POST_NOTIFICATIONS '
+ 'permission before launching the app',
+ default=False,
+ action='store_true')
+ result.add_argument('--hot-startup',
+ help='Measure hot startup instead of cold startup',
+ default=False,
+ action='store_true')
+ result.add_argument('--intent-data-uri',
+ help='Value to use for the -d argument to the intent '
+ 'that is used to launch the app')
+ result.add_argument('--iterations',
+ help='Number of traces to generate',
+ default=1,
+ type=int)
+ result.add_argument('--main-activity',
+ help='Main activity class name',
+ required=True)
+ result.add_argument('--no-perfetto',
+ help='Disables perfetto trace generation',
+ action='store_true',
+ default=False)
+ result.add_argument('--out', help='File to store result in')
+ result.add_argument('--out-dir',
+ help='Directory to store trace files in',
+ required=True)
+ result.add_argument('--baseline-profile',
+ help='Baseline profile (.prof) in binary format')
+ result.add_argument('--baseline-profile-metadata',
+ help='Baseline profile metadata (.profm) in binary '
+ 'format')
+ result.add_argument('--baseline-profile-install',
+ help='Whether to install profile using adb or '
+ 'profileinstaller',
+ choices=['adb', 'profileinstaller'],
+ default='profileinstaller')
+ result.add_argument('--startup-duration',
+ help='Duration in seconds before shutting down app',
+ default=15,
+ type=int)
+ options, args = result.parse_known_args(argv)
+ setattr(options, 'perfetto', not options.no_perfetto)
- paths = [
- path for path in [options.apk, options.apks, options.bundle]
- if path is not None]
- assert len(paths) == 1, 'Expected exactly one .apk, .apks, or .aab file.'
+ paths = [
+ path for path in [options.apk, options.apks, options.bundle]
+ if path is not None
+ ]
+ assert len(paths) == 1, 'Expected exactly one .apk, .apks, or .aab file.'
- # Build .apks file up front to avoid building the bundle upon each install.
- if options.bundle:
- os.makedirs(options.out_dir, exist_ok=True)
- options.apks = os.path.join(options.out_dir, 'Bundle.apks')
- adb_utils.build_apks_from_bundle(
- options.bundle, options.apks, overwrite=True)
- del options.bundle
+ # Build .apks file up front to avoid building the bundle upon each install.
+ if options.bundle:
+ os.makedirs(options.out_dir, exist_ok=True)
+ options.apks = os.path.join(options.out_dir, 'Bundle.apks')
+ adb_utils.build_apks_from_bundle(options.bundle,
+ options.apks,
+ overwrite=True)
+ del options.bundle
- # Profile is only used with --aot.
- assert options.aot or not options.baseline_profile
+ # Profile is only used with --aot.
+ assert options.aot or not options.baseline_profile
- # Fully drawn logcat filter and message is absent or both present.
- assert (options.fully_drawn_logcat_filter is None) == \
- (options.fully_drawn_logcat_message is None)
+ # Fully drawn logcat filter and message is absent or both present.
+ assert (options.fully_drawn_logcat_filter is None) == \
+ (options.fully_drawn_logcat_message is None)
- return options, args
+ return options, args
+
def global_setup(options):
- # If there is no cooldown then unlock the screen once. Otherwise we turn off
- # the screen during the cooldown and unlock the screen before each iteration.
- teardown_options = None
- if options.cooldown == 0:
- teardown_options = adb_utils.prepare_for_interaction_with_device(
- options.device_id, options.device_pin)
- assert adb_utils.get_screen_state(options.device_id).is_on()
- else:
- adb_utils.ensure_screen_off(options.device_id)
- return teardown_options
+ # If there is no cooldown then unlock the screen once. Otherwise we turn off
+ # the screen during the cooldown and unlock the screen before each iteration.
+ teardown_options = None
+ if options.cooldown == 0:
+ teardown_options = adb_utils.prepare_for_interaction_with_device(
+ options.device_id, options.device_pin)
+ assert adb_utils.get_screen_state(options.device_id).is_on()
+ else:
+ adb_utils.ensure_screen_off(options.device_id)
+ return teardown_options
+
def global_teardown(options, teardown_options):
- if options.cooldown == 0:
- adb_utils.teardown_after_interaction_with_device(
- teardown_options, options.device_id)
- else:
- assert teardown_options is None
+ if options.cooldown == 0:
+ adb_utils.teardown_after_interaction_with_device(
+ teardown_options, options.device_id)
+ else:
+ assert teardown_options is None
+
def main(argv):
- (options, args) = parse_options(argv)
- with utils.TempDir() as tmp_dir:
- apk_or_apks = { 'apk': options.apk, 'apks': options.apks }
- if options.baseline_profile \
- and options.baseline_profile_install == 'profileinstaller':
- assert not options.apks, 'Unimplemented'
- apk_or_apks['apk'] = apk_utils.add_baseline_profile_to_apk(
- options.apk,
- options.baseline_profile,
- options.baseline_profile_metadata,
- tmp_dir)
- teardown_options = global_setup(options)
- run_all(apk_or_apks, options, tmp_dir)
- global_teardown(options, teardown_options)
+ (options, args) = parse_options(argv)
+ with utils.TempDir() as tmp_dir:
+ apk_or_apks = {'apk': options.apk, 'apks': options.apks}
+ if options.baseline_profile \
+ and options.baseline_profile_install == 'profileinstaller':
+ assert not options.apks, 'Unimplemented'
+ apk_or_apks['apk'] = apk_utils.add_baseline_profile_to_apk(
+ options.apk, options.baseline_profile,
+ options.baseline_profile_metadata, tmp_dir)
+ teardown_options = global_setup(options)
+ run_all(apk_or_apks, options, tmp_dir)
+ global_teardown(options, teardown_options)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
\ No newline at end of file
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/perfetto_utils.py b/tools/startup/perfetto_utils.py
index eb32722..4cf8613 100644
--- a/tools/startup/perfetto_utils.py
+++ b/tools/startup/perfetto_utils.py
@@ -7,85 +7,85 @@
import subprocess
import sys
+
def get_trace_processor():
- try:
- from perfetto.trace_processor import TraceProcessor
- except ImportError:
- sys.exit(
- 'Unable to analyze perfetto trace without the perfetto library. '
- 'Install instructions:\n'
- ' sudo apt install python3-pip\n'
- ' pip3 install perfetto')
- return TraceProcessor
+ try:
+ from perfetto.trace_processor import TraceProcessor
+ except ImportError:
+ sys.exit(
+ 'Unable to analyze perfetto trace without the perfetto library. '
+ 'Install instructions:\n'
+ ' sudo apt install python3-pip\n'
+ ' pip3 install perfetto')
+ return TraceProcessor
+
def ensure_record_android_trace(tmp_dir):
- record_android_trace_path = os.path.join(tmp_dir, 'record_android_trace')
- if not os.path.exists(record_android_trace_path):
- cmd = [
- 'curl',
- '--output',
- record_android_trace_path,
- '--silent',
- 'https://raw.githubusercontent.com/google/perfetto/master/tools/'
+ record_android_trace_path = os.path.join(tmp_dir, 'record_android_trace')
+ if not os.path.exists(record_android_trace_path):
+ cmd = [
+ 'curl', '--output', record_android_trace_path, '--silent',
+ 'https://raw.githubusercontent.com/google/perfetto/master/tools/'
'record_android_trace'
- ]
- subprocess.check_call(cmd)
- assert os.path.exists(record_android_trace_path)
- return record_android_trace_path
+ ]
+ subprocess.check_call(cmd)
+ assert os.path.exists(record_android_trace_path)
+ return record_android_trace_path
+
def record_android_trace(out_dir, tmp_dir, device_id=None):
- record_android_trace_path = ensure_record_android_trace(tmp_dir)
- config_path = os.path.join(os.path.dirname(__file__), 'config.pbtx')
- perfetto_trace_path = os.path.join(out_dir, 'trace.perfetto-trace')
- cmd = [
- sys.executable,
- record_android_trace_path,
- '--config',
- config_path,
- '--out',
- perfetto_trace_path,
- '--no-open']
- if device_id is not None:
- cmd.extend(['--serial', device_id])
- perfetto_process = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- lines = []
- for line in perfetto_process.stdout:
- line = line.decode('utf-8')
- lines.append(line)
- if 'enabled ftrace' in line.strip():
- return perfetto_process, perfetto_trace_path
- raise ValueError(
- 'Expected to find line containing: enabled ftrace, got: %s' % lines)
+ record_android_trace_path = ensure_record_android_trace(tmp_dir)
+ config_path = os.path.join(os.path.dirname(__file__), 'config.pbtx')
+ perfetto_trace_path = os.path.join(out_dir, 'trace.perfetto-trace')
+ cmd = [
+ sys.executable, record_android_trace_path, '--config', config_path,
+ '--out', perfetto_trace_path, '--no-open'
+ ]
+ if device_id is not None:
+ cmd.extend(['--serial', device_id])
+ perfetto_process = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ lines = []
+ for line in perfetto_process.stdout:
+ line = line.decode('utf-8')
+ lines.append(line)
+ if 'enabled ftrace' in line.strip():
+ return perfetto_process, perfetto_trace_path
+ raise ValueError(
+ 'Expected to find line containing: enabled ftrace, got: %s' % lines)
+
def stop_record_android_trace(perfetto_process, out_dir):
- if perfetto_process.poll() is not None:
- raise ValueError('Expected perfetto process to be running')
- # perfetto should terminate in at most 15 seconds,
- perfetto_config_duration=15
- stdout, stderr = perfetto_process.communicate(
- timeout=perfetto_config_duration*2)
- stdout = stdout.decode('utf-8')
- stderr = stderr.decode('utf-8')
- assert perfetto_process.returncode == 0
- assert os.path.exists(os.path.join(out_dir, 'trace.perfetto-trace'))
+ if perfetto_process.poll() is not None:
+ raise ValueError('Expected perfetto process to be running')
+ # perfetto should terminate in at most 15 seconds,
+ perfetto_config_duration = 15
+ stdout, stderr = perfetto_process.communicate(
+ timeout=perfetto_config_duration * 2)
+ stdout = stdout.decode('utf-8')
+ stderr = stderr.decode('utf-8')
+ assert perfetto_process.returncode == 0
+ assert os.path.exists(os.path.join(out_dir, 'trace.perfetto-trace'))
+
# https://perfetto.dev/docs/analysis/sql-tables
def find_slices_by_name(slice_name, options, trace_processor):
- return trace_processor.query(
- 'SELECT slice.dur, slice.ts FROM slice'
- ' INNER JOIN thread_track ON (slice.track_id = thread_track.id)'
- ' INNER JOIN thread using (utid)'
- ' INNER JOIN process using (upid)'
- ' WHERE slice.name = "%s"'
- ' AND process.name = "%s"'
- ' ORDER BY slice.ts ASC'
- % (slice_name, options.app_id))
+ return trace_processor.query(
+ 'SELECT slice.dur, slice.ts FROM slice'
+ ' INNER JOIN thread_track ON (slice.track_id = thread_track.id)'
+ ' INNER JOIN thread using (utid)'
+ ' INNER JOIN process using (upid)'
+ ' WHERE slice.name = "%s"'
+ ' AND process.name = "%s"'
+ ' ORDER BY slice.ts ASC' % (slice_name, options.app_id))
+
def find_unique_slice_by_name(slice_name, options, trace_processor):
- query_it = find_slices_by_name(slice_name, options, trace_processor)
- assert len(query_it) == 1
- return next(query_it)
+ query_it = find_slices_by_name(slice_name, options, trace_processor)
+ assert len(query_it) == 1
+ return next(query_it)
+
def get_slice_end_since_start(slice, initial_slice):
- return (slice.ts + slice.dur - initial_slice.ts) / 1000000
+ return (slice.ts + slice.dur - initial_slice.ts) / 1000000
diff --git a/tools/startup/profile_utils.py b/tools/startup/profile_utils.py
index 5c74a5c..7a8de4a 100755
--- a/tools/startup/profile_utils.py
+++ b/tools/startup/profile_utils.py
@@ -10,6 +10,7 @@
EXTERNAL_SYNTHETIC_SUFFIX = '$$ExternalSynthetic'
SYNTHETIC_PREFIX = 'S'
+
# Parses a list of class and method descriptors, prefixed with one or more flags
# 'H' (hot), 'S' (startup), 'P' (post startup).
#
@@ -24,86 +25,93 @@
#
# See also https://developer.android.com/studio/profile/baselineprofiles.
def parse_art_profile(lines):
- art_profile = {}
- flags_to_name = { 'H': 'hot', 'S': 'startup', 'P': 'post_startup' }
- for line in lines:
- line = line.strip()
- if not line:
- continue
- flags = { 'hot': False, 'startup': False, 'post_startup': False }
- while line[0] in flags_to_name:
- flag_abbreviation = line[0]
- flag_name = flags_to_name.get(flag_abbreviation)
- flags[flag_name] = True
- line = line[1:]
- while line.startswith('['):
- line = line[1:]
- assert line.startswith('L'), line
- descriptor = line
- art_profile[descriptor] = flags
- return art_profile
+ art_profile = {}
+ flags_to_name = {'H': 'hot', 'S': 'startup', 'P': 'post_startup'}
+ for line in lines:
+ line = line.strip()
+ if not line:
+ continue
+ flags = {'hot': False, 'startup': False, 'post_startup': False}
+ while line[0] in flags_to_name:
+ flag_abbreviation = line[0]
+ flag_name = flags_to_name.get(flag_abbreviation)
+ flags[flag_name] = True
+ line = line[1:]
+ while line.startswith('['):
+ line = line[1:]
+ assert line.startswith('L'), line
+ descriptor = line
+ art_profile[descriptor] = flags
+ return art_profile
-def transform_art_profile_to_r8_startup_list(
- art_profile, generalize_synthetics=False):
- r8_startup_list = {}
- for startup_descriptor, flags in art_profile.items():
- transformed_startup_descriptor = transform_synthetic_descriptor(
- startup_descriptor) if generalize_synthetics else startup_descriptor
- r8_startup_list[transformed_startup_descriptor] = {
- 'conditional_startup': False,
- 'hot': flags['hot'],
- 'startup': flags['startup'],
- 'post_startup': flags['post_startup']
- }
- return r8_startup_list
+
+def transform_art_profile_to_r8_startup_list(art_profile,
+ generalize_synthetics=False):
+ r8_startup_list = {}
+ for startup_descriptor, flags in art_profile.items():
+ transformed_startup_descriptor = transform_synthetic_descriptor(
+ startup_descriptor) if generalize_synthetics else startup_descriptor
+ r8_startup_list[transformed_startup_descriptor] = {
+ 'conditional_startup': False,
+ 'hot': flags['hot'],
+ 'startup': flags['startup'],
+ 'post_startup': flags['post_startup']
+ }
+ return r8_startup_list
+
def transform_synthetic_descriptor(descriptor):
- companion_class_index = descriptor.find(COMPANION_CLASS_SUFFIX)
- if companion_class_index >= 0:
- return SYNTHETIC_PREFIX + descriptor[0:companion_class_index] + ';'
- external_synthetic_index = descriptor.find(EXTERNAL_SYNTHETIC_SUFFIX)
- if external_synthetic_index >= 0:
- return SYNTHETIC_PREFIX + descriptor[0:external_synthetic_index] + ';'
- return descriptor
+ companion_class_index = descriptor.find(COMPANION_CLASS_SUFFIX)
+ if companion_class_index >= 0:
+ return SYNTHETIC_PREFIX + descriptor[0:companion_class_index] + ';'
+ external_synthetic_index = descriptor.find(EXTERNAL_SYNTHETIC_SUFFIX)
+ if external_synthetic_index >= 0:
+ return SYNTHETIC_PREFIX + descriptor[0:external_synthetic_index] + ';'
+ return descriptor
+
def filter_r8_startup_list(r8_startup_list, options):
- filtered_r8_startup_list = {}
- for startup_descriptor, flags in r8_startup_list.items():
- if not options.include_post_startup \
- and flags.get('post_startup') \
- and not flags.get('startup'):
- continue
- filtered_r8_startup_list[startup_descriptor] = flags
- return filtered_r8_startup_list
+ filtered_r8_startup_list = {}
+ for startup_descriptor, flags in r8_startup_list.items():
+ if not options.include_post_startup \
+ and flags.get('post_startup') \
+ and not flags.get('startup'):
+ continue
+ filtered_r8_startup_list[startup_descriptor] = flags
+ return filtered_r8_startup_list
+
def parse_options(argv):
- result = argparse.ArgumentParser(
- description='Utilities for converting an ART profile into an R8 startup '
- 'list.')
- result.add_argument('--art-profile', help='Path to the ART profile')
- result.add_argument('--include-post-startup',
- help='Include post startup classes and methods in the R8 '
- 'startup list',
- action='store_true',
- default=False)
- result.add_argument('--out', help='Where to store the R8 startup list')
- options, args = result.parse_known_args(argv)
- return options, args
+ result = argparse.ArgumentParser(
+ description='Utilities for converting an ART profile into an R8 startup '
+ 'list.')
+ result.add_argument('--art-profile', help='Path to the ART profile')
+ result.add_argument(
+ '--include-post-startup',
+ help='Include post startup classes and methods in the R8 '
+ 'startup list',
+ action='store_true',
+ default=False)
+ result.add_argument('--out', help='Where to store the R8 startup list')
+ options, args = result.parse_known_args(argv)
+ return options, args
+
def main(argv):
- (options, args) = parse_options(argv)
- with open(options.art_profile, 'r') as f:
- art_profile = parse_art_profile(f.read().splitlines())
- r8_startup_list = transform_art_profile_to_r8_startup_list(art_profile)
- filtered_r8_startup_list = filter_r8_startup_list(r8_startup_list, options)
- if options.out is not None:
- with open(options.out, 'w') as f:
- for startup_descriptor, flags in filtered_r8_startup_list.items():
- f.write(startup_descriptor)
- f.write('\n')
- else:
- for startup_descriptor, flags in filtered_r8_startup_list.items():
- print(startup_descriptor)
+ (options, args) = parse_options(argv)
+ with open(options.art_profile, 'r') as f:
+ art_profile = parse_art_profile(f.read().splitlines())
+ r8_startup_list = transform_art_profile_to_r8_startup_list(art_profile)
+ filtered_r8_startup_list = filter_r8_startup_list(r8_startup_list, options)
+ if options.out is not None:
+ with open(options.out, 'w') as f:
+ for startup_descriptor, flags in filtered_r8_startup_list.items():
+ f.write(startup_descriptor)
+ f.write('\n')
+ else:
+ for startup_descriptor, flags in filtered_r8_startup_list.items():
+ print(startup_descriptor)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/startup/relayout.py b/tools/startup/relayout.py
index b0f8aac..3d0cd94 100755
--- a/tools/startup/relayout.py
+++ b/tools/startup/relayout.py
@@ -17,79 +17,85 @@
import utils
import zip_utils
-LOWEST_SUPPORTED_MIN_API = 21 # Android L (native multi dex)
+LOWEST_SUPPORTED_MIN_API = 21 # Android L (native multi dex)
+
def parse_options(argv):
- result = argparse.ArgumentParser(
- description='Relayout a given APK using a startup profile.')
- result.add_argument('--apk',
- help='Path to the .apk',
- required=True)
- result.add_argument('--desugared-library',
- choices=['auto', 'true', 'false'],
- default='auto',
- help='Whether the last dex file of the app is desugared '
- 'library')
- result.add_argument('--no-build',
- action='store_true',
- default=False,
- help='To disable building using gradle')
- result.add_argument('--out',
- help='Destination of resulting apk',
- required=True)
- result.add_argument('--profile',
- help='Path to the startup profile')
- options, args = result.parse_known_args(argv)
- return options, args
+ result = argparse.ArgumentParser(
+ description='Relayout a given APK using a startup profile.')
+ result.add_argument('--apk', help='Path to the .apk', required=True)
+ result.add_argument(
+ '--desugared-library',
+ choices=['auto', 'true', 'false'],
+ default='auto',
+ help='Whether the last dex file of the app is desugared '
+ 'library')
+ result.add_argument('--no-build',
+ action='store_true',
+ default=False,
+ help='To disable building using gradle')
+ result.add_argument('--out',
+ help='Destination of resulting apk',
+ required=True)
+ result.add_argument('--profile', help='Path to the startup profile')
+ options, args = result.parse_known_args(argv)
+ return options, args
+
def get_dex_to_relayout(options, temp):
- marker = extractmarker.extractmarker(options.apk, build=not options.no_build)
- if '~~L8' not in marker:
- return [options.apk], None
- dex_dir = os.path.join(temp, 'dex')
- dex_predicate = \
- lambda name : name.startswith('classes') and name.endswith('.dex')
- extracted_dex_files = \
- zip_utils.extract_all_that_matches(options.apk, dex_dir, dex_predicate)
- desugared_library_dex = 'classes%s.dex' % len(extracted_dex_files)
- assert desugared_library_dex in extracted_dex_files
- return [
- os.path.join(dex_dir, name) \
- for name in extracted_dex_files if name != desugared_library_dex], \
- os.path.join(dex_dir, desugared_library_dex)
+ marker = extractmarker.extractmarker(options.apk,
+ build=not options.no_build)
+ if '~~L8' not in marker:
+ return [options.apk], None
+ dex_dir = os.path.join(temp, 'dex')
+ dex_predicate = \
+ lambda name : name.startswith('classes') and name.endswith('.dex')
+ extracted_dex_files = \
+ zip_utils.extract_all_that_matches(options.apk, dex_dir, dex_predicate)
+ desugared_library_dex = 'classes%s.dex' % len(extracted_dex_files)
+ assert desugared_library_dex in extracted_dex_files
+ return [
+ os.path.join(dex_dir, name) \
+ for name in extracted_dex_files if name != desugared_library_dex], \
+ os.path.join(dex_dir, desugared_library_dex)
+
def has_desugared_library_dex(options):
- if options.desugared_library == 'auto':
- marker = extractmarker.extractmarker(
- options.apk, build=not options.no_build)
- return '~~L8' in marker
- return options.desugared_library == 'true'
+ if options.desugared_library == 'auto':
+ marker = extractmarker.extractmarker(options.apk,
+ build=not options.no_build)
+ return '~~L8' in marker
+ return options.desugared_library == 'true'
+
def main(argv):
- (options, args) = parse_options(argv)
- with utils.TempDir() as temp:
- dex = os.path.join(temp, 'dex.zip')
- d8_args = [
- '--min-api',
- str(max(apk_utils.get_min_api(options.apk), LOWEST_SUPPORTED_MIN_API)),
- '--output', dex,
- '--no-desugaring',
- '--release']
- if options.profile:
- d8_args.extend(['--startup-profile', options.profile])
- dex_to_relayout, desugared_library_dex = get_dex_to_relayout(options, temp)
- d8_args.extend(dex_to_relayout)
- toolhelper.run(
- 'd8',
- d8_args,
- build=not options.no_build,
- main='com.android.tools.r8.D8')
- if desugared_library_dex is not None:
- dex_files = [name for name in \
- zip_utils.get_names_that_matches(dex, lambda x : True)]
- zip_utils.add_file_to_zip(
- desugared_library_dex, 'classes%s.dex' % str(len(dex_files) + 1), dex)
- apk_masseur.masseur(options.apk, dex=dex, out=options.out)
+ (options, args) = parse_options(argv)
+ with utils.TempDir() as temp:
+ dex = os.path.join(temp, 'dex.zip')
+ d8_args = [
+ '--min-api',
+ str(
+ max(apk_utils.get_min_api(options.apk),
+ LOWEST_SUPPORTED_MIN_API)), '--output', dex,
+ '--no-desugaring', '--release'
+ ]
+ if options.profile:
+ d8_args.extend(['--startup-profile', options.profile])
+ dex_to_relayout, desugared_library_dex = get_dex_to_relayout(
+ options, temp)
+ d8_args.extend(dex_to_relayout)
+ toolhelper.run('d8',
+ d8_args,
+ build=not options.no_build,
+ main='com.android.tools.r8.D8')
+ if desugared_library_dex is not None:
+ dex_files = [name for name in \
+ zip_utils.get_names_that_matches(dex, lambda x : True)]
+ zip_utils.add_file_to_zip(desugared_library_dex,
+ 'classes%s.dex' % str(len(dex_files) + 1),
+ dex)
+ apk_masseur.masseur(options.apk, dex=dex, out=options.out)
+
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main(sys.argv[1:]))
diff --git a/tools/tag_versions.py b/tools/tag_versions.py
index 28a1cae..243625a 100755
--- a/tools/tag_versions.py
+++ b/tools/tag_versions.py
@@ -14,134 +14,141 @@
import utils
# Grep match string for 'Version X.Y.Z[-dev]'
-VERSION_EXP='^Version [[:digit:]]\+.[[:digit:]]\+.[[:digit:]]\+\(\|-dev\)$'
+VERSION_EXP = '^Version [[:digit:]]\+.[[:digit:]]\+.[[:digit:]]\+\(\|-dev\)$'
# R8 is located in the 'builder' library.
-AGP_MAVEN="https://dl.google.com/android/maven2/com/android/tools/build/builder"
+AGP_MAVEN = "https://dl.google.com/android/maven2/com/android/tools/build/builder"
+
def parse_options():
- parser = argparse.ArgumentParser(description='Tag R8 Versions')
- parser.add_argument(
- '--branch',
- help='The R8 branch to tag versions on, eg, origin/3.0')
- parser.add_argument(
- '--agp',
- help='The AGP to compute the tag for, eg, 4.2.0-beta03')
- parser.add_argument(
- '--dry-run',
- default=False,
- action='store_true',
- help='Print the state changing commands without running them.')
- return parser.parse_args()
+ parser = argparse.ArgumentParser(description='Tag R8 Versions')
+ parser.add_argument('--branch',
+ help='The R8 branch to tag versions on, eg, origin/3.0')
+ parser.add_argument('--agp',
+ help='The AGP to compute the tag for, eg, 4.2.0-beta03')
+ parser.add_argument(
+ '--dry-run',
+ default=False,
+ action='store_true',
+ help='Print the state changing commands without running them.')
+ return parser.parse_args()
+
def run(options, cmd):
- print(' '.join(cmd))
- if not options.dry_run:
- subprocess.check_call(cmd)
+ print(' '.join(cmd))
+ if not options.dry_run:
+ subprocess.check_call(cmd)
+
def main():
- args = parse_options()
- if args.branch:
- tag_r8_branch(args.branch, args)
- elif args.agp:
- if (args.agp == 'all'):
- tag_all_agp_versions(args)
+ args = parse_options()
+ if args.branch:
+ tag_r8_branch(args.branch, args)
+ elif args.agp:
+ if (args.agp == 'all'):
+ tag_all_agp_versions(args)
+ else:
+ tag_agp_version(args.agp, args)
else:
- tag_agp_version(args.agp, args)
- else:
- print("Should use a top-level option, such as --branch or --agp.")
- return 1
- return 0
+ print("Should use a top-level option, such as --branch or --agp.")
+ return 1
+ return 0
+
def prepare_print_version(dist, temp):
- wrapper_file = os.path.join(
- utils.REPO_ROOT,
- 'src/main/java/com/android/tools/r8/utils/PrintR8Version.java')
- cmd = [
- jdk.GetJavacExecutable(),
- wrapper_file,
- '-d', temp,
- '-cp', dist,
- ]
- utils.PrintCmd(cmd)
- subprocess.check_output(cmd)
- return temp
+ wrapper_file = os.path.join(
+ utils.REPO_ROOT,
+ 'src/main/java/com/android/tools/r8/utils/PrintR8Version.java')
+ cmd = [
+ jdk.GetJavacExecutable(),
+ wrapper_file,
+ '-d',
+ temp,
+ '-cp',
+ dist,
+ ]
+ utils.PrintCmd(cmd)
+ subprocess.check_output(cmd)
+ return temp
+
def get_tag_info_on_origin(tag):
- output = subprocess.check_output(
- ['git', 'ls-remote', '--tags', 'origin', tag]).decode('utf-8')
- if len(output.strip()) == 0:
- return None
- return output
+ output = subprocess.check_output(
+ ['git', 'ls-remote', '--tags', 'origin', tag]).decode('utf-8')
+ if len(output.strip()) == 0:
+ return None
+ return output
+
def tag_all_agp_versions(args):
- with utils.TempDir() as temp:
- url = "%s/maven-metadata.xml" % AGP_MAVEN
- metadata = os.path.join(temp, "maven-metadata.xml")
- try:
- urllib.request.urlretrieve(url, metadata)
- except urllib.error.HTTPError as e:
- print('Could not find maven-metadata.xml for agp')
- print(e)
- return 1
- with open(metadata, 'r') as file:
- data = file.read()
- pattern = r'<version>(.+)</version>'
- matches = re.findall(pattern, data)
- matches.reverse()
- for version in matches:
- print('Tagging agp version ' + version)
- tag_agp_version(version, args)
+ with utils.TempDir() as temp:
+ url = "%s/maven-metadata.xml" % AGP_MAVEN
+ metadata = os.path.join(temp, "maven-metadata.xml")
+ try:
+ urllib.request.urlretrieve(url, metadata)
+ except urllib.error.HTTPError as e:
+ print('Could not find maven-metadata.xml for agp')
+ print(e)
+ return 1
+ with open(metadata, 'r') as file:
+ data = file.read()
+ pattern = r'<version>(.+)</version>'
+ matches = re.findall(pattern, data)
+ matches.reverse()
+ for version in matches:
+ print('Tagging agp version ' + version)
+ tag_agp_version(version, args)
def tag_agp_version(agp, args):
- tag = 'agp-%s' % agp
- result = get_tag_info_on_origin(tag)
- if result:
- print('Tag %s is already present' % tag)
- print(result)
- subprocess.call(['git', 'show', '--oneline', '-s', tag])
- return 0
- with utils.TempDir() as temp:
- url = "%s/%s/builder-%s.jar" % (AGP_MAVEN, agp, agp)
- jar = os.path.join(temp, "agp.jar")
- try:
- urllib.request.urlretrieve(url, jar)
- except urllib.error.HTTPError as e:
- print('Could not find jar for agp %s' % agp)
- print(e)
- return 1
- print_version_helper = prepare_print_version(utils.R8_JAR, temp)
- output = subprocess.check_output([
- jdk.GetJavaExecutable(),
- '-cp', ':'.join([jar, print_version_helper]),
- 'com.android.tools.r8.utils.PrintR8Version'
- ]).decode('utf-8')
- version = output.split(' ')[0]
- run(args, ['git', 'tag', '-f', tag, '-m', tag, '%s^{}' % version])
- run(args, ['git', 'push', 'origin', tag])
+ tag = 'agp-%s' % agp
+ result = get_tag_info_on_origin(tag)
+ if result:
+ print('Tag %s is already present' % tag)
+ print(result)
+ subprocess.call(['git', 'show', '--oneline', '-s', tag])
+ return 0
+ with utils.TempDir() as temp:
+ url = "%s/%s/builder-%s.jar" % (AGP_MAVEN, agp, agp)
+ jar = os.path.join(temp, "agp.jar")
+ try:
+ urllib.request.urlretrieve(url, jar)
+ except urllib.error.HTTPError as e:
+ print('Could not find jar for agp %s' % agp)
+ print(e)
+ return 1
+ print_version_helper = prepare_print_version(utils.R8_JAR, temp)
+ output = subprocess.check_output([
+ jdk.GetJavaExecutable(), '-cp',
+ ':'.join([jar, print_version_helper]),
+ 'com.android.tools.r8.utils.PrintR8Version'
+ ]).decode('utf-8')
+ version = output.split(' ')[0]
+ run(args, ['git', 'tag', '-f', tag, '-m', tag, '%s^{}' % version])
+ run(args, ['git', 'push', 'origin', tag])
+
def tag_r8_branch(branch, args):
- if not branch.startswith('origin/'):
- print('Expected branch to start with origin/')
- return 1
- output = subprocess.check_output([
- 'git', 'log', '--pretty=format:%H\t%s', '--grep', VERSION_EXP, branch
- ]).decode('utf-8')
- for l in output.split('\n'):
- (hash, subject) = l.split('\t')
- m = re.search('Version (.+)', subject)
- if not m:
- print('Unable to find a version for line: %s' % l)
- continue
- version = m.group(1)
- result = get_tag_info_on_origin(version)
- if not result:
- run(args, ['git', 'tag', '-a', version, '-m', version, hash])
- run(args, ['git', 'push', 'origin', version])
- if args.dry_run:
- print('Dry run complete. None of the above have been executed.')
+ if not branch.startswith('origin/'):
+ print('Expected branch to start with origin/')
+ return 1
+ output = subprocess.check_output(
+ ['git', 'log', '--pretty=format:%H\t%s', '--grep', VERSION_EXP,
+ branch]).decode('utf-8')
+ for l in output.split('\n'):
+ (hash, subject) = l.split('\t')
+ m = re.search('Version (.+)', subject)
+ if not m:
+ print('Unable to find a version for line: %s' % l)
+ continue
+ version = m.group(1)
+ result = get_tag_info_on_origin(version)
+ if not result:
+ run(args, ['git', 'tag', '-a', version, '-m', version, hash])
+ run(args, ['git', 'push', 'origin', version])
+ if args.dry_run:
+ print('Dry run complete. None of the above have been executed.')
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools/test.py b/tools/test.py
index b106d33..efbe5a1 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -23,23 +23,14 @@
import utils
if utils.is_python3():
- import threading
+ import threading
else:
- import thread
+ import thread
ALL_ART_VMS = [
- "default",
- "14.0.0",
- "13.0.0",
- "12.0.0",
- "10.0.0",
- "9.0.0",
- "8.1.0",
- "7.0.0",
- "6.0.1",
- "5.1.1",
- "4.4.4",
- "4.0.4"]
+ "default", "14.0.0", "13.0.0", "12.0.0", "10.0.0", "9.0.0", "8.1.0",
+ "7.0.0", "6.0.1", "5.1.1", "4.4.4", "4.0.4"
+]
# How often do we check for progress on the bots:
# Should be long enough that a normal run would always have med progress
@@ -55,562 +46,666 @@
REPORTS_PATH = os.path.join(utils.BUILD, 'reports')
REPORT_INDEX = ['tests', 'test', 'index.html']
VALID_RUNTIMES = [
- 'none',
- 'jdk8',
- 'jdk9',
- 'jdk11',
- 'jdk17',
- 'jdk20',
-] + [ 'dex-%s' % dexvm for dexvm in ALL_ART_VMS ]
+ 'none',
+ 'jdk8',
+ 'jdk9',
+ 'jdk11',
+ 'jdk17',
+ 'jdk20',
+] + ['dex-%s' % dexvm for dexvm in ALL_ART_VMS]
+
def ParseOptions():
- result = argparse.ArgumentParser()
- result.add_argument('--no-internal', '--no_internal',
- help='Do not run Google internal tests.',
- default=False, action='store_true')
- result.add_argument('--archive-failures', '--archive_failures',
- help='Upload test results to cloud storage on failure.',
- default=False, action='store_true')
- result.add_argument('--archive-failures-file-name',
- '--archive_failures_file_name',
- help='Set file name for the archived failures file name',
- default=uuid.uuid4())
- result.add_argument('--only-internal', '--only_internal',
- help='Only run Google internal tests.',
- default=False, action='store_true')
- result.add_argument('--all-tests', '--all_tests',
- help='Run tests in all configurations.',
- default=False, action='store_true')
- result.add_argument('--slow-tests', '--slow_tests',
- help='Also run slow tests.',
- default=False, action='store_true')
- result.add_argument('-v', '--verbose',
- help='Print test stdout to, well, stdout.',
- default=False, action='store_true')
- result.add_argument('--dex-vm', '--dex_vm',
- help='The android version of the vm to use. "all" will run the tests on '
- 'all art vm versions (stopping after first failed execution)',
- default="default",
- choices=ALL_ART_VMS + ["all"])
- result.add_argument('--dex-vm-kind', '--dex_vm_kind',
- help='Whether to use host or target version of runtime',
- default="host",
- nargs=1,
- choices=["host", "target"])
- result.add_argument('--one-line-per-test', '--one_line_per_test',
- help='Print a line before a tests starts and after it ends to stdout.',
- default=False, action='store_true')
- result.add_argument('--tool',
- help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
- ' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
- default=None, choices=["r8", "d8", "r8cf"])
- result.add_argument('--disable-assertions', '--disable_assertions', '-da',
- help='Disable Java assertions when running the compiler '
- '(default enabled)',
- default=False, action='store_true')
- result.add_argument('--with-code-coverage', '--with_code_coverage',
- help='Enable code coverage with Jacoco.',
- default=False, action='store_true')
- result.add_argument('--test-dir', '--test_dir',
- help='Use a custom directory for the test artifacts instead of a'
- ' temporary (which is automatically removed after the test).'
- ' Note that the directory will not be cleared before the test.')
- result.add_argument('--command-cache-dir', '--command_cache_dir',
- help='Cache command invocations to this directory, speeds up test runs',
- default=os.environ.get('R8_COMMAND_CACHE_DIR'))
- result.add_argument('--command-cache-stats', '--command_cache_stats',
- help='Collect and print statistics about the command cache.',
- default=False, action='store_true')
- result.add_argument('--java-home', '--java_home',
- help='Use a custom java version to run tests.')
- result.add_argument('--java-max-memory-size', '--java_max_memory_size',
- help='Set memory for running tests, default 4G',
- default=os.environ.get('R8_JAVA_MAX_MEMORY_SIZE', '4G'))
- result.add_argument('--test-namespace', '--test_namespace',
- help='Only run tests in this namespace. The namespace is relative to '
- 'com/android/tools/r8, e.g., desugar/desugaredlibrary',
- default=None)
- result.add_argument('--shard-count', '--shard_count',
- help='We are running this many shards.')
- result.add_argument('--shard-number', '--shard_number',
- help='We are running this shard.')
- result.add_argument('--generate-golden-files-to', '--generate_golden_files_to',
- help='Store dex files produced by tests in the specified directory.'
- ' It is aimed to be read on platforms with no host runtime available'
- ' for comparison.')
- result.add_argument('--use-golden-files-in', '--use_golden_files_in',
- help='Download golden files hierarchy for this commit in the specified'
- ' location and use them instead of executing on host runtime.')
- result.add_argument('--no-r8lib', '--no_r8lib',
- default=False, action='store_true',
- help='Run the tests on R8 full with relocated dependencies.')
- result.add_argument('--no-arttests', '--no_arttests',
- default=False, action='store_true',
- help='Do not run the art tests.')
- result.add_argument('--r8lib-no-deps', '--r8lib_no_deps',
- default=False, action='store_true',
- help='Run the tests on r8lib without relocated dependencies.')
- result.add_argument('--failed',
- default=False, action='store_true',
- help='Run the tests that failed last execution.')
- result.add_argument('--fail-fast', '--fail_fast',
- default=False, action='store_true',
- help='Stop on first failure. Passes --fail-fast to gradle test runner.')
- result.add_argument('--worktree',
- default=False, action='store_true',
- help='Tests are run in worktree and should not use gradle user home.')
- result.add_argument('--runtimes',
- default=None,
- help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
- ' Special values include: all (for all runtimes)'
- ' and empty (for no runtimes).')
- result.add_argument('--print-hanging-stacks', '--print_hanging_stacks',
- default=-1, type=int, help='Print hanging stacks after timeout in seconds')
- result.add_argument('--print-full-stacktraces', '--print_full_stacktraces',
- default=False, action='store_true',
- help='Print the full stacktraces without any filtering applied')
- result.add_argument(
- '--print-obfuscated-stacktraces', '--print_obfuscated_stacktraces',
- default=False, action='store_true',
- help='Print the obfuscated stacktraces')
- result.add_argument(
- '--debug-agent', '--debug_agent',
- help='Enable Java debug agent and suspend compilation (default disabled)',
- default=False,
- action='store_true')
- result.add_argument('--desugared-library-configuration',
- '--desugared_library-configuration',
- help='Use alternative desugared library configuration.')
- result.add_argument('--desugared-library', '--desugared_library',
- help='Build and use desugared library from GitHub.')
- result.add_argument('--print-times', '--print_times',
- help='Print the execution time of the slowest tests..',
- default=False, action='store_true')
- result.add_argument(
- '--testing-state-dir',
- help='Explicitly set the testing state directory '
- '(defaults to build/test-state/<git-branch>).')
- result.add_argument(
- '--rerun',
- help='Rerun tests (implicitly enables testing state).',
- choices=testing_state.CHOICES)
- result.add_argument(
- '--stacktrace',
- help='Pass --stacktrace to the gradle run',
- default=False, action='store_true')
- result.add_argument('--kotlin-compiler-dev',
- help='Specify to download a kotlin dev compiler and run '
- 'tests with that',
- default=False, action='store_true')
- result.add_argument('--kotlin-compiler-old',
- help='Specify to run tests on older kotlin compilers',
- default=False, action='store_true')
- return result.parse_known_args()
+ result = argparse.ArgumentParser()
+ result.add_argument('--no-internal',
+ '--no_internal',
+ help='Do not run Google internal tests.',
+ default=False,
+ action='store_true')
+ result.add_argument('--archive-failures',
+ '--archive_failures',
+ help='Upload test results to cloud storage on failure.',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--archive-failures-file-name',
+ '--archive_failures_file_name',
+ help='Set file name for the archived failures file name',
+ default=uuid.uuid4())
+ result.add_argument('--only-internal',
+ '--only_internal',
+ help='Only run Google internal tests.',
+ default=False,
+ action='store_true')
+ result.add_argument('--all-tests',
+ '--all_tests',
+ help='Run tests in all configurations.',
+ default=False,
+ action='store_true')
+ result.add_argument('--slow-tests',
+ '--slow_tests',
+ help='Also run slow tests.',
+ default=False,
+ action='store_true')
+ result.add_argument('-v',
+ '--verbose',
+ help='Print test stdout to, well, stdout.',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--dex-vm',
+ '--dex_vm',
+ help='The android version of the vm to use. "all" will run the tests on '
+ 'all art vm versions (stopping after first failed execution)',
+ default="default",
+ choices=ALL_ART_VMS + ["all"])
+ result.add_argument('--dex-vm-kind',
+ '--dex_vm_kind',
+ help='Whether to use host or target version of runtime',
+ default="host",
+ nargs=1,
+ choices=["host", "target"])
+ result.add_argument(
+ '--one-line-per-test',
+ '--one_line_per_test',
+ help='Print a line before a tests starts and after it ends to stdout.',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--tool',
+ help='Tool to run ART tests with: "r8" (default) or "d8" or "r8cf"'
+ ' (r8 w/CF-backend). Ignored if "--all_tests" enabled.',
+ default=None,
+ choices=["r8", "d8", "r8cf"])
+ result.add_argument(
+ '--disable-assertions',
+ '--disable_assertions',
+ '-da',
+ help='Disable Java assertions when running the compiler '
+ '(default enabled)',
+ default=False,
+ action='store_true')
+ result.add_argument('--with-code-coverage',
+ '--with_code_coverage',
+ help='Enable code coverage with Jacoco.',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--test-dir',
+ '--test_dir',
+ help='Use a custom directory for the test artifacts instead of a'
+ ' temporary (which is automatically removed after the test).'
+ ' Note that the directory will not be cleared before the test.')
+ result.add_argument(
+ '--command-cache-dir',
+ '--command_cache_dir',
+ help='Cache command invocations to this directory, speeds up test runs',
+ default=os.environ.get('R8_COMMAND_CACHE_DIR'))
+ result.add_argument(
+ '--command-cache-stats',
+ '--command_cache_stats',
+ help='Collect and print statistics about the command cache.',
+ default=False,
+ action='store_true')
+ result.add_argument('--java-home',
+ '--java_home',
+ help='Use a custom java version to run tests.')
+ result.add_argument('--java-max-memory-size',
+ '--java_max_memory_size',
+ help='Set memory for running tests, default 4G',
+ default=os.environ.get('R8_JAVA_MAX_MEMORY_SIZE', '4G'))
+ result.add_argument(
+ '--test-namespace',
+ '--test_namespace',
+ help='Only run tests in this namespace. The namespace is relative to '
+ 'com/android/tools/r8, e.g., desugar/desugaredlibrary',
+ default=None)
+ result.add_argument('--shard-count',
+ '--shard_count',
+ help='We are running this many shards.')
+ result.add_argument('--shard-number',
+ '--shard_number',
+ help='We are running this shard.')
+ result.add_argument(
+ '--generate-golden-files-to',
+ '--generate_golden_files_to',
+ help='Store dex files produced by tests in the specified directory.'
+ ' It is aimed to be read on platforms with no host runtime available'
+ ' for comparison.')
+ result.add_argument(
+ '--use-golden-files-in',
+ '--use_golden_files_in',
+ help='Download golden files hierarchy for this commit in the specified'
+ ' location and use them instead of executing on host runtime.')
+ result.add_argument(
+ '--no-r8lib',
+ '--no_r8lib',
+ default=False,
+ action='store_true',
+ help='Run the tests on R8 full with relocated dependencies.')
+ result.add_argument('--no-arttests',
+ '--no_arttests',
+ default=False,
+ action='store_true',
+ help='Do not run the art tests.')
+ result.add_argument(
+ '--r8lib-no-deps',
+ '--r8lib_no_deps',
+ default=False,
+ action='store_true',
+ help='Run the tests on r8lib without relocated dependencies.')
+ result.add_argument('--failed',
+ default=False,
+ action='store_true',
+ help='Run the tests that failed last execution.')
+ result.add_argument(
+ '--fail-fast',
+ '--fail_fast',
+ default=False,
+ action='store_true',
+ help='Stop on first failure. Passes --fail-fast to gradle test runner.')
+ result.add_argument(
+ '--worktree',
+ default=False,
+ action='store_true',
+ help='Tests are run in worktree and should not use gradle user home.')
+ result.add_argument(
+ '--runtimes',
+ default=None,
+ help='Test parameter runtimes to use, separated by : (eg, none:jdk9).'
+ ' Special values include: all (for all runtimes)'
+ ' and empty (for no runtimes).')
+ result.add_argument('--print-hanging-stacks',
+ '--print_hanging_stacks',
+ default=-1,
+ type=int,
+ help='Print hanging stacks after timeout in seconds')
+ result.add_argument(
+ '--print-full-stacktraces',
+ '--print_full_stacktraces',
+ default=False,
+ action='store_true',
+ help='Print the full stacktraces without any filtering applied')
+ result.add_argument('--print-obfuscated-stacktraces',
+ '--print_obfuscated_stacktraces',
+ default=False,
+ action='store_true',
+ help='Print the obfuscated stacktraces')
+ result.add_argument(
+ '--debug-agent',
+ '--debug_agent',
+ help=
+ 'Enable Java debug agent and suspend compilation (default disabled)',
+ default=False,
+ action='store_true')
+ result.add_argument('--desugared-library-configuration',
+ '--desugared_library-configuration',
+ help='Use alternative desugared library configuration.')
+ result.add_argument('--desugared-library',
+ '--desugared_library',
+ help='Build and use desugared library from GitHub.')
+ result.add_argument('--print-times',
+ '--print_times',
+ help='Print the execution time of the slowest tests..',
+ default=False,
+ action='store_true')
+ result.add_argument('--testing-state-dir',
+ help='Explicitly set the testing state directory '
+ '(defaults to build/test-state/<git-branch>).')
+ result.add_argument('--rerun',
+ help='Rerun tests (implicitly enables testing state).',
+ choices=testing_state.CHOICES)
+ result.add_argument('--stacktrace',
+ help='Pass --stacktrace to the gradle run',
+ default=False,
+ action='store_true')
+ result.add_argument(
+ '--kotlin-compiler-dev',
+ help='Specify to download a kotlin dev compiler and run '
+ 'tests with that',
+ default=False,
+ action='store_true')
+ result.add_argument('--kotlin-compiler-old',
+ help='Specify to run tests on older kotlin compilers',
+ default=False,
+ action='store_true')
+ return result.parse_known_args()
+
def has_failures(classes_file):
- with open(classes_file) as f:
- contents = f.read()
- # The report has a div tag with the percentage of tests that succeeded.
- assert '<div class="percent">' in contents
- return '<div class="percent">100%</div>' not in contents
+ with open(classes_file) as f:
+ contents = f.read()
+ # The report has a div tag with the percentage of tests that succeeded.
+ assert '<div class="percent">' in contents
+ return '<div class="percent">100%</div>' not in contents
+
def should_upload(filename, absolute_filename):
- # filename is relative to REPO_ROOT/build/reports/tests
- if filename.startswith('test/packages'):
- # We don't upload the package overview
- return False
- if filename.startswith('test/classes'):
- return has_failures(absolute_filename)
- # Always upload index, css and js
- return True
+ # filename is relative to REPO_ROOT/build/reports/tests
+ if filename.startswith('test/packages'):
+ # We don't upload the package overview
+ return False
+ if filename.startswith('test/classes'):
+ return has_failures(absolute_filename)
+ # Always upload index, css and js
+ return True
+
def archive_failures(options):
- upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
- file_name = options.archive_failures_file_name
- destination_dir = 'gs://%s/%s/' % (BUCKET, file_name)
- for (dir_path, dir_names, file_names) in os.walk(upload_dir):
- for f in file_names:
- absolute_file = os.path.join(dir_path, f)
- relative_file = absolute_file[len(upload_dir)+1:]
- if (should_upload(relative_file, absolute_file)):
- utils.upload_file_to_cloud_storage(absolute_file,
- destination_dir + relative_file)
- url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET, file_name)
- print('Test results available at: %s' % url)
+ upload_dir = os.path.join(utils.REPO_ROOT, 'build', 'reports', 'tests')
+ file_name = options.archive_failures_file_name
+ destination_dir = 'gs://%s/%s/' % (BUCKET, file_name)
+ for (dir_path, dir_names, file_names) in os.walk(upload_dir):
+ for f in file_names:
+ absolute_file = os.path.join(dir_path, f)
+ relative_file = absolute_file[len(upload_dir) + 1:]
+ if (should_upload(relative_file, absolute_file)):
+ utils.upload_file_to_cloud_storage(
+ absolute_file, destination_dir + relative_file)
+ url = 'https://storage.googleapis.com/%s/%s/test/index.html' % (BUCKET,
+ file_name)
+ print('Test results available at: %s' % url)
+
def Main():
- (options, args) = ParseOptions()
- if utils.is_bot():
- gradle.RunGradle(['--no-daemon', 'clean'])
- print('Running with python ' + str(sys.version_info))
- # Always print stats on bots if command cache is enabled
- options.command_cache_stats = options.command_cache_dir is not None
+ (options, args) = ParseOptions()
+ if utils.is_bot():
+ gradle.RunGradle(['--no-daemon', 'clean'])
+ print('Running with python ' + str(sys.version_info))
+ # Always print stats on bots if command cache is enabled
+ options.command_cache_stats = options.command_cache_dir is not None
- desugar_jdk_json_dir = None
- if options.desugared_library_configuration:
- if options.desugared_library_configuration != 'jdk11':
- print("Only value supported for --desugared-library is 'jdk11'")
- exit(1)
- desugar_jdk_json_dir = 'src/library_desugar/jdk11'
+ desugar_jdk_json_dir = None
+ if options.desugared_library_configuration:
+ if options.desugared_library_configuration != 'jdk11':
+ print("Only value supported for --desugared-library is 'jdk11'")
+ exit(1)
+ desugar_jdk_json_dir = 'src/library_desugar/jdk11'
- desugar_jdk_libs = None
- if options.desugared_library:
- if options.desugared_library != 'HEAD':
- print("Only value supported for --desugared-library is 'HEAD'")
- exit(1)
- desugar_jdk_libs_dir = 'build/desugar_jdk_libs'
- shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True)
- os.makedirs(desugar_jdk_libs_dir)
- print('Building desugared library.')
- with utils.TempDir() as checkout_dir:
- archive_desugar_jdk_libs.CloneDesugaredLibrary('google', checkout_dir, 'HEAD')
- # Make sure bazel is extracted in third_party.
- utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
- utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
- utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
- (library_jar, maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(checkout_dir, 'jdk11_legacy' if options.desugared_library_configuration == 'jdk11' else 'jdk8')
- desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir, os.path.basename(library_jar))
- shutil.copyfile(library_jar, desugar_jdk_libs)
- print('Desugared library for test in ' + desugar_jdk_libs)
+ desugar_jdk_libs = None
+ if options.desugared_library:
+ if options.desugared_library != 'HEAD':
+ print("Only value supported for --desugared-library is 'HEAD'")
+ exit(1)
+ desugar_jdk_libs_dir = 'build/desugar_jdk_libs'
+ shutil.rmtree(desugar_jdk_libs_dir, ignore_errors=True)
+ os.makedirs(desugar_jdk_libs_dir)
+ print('Building desugared library.')
+ with utils.TempDir() as checkout_dir:
+ archive_desugar_jdk_libs.CloneDesugaredLibrary(
+ 'google', checkout_dir, 'HEAD')
+ # Make sure bazel is extracted in third_party.
+ utils.DownloadFromGoogleCloudStorage(utils.BAZEL_SHA_FILE)
+ utils.DownloadFromGoogleCloudStorage(utils.JAVA8_SHA_FILE)
+ utils.DownloadFromGoogleCloudStorage(utils.JAVA11_SHA_FILE)
+ (library_jar,
+ maven_zip) = archive_desugar_jdk_libs.BuildDesugaredLibrary(
+ checkout_dir, 'jdk11_legacy' if
+ options.desugared_library_configuration == 'jdk11' else 'jdk8')
+ desugar_jdk_libs = os.path.join(desugar_jdk_libs_dir,
+ os.path.basename(library_jar))
+ shutil.copyfile(library_jar, desugar_jdk_libs)
+ print('Desugared library for test in ' + desugar_jdk_libs)
- gradle_args = []
+ gradle_args = []
- if options.stacktrace or utils.is_bot():
- gradle_args.append('--stacktrace')
+ if options.stacktrace or utils.is_bot():
+ gradle_args.append('--stacktrace')
- if utils.is_bot():
- # Bots don't like dangling processes.
- gradle_args.append('--no-daemon')
+ if utils.is_bot():
+ # Bots don't like dangling processes.
+ gradle_args.append('--no-daemon')
- # Set all necessary Gradle properties and options first.
- if options.shard_count:
- assert options.shard_number
- gradle_args.append('-Pshard_count=%s' % options.shard_count)
- gradle_args.append('-Pshard_number=%s' % options.shard_number)
- if options.verbose:
- gradle_args.append('-Pprint_test_stdout')
- if options.no_internal:
- gradle_args.append('-Pno_internal')
- if options.only_internal:
- gradle_args.append('-Ponly_internal')
- if options.all_tests:
- gradle_args.append('-Pall_tests')
- if options.slow_tests:
- gradle_args.append('-Pslow_tests=1')
- if options.tool:
- gradle_args.append('-Ptool=%s' % options.tool)
- if options.one_line_per_test:
- gradle_args.append('-Pone_line_per_test')
- if options.test_namespace:
- gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
- if options.disable_assertions:
- gradle_args.append('-Pdisable_assertions')
- if options.with_code_coverage:
- gradle_args.append('-Pwith_code_coverage')
- if options.print_full_stacktraces:
- gradle_args.append('-Pprint_full_stacktraces')
- if options.print_obfuscated_stacktraces:
- gradle_args.append('-Pprint_obfuscated_stacktraces')
- if options.kotlin_compiler_old:
- gradle_args.append('-Pkotlin_compiler_old')
- if options.kotlin_compiler_dev:
- gradle_args.append('-Pkotlin_compiler_dev')
- download_kotlin_dev.download_newest()
- if os.name == 'nt':
- gradle_args.append('-Pno_internal')
- if options.test_dir:
- gradle_args.append('-Ptest_dir=' + options.test_dir)
- if not os.path.exists(options.test_dir):
- os.makedirs(options.test_dir)
- if options.command_cache_dir:
- gradle_args.append('-Pcommand_cache_dir=' + options.command_cache_dir)
- if not os.path.exists(options.command_cache_dir):
- os.makedirs(options.command_cache_dir)
- if options.command_cache_stats:
- stats_dir = os.path.join(options.command_cache_dir, 'stats')
- gradle_args.append('-Pcommand_cache_stats_dir=' + stats_dir)
- if not os.path.exists(stats_dir):
- os.makedirs(stats_dir)
- # Clean out old stats files
- for (_, _, file_names) in os.walk(stats_dir):
- for f in file_names:
- os.remove(os.path.join(stats_dir, f))
- if options.java_home:
- gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
- if options.java_max_memory_size:
- gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
- if options.generate_golden_files_to:
- gradle_args.append('-Pgenerate_golden_files_to=' + options.generate_golden_files_to)
- if not os.path.exists(options.generate_golden_files_to):
- os.makedirs(options.generate_golden_files_to)
- gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
- if options.use_golden_files_in:
- gradle_args.append('-Puse_golden_files_in=' + options.use_golden_files_in)
- if not os.path.exists(options.use_golden_files_in):
- os.makedirs(options.use_golden_files_in)
- gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
- if options.r8lib_no_deps and options.no_r8lib:
- print('Inconsistent arguments: both --no-r8lib and --r8lib-no-deps specified.')
- exit(1)
- if options.r8lib_no_deps:
- gradle_args.append('-Pr8lib_no_deps')
- elif not options.no_r8lib:
- gradle_args.append('-Pr8lib')
- if options.worktree:
- gradle_args.append('-g=' + os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
- gradle_args.append('--no-daemon')
- if options.debug_agent:
- gradle_args.append('--no-daemon')
- if desugar_jdk_json_dir:
- gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir)
- if desugar_jdk_libs:
- gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
- if options.no_arttests:
- gradle_args.append('-Pno_arttests=true')
-
- if options.rerun:
- testing_state.set_up_test_state(gradle_args, options.rerun, options.testing_state_dir)
-
- # Enable completeness testing of ART profile rewriting.
- gradle_args.append('-Part_profile_rewriting_completeness_check=true')
-
- # Build an R8 with dependencies for bootstrapping tests before adding test sources.
- gradle_args.append(utils.GRADLE_TASK_R8)
- gradle_args.append(utils.GRADLE_TASK_CLEAN_TEST)
- gradle_args.append(utils.GRADLE_TASK_TEST)
- gradle_args.append('--stacktrace')
- gradle_args.append('-Pprint_full_stacktraces')
-
- if options.debug_agent:
- gradle_args.append('--debug-jvm')
- if options.fail_fast:
- gradle_args.append('--fail-fast')
- if options.failed:
- args = compute_failed_tests(args)
- if args is None:
- return 1
- if len(args) == 0:
- print("No failing tests")
- return 0
- # Test filtering. Must always follow the 'test' task.
- testFilterProperty = []
- for testFilter in args:
- gradle_args.append('--tests')
- gradle_args.append(testFilter)
- testFilterProperty.append(testFilter)
- assert not ("|" in testFilter), "| is used as separating character"
- if len(testFilterProperty) > 0:
- gradle_args.append("-Ptestfilter=" + "|".join(testFilterProperty))
- if options.with_code_coverage:
- # Create Jacoco report after tests.
- gradle_args.append('jacocoTestReport')
-
- if options.use_golden_files_in:
- sha1 = '%s' % utils.get_HEAD_sha1()
- with utils.ChangedWorkingDirectory(options.use_golden_files_in):
- utils.download_file_from_cloud_storage(
- 'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
- '%s.tar.gz' % sha1)
- utils.unpack_archive('%s.tar.gz' % sha1)
-
- print_stacks_timeout = options.print_hanging_stacks
- if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
- timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
- if os.path.exists(timestamp_file):
- os.remove(timestamp_file)
- gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
- print_stacks_timeout = (print_stacks_timeout
- if print_stacks_timeout != -1
- else TIMEOUT_HANDLER_PERIOD)
- if utils.is_python3():
- threading.Thread(
- target=timeout_handler,
- args=(timestamp_file, print_stacks_timeout),
- daemon=True).start()
- else:
- thread.start_new_thread(
- timeout_handler, (timestamp_file, print_stacks_timeout,))
- rotate_test_reports()
-
- # Now run tests on selected runtime(s).
- if options.runtimes:
- if options.dex_vm != 'default':
- print('Unexpected runtimes and dex_vm argument: ' + options.dex_vm)
- sys.exit(1)
- if options.runtimes == 'empty':
- # Set runtimes with no content will configure no runtimes.
- gradle_args.append('-Pruntimes=')
- elif options.runtimes == 'all':
- # An unset runtimes will configure all runtimes
- pass
- else:
- prefixes = [prefix.strip() for prefix in options.runtimes.split(':')]
- runtimes = []
- for prefix in prefixes:
- matches = [ rt for rt in VALID_RUNTIMES if rt.startswith(prefix) ]
- if len(matches) == 0:
- print("Invalid runtime prefix '%s'." % prefix)
- print("Must be just 'all', 'empty'," \
- " or a prefix of %s" % ', '.join(VALID_RUNTIMES))
- sys.exit(1)
- runtimes.extend(matches)
- gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
-
- return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
- return archive_and_return(return_code, options)
-
- # Legacy testing populates the runtimes based on dex_vm.
- vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
-
- if options.print_times:
- gradle_args.append('-Pprint_times=true')
- for art_vm in vms_to_test:
- vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
- runtimes = ['dex-' + art_vm]
- # Append the "none" runtime and default JVM if running the "default" DEX VM.
- if art_vm == "default":
- runtimes.extend(['jdk11', 'none'])
- return_code = gradle.RunGradle(
- gradle_args + [
- '-Pdex_vm=%s' % art_vm + vm_suffix,
- '-Pruntimes=%s' % ':'.join(runtimes),
- ],
- throw_on_failure=False)
+ # Set all necessary Gradle properties and options first.
+ if options.shard_count:
+ assert options.shard_number
+ gradle_args.append('-Pshard_count=%s' % options.shard_count)
+ gradle_args.append('-Pshard_number=%s' % options.shard_number)
+ if options.verbose:
+ gradle_args.append('-Pprint_test_stdout')
+ if options.no_internal:
+ gradle_args.append('-Pno_internal')
+ if options.only_internal:
+ gradle_args.append('-Ponly_internal')
+ if options.all_tests:
+ gradle_args.append('-Pall_tests')
+ if options.slow_tests:
+ gradle_args.append('-Pslow_tests=1')
+ if options.tool:
+ gradle_args.append('-Ptool=%s' % options.tool)
+ if options.one_line_per_test:
+ gradle_args.append('-Pone_line_per_test')
+ if options.test_namespace:
+ gradle_args.append('-Ptest_namespace=%s' % options.test_namespace)
+ if options.disable_assertions:
+ gradle_args.append('-Pdisable_assertions')
+ if options.with_code_coverage:
+ gradle_args.append('-Pwith_code_coverage')
+ if options.print_full_stacktraces:
+ gradle_args.append('-Pprint_full_stacktraces')
+ if options.print_obfuscated_stacktraces:
+ gradle_args.append('-Pprint_obfuscated_stacktraces')
+ if options.kotlin_compiler_old:
+ gradle_args.append('-Pkotlin_compiler_old')
+ if options.kotlin_compiler_dev:
+ gradle_args.append('-Pkotlin_compiler_dev')
+ download_kotlin_dev.download_newest()
+ if os.name == 'nt':
+ gradle_args.append('-Pno_internal')
+ if options.test_dir:
+ gradle_args.append('-Ptest_dir=' + options.test_dir)
+ if not os.path.exists(options.test_dir):
+ os.makedirs(options.test_dir)
+ if options.command_cache_dir:
+ gradle_args.append('-Pcommand_cache_dir=' + options.command_cache_dir)
+ if not os.path.exists(options.command_cache_dir):
+ os.makedirs(options.command_cache_dir)
+ if options.command_cache_stats:
+ stats_dir = os.path.join(options.command_cache_dir, 'stats')
+ gradle_args.append('-Pcommand_cache_stats_dir=' + stats_dir)
+ if not os.path.exists(stats_dir):
+ os.makedirs(stats_dir)
+ # Clean out old stats files
+ for (_, _, file_names) in os.walk(stats_dir):
+ for f in file_names:
+ os.remove(os.path.join(stats_dir, f))
+ if options.java_home:
+ gradle_args.append('-Dorg.gradle.java.home=' + options.java_home)
+ if options.java_max_memory_size:
+ gradle_args.append('-Ptest_xmx=' + options.java_max_memory_size)
if options.generate_golden_files_to:
- sha1 = '%s' % utils.get_HEAD_sha1()
- with utils.ChangedWorkingDirectory(options.generate_golden_files_to):
- archive = utils.create_archive(sha1)
- utils.upload_file_to_cloud_storage(archive,
- 'gs://r8-test-results/golden-files/' + archive)
+ gradle_args.append('-Pgenerate_golden_files_to=' +
+ options.generate_golden_files_to)
+ if not os.path.exists(options.generate_golden_files_to):
+ os.makedirs(options.generate_golden_files_to)
+ gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
+ if options.use_golden_files_in:
+ gradle_args.append('-Puse_golden_files_in=' +
+ options.use_golden_files_in)
+ if not os.path.exists(options.use_golden_files_in):
+ os.makedirs(options.use_golden_files_in)
+ gradle_args.append('-PHEAD_sha1=' + utils.get_HEAD_sha1())
+ if options.r8lib_no_deps and options.no_r8lib:
+ print(
+ 'Inconsistent arguments: both --no-r8lib and --r8lib-no-deps specified.'
+ )
+ exit(1)
+ if options.r8lib_no_deps:
+ gradle_args.append('-Pr8lib_no_deps')
+ elif not options.no_r8lib:
+ gradle_args.append('-Pr8lib')
+ if options.worktree:
+ gradle_args.append('-g=' +
+ os.path.join(utils.REPO_ROOT, ".gradle_user_home"))
+ gradle_args.append('--no-daemon')
+ if options.debug_agent:
+ gradle_args.append('--no-daemon')
+ if desugar_jdk_json_dir:
+ gradle_args.append('-Pdesugar_jdk_json_dir=' + desugar_jdk_json_dir)
+ if desugar_jdk_libs:
+ gradle_args.append('-Pdesugar_jdk_libs=' + desugar_jdk_libs)
+ if options.no_arttests:
+ gradle_args.append('-Pno_arttests=true')
- return archive_and_return(return_code, options)
+ if options.rerun:
+ testing_state.set_up_test_state(gradle_args, options.rerun,
+ options.testing_state_dir)
- return 0
+ # Enable completeness testing of ART profile rewriting.
+ gradle_args.append('-Part_profile_rewriting_completeness_check=true')
+
+ # Build an R8 with dependencies for bootstrapping tests before adding test sources.
+ gradle_args.append(utils.GRADLE_TASK_R8)
+ gradle_args.append(utils.GRADLE_TASK_CLEAN_TEST)
+ gradle_args.append(utils.GRADLE_TASK_TEST)
+ gradle_args.append('--stacktrace')
+ gradle_args.append('-Pprint_full_stacktraces')
+
+ if options.debug_agent:
+ gradle_args.append('--debug-jvm')
+ if options.fail_fast:
+ gradle_args.append('--fail-fast')
+ if options.failed:
+ args = compute_failed_tests(args)
+ if args is None:
+ return 1
+ if len(args) == 0:
+ print("No failing tests")
+ return 0
+ # Test filtering. Must always follow the 'test' task.
+ testFilterProperty = []
+ for testFilter in args:
+ gradle_args.append('--tests')
+ gradle_args.append(testFilter)
+ testFilterProperty.append(testFilter)
+ assert not ("|" in testFilter), "| is used as separating character"
+ if len(testFilterProperty) > 0:
+ gradle_args.append("-Ptestfilter=" + "|".join(testFilterProperty))
+ if options.with_code_coverage:
+ # Create Jacoco report after tests.
+ gradle_args.append('jacocoTestReport')
+
+ if options.use_golden_files_in:
+ sha1 = '%s' % utils.get_HEAD_sha1()
+ with utils.ChangedWorkingDirectory(options.use_golden_files_in):
+ utils.download_file_from_cloud_storage(
+ 'gs://r8-test-results/golden-files/%s.tar.gz' % sha1,
+ '%s.tar.gz' % sha1)
+ utils.unpack_archive('%s.tar.gz' % sha1)
+
+ print_stacks_timeout = options.print_hanging_stacks
+ if (utils.is_bot() and not utils.IsWindows()) or print_stacks_timeout > -1:
+ timestamp_file = os.path.join(utils.BUILD, 'last_test_time')
+ if os.path.exists(timestamp_file):
+ os.remove(timestamp_file)
+ gradle_args.append('-Pupdate_test_timestamp=' + timestamp_file)
+ print_stacks_timeout = (print_stacks_timeout if print_stacks_timeout
+ != -1 else TIMEOUT_HANDLER_PERIOD)
+ if utils.is_python3():
+ threading.Thread(target=timeout_handler,
+ args=(timestamp_file, print_stacks_timeout),
+ daemon=True).start()
+ else:
+ thread.start_new_thread(timeout_handler, (
+ timestamp_file,
+ print_stacks_timeout,
+ ))
+ rotate_test_reports()
+
+ # Now run tests on selected runtime(s).
+ if options.runtimes:
+ if options.dex_vm != 'default':
+ print('Unexpected runtimes and dex_vm argument: ' + options.dex_vm)
+ sys.exit(1)
+ if options.runtimes == 'empty':
+ # Set runtimes with no content will configure no runtimes.
+ gradle_args.append('-Pruntimes=')
+ elif options.runtimes == 'all':
+ # An unset runtimes will configure all runtimes
+ pass
+ else:
+ prefixes = [
+ prefix.strip() for prefix in options.runtimes.split(':')
+ ]
+ runtimes = []
+ for prefix in prefixes:
+ matches = [rt for rt in VALID_RUNTIMES if rt.startswith(prefix)]
+ if len(matches) == 0:
+ print("Invalid runtime prefix '%s'." % prefix)
+ print("Must be just 'all', 'empty'," \
+ " or a prefix of %s" % ', '.join(VALID_RUNTIMES))
+ sys.exit(1)
+ runtimes.extend(matches)
+ gradle_args.append('-Pruntimes=%s' % ':'.join(runtimes))
+
+ return_code = gradle.RunGradle(gradle_args, throw_on_failure=False)
+ return archive_and_return(return_code, options)
+
+ # Legacy testing populates the runtimes based on dex_vm.
+ vms_to_test = [options.dex_vm] if options.dex_vm != "all" else ALL_ART_VMS
+
+ if options.print_times:
+ gradle_args.append('-Pprint_times=true')
+ for art_vm in vms_to_test:
+ vm_suffix = "_" + options.dex_vm_kind if art_vm != "default" else ""
+ runtimes = ['dex-' + art_vm]
+ # Append the "none" runtime and default JVM if running the "default" DEX VM.
+ if art_vm == "default":
+ runtimes.extend(['jdk11', 'none'])
+ return_code = gradle.RunGradle(gradle_args + [
+ '-Pdex_vm=%s' % art_vm + vm_suffix,
+ '-Pruntimes=%s' % ':'.join(runtimes),
+ ],
+ throw_on_failure=False)
+ if options.generate_golden_files_to:
+ sha1 = '%s' % utils.get_HEAD_sha1()
+ with utils.ChangedWorkingDirectory(
+ options.generate_golden_files_to):
+ archive = utils.create_archive(sha1)
+ utils.upload_file_to_cloud_storage(
+ archive, 'gs://r8-test-results/golden-files/' + archive)
+
+ return archive_and_return(return_code, options)
+
+ return 0
+
def archive_and_return(return_code, options):
- if return_code != 0:
- if options.archive_failures:
- archive_failures(options)
- if options.command_cache_stats:
- stats_dir = os.path.join(options.command_cache_dir, 'stats')
- cache_hit = 0
- cache_miss = 0
- cache_put = 0
- for (_, _, file_names) in os.walk(stats_dir):
- for f in file_names:
- if f.endswith('CACHEHIT'):
- cache_hit += os.stat(os.path.join(stats_dir, f)).st_size
- if f.endswith('CACHEMISS'):
- cache_miss += os.stat(os.path.join(stats_dir, f)).st_size
- if f.endswith('CACHEPUT'):
- cache_put += os.stat(os.path.join(stats_dir, f)).st_size
- print('Command cache stats')
- print(' Cache hits: ' + str(cache_hit))
- print(' Cache miss: ' + str(cache_miss))
- print(' Cache puts: ' + str(cache_put))
- return return_code
+ if return_code != 0:
+ if options.archive_failures:
+ archive_failures(options)
+ if options.command_cache_stats:
+ stats_dir = os.path.join(options.command_cache_dir, 'stats')
+ cache_hit = 0
+ cache_miss = 0
+ cache_put = 0
+ for (_, _, file_names) in os.walk(stats_dir):
+ for f in file_names:
+ if f.endswith('CACHEHIT'):
+ cache_hit += os.stat(os.path.join(stats_dir, f)).st_size
+ if f.endswith('CACHEMISS'):
+ cache_miss += os.stat(os.path.join(stats_dir, f)).st_size
+ if f.endswith('CACHEPUT'):
+ cache_put += os.stat(os.path.join(stats_dir, f)).st_size
+ print('Command cache stats')
+ print(' Cache hits: ' + str(cache_hit))
+ print(' Cache miss: ' + str(cache_miss))
+ print(' Cache puts: ' + str(cache_put))
+ return return_code
+
def print_jstacks():
- processes = subprocess.check_output(['ps', 'aux']).decode('utf-8')
- for l in processes.splitlines():
- if 'art' in l or 'dalvik' in l:
- print('Running art of dalvik process: \n%s' % l)
- if 'java' in l and 'openjdk' in l:
- print('Running jstack on process: \n%s' % l)
- # Example line:
- # ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
- columns = l.split()
- pid = columns[1]
- return_value = subprocess.call(['jstack', pid])
- if return_value:
- print('Could not jstack %s' % l)
+ processes = subprocess.check_output(['ps', 'aux']).decode('utf-8')
+ for l in processes.splitlines():
+ if 'art' in l or 'dalvik' in l:
+ print('Running art of dalvik process: \n%s' % l)
+ if 'java' in l and 'openjdk' in l:
+ print('Running jstack on process: \n%s' % l)
+ # Example line:
+ # ricow 184313 2.6 0.0 36839068 31808 ? Sl 09:53 0:00 /us..
+ columns = l.split()
+ pid = columns[1]
+ return_value = subprocess.call(['jstack', pid])
+ if return_value:
+ print('Could not jstack %s' % l)
+
def get_time_from_file(timestamp_file):
- if os.path.exists(timestamp_file):
- timestamp = os.stat(timestamp_file).st_mtime
- print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
- sys.stdout.flush()
- return timestamp
- else:
- print('TIMEOUT HANDLER no timestamp file yet')
- sys.stdout.flush()
- return None
+ if os.path.exists(timestamp_file):
+ timestamp = os.stat(timestamp_file).st_mtime
+ print('TIMEOUT HANDLER timestamp: %s' % (timestamp))
+ sys.stdout.flush()
+ return timestamp
+ else:
+ print('TIMEOUT HANDLER no timestamp file yet')
+ sys.stdout.flush()
+ return None
+
def timeout_handler(timestamp_file, timeout_handler_period):
- last_timestamp = None
- while True:
- time.sleep(timeout_handler_period)
- new_timestamp = get_time_from_file(timestamp_file)
- if last_timestamp and new_timestamp == last_timestamp:
- print_jstacks()
- last_timestamp = new_timestamp
+ last_timestamp = None
+ while True:
+ time.sleep(timeout_handler_period)
+ new_timestamp = get_time_from_file(timestamp_file)
+ if last_timestamp and new_timestamp == last_timestamp:
+ print_jstacks()
+ last_timestamp = new_timestamp
+
def report_dir_path(index):
- if index == 0:
- return REPORTS_PATH
- return '%s%d' % (REPORTS_PATH, index)
+ if index == 0:
+ return REPORTS_PATH
+ return '%s%d' % (REPORTS_PATH, index)
+
def report_index_path(index):
- return os.path.join(report_dir_path(index), *REPORT_INDEX)
+ return os.path.join(report_dir_path(index), *REPORT_INDEX)
+
# Rotate test results so previous results are still accessible.
def rotate_test_reports():
- if not os.path.exists(report_dir_path(0)):
- return
- i = 1
- while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
- i += 1
- if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
- shutil.rmtree(report_dir_path(i))
- while i > 0:
- shutil.move(report_dir_path(i - 1), report_dir_path(i))
- i -= 1
+ if not os.path.exists(report_dir_path(0)):
+ return
+ i = 1
+ while i < NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
+ i += 1
+ if i == NUMBER_OF_TEST_REPORTS and os.path.exists(report_dir_path(i)):
+ shutil.rmtree(report_dir_path(i))
+ while i > 0:
+ shutil.move(report_dir_path(i - 1), report_dir_path(i))
+ i -= 1
+
def compute_failed_tests(args):
- if len(args) > 1:
- print("Running with --failed can take an optional path to a report index (or report number).")
- return None
- report = report_index_path(0)
- # If the default report does not exist, fall back to the previous report as it may be a failed
- # gradle run which has already moved the report to report1, but did not produce a new report.
- if not os.path.exists(report):
- report1 = report_index_path(1)
- if os.path.exists(report1):
- report = report1
- if len(args) == 1:
- try:
- # try to parse the arg as a report index.
- index = int(args[0])
- report = report_index_path(index)
- except ValueError:
- # if integer parsing failed assume it is a report file path.
- report = args[0]
- if not os.path.exists(report):
- print("Can't re-run failing, no report at:", report)
- return None
- print("Reading failed tests in", report)
- failing = set()
- inFailedSection = False
- for line in open(report):
- l = line.strip()
- if l == "<h2>Failed tests</h2>":
- inFailedSection = True
- elif l.startswith("<h2>"):
- inFailedSection = False
- prefix = '<a href="classes/'
- if inFailedSection and l.startswith(prefix):
- href = l[len(prefix):l.index('">')]
- # Ignore enties ending with .html which are test classes, not test methods.
- if not href.endswith('.html'):
- # Remove the .html and anchor separateor, also, a classMethod test is the static
- # setup failing so rerun the full class of tests.
- test = href.replace('.html','').replace('#', '.').replace('.classMethod', '')
- failing.add(test)
- return list(failing)
+ if len(args) > 1:
+ print(
+ "Running with --failed can take an optional path to a report index (or report number)."
+ )
+ return None
+ report = report_index_path(0)
+ # If the default report does not exist, fall back to the previous report as it may be a failed
+ # gradle run which has already moved the report to report1, but did not produce a new report.
+ if not os.path.exists(report):
+ report1 = report_index_path(1)
+ if os.path.exists(report1):
+ report = report1
+ if len(args) == 1:
+ try:
+ # try to parse the arg as a report index.
+ index = int(args[0])
+ report = report_index_path(index)
+ except ValueError:
+ # if integer parsing failed assume it is a report file path.
+ report = args[0]
+ if not os.path.exists(report):
+ print("Can't re-run failing, no report at:", report)
+ return None
+ print("Reading failed tests in", report)
+ failing = set()
+ inFailedSection = False
+ for line in open(report):
+ l = line.strip()
+ if l == "<h2>Failed tests</h2>":
+ inFailedSection = True
+ elif l.startswith("<h2>"):
+ inFailedSection = False
+ prefix = '<a href="classes/'
+ if inFailedSection and l.startswith(prefix):
+ href = l[len(prefix):l.index('">')]
+ # Ignore enties ending with .html which are test classes, not test methods.
+ if not href.endswith('.html'):
+ # Remove the .html and anchor separateor, also, a classMethod test is the static
+ # setup failing so rerun the full class of tests.
+ test = href.replace('.html', '').replace('#', '.').replace(
+ '.classMethod', '')
+ failing.add(test)
+ return list(failing)
+
+
if __name__ == '__main__':
- return_code = Main()
- if return_code != 0:
- notify.notify("Tests failed.")
- else:
- notify.notify("Tests passed.")
- sys.exit(return_code)
+ return_code = Main()
+ if return_code != 0:
+ notify.notify("Tests failed.")
+ else:
+ notify.notify("Tests passed.")
+ sys.exit(return_code)
diff --git a/tools/test_r8cfsegments.py b/tools/test_r8cfsegments.py
index 2081ee2..7b21fa5 100755
--- a/tools/test_r8cfsegments.py
+++ b/tools/test_r8cfsegments.py
@@ -27,57 +27,60 @@
import sys
import utils
+
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Run R8 or PG on'
- ' third_party/r8/r8.jar.'
- ' Report Golem-compatible CodeSize and RunTimeRaw values.')
- parser.add_argument('--tool',
- choices = ['pg', 'r8'],
- required = True,
- help = 'Compiler tool to use.')
- parser.add_argument('--name',
- required = True,
- help = 'Results will be printed using the specified benchmark name (e.g.'
- ' <NAME>-<segment>(CodeSize): <bytes>), the full size is reported'
- ' with <NAME>-Total(CodeSize)')
- parser.add_argument('--print-memoryuse',
- help = 'Prints the line \'<NAME>-Total(MemoryUse):'
- ' <mem>\' at the end where <mem> is the peak'
- ' peak resident set size (VmHWM) in bytes.',
- default = False,
- action = 'store_true')
- parser.add_argument('--output',
- help = 'Output directory to keep the generated files')
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='Run R8 or PG on'
+ ' third_party/r8/r8.jar.'
+ ' Report Golem-compatible CodeSize and RunTimeRaw values.')
+ parser.add_argument('--tool',
+ choices=['pg', 'r8'],
+ required=True,
+ help='Compiler tool to use.')
+ parser.add_argument(
+ '--name',
+ required=True,
+ help='Results will be printed using the specified benchmark name (e.g.'
+ ' <NAME>-<segment>(CodeSize): <bytes>), the full size is reported'
+ ' with <NAME>-Total(CodeSize)')
+ parser.add_argument('--print-memoryuse',
+ help='Prints the line \'<NAME>-Total(MemoryUse):'
+ ' <mem>\' at the end where <mem> is the peak'
+ ' peak resident set size (VmHWM) in bytes.',
+ default=False,
+ action='store_true')
+ parser.add_argument('--output',
+ help='Output directory to keep the generated files')
+ return parser.parse_args()
def Main():
- args = parse_arguments()
- utils.check_java_version()
- output_dir = args.output
- with utils.TempDir() as temp_dir:
- if not output_dir:
- output_dir = temp_dir
- track_memory_file = None
- if args.print_memoryuse:
- track_memory_file = os.path.join(output_dir, utils.MEMORY_USE_TMP_FILE)
- if args.tool == 'pg':
- utils.print_cfsegments(args.name, [utils.PINNED_PGR8_JAR])
- else:
- out_file = os.path.join(output_dir, 'out.jar')
- return_code = minify_tool.minify_tool(
- input_jar=utils.PINNED_R8_JAR,
- output_jar=out_file,
- debug=False,
- build=False,
- track_memory_file=track_memory_file,
- benchmark_name=args.name + "-Total")
- if return_code != 0:
- sys.exit(return_code)
+ args = parse_arguments()
+ utils.check_java_version()
+ output_dir = args.output
+ with utils.TempDir() as temp_dir:
+ if not output_dir:
+ output_dir = temp_dir
+ track_memory_file = None
+ if args.print_memoryuse:
+ track_memory_file = os.path.join(output_dir,
+ utils.MEMORY_USE_TMP_FILE)
+ if args.tool == 'pg':
+ utils.print_cfsegments(args.name, [utils.PINNED_PGR8_JAR])
+ else:
+ out_file = os.path.join(output_dir, 'out.jar')
+ return_code = minify_tool.minify_tool(
+ input_jar=utils.PINNED_R8_JAR,
+ output_jar=out_file,
+ debug=False,
+ build=False,
+ track_memory_file=track_memory_file,
+ benchmark_name=args.name + "-Total")
+ if return_code != 0:
+ sys.exit(return_code)
- utils.print_cfsegments(args.name, [out_file])
+ utils.print_cfsegments(args.name, [out_file])
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/testing_state.py b/tools/testing_state.py
index 6cda6ca..b3a4ed4 100644
--- a/tools/testing_state.py
+++ b/tools/testing_state.py
@@ -11,58 +11,70 @@
CHOICES = ["all", "failing", "past-failing", "outstanding"]
DEFAULT_REPORTS_ROOT = os.path.join(utils.BUILD, "testing-state")
+
def set_up_test_state(gradle_args, testing_state_mode, testing_state_path):
- if not testing_state_mode:
- return
- if not testing_state_path:
- testing_state_path = os.path.join(DEFAULT_REPORTS_ROOT, utils.get_HEAD_branch())
- testing_state_path = os.path.abspath(testing_state_path)
- gradle_args.append('-Ptesting-state-mode=%s' % testing_state_mode)
- gradle_args.append('-Ptesting-state-path=%s' % testing_state_path)
- prepare_testing_index(testing_state_mode, testing_state_path)
+ if not testing_state_mode:
+ return
+ if not testing_state_path:
+ testing_state_path = os.path.join(DEFAULT_REPORTS_ROOT,
+ utils.get_HEAD_branch())
+ testing_state_path = os.path.abspath(testing_state_path)
+ gradle_args.append('-Ptesting-state-mode=%s' % testing_state_mode)
+ gradle_args.append('-Ptesting-state-path=%s' % testing_state_path)
+ prepare_testing_index(testing_state_mode, testing_state_path)
+
def fresh_testing_index(testing_state_dir):
- number = 0
- while True:
- freshIndex = os.path.join(testing_state_dir, "index.%d.html" % number)
- number += 1
- if not os.path.exists(freshIndex):
- return freshIndex
+ number = 0
+ while True:
+ freshIndex = os.path.join(testing_state_dir, "index.%d.html" % number)
+ number += 1
+ if not os.path.exists(freshIndex):
+ return freshIndex
+
def prepare_testing_index(testing_state_mode, testing_state_dir):
- if not os.path.exists(testing_state_dir):
- os.makedirs(testing_state_dir)
- index_path = os.path.join(testing_state_dir, "index.html")
- parent_report = None
- resuming = os.path.exists(index_path)
- mode = testing_state_mode if resuming else f"starting (flag: {testing_state_mode})"
- if (resuming):
- parent_report = fresh_testing_index(testing_state_dir)
- os.rename(index_path, parent_report)
- index = open(index_path, "a")
- title = f"Testing: {os.path.basename(testing_state_dir)}"
- # Print a console link to the test report for easy access.
- print("=" * 70)
- print("Test report written to:")
- print(f" file://{index_path}")
- print("=" * 70)
- # Print the new index content.
- index.write(f"<html><head><title>{title}</title>")
- index.write("<style> * { font-family: monospace; }</style>")
- index.write("<meta http-equiv='refresh' content='10' />")
- index.write(f"</head><body><h1>{title}</h1>")
- index.write(f"<h2>Mode: {mode}</h2>")
- # write index links first to avoid jumping when browsing.
- if parent_report:
- index.write(f"<p><a href=\"file://{parent_report}\">Previous result index</a></p>")
- index.write(f"<p><a href=\"file://{index_path}\">Most recent result index</a></p>")
- index.write(f"<p><a href=\"file://{testing_state_dir}\">Test directories</a></p>")
- # git branch/hash and diff for future reference
- index.write(f"<p>Run on: {datetime.datetime.now()}</p>")
- index.write(f"<p>State path: {testing_state_dir}</p>")
- index.write(f"<p>Git branch: {utils.get_HEAD_branch()}")
- index.write(f"</br>Git SHA: {utils.get_HEAD_sha1()}")
- index.write(f'</br>Git diff summary:\n')
- index.write(f'<pre style="background-color: lightgray">{utils.get_HEAD_diff_stat()}</pre></p>')
- # header for the failing tests
- index.write("<h2>Failing tests (refreshing automatically every 10 seconds)</h2><ul>")
+ if not os.path.exists(testing_state_dir):
+ os.makedirs(testing_state_dir)
+ index_path = os.path.join(testing_state_dir, "index.html")
+ parent_report = None
+ resuming = os.path.exists(index_path)
+ mode = testing_state_mode if resuming else f"starting (flag: {testing_state_mode})"
+ if (resuming):
+ parent_report = fresh_testing_index(testing_state_dir)
+ os.rename(index_path, parent_report)
+ index = open(index_path, "a")
+ title = f"Testing: {os.path.basename(testing_state_dir)}"
+ # Print a console link to the test report for easy access.
+ print("=" * 70)
+ print("Test report written to:")
+ print(f" file://{index_path}")
+ print("=" * 70)
+ # Print the new index content.
+ index.write(f"<html><head><title>{title}</title>")
+ index.write("<style> * { font-family: monospace; }</style>")
+ index.write("<meta http-equiv='refresh' content='10' />")
+ index.write(f"</head><body><h1>{title}</h1>")
+ index.write(f"<h2>Mode: {mode}</h2>")
+ # write index links first to avoid jumping when browsing.
+ if parent_report:
+ index.write(
+ f"<p><a href=\"file://{parent_report}\">Previous result index</a></p>"
+ )
+ index.write(
+ f"<p><a href=\"file://{index_path}\">Most recent result index</a></p>")
+ index.write(
+ f"<p><a href=\"file://{testing_state_dir}\">Test directories</a></p>")
+ # git branch/hash and diff for future reference
+ index.write(f"<p>Run on: {datetime.datetime.now()}</p>")
+ index.write(f"<p>State path: {testing_state_dir}</p>")
+ index.write(f"<p>Git branch: {utils.get_HEAD_branch()}")
+ index.write(f"</br>Git SHA: {utils.get_HEAD_sha1()}")
+ index.write(f'</br>Git diff summary:\n')
+ index.write(
+ f'<pre style="background-color: lightgray">{utils.get_HEAD_diff_stat()}</pre></p>'
+ )
+ # header for the failing tests
+ index.write(
+ "<h2>Failing tests (refreshing automatically every 10 seconds)</h2><ul>"
+ )
diff --git a/tools/thread_utils.py b/tools/thread_utils.py
index 5810332..e92d2c8 100755
--- a/tools/thread_utils.py
+++ b/tools/thread_utils.py
@@ -8,6 +8,7 @@
from threading import Thread
import traceback
+
# A thread that is given a list of jobs. The thread will repeatedly take a job
# from the list of jobs (which is shared with other threads) and execute it
# until there is no more jobs.
@@ -21,99 +22,101 @@
# setting an appropriate timeout).
class WorkerThread(Thread):
- # The initialization of a WorkerThread is never run concurrently with the
- # initialization of other WorkerThreads.
- def __init__(self, jobs, jobs_lock, stop_on_first_failure, worker_id):
- Thread.__init__(self)
- self.completed = False
- self.jobs = jobs
- self.jobs_lock = jobs_lock
- self.number_of_jobs = len(jobs)
- self.stop_on_first_failure = stop_on_first_failure
- self.success = True
- self.worker_id = worker_id
+ # The initialization of a WorkerThread is never run concurrently with the
+ # initialization of other WorkerThreads.
+ def __init__(self, jobs, jobs_lock, stop_on_first_failure, worker_id):
+ Thread.__init__(self)
+ self.completed = False
+ self.jobs = jobs
+ self.jobs_lock = jobs_lock
+ self.number_of_jobs = len(jobs)
+ self.stop_on_first_failure = stop_on_first_failure
+ self.success = True
+ self.worker_id = worker_id
- def run(self):
- print_thread("Starting worker", self.worker_id)
- while True:
- (job, job_id) = self.take_job(self.jobs, self.jobs_lock)
- if job is None:
- break
- try:
- print_thread(
- "Starting job %s/%s" % (job_id, self.number_of_jobs),
- self.worker_id)
- exit_code = job(self.worker_id)
- print_thread(
- "Job %s finished with exit code %s"
- % (job_id, exit_code),
- self.worker_id)
- if exit_code:
- self.success = False
- if self.stop_on_first_failure:
- self.clear_jobs(self.jobs, self.jobs_lock)
- break
- except:
- print_thread("Job %s crashed" % job_id, self.worker_id)
- print_thread(traceback.format_exc(), self.worker_id)
- self.success = False
- if self.stop_on_first_failure:
- self.clear_jobs(self.jobs, self.jobs_lock)
- break
- print_thread("Exiting", self.worker_id)
- self.completed = True
+ def run(self):
+ print_thread("Starting worker", self.worker_id)
+ while True:
+ (job, job_id) = self.take_job(self.jobs, self.jobs_lock)
+ if job is None:
+ break
+ try:
+ print_thread(
+ "Starting job %s/%s" % (job_id, self.number_of_jobs),
+ self.worker_id)
+ exit_code = job(self.worker_id)
+ print_thread(
+ "Job %s finished with exit code %s" % (job_id, exit_code),
+ self.worker_id)
+ if exit_code:
+ self.success = False
+ if self.stop_on_first_failure:
+ self.clear_jobs(self.jobs, self.jobs_lock)
+ break
+ except:
+ print_thread("Job %s crashed" % job_id, self.worker_id)
+ print_thread(traceback.format_exc(), self.worker_id)
+ self.success = False
+ if self.stop_on_first_failure:
+ self.clear_jobs(self.jobs, self.jobs_lock)
+ break
+ print_thread("Exiting", self.worker_id)
+ self.completed = True
- def take_job(self, jobs, jobs_lock):
- jobs_lock.acquire()
- job_id = self.number_of_jobs - len(jobs) + 1
- job = jobs.pop(0) if jobs else None
- jobs_lock.release()
- return (job, job_id)
+ def take_job(self, jobs, jobs_lock):
+ jobs_lock.acquire()
+ job_id = self.number_of_jobs - len(jobs) + 1
+ job = jobs.pop(0) if jobs else None
+ jobs_lock.release()
+ return (job, job_id)
- def clear_jobs(self, jobs, jobs_lock):
- jobs_lock.acquire()
- jobs.clear()
- jobs_lock.release()
+ def clear_jobs(self, jobs, jobs_lock):
+ jobs_lock.acquire()
+ jobs.clear()
+ jobs_lock.release()
+
def run_in_parallel(jobs, number_of_workers, stop_on_first_failure):
- assert number_of_workers > 0
- if number_of_workers > len(jobs):
- number_of_workers = len(jobs)
- if number_of_workers == 1:
- return run_in_sequence(jobs, stop_on_first_failure)
- jobs_lock = threading.Lock()
- threads = []
- for worker_id in range(1, number_of_workers + 1):
- threads.append(
- WorkerThread(jobs, jobs_lock, stop_on_first_failure, worker_id))
- for thread in threads:
- thread.start()
- for thread in threads:
- thread.join()
- for thread in threads:
- if not thread.completed or not thread.success:
- return 1
- return 0
+ assert number_of_workers > 0
+ if number_of_workers > len(jobs):
+ number_of_workers = len(jobs)
+ if number_of_workers == 1:
+ return run_in_sequence(jobs, stop_on_first_failure)
+ jobs_lock = threading.Lock()
+ threads = []
+ for worker_id in range(1, number_of_workers + 1):
+ threads.append(
+ WorkerThread(jobs, jobs_lock, stop_on_first_failure, worker_id))
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ for thread in threads:
+ if not thread.completed or not thread.success:
+ return 1
+ return 0
+
def run_in_sequence(jobs, stop_on_first_failure):
- combined_exit_code = 0
- worker_id = None
- for job in jobs:
- try:
- exit_code = job(worker_id)
- if exit_code:
- combined_exit_code = exit_code
- if stop_on_first_failure:
- break
- except:
- print(traceback.format_exc())
- combined_exit_code = 1
- if stop_on_first_failure:
- break
- return combined_exit_code
+ combined_exit_code = 0
+ worker_id = None
+ for job in jobs:
+ try:
+ exit_code = job(worker_id)
+ if exit_code:
+ combined_exit_code = exit_code
+ if stop_on_first_failure:
+ break
+ except:
+ print(traceback.format_exc())
+ combined_exit_code = 1
+ if stop_on_first_failure:
+ break
+ return combined_exit_code
+
def print_thread(msg, worker_id):
- if worker_id is None:
- print(msg)
- else:
- print('WORKER %s: %s' % (worker_id, msg))
\ No newline at end of file
+ if worker_id is None:
+ print(msg)
+ else:
+ print('WORKER %s: %s' % (worker_id, msg))
diff --git a/tools/toolhelper.py b/tools/toolhelper.py
index f63b3ef..5a6b8ee 100644
--- a/tools/toolhelper.py
+++ b/tools/toolhelper.py
@@ -12,100 +12,121 @@
import utils
-def run(tool, args, build=None, debug=True,
- profile=False, track_memory_file=None, extra_args=None,
- stderr=None, stdout=None, return_stdout=False, timeout=0, quiet=False,
- cmd_prefix=None, jar=None, main=None, time_consumer=None,
- debug_agent=None, worker_id=None):
- cmd = []
- if cmd_prefix:
- cmd.extend(cmd_prefix)
- if build is None:
- build, args = extract_build_from_args(args)
- if build:
- gradle.RunGradle([
- utils.GRADLE_TASK_R8LIB if tool.startswith('r8lib')
- else utils.GRADLE_TASK_R8])
- if track_memory_file:
- cmd.extend(['tools/track_memory.sh', track_memory_file])
- cmd.append(jdk.GetJavaExecutable())
- if extra_args:
- cmd.extend(extra_args)
- if debug_agent is None:
- debug_agent, args = extract_debug_agent_from_args(args)
- if debug_agent:
- cmd.append(
- '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005')
- if debug:
- cmd.append('-ea')
- if profile:
- cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
- if jar:
- cmd.extend(['-cp', jar, main])
- elif tool == 'r8lib-d8':
- cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.D8'])
- elif tool == 'r8lib-l8':
- cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.L8'])
- elif tool == 'r8lib-r8':
- cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.R8'])
- elif tool == 'r8lib-tracereferences':
- cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.tracereferences.TraceReferences'])
- else:
- cmd.extend(['-jar', utils.R8_JAR, tool])
- lib, args = extract_lib_from_args(args)
- if lib:
- cmd.extend(["--lib", lib])
- cmd.extend(args)
- utils.PrintCmd(cmd, quiet=quiet, worker_id=worker_id)
- start = time.time()
- if timeout > 0:
- kill = lambda process: process.kill()
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- timer = Timer(timeout, kill, [proc])
- try:
- timer.start()
- stdout, stderr = proc.communicate()
- finally:
- timer.cancel()
- result = stdout.decode('utf-8') if return_stdout else proc.returncode
- else:
- result = (
- subprocess.check_output(cmd).decode('utf-8')
- if return_stdout
- else subprocess.call(cmd, stdout=stdout, stderr=stderr))
- duration = int((time.time() - start) * 1000)
- if time_consumer:
- time_consumer(duration)
- return result
+def run(tool,
+ args,
+ build=None,
+ debug=True,
+ profile=False,
+ track_memory_file=None,
+ extra_args=None,
+ stderr=None,
+ stdout=None,
+ return_stdout=False,
+ timeout=0,
+ quiet=False,
+ cmd_prefix=None,
+ jar=None,
+ main=None,
+ time_consumer=None,
+ debug_agent=None,
+ worker_id=None):
+ cmd = []
+ if cmd_prefix:
+ cmd.extend(cmd_prefix)
+ if build is None:
+ build, args = extract_build_from_args(args)
+ if build:
+ gradle.RunGradle([
+ utils.GRADLE_TASK_R8LIB
+ if tool.startswith('r8lib') else utils.GRADLE_TASK_R8
+ ])
+ if track_memory_file:
+ cmd.extend(['tools/track_memory.sh', track_memory_file])
+ cmd.append(jdk.GetJavaExecutable())
+ if extra_args:
+ cmd.extend(extra_args)
+ if debug_agent is None:
+ debug_agent, args = extract_debug_agent_from_args(args)
+ if debug_agent:
+ cmd.append(
+ '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
+ )
+ if debug:
+ cmd.append('-ea')
+ if profile:
+ cmd.append('-agentlib:hprof=cpu=samples,interval=1,depth=8')
+ if jar:
+ cmd.extend(['-cp', jar, main])
+ elif tool == 'r8lib-d8':
+ cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.D8'])
+ elif tool == 'r8lib-l8':
+ cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.L8'])
+ elif tool == 'r8lib-r8':
+ cmd.extend(['-cp', utils.R8LIB_JAR, 'com.android.tools.r8.R8'])
+ elif tool == 'r8lib-tracereferences':
+ cmd.extend([
+ '-cp', utils.R8LIB_JAR,
+ 'com.android.tools.r8.tracereferences.TraceReferences'
+ ])
+ else:
+ cmd.extend(['-jar', utils.R8_JAR, tool])
+ lib, args = extract_lib_from_args(args)
+ if lib:
+ cmd.extend(["--lib", lib])
+ cmd.extend(args)
+ utils.PrintCmd(cmd, quiet=quiet, worker_id=worker_id)
+ start = time.time()
+ if timeout > 0:
+ kill = lambda process: process.kill()
+ proc = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ timer = Timer(timeout, kill, [proc])
+ try:
+ timer.start()
+ stdout, stderr = proc.communicate()
+ finally:
+ timer.cancel()
+ result = stdout.decode('utf-8') if return_stdout else proc.returncode
+ else:
+ result = (subprocess.check_output(cmd).decode('utf-8') if return_stdout
+ else subprocess.call(cmd, stdout=stdout, stderr=stderr))
+ duration = int((time.time() - start) * 1000)
+ if time_consumer:
+ time_consumer(duration)
+ return result
+
def extract_build_from_args(input_args):
- build = True
- args = []
- for arg in input_args:
- if arg in ("--build", "--no-build"):
- build = arg == "--build"
- else:
- args.append(arg)
- return build, args
+ build = True
+ args = []
+ for arg in input_args:
+ if arg in ("--build", "--no-build"):
+ build = arg == "--build"
+ else:
+ args.append(arg)
+ return build, args
+
def extract_lib_from_args(input_args):
- lib = None
- args = []
- for arg in input_args:
- if arg == '--lib-android':
- lib = utils.get_android_jar(28)
- elif arg == '--lib-java':
- lib = utils.RT_JAR
- else:
- args.append(arg)
- return lib, args
+ lib = None
+ args = []
+ for arg in input_args:
+ if arg == '--lib-android':
+ lib = utils.get_android_jar(28)
+ elif arg == '--lib-java':
+ lib = utils.RT_JAR
+ else:
+ args.append(arg)
+ return lib, args
+
def extract_debug_agent_from_args(input_args):
- agent = False
- args = []
- for arg in input_args:
- if arg in ('--debug-agent', '--debug_agent'):
- agent = True
- else:
- args.append(arg)
- return agent, args
+ agent = False
+ args = []
+ for arg in input_args:
+ if arg in ('--debug-agent', '--debug_agent'):
+ agent = True
+ else:
+ args.append(arg)
+ return agent, args
diff --git a/tools/trigger.py b/tools/trigger.py
index b3e09f2..0c44d24 100755
--- a/tools/trigger.py
+++ b/tools/trigger.py
@@ -27,141 +27,152 @@
DESUGAR_JDK8_BOT = 'lib_desugar-archive-jdk8'
SMALI_BOT = 'smali'
-def ParseOptions():
- result = optparse.OptionParser()
- result.add_option('--release',
- help='Run on the release branch builders.',
- default=False, action='store_true')
- result.add_option('--cl',
- metavar=('<url>'),
- help='Run the specified cl on the bots. This should be '
- 'the full url, e.g., '
- 'https://r8-review.googlesource.com/c/r8/+/37420/1')
- result.add_option('--desugar-jdk11',
- help='Run the jdk11 library desugar and archiving bot.',
- default=False, action='store_true')
- result.add_option('--desugar-jdk11-legacy',
- help='Run the jdk11 legacy library desugar and archiving bot.',
- default=False, action='store_true')
- result.add_option('--desugar-jdk8',
- help='Run the jdk8 library desugar and archiving bot.',
- default=False, action='store_true')
- result.add_option('--smali',
- metavar=('<version>'),
- help='Build smali version <version>.')
- result.add_option('--builder', help='Trigger specific builder')
- return result.parse_args()
+def ParseOptions():
+ result = optparse.OptionParser()
+ result.add_option('--release',
+ help='Run on the release branch builders.',
+ default=False,
+ action='store_true')
+ result.add_option('--cl',
+ metavar=('<url>'),
+ help='Run the specified cl on the bots. This should be '
+ 'the full url, e.g., '
+ 'https://r8-review.googlesource.com/c/r8/+/37420/1')
+ result.add_option('--desugar-jdk11',
+ help='Run the jdk11 library desugar and archiving bot.',
+ default=False,
+ action='store_true')
+ result.add_option(
+ '--desugar-jdk11-legacy',
+ help='Run the jdk11 legacy library desugar and archiving bot.',
+ default=False,
+ action='store_true')
+ result.add_option('--desugar-jdk8',
+ help='Run the jdk8 library desugar and archiving bot.',
+ default=False,
+ action='store_true')
+ result.add_option('--smali',
+ metavar=('<version>'),
+ help='Build smali version <version>.')
+
+ result.add_option('--builder', help='Trigger specific builder')
+ return result.parse_args()
+
def get_builders():
- is_release = False
- main_builders = []
- release_builders = []
- with open(LUCI_SCHEDULE, 'r') as fp:
- lines = fp.readlines()
- for line in lines:
- if 'branch-gitiles' in line:
- is_release = True
- if 'main-gitiles-trigger' in line:
- is_release = False
- match = re.match(TRIGGERS_RE, line)
- if match:
- builder = match.group(1)
- if is_release:
- assert 'release' in builder, builder
- release_builders.append(builder)
- else:
- assert 'release' not in builder, builder
- main_builders.append(builder)
- print('Desugar jdk11 builder:\n ' + DESUGAR_JDK11_BOT)
- print('Desugar jdk11 legacy builder:\n ' + DESUGAR_JDK11_LEGACY_BOT)
- print('Desugar jdk8 builder:\n ' + DESUGAR_JDK8_BOT)
- print('Smali builder:\n ' + SMALI_BOT)
- print('Main builders:\n ' + '\n '.join(main_builders))
- print('Release builders:\n ' + '\n '.join(release_builders))
- return (main_builders, release_builders)
+ is_release = False
+ main_builders = []
+ release_builders = []
+ with open(LUCI_SCHEDULE, 'r') as fp:
+ lines = fp.readlines()
+ for line in lines:
+ if 'branch-gitiles' in line:
+ is_release = True
+ if 'main-gitiles-trigger' in line:
+ is_release = False
+ match = re.match(TRIGGERS_RE, line)
+ if match:
+ builder = match.group(1)
+ if is_release:
+ assert 'release' in builder, builder
+ release_builders.append(builder)
+ else:
+ assert 'release' not in builder, builder
+ main_builders.append(builder)
+ print('Desugar jdk11 builder:\n ' + DESUGAR_JDK11_BOT)
+ print('Desugar jdk11 legacy builder:\n ' + DESUGAR_JDK11_LEGACY_BOT)
+ print('Desugar jdk8 builder:\n ' + DESUGAR_JDK8_BOT)
+ print('Smali builder:\n ' + SMALI_BOT)
+ print('Main builders:\n ' + '\n '.join(main_builders))
+ print('Release builders:\n ' + '\n '.join(release_builders))
+ return (main_builders, release_builders)
+
def sanity_check_url(url):
- a = urlopen(url)
- if a.getcode() != 200:
- raise Exception('Url: %s \n returned %s' % (url, a.getcode()))
+ a = urlopen(url)
+ if a.getcode() != 200:
+ raise Exception('Url: %s \n returned %s' % (url, a.getcode()))
+
def trigger_builders(builders, commit):
- commit_url = 'https://r8.googlesource.com/r8/+/%s' % commit
- sanity_check_url(commit_url)
- for builder in builders:
- cmd = ['bb', 'add', 'r8/ci/%s' % builder , '-commit', commit_url]
- subprocess.check_call(cmd)
+ commit_url = 'https://r8.googlesource.com/r8/+/%s' % commit
+ sanity_check_url(commit_url)
+ for builder in builders:
+ cmd = ['bb', 'add', 'r8/ci/%s' % builder, '-commit', commit_url]
+ subprocess.check_call(cmd)
+
def trigger_smali_builder(version):
- utils.check_basic_semver_version(
- version,
- 'use semantic version of the smali version to built (pre-releases are not supported)',
- allowPrerelease = False)
- cmd = [
- 'bb',
- 'add',
- 'r8/ci/%s' % SMALI_BOT,
- '-p',
- 'test_options=["--version", "%s"]' % version
- ]
- subprocess.check_call(cmd)
-
-def trigger_cl(builders, cl_url):
- for builder in builders:
- cmd = ['bb', 'add', 'r8/ci/%s' % builder , '-cl', cl_url]
+ utils.check_basic_semver_version(
+ version,
+ 'use semantic version of the smali version to built (pre-releases are not supported)',
+ allowPrerelease=False)
+ cmd = [
+ 'bb', 'add',
+ 'r8/ci/%s' % SMALI_BOT, '-p',
+ 'test_options=["--version", "%s"]' % version
+ ]
subprocess.check_call(cmd)
+
+def trigger_cl(builders, cl_url):
+ for builder in builders:
+ cmd = ['bb', 'add', 'r8/ci/%s' % builder, '-cl', cl_url]
+ subprocess.check_call(cmd)
+
+
def Main():
- (options, args) = ParseOptions()
- desugar = options.desugar_jdk11 or options.desugar_jdk11_legacy or options.desugar_jdk8
- requires_commit = not options.cl and not desugar and not options.smali
- if len(args) != 1 and requires_commit:
- print('Takes exactly one argument, the commit to run')
- return 1
+ (options, args) = ParseOptions()
+ desugar = options.desugar_jdk11 or options.desugar_jdk11_legacy or options.desugar_jdk8
+ requires_commit = not options.cl and not desugar and not options.smali
+ if len(args) != 1 and requires_commit:
+ print('Takes exactly one argument, the commit to run')
+ return 1
- if options.cl and options.release:
- print('You can\'t run cls on the release bots')
- return 1
+ if options.cl and options.release:
+ print('You can\'t run cls on the release bots')
+ return 1
- if options.cl and desugar:
- print('You can\'t run cls on the desugar bot')
- return 1
+ if options.cl and desugar:
+ print('You can\'t run cls on the desugar bot')
+ return 1
- if options.cl and options.smali:
- print('You can\'t run cls on the smali bot')
- return 1
+ if options.cl and options.smali:
+ print('You can\'t run cls on the smali bot')
+ return 1
- if options.smali:
- if not options.release:
- print('Only release versions of smali can be built')
- return 1
+ if options.smali:
+ if not options.release:
+ print('Only release versions of smali can be built')
+ return 1
- trigger_smali_builder(options.smali)
- return
+ trigger_smali_builder(options.smali)
+ return
- commit = None if not requires_commit else args[0]
- (main_builders, release_builders) = get_builders()
- builders = release_builders if options.release else main_builders
- if options.builder:
- builder = options.builder
- assert builder in main_builders or builder in release_builders
- builders = [options.builder]
- if desugar:
- assert options.desugar_jdk11 or options.desugar_jdk11_legacy or options.desugar_jdk8
- if options.desugar_jdk11:
- builders = [DESUGAR_JDK11_BOT]
- elif options.desugar_jdk11_legacy:
- builders = [DESUGAR_JDK11_LEGACY_BOT]
+ commit = None if not requires_commit else args[0]
+ (main_builders, release_builders) = get_builders()
+ builders = release_builders if options.release else main_builders
+ if options.builder:
+ builder = options.builder
+ assert builder in main_builders or builder in release_builders
+ builders = [options.builder]
+ if desugar:
+ assert options.desugar_jdk11 or options.desugar_jdk11_legacy or options.desugar_jdk8
+ if options.desugar_jdk11:
+ builders = [DESUGAR_JDK11_BOT]
+ elif options.desugar_jdk11_legacy:
+ builders = [DESUGAR_JDK11_LEGACY_BOT]
+ else:
+ builders = [DESUGAR_JDK8_BOT]
+ commit = git_utils.GetHeadRevision(utils.REPO_ROOT, use_main=True)
+ if options.cl:
+ trigger_cl(builders, options.cl)
else:
- builders = [DESUGAR_JDK8_BOT]
- commit = git_utils.GetHeadRevision(utils.REPO_ROOT, use_main=True)
- if options.cl:
- trigger_cl(builders, options.cl)
- else:
- assert commit
- trigger_builders(builders, commit)
+ assert commit
+ trigger_builders(builders, commit)
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/update_prebuilds_in_android.py b/tools/update_prebuilds_in_android.py
index ea2c32e..31ee23e 100755
--- a/tools/update_prebuilds_in_android.py
+++ b/tools/update_prebuilds_in_android.py
@@ -12,129 +12,138 @@
import archive
JAR_TARGETS_MAP = {
- 'full': [
- (utils.R8, 'r8'),
- ],
- 'lib': [
- (utils.R8LIB, 'r8'),
- ],
+ 'full': [(utils.R8, 'r8'),],
+ 'lib': [(utils.R8LIB, 'r8'),],
}
OTHER_TARGETS = ["LICENSE"]
KEEPANNO_JAR = 'keepanno-annotations.jar'
+
def parse_arguments():
- parser = argparse.ArgumentParser(
- description = 'Build and copy jars to an Android tree.')
- parser.add_argument('android_root', nargs=1,
- help='Android checkout root.')
- parser.add_argument('--commit_hash', default=None, help='Commit hash')
- parser.add_argument('--version', default=None, help='The version to download')
- parser.add_argument(
- '--targets',
- required=True,
- choices=['full', 'lib'],
- help="Use 'full' to download the full, non-optimized jars (legacy" +
- " behaviour) and 'lib' for the R8-processed, optimized jars.",
- )
- parser.add_argument(
- '--maps',
- action='store_true',
- help="Download proguard maps for jars, use only with '--target lib'.",
- )
- parser.add_argument(
- '--keepanno',
- action='store_true',
- help="Download keepanno-annotations library.",
- )
- parser.add_argument(
- '--java-max-memory-size',
- '--java_max_memory_size',
- help='Use a custom max memory size for the gradle java instance, eg, 4g')
- return parser.parse_args()
+ parser = argparse.ArgumentParser(
+ description='Build and copy jars to an Android tree.')
+ parser.add_argument('android_root', nargs=1, help='Android checkout root.')
+ parser.add_argument('--commit_hash', default=None, help='Commit hash')
+ parser.add_argument('--version',
+ default=None,
+ help='The version to download')
+ parser.add_argument(
+ '--targets',
+ required=True,
+ choices=['full', 'lib'],
+ help="Use 'full' to download the full, non-optimized jars (legacy" +
+ " behaviour) and 'lib' for the R8-processed, optimized jars.",
+ )
+ parser.add_argument(
+ '--maps',
+ action='store_true',
+ help="Download proguard maps for jars, use only with '--target lib'.",
+ )
+ parser.add_argument(
+ '--keepanno',
+ action='store_true',
+ help="Download keepanno-annotations library.",
+ )
+ parser.add_argument(
+ '--java-max-memory-size',
+ '--java_max_memory_size',
+ help='Use a custom max memory size for the gradle java instance, eg, 4g'
+ )
+ return parser.parse_args()
+
def copy_targets(root, target_root, srcs, dests, maps=False):
- assert len(srcs) == len(dests)
- for i in range(len(srcs)):
- src = os.path.join(root, srcs[i])
- dest = os.path.join(target_root, 'prebuilts', 'r8', dests[i])
- if os.path.exists(dest):
- print('Copying: ' + src + ' -> ' + dest)
- copyfile(src, dest)
- if maps:
- print('Copying: ' + src + '.map -> ' + dest + '.map')
- copyfile(src + '.map', dest + '.map')
- else:
- print('WARNING: Not copying ' + src + ' -> ' + dest +
- ', as' + dest + ' does not exist already')
+ assert len(srcs) == len(dests)
+ for i in range(len(srcs)):
+ src = os.path.join(root, srcs[i])
+ dest = os.path.join(target_root, 'prebuilts', 'r8', dests[i])
+ if os.path.exists(dest):
+ print('Copying: ' + src + ' -> ' + dest)
+ copyfile(src, dest)
+ if maps:
+ print('Copying: ' + src + '.map -> ' + dest + '.map')
+ copyfile(src + '.map', dest + '.map')
+ else:
+ print('WARNING: Not copying ' + src + ' -> ' + dest + ', as' +
+ dest + ' does not exist already')
+
def copy_jar_targets(root, target_root, jar_targets, maps):
- srcs = list(map((lambda t: t[0] + '.jar'), jar_targets))
- dests = list(map((lambda t: t[1] + '.jar'), jar_targets))
- copy_targets(root, target_root, srcs, dests, maps=maps)
+ srcs = list(map((lambda t: t[0] + '.jar'), jar_targets))
+ dests = list(map((lambda t: t[1] + '.jar'), jar_targets))
+ copy_targets(root, target_root, srcs, dests, maps=maps)
+
def copy_other_targets(root, target_root):
- copy_targets(root, target_root, OTHER_TARGETS, OTHER_TARGETS)
+ copy_targets(root, target_root, OTHER_TARGETS, OTHER_TARGETS)
+
def download_hash(root, commit_hash, target, quiet=False):
- download_target(root, target, commit_hash, True, quiet=quiet)
+ download_target(root, target, commit_hash, True, quiet=quiet)
+
def download_version(root, version, target):
- download_target(root, target, version, False)
+ download_target(root, target, version, False)
+
def download_target(root, target, hash_or_version, is_hash, quiet=False):
- download_path = os.path.join(root, target)
- url = archive.GetUploadDestination(
- hash_or_version,
- target,
- is_hash)
- if not quiet:
- print('Downloading: ' + url + ' -> ' + download_path)
- utils.download_file_from_cloud_storage(url, download_path, quiet=quiet)
+ download_path = os.path.join(root, target)
+ url = archive.GetUploadDestination(hash_or_version, target, is_hash)
+ if not quiet:
+ print('Downloading: ' + url + ' -> ' + download_path)
+ utils.download_file_from_cloud_storage(url, download_path, quiet=quiet)
+
def main_download(hash, maps, targets, target_root, version, keepanno=False):
- jar_targets = JAR_TARGETS_MAP[targets]
- final_targets = list(map((lambda t: t[0] + '.jar'), jar_targets)) + OTHER_TARGETS
- with utils.TempDir() as root:
- for target in final_targets:
- if hash:
- download_hash(root, hash, target)
- if maps and target not in OTHER_TARGETS:
- download_hash(root, hash, target + '.map')
+ jar_targets = JAR_TARGETS_MAP[targets]
+ final_targets = list(map(
+ (lambda t: t[0] + '.jar'), jar_targets)) + OTHER_TARGETS
+ with utils.TempDir() as root:
+ for target in final_targets:
+ if hash:
+ download_hash(root, hash, target)
+ if maps and target not in OTHER_TARGETS:
+ download_hash(root, hash, target + '.map')
+ if keepanno:
+ download_hash(root, hash, KEEPANNO_JAR)
+ else:
+ assert version
+ download_version(root, version, target)
+ if maps and target not in OTHER_TARGETS:
+ download_version(root, version, target + '.map')
+ if keepanno:
+ download_version(root, version, KEEPANNO_JAR)
+ copy_jar_targets(root, target_root, jar_targets, maps)
+ copy_other_targets(root, target_root)
if keepanno:
- download_hash(root, hash, KEEPANNO_JAR)
- else:
- assert version
- download_version(root, version, target)
- if maps and target not in OTHER_TARGETS:
- download_version(root, version, target + '.map')
- if keepanno:
- download_version(root, version, KEEPANNO_JAR)
- copy_jar_targets(root, target_root, jar_targets, maps)
- copy_other_targets(root, target_root)
- if keepanno:
- copy_targets(root, target_root, [KEEPANNO_JAR], [KEEPANNO_JAR])
+ copy_targets(root, target_root, [KEEPANNO_JAR], [KEEPANNO_JAR])
+
def main_build(maps, max_memory_size, targets, target_root):
- jar_targets = JAR_TARGETS_MAP[targets]
- gradle_args = [utils.GRADLE_TASK_R8LIB if targets == 'lib'
- else utils.GRADLE_TASK_R8]
- if max_memory_size:
- gradle_args.append('-Dorg.gradle.jvmargs=-Xmx' + max_memory_size)
- gradle.RunGradle(gradle_args)
- copy_jar_targets(utils.LIBS, target_root, jar_targets, maps)
- copy_other_targets(utils.GENERATED_LICENSE_DIR, target_root)
+ jar_targets = JAR_TARGETS_MAP[targets]
+ gradle_args = [
+ utils.GRADLE_TASK_R8LIB if targets == 'lib' else utils.GRADLE_TASK_R8
+ ]
+ if max_memory_size:
+ gradle_args.append('-Dorg.gradle.jvmargs=-Xmx' + max_memory_size)
+ gradle.RunGradle(gradle_args)
+ copy_jar_targets(utils.LIBS, target_root, jar_targets, maps)
+ copy_other_targets(utils.GENERATED_LICENSE_DIR, target_root)
+
def main(args):
- if args.maps and args.targets != 'lib':
- raise Exception("Use '--maps' only with '--targets lib.")
- target_root = args.android_root[0]
- if args.commit_hash == None and args.version == None:
- main_build(args.maps, args.java_max_memory_size, args.targets, target_root)
- else:
- assert args.commit_hash == None or args.version == None
- main_download(
- args.commit_hash, args.maps, args.targets, target_root, args.version, args.keepanno)
+ if args.maps and args.targets != 'lib':
+ raise Exception("Use '--maps' only with '--targets lib.")
+ target_root = args.android_root[0]
+ if args.commit_hash == None and args.version == None:
+ main_build(args.maps, args.java_max_memory_size, args.targets,
+ target_root)
+ else:
+ assert args.commit_hash == None or args.version == None
+ main_download(args.commit_hash, args.maps, args.targets, target_root,
+ args.version, args.keepanno)
+
if __name__ == '__main__':
- sys.exit(main(parse_arguments()))
+ sys.exit(main(parse_arguments()))
diff --git a/tools/upload_to_x20.py b/tools/upload_to_x20.py
index e6c293d..3ef7b73 100755
--- a/tools/upload_to_x20.py
+++ b/tools/upload_to_x20.py
@@ -17,30 +17,36 @@
GMSCORE_DEPS = '/google/data/rw/teams/r8/deps'
+
def parse_options():
- return optparse.OptionParser().parse_args()
+ return optparse.OptionParser().parse_args()
+
def uploadFile(filename, dest):
- print('Uploading to %s' % dest)
- shutil.copyfile(filename, dest)
- subprocess.check_call(['chmod', '664', dest])
+ print('Uploading to %s' % dest)
+ shutil.copyfile(filename, dest)
+ subprocess.check_call(['chmod', '664', dest])
+
def Main():
- (options, args) = parse_options()
- assert len(args) == 1
- name = args[0]
- print('Creating archive for %s' % name)
- if not name in os.listdir('.'):
- print('You must be standing directly below the directory you are uploading')
- return 1
- filename = utils.create_archive(name)
- sha1 = utils.get_sha1(filename)
- dest = os.path.join(GMSCORE_DEPS, sha1)
- uploadFile(filename, dest)
- sha1_file = '%s.sha1' % filename
- with open(sha1_file, 'w') as output:
- output.write(sha1)
- print('Sha (%s) written to: %s' % (sha1, sha1_file))
+ (options, args) = parse_options()
+ assert len(args) == 1
+ name = args[0]
+ print('Creating archive for %s' % name)
+ if not name in os.listdir('.'):
+ print(
+ 'You must be standing directly below the directory you are uploading'
+ )
+ return 1
+ filename = utils.create_archive(name)
+ sha1 = utils.get_sha1(filename)
+ dest = os.path.join(GMSCORE_DEPS, sha1)
+ uploadFile(filename, dest)
+ sha1_file = '%s.sha1' % filename
+ with open(sha1_file, 'w') as output:
+ output.write(sha1)
+ print('Sha (%s) written to: %s' % (sha1, sha1_file))
+
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
diff --git a/tools/utils.py b/tools/utils.py
index 2fd1140..db9844e 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -34,8 +34,8 @@
BUILD = os.path.join(REPO_ROOT, 'build')
BUILD_JAVA_MAIN_DIR = os.path.join(BUILD, 'classes', 'java', 'main')
LIBS = os.path.join(BUILD, 'libs')
-CUSTOM_CONVERSION_DIR = os.path.join(
- THIRD_PARTY, 'openjdk', 'custom_conversion')
+CUSTOM_CONVERSION_DIR = os.path.join(THIRD_PARTY, 'openjdk',
+ 'custom_conversion')
GENERATED_LICENSE_DIR = os.path.join(BUILD, 'generatedLicense')
SRC_ROOT = os.path.join(REPO_ROOT, 'src', 'main', 'java')
REPO_SOURCE = 'https://r8.googlesource.com/r8'
@@ -78,30 +78,33 @@
CUSTOM_CONVERSION_DIR, 'library_desugar_conversions.jar')
KEEPANNO_ANNOTATIONS_JAR = os.path.join(LIBS, 'keepanno-annotations.jar')
-DESUGAR_CONFIGURATION = os.path.join(
- 'src', 'library_desugar', 'desugar_jdk_libs.json')
-DESUGAR_IMPLEMENTATION = os.path.join(
- 'third_party', 'openjdk', 'desugar_jdk_libs', 'desugar_jdk_libs.jar')
+DESUGAR_CONFIGURATION = os.path.join('src', 'library_desugar',
+ 'desugar_jdk_libs.json')
+DESUGAR_IMPLEMENTATION = os.path.join('third_party', 'openjdk',
+ 'desugar_jdk_libs',
+ 'desugar_jdk_libs.jar')
DESUGAR_CONFIGURATION_JDK11_LEGACY = os.path.join(
- 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
+ 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_legacy.json')
DESUGAR_CONFIGURATION_JDK11_MINIMAL = os.path.join(
- 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json')
-DESUGAR_CONFIGURATION_JDK11 = os.path.join(
- 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs.json')
-DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join(
- 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_nio.json')
-DESUGAR_IMPLEMENTATION_JDK11 = os.path.join(
- 'third_party', 'openjdk', 'desugar_jdk_libs_11', 'desugar_jdk_libs.jar')
+ 'src', 'library_desugar', 'jdk11', 'desugar_jdk_libs_minimal.json')
+DESUGAR_CONFIGURATION_JDK11 = os.path.join('src', 'library_desugar', 'jdk11',
+ 'desugar_jdk_libs.json')
+DESUGAR_CONFIGURATION_JDK11_NIO = os.path.join('src', 'library_desugar',
+ 'jdk11',
+ 'desugar_jdk_libs_nio.json')
+DESUGAR_IMPLEMENTATION_JDK11 = os.path.join('third_party', 'openjdk',
+ 'desugar_jdk_libs_11',
+ 'desugar_jdk_libs.jar')
DESUGAR_CONFIGURATION_MAVEN_ZIP = os.path.join(
- LIBS, 'desugar_jdk_libs_configuration.zip')
+ LIBS, 'desugar_jdk_libs_configuration.zip')
DESUGAR_CONFIGURATION_JDK11_LEGACY_MAVEN_ZIP = os.path.join(
- LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip')
+ LIBS, 'desugar_jdk_libs_configuration_jdk11_legacy.zip')
DESUGAR_CONFIGURATION_JDK11_MINIMAL_MAVEN_ZIP = os.path.join(
- LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip')
+ LIBS, 'desugar_jdk_libs_configuration_jdk11_minimal.zip')
DESUGAR_CONFIGURATION_JDK11_MAVEN_ZIP = os.path.join(
- LIBS, 'desugar_jdk_libs_configuration_jdk11.zip')
+ LIBS, 'desugar_jdk_libs_configuration_jdk11.zip')
DESUGAR_CONFIGURATION_JDK11_NIO_MAVEN_ZIP = os.path.join(
- LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip')
+ LIBS, 'desugar_jdk_libs_configuration_jdk11_nio.zip')
GENERATED_LICENSE = os.path.join(GENERATED_LICENSE_DIR, 'LICENSE')
RT_JAR = os.path.join(REPO_ROOT, 'third_party/openjdk/openjdk-rt-1.8/rt.jar')
R8LIB_KEEP_RULES = os.path.join(REPO_ROOT, 'src/main/keep.txt')
@@ -113,10 +116,14 @@
INTERNAL_DUMPS_DIR = os.path.join(THIRD_PARTY, 'internal-apps')
BAZEL_SHA_FILE = os.path.join(THIRD_PARTY, 'bazel.tar.gz.sha1')
BAZEL_TOOL = os.path.join(THIRD_PARTY, 'bazel')
-JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8', 'linux-x86.tar.gz.sha1')
-JAVA11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11', 'linux.tar.gz.sha1')
-DESUGAR_JDK_LIBS_11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'desugar_jdk_libs_11.tar.gz.sha1')
-IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test', 'ignorewarnings.rules')
+JAVA8_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk8',
+ 'linux-x86.tar.gz.sha1')
+JAVA11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk', 'jdk-11',
+ 'linux.tar.gz.sha1')
+DESUGAR_JDK_LIBS_11_SHA_FILE = os.path.join(THIRD_PARTY, 'openjdk',
+ 'desugar_jdk_libs_11.tar.gz.sha1')
+IGNORE_WARNINGS_RULES = os.path.join(REPO_ROOT, 'src', 'test',
+ 'ignorewarnings.rules')
ANDROID_HOME_ENVIROMENT_NAME = "ANDROID_HOME"
ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME = "ANDROID_TOOLS_VERSION"
USER_HOME = os.path.expanduser('~')
@@ -124,573 +131,639 @@
R8_TEST_RESULTS_BUCKET = 'r8-test-results'
R8_INTERNAL_TEST_RESULTS_BUCKET = 'r8-internal-test-results'
+
def archive_file(name, gs_dir, src_file):
- gs_file = '%s/%s' % (gs_dir, name)
- upload_file_to_cloud_storage(src_file, gs_file)
+ gs_file = '%s/%s' % (gs_dir, name)
+ upload_file_to_cloud_storage(src_file, gs_file)
+
def archive_value(name, gs_dir, value):
- with TempDir() as temp:
- temparchive = os.path.join(temp, name)
- with open(temparchive, 'w') as f:
- f.write(str(value))
- archive_file(name, gs_dir, temparchive)
+ with TempDir() as temp:
+ temparchive = os.path.join(temp, name)
+ with open(temparchive, 'w') as f:
+ f.write(str(value))
+ archive_file(name, gs_dir, temparchive)
+
def find_cloud_storage_file_from_options(name, options, orElse=None):
- # Import archive on-demand since archive depends on utils.
- from archive import GetUploadDestination
- hash_or_version = find_hash_or_version_from_options(options)
- if not hash_or_version:
- return orElse
- is_hash = options.commit_hash is not None
- download_path = GetUploadDestination(hash_or_version, name, is_hash)
- if file_exists_on_cloud_storage(download_path):
- out = tempfile.NamedTemporaryFile().name
- download_file_from_cloud_storage(download_path, out)
- return out
- else:
- raise Exception('Could not find file {} from hash/version: {}.'
- .format(name, hash_or_version))
+ # Import archive on-demand since archive depends on utils.
+ from archive import GetUploadDestination
+ hash_or_version = find_hash_or_version_from_options(options)
+ if not hash_or_version:
+ return orElse
+ is_hash = options.commit_hash is not None
+ download_path = GetUploadDestination(hash_or_version, name, is_hash)
+ if file_exists_on_cloud_storage(download_path):
+ out = tempfile.NamedTemporaryFile().name
+ download_file_from_cloud_storage(download_path, out)
+ return out
+ else:
+ raise Exception('Could not find file {} from hash/version: {}.'.format(
+ name, hash_or_version))
+
def find_r8_jar_from_options(options):
- return find_cloud_storage_file_from_options('r8.jar', options)
+ return find_cloud_storage_file_from_options('r8.jar', options)
def find_hash_or_version_from_options(options):
- if options.tag:
- return find_hash_or_version_from_tag(options.tag)
- else:
- return options.commit_hash or options.version
+ if options.tag:
+ return find_hash_or_version_from_tag(options.tag)
+ else:
+ return options.commit_hash or options.version
+
def find_hash_or_version_from_tag(tag_or_hash):
- info = subprocess.check_output([
- 'git',
- 'show',
- tag_or_hash,
- '-s',
- '--format=oneline']).decode('utf-8').splitlines()[-1].split()
- # The info should be on the following form [hash,"Version",version]
- if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version":
- return info[2]
- return None
+ info = subprocess.check_output(
+ ['git', 'show', tag_or_hash, '-s',
+ '--format=oneline']).decode('utf-8').splitlines()[-1].split()
+ # The info should be on the following form [hash,"Version",version]
+ if len(info) == 3 and len(info[0]) == 40 and info[1] == "Version":
+ return info[2]
+ return None
+
def getAndroidHome():
- return os.environ.get(
- ANDROID_HOME_ENVIROMENT_NAME, os.path.join(USER_HOME, 'Android', 'Sdk'))
+ return os.environ.get(ANDROID_HOME_ENVIROMENT_NAME,
+ os.path.join(USER_HOME, 'Android', 'Sdk'))
+
def getAndroidBuildTools():
- if ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME in os.environ:
- version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME)
- build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
- assert os.path.exists(build_tools_dir)
- return build_tools_dir
- else:
- versions = ['33.0.1', '32.0.0']
- for version in versions:
- build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
- if os.path.exists(build_tools_dir):
+ if ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME in os.environ:
+ version = os.environ.get(ANDROID_TOOLS_VERSION_ENVIRONMENT_NAME)
+ build_tools_dir = os.path.join(getAndroidHome(), 'build-tools', version)
+ assert os.path.exists(build_tools_dir)
return build_tools_dir
- raise Exception('Unable to find Android build-tools')
+ else:
+ versions = ['33.0.1', '32.0.0']
+ for version in versions:
+ build_tools_dir = os.path.join(getAndroidHome(), 'build-tools',
+ version)
+ if os.path.exists(build_tools_dir):
+ return build_tools_dir
+ raise Exception('Unable to find Android build-tools')
+
def is_python3():
- return sys.version_info.major == 3
+ return sys.version_info.major == 3
+
def Print(s, quiet=False):
- if quiet:
- return
- print(s)
+ if quiet:
+ return
+ print(s)
+
def Warn(message):
- CRED = '\033[91m'
- CEND = '\033[0m'
- print(CRED + message + CEND)
+ CRED = '\033[91m'
+ CEND = '\033[0m'
+ print(CRED + message + CEND)
+
def PrintCmd(cmd, env=None, quiet=False, worker_id=None):
- if quiet:
- return
- if type(cmd) is list:
- cmd = ' '.join(cmd)
- if env:
- env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
- print_thread('Running: {} {}'.format(env, cmd), worker_id)
- else:
- print_thread('Running: {}'.format(cmd), worker_id)
- # I know this will hit os on windows eventually if we don't do this.
- sys.stdout.flush()
+ if quiet:
+ return
+ if type(cmd) is list:
+ cmd = ' '.join(cmd)
+ if env:
+ env = ' '.join(['{}=\"{}\"'.format(x, y) for x, y in env.iteritems()])
+ print_thread('Running: {} {}'.format(env, cmd), worker_id)
+ else:
+ print_thread('Running: {}'.format(cmd), worker_id)
+ # I know this will hit os on windows eventually if we don't do this.
+ sys.stdout.flush()
+
class ProgressLogger(object):
- CLEAR_LINE = '\033[K'
- UP = '\033[F'
+ CLEAR_LINE = '\033[K'
+ UP = '\033[F'
- def __init__(self, quiet=False):
- self._count = 0
- self._has_printed = False
- self._quiet = quiet
+ def __init__(self, quiet=False):
+ self._count = 0
+ self._has_printed = False
+ self._quiet = quiet
- def log(self, text):
- if len(text.strip()) == 0:
- return
- if self._quiet:
- if self._has_printed:
- sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
- if len(text) > 140:
- text = text[0:140] + '...'
- print(text)
- self._has_printed = True
+ def log(self, text):
+ if len(text.strip()) == 0:
+ return
+ if self._quiet:
+ if self._has_printed:
+ sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
+ if len(text) > 140:
+ text = text[0:140] + '...'
+ print(text)
+ self._has_printed = True
- def done(self):
- if self._quiet and self._has_printed:
- sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
- print('')
- sys.stdout.write(ProgressLogger.UP)
+ def done(self):
+ if self._quiet and self._has_printed:
+ sys.stdout.write(ProgressLogger.UP + ProgressLogger.CLEAR_LINE)
+ print('')
+ sys.stdout.write(ProgressLogger.UP)
+
def RunCmd(cmd, env_vars=None, quiet=False, fail=True, logging=True):
- PrintCmd(cmd, env=env_vars, quiet=quiet)
- env = os.environ.copy()
- if env_vars:
- env.update(env_vars)
- process = subprocess.Popen(
- cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout = []
- logger = ProgressLogger(quiet=quiet) if logging else None
- failed = False
- while True:
- line = process.stdout.readline().decode('utf-8')
- if line != '':
- stripped = line.rstrip()
- stdout.append(stripped)
- if logger:
- logger.log(stripped)
- # TODO(christofferqa): r8 should fail with non-zero exit code.
- if ('AssertionError:' in stripped
- or 'CompilationError:' in stripped
- or 'CompilationFailedException:' in stripped
- or 'Compilation failed' in stripped
- or 'FAILURE:' in stripped
- or 'org.gradle.api.ProjectConfigurationException' in stripped
- or 'BUILD FAILED' in stripped):
- failed = True
- else:
- if logger:
- logger.done()
- exit_code = process.poll()
- if exit_code or failed:
- for line in stdout:
- Warn(line)
- if fail:
- raise subprocess.CalledProcessError(
- exit_code or -1, cmd, output='\n'.join(stdout))
- return stdout
+ PrintCmd(cmd, env=env_vars, quiet=quiet)
+ env = os.environ.copy()
+ if env_vars:
+ env.update(env_vars)
+ process = subprocess.Popen(cmd,
+ env=env,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ stdout = []
+ logger = ProgressLogger(quiet=quiet) if logging else None
+ failed = False
+ while True:
+ line = process.stdout.readline().decode('utf-8')
+ if line != '':
+ stripped = line.rstrip()
+ stdout.append(stripped)
+ if logger:
+ logger.log(stripped)
+ # TODO(christofferqa): r8 should fail with non-zero exit code.
+ if ('AssertionError:' in stripped or
+ 'CompilationError:' in stripped or
+ 'CompilationFailedException:' in stripped or
+ 'Compilation failed' in stripped or
+ 'FAILURE:' in stripped or
+ 'org.gradle.api.ProjectConfigurationException' in stripped
+ or 'BUILD FAILED' in stripped):
+ failed = True
+ else:
+ if logger:
+ logger.done()
+ exit_code = process.poll()
+ if exit_code or failed:
+ for line in stdout:
+ Warn(line)
+ if fail:
+ raise subprocess.CalledProcessError(
+ exit_code or -1, cmd, output='\n'.join(stdout))
+ return stdout
+
def IsWindows():
- return defines.IsWindows()
+ return defines.IsWindows()
+
def EnsureDepFromGoogleCloudStorage(dep, tgz, sha1, msg):
- if (not os.path.exists(dep)
- or not os.path.exists(tgz)
- or os.path.getmtime(tgz) < os.path.getmtime(sha1)):
- DownloadFromGoogleCloudStorage(sha1)
- # Update the mtime of the tar file to make sure we do not run again unless
- # there is an update.
- os.utime(tgz, None)
- else:
- print('Ensure cloud dependency:', msg, 'present')
+ if (not os.path.exists(dep) or not os.path.exists(tgz) or
+ os.path.getmtime(tgz) < os.path.getmtime(sha1)):
+ DownloadFromGoogleCloudStorage(sha1)
+ # Update the mtime of the tar file to make sure we do not run again unless
+ # there is an update.
+ os.utime(tgz, None)
+ else:
+ print('Ensure cloud dependency:', msg, 'present')
+
def DownloadFromX20(sha1_file):
- download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
- cmd = [download_script, sha1_file]
- PrintCmd(cmd)
- subprocess.check_call(cmd)
-
-def DownloadFromGoogleCloudStorage(sha1_file, bucket='r8-deps', auth=False,
- quiet=False):
- suffix = '.bat' if IsWindows() else ''
- download_script = 'download_from_google_storage%s' % suffix
- cmd = [download_script]
- if not auth:
- cmd.append('-n')
- cmd.extend(['-b', bucket, '-u', '-s', sha1_file])
- if not quiet:
+ download_script = os.path.join(REPO_ROOT, 'tools', 'download_from_x20.py')
+ cmd = [download_script, sha1_file]
PrintCmd(cmd)
subprocess.check_call(cmd)
- else:
- subprocess.check_output(cmd)
+
+
+def DownloadFromGoogleCloudStorage(sha1_file,
+ bucket='r8-deps',
+ auth=False,
+ quiet=False):
+ suffix = '.bat' if IsWindows() else ''
+ download_script = 'download_from_google_storage%s' % suffix
+ cmd = [download_script]
+ if not auth:
+ cmd.append('-n')
+ cmd.extend(['-b', bucket, '-u', '-s', sha1_file])
+ if not quiet:
+ PrintCmd(cmd)
+ subprocess.check_call(cmd)
+ else:
+ subprocess.check_output(cmd)
+
def get_sha1(filename):
- sha1 = hashlib.sha1()
- with open(filename, 'rb') as f:
- while True:
- chunk = f.read(1024*1024)
- if not chunk:
- break
- sha1.update(chunk)
- return sha1.hexdigest()
+ sha1 = hashlib.sha1()
+ with open(filename, 'rb') as f:
+ while True:
+ chunk = f.read(1024 * 1024)
+ if not chunk:
+ break
+ sha1.update(chunk)
+ return sha1.hexdigest()
+
def get_HEAD_branch():
- result = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8')
- return result.strip()
+ result = subprocess.check_output(
+ ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8')
+ return result.strip()
+
def get_HEAD_sha1():
- return get_HEAD_sha1_for_checkout(REPO_ROOT)
+ return get_HEAD_sha1_for_checkout(REPO_ROOT)
+
def get_HEAD_diff_stat():
- return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8')
+ return subprocess.check_output(['git', 'diff', '--stat']).decode('utf-8')
+
def get_HEAD_sha1_for_checkout(checkout):
- cmd = ['git', 'rev-parse', 'HEAD']
- PrintCmd(cmd)
- with ChangedWorkingDirectory(checkout):
- return subprocess.check_output(cmd).decode('utf-8').strip()
+ cmd = ['git', 'rev-parse', 'HEAD']
+ PrintCmd(cmd)
+ with ChangedWorkingDirectory(checkout):
+ return subprocess.check_output(cmd).decode('utf-8').strip()
+
def makedirs_if_needed(path):
- try:
- os.makedirs(path)
- except OSError:
- if not os.path.isdir(path):
- raise
+ try:
+ os.makedirs(path)
+ except OSError:
+ if not os.path.isdir(path):
+ raise
+
def get_gsutil():
- return 'gsutil.py' if os.name != 'nt' else 'gsutil.py.bat'
+ return 'gsutil.py' if os.name != 'nt' else 'gsutil.py.bat'
+
def upload_file_to_cloud_storage(source, destination):
- cmd = [get_gsutil(), 'cp']
- cmd += [source, destination]
- PrintCmd(cmd)
- subprocess.check_call(cmd)
+ cmd = [get_gsutil(), 'cp']
+ cmd += [source, destination]
+ PrintCmd(cmd)
+ subprocess.check_call(cmd)
+
def delete_file_from_cloud_storage(destination):
- cmd = [get_gsutil(), 'rm', destination]
- PrintCmd(cmd)
- subprocess.check_call(cmd)
+ cmd = [get_gsutil(), 'rm', destination]
+ PrintCmd(cmd)
+ subprocess.check_call(cmd)
+
def ls_files_on_cloud_storage(destination):
- cmd = [get_gsutil(), 'ls', destination]
- PrintCmd(cmd)
- return subprocess.check_output(cmd).decode('utf-8')
+ cmd = [get_gsutil(), 'ls', destination]
+ PrintCmd(cmd)
+ return subprocess.check_output(cmd).decode('utf-8')
+
def cat_file_on_cloud_storage(destination, ignore_errors=False):
- cmd = [get_gsutil(), 'cat', destination]
- PrintCmd(cmd)
- try:
- return subprocess.check_output(cmd).decode('utf-8').strip()
- except subprocess.CalledProcessError as e:
- if ignore_errors:
- return ''
- else:
- raise e
+ cmd = [get_gsutil(), 'cat', destination]
+ PrintCmd(cmd)
+ try:
+ return subprocess.check_output(cmd).decode('utf-8').strip()
+ except subprocess.CalledProcessError as e:
+ if ignore_errors:
+ return ''
+ else:
+ raise e
+
def file_exists_on_cloud_storage(destination):
- cmd = [get_gsutil(), 'ls', destination]
- PrintCmd(cmd)
- return subprocess.call(cmd) == 0
+ cmd = [get_gsutil(), 'ls', destination]
+ PrintCmd(cmd)
+ return subprocess.call(cmd) == 0
+
def download_file_from_cloud_storage(source, destination, quiet=False):
- cmd = [get_gsutil(), 'cp', source, destination]
- PrintCmd(cmd, quiet=quiet)
- subprocess.check_call(cmd)
+ cmd = [get_gsutil(), 'cp', source, destination]
+ PrintCmd(cmd, quiet=quiet)
+ subprocess.check_call(cmd)
+
def create_archive(name, sources=None):
- if not sources:
- sources = [name]
- tarname = '%s.tar.gz' % name
- with tarfile.open(tarname, 'w:gz') as tar:
- for source in sources:
- tar.add(source)
- return tarname
+ if not sources:
+ sources = [name]
+ tarname = '%s.tar.gz' % name
+ with tarfile.open(tarname, 'w:gz') as tar:
+ for source in sources:
+ tar.add(source)
+ return tarname
+
def extract_dir(filename):
- return filename[0:len(filename) - len('.tar.gz')]
+ return filename[0:len(filename) - len('.tar.gz')]
+
def unpack_archive(filename):
- dest_dir = extract_dir(filename)
- if os.path.exists(dest_dir):
- print('Deleting existing dir %s' % dest_dir)
- shutil.rmtree(dest_dir)
- dirname = os.path.dirname(os.path.abspath(filename))
- with tarfile.open(filename, 'r:gz') as tar:
- tar.extractall(path=dirname)
+ dest_dir = extract_dir(filename)
+ if os.path.exists(dest_dir):
+ print('Deleting existing dir %s' % dest_dir)
+ shutil.rmtree(dest_dir)
+ dirname = os.path.dirname(os.path.abspath(filename))
+ with tarfile.open(filename, 'r:gz') as tar:
+ tar.extractall(path=dirname)
+
def check_gcert():
- status = subprocess.call(['gcertstatus'])
- if status != 0:
- subprocess.check_call(['gcert'])
+ status = subprocess.call(['gcertstatus'])
+ if status != 0:
+ subprocess.check_call(['gcert'])
+
# Note that gcs is eventually consistent with regards to list operations.
# This is not a problem in our case, but don't ever use this method
# for synchronization.
def cloud_storage_exists(destination):
- cmd = [get_gsutil(), 'ls', destination]
- PrintCmd(cmd)
- exit_code = subprocess.call(cmd)
- return exit_code == 0
+ cmd = [get_gsutil(), 'ls', destination]
+ PrintCmd(cmd)
+ exit_code = subprocess.call(cmd)
+ return exit_code == 0
+
class TempDir(object):
- def __init__(self, prefix='', delete=True):
- self._temp_dir = None
- self._prefix = prefix
- self._delete = delete
- def __enter__(self):
- self._temp_dir = tempfile.mkdtemp(self._prefix)
- return self._temp_dir
+ def __init__(self, prefix='', delete=True):
+ self._temp_dir = None
+ self._prefix = prefix
+ self._delete = delete
- def __exit__(self, *_):
- if self._delete:
- shutil.rmtree(self._temp_dir, ignore_errors=True)
+ def __enter__(self):
+ self._temp_dir = tempfile.mkdtemp(self._prefix)
+ return self._temp_dir
+
+ def __exit__(self, *_):
+ if self._delete:
+ shutil.rmtree(self._temp_dir, ignore_errors=True)
+
class ChangedWorkingDirectory(object):
- def __init__(self, working_directory, quiet=False):
- self._quiet = quiet
- self._working_directory = working_directory
- def __enter__(self):
- self._old_cwd = os.getcwd()
- if not self._quiet:
- print('Enter directory:', self._working_directory)
- os.chdir(self._working_directory)
+ def __init__(self, working_directory, quiet=False):
+ self._quiet = quiet
+ self._working_directory = working_directory
- def __exit__(self, *_):
- if not self._quiet:
- print('Enter directory:', self._old_cwd)
- os.chdir(self._old_cwd)
+ def __enter__(self):
+ self._old_cwd = os.getcwd()
+ if not self._quiet:
+ print('Enter directory:', self._working_directory)
+ os.chdir(self._working_directory)
+
+ def __exit__(self, *_):
+ if not self._quiet:
+ print('Enter directory:', self._old_cwd)
+ os.chdir(self._old_cwd)
+
# Reading Android CTS test_result.xml
+
class CtsModule(object):
- def __init__(self, module_name):
- self.name = module_name
+
+ def __init__(self, module_name):
+ self.name = module_name
+
class CtsTestCase(object):
- def __init__(self, test_case_name):
- self.name = test_case_name
+
+ def __init__(self, test_case_name):
+ self.name = test_case_name
+
class CtsTest(object):
- def __init__(self, test_name, outcome):
- self.name = test_name
- self.outcome = outcome
+
+ def __init__(self, test_name, outcome):
+ self.name = test_name
+ self.outcome = outcome
+
# Generator yielding CtsModule, CtsTestCase or CtsTest from
# reading through a CTS test_result.xml file.
def read_cts_test_result(file_xml):
- re_module = re.compile('<Module name="([^"]*)"')
- re_test_case = re.compile('<TestCase name="([^"]*)"')
- re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
- with open(file_xml) as f:
- for line in f:
- m = re_module.search(line)
- if m:
- yield CtsModule(m.groups()[0])
- continue
- m = re_test_case.search(line)
- if m:
- yield CtsTestCase(m.groups()[0])
- continue
- m = re_test.search(line)
- if m:
- outcome = m.groups()[0]
- assert outcome in ['fail', 'pass']
- yield CtsTest(m.groups()[1], outcome == 'pass')
+ re_module = re.compile('<Module name="([^"]*)"')
+ re_test_case = re.compile('<TestCase name="([^"]*)"')
+ re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
+ with open(file_xml) as f:
+ for line in f:
+ m = re_module.search(line)
+ if m:
+ yield CtsModule(m.groups()[0])
+ continue
+ m = re_test_case.search(line)
+ if m:
+ yield CtsTestCase(m.groups()[0])
+ continue
+ m = re_test.search(line)
+ if m:
+ outcome = m.groups()[0]
+ assert outcome in ['fail', 'pass']
+ yield CtsTest(m.groups()[1], outcome == 'pass')
+
def grep_memoryuse(logfile):
- re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
- result = None
- with open(logfile) as f:
- for line in f:
- m = re_vmhwm.search(line)
- if m:
- groups = m.groups()
- s = len(groups)
- if s >= 1:
- result = int(groups[0])
- if s >= 2:
- unit = groups[1]
- if unit == 'kB':
- result *= 1024
- elif unit != '':
- raise Exception('Unrecognized unit in memory usage log: {}'
- .format(unit))
- if result is None:
- raise Exception('No memory usage found in log: {}'.format(logfile))
- return result
+ re_vmhwm = re.compile('^VmHWM:[ \t]*([0-9]+)[ \t]*([a-zA-Z]*)')
+ result = None
+ with open(logfile) as f:
+ for line in f:
+ m = re_vmhwm.search(line)
+ if m:
+ groups = m.groups()
+ s = len(groups)
+ if s >= 1:
+ result = int(groups[0])
+ if s >= 2:
+ unit = groups[1]
+ if unit == 'kB':
+ result *= 1024
+ elif unit != '':
+ raise Exception(
+ 'Unrecognized unit in memory usage log: {}'.
+ format(unit))
+ if result is None:
+ raise Exception('No memory usage found in log: {}'.format(logfile))
+ return result
+
# Return a dictionary: {segment_name -> segments_size}
def getDexSegmentSizes(dex_files):
- assert len(dex_files) > 0
- cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments']
- cmd.extend(dex_files)
- PrintCmd(cmd)
- output = subprocess.check_output(cmd).decode('utf-8')
+ assert len(dex_files) > 0
+ cmd = [jdk.GetJavaExecutable(), '-jar', R8_JAR, 'dexsegments']
+ cmd.extend(dex_files)
+ PrintCmd(cmd)
+ output = subprocess.check_output(cmd).decode('utf-8')
- matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
+ matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
- if matches is None or len(matches) == 0:
- raise Exception('DexSegments failed to return any output for' \
- ' these files: {}'.format(dex_files))
+ if matches is None or len(matches) == 0:
+ raise Exception('DexSegments failed to return any output for' \
+ ' these files: {}'.format(dex_files))
- result = {}
+ result = {}
- for match in matches:
- result[match[0]] = int(match[1])
+ for match in matches:
+ result[match[0]] = int(match[1])
- return result
+ return result
+
# Return a dictionary: {segment_name -> segments_size}
def getCfSegmentSizes(cfFile):
- cmd = [jdk.GetJavaExecutable(),
- '-cp',
- CF_SEGMENTS_TOOL,
- 'com.android.tools.r8.cf_segments.MeasureLib',
- cfFile]
- PrintCmd(cmd)
- output = subprocess.check_output(cmd).decode('utf-8')
+ cmd = [
+ jdk.GetJavaExecutable(), '-cp', CF_SEGMENTS_TOOL,
+ 'com.android.tools.r8.cf_segments.MeasureLib', cfFile
+ ]
+ PrintCmd(cmd)
+ output = subprocess.check_output(cmd).decode('utf-8')
- matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
+ matches = DEX_SEGMENTS_RESULT_PATTERN.findall(output)
- if matches is None or len(matches) == 0:
- raise Exception('CfSegments failed to return any output for' \
- ' the file: ' + cfFile)
+ if matches is None or len(matches) == 0:
+ raise Exception('CfSegments failed to return any output for' \
+ ' the file: ' + cfFile)
- result = {}
+ result = {}
- for match in matches:
- result[match[0]] = int(match[1])
+ for match in matches:
+ result[match[0]] = int(match[1])
- return result
+ return result
+
def get_maven_path(artifact, version):
- return os.path.join('com', 'android', 'tools', artifact, version)
+ return os.path.join('com', 'android', 'tools', artifact, version)
+
def print_cfsegments(prefix, cf_files):
- for cf_file in cf_files:
- for segment_name, size in getCfSegmentSizes(cf_file).items():
- print('{}-{}(CodeSize): {}'
- .format(prefix, segment_name, size))
+ for cf_file in cf_files:
+ for segment_name, size in getCfSegmentSizes(cf_file).items():
+ print('{}-{}(CodeSize): {}'.format(prefix, segment_name, size))
+
def print_dexsegments(prefix, dex_files, worker_id=None):
- for segment_name, size in getDexSegmentSizes(dex_files).items():
- print_thread(
- '{}-{}(CodeSize): {}'.format(prefix, segment_name, size),
- worker_id)
+ for segment_name, size in getDexSegmentSizes(dex_files).items():
+ print_thread('{}-{}(CodeSize): {}'.format(prefix, segment_name, size),
+ worker_id)
+
# Ensure that we are not benchmarking with a google jvm.
def check_java_version():
- cmd= [jdk.GetJavaExecutable(), '-version']
- output = subprocess.check_output(cmd, stderr = subprocess.STDOUT).decode('utf-8')
- m = re.search('openjdk version "([^"]*)"', output)
- if m is None:
- raise Exception("Can't check java version: no version string in output"
- " of 'java -version': '{}'".format(output))
- version = m.groups(0)[0]
- m = re.search('google', version)
- if m is not None:
- raise Exception("Do not use google JVM for benchmarking: " + version)
+ cmd = [jdk.GetJavaExecutable(), '-version']
+ output = subprocess.check_output(cmd,
+ stderr=subprocess.STDOUT).decode('utf-8')
+ m = re.search('openjdk version "([^"]*)"', output)
+ if m is None:
+ raise Exception("Can't check java version: no version string in output"
+ " of 'java -version': '{}'".format(output))
+ version = m.groups(0)[0]
+ m = re.search('google', version)
+ if m is not None:
+ raise Exception("Do not use google JVM for benchmarking: " + version)
+
def get_android_jar_dir(api):
- return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api))
+ return os.path.join(REPO_ROOT, ANDROID_JAR_DIR.format(api=api))
+
def get_android_jar(api):
- return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
+ return os.path.join(REPO_ROOT, ANDROID_JAR.format(api=api))
+
def is_bot():
- return 'SWARMING_BOT_ID' in os.environ
+ return 'SWARMING_BOT_ID' in os.environ
+
def uncompressed_size(path):
- return sum(z.file_size for z in zipfile.ZipFile(path).infolist())
+ return sum(z.file_size for z in zipfile.ZipFile(path).infolist())
+
def desugar_configuration_name_and_version(configuration, is_for_maven):
- name = 'desugar_jdk_libs_configuration'
- with open(configuration, 'r') as f:
- configuration_json = json.loads(f.read())
- configuration_format_version = \
- configuration_json.get('configuration_format_version')
- if (not configuration_format_version):
- raise Exception(
- 'No "configuration_format_version" found in ' + configuration)
- if (configuration_format_version != 3
- and configuration_format_version != 5
- and configuration_format_version != (200 if is_for_maven else 100)):
- raise Exception(
- 'Unsupported "configuration_format_version" "%s" found in %s'
- % (configuration_format_version, configuration))
- version = configuration_json.get('version')
- if not version:
- if configuration_format_version == (200 if is_for_maven else 100):
- identifier = configuration_json.get('identifier')
- if not identifier:
- raise Exception(
- 'No "identifier" found in ' + configuration)
- identifier_split = identifier.split(':')
- if (len(identifier_split) != 3):
- raise Exception('Invalid "identifier" found in ' + configuration)
- if (identifier_split[0] != 'com.tools.android'):
- raise Exception('Invalid "identifier" found in ' + configuration)
- if not identifier_split[1].startswith('desugar_jdk_libs_configuration'):
- raise Exception('Invalid "identifier" found in ' + configuration)
- name = identifier_split[1]
- version = identifier_split[2]
- else:
- raise Exception(
- 'No "version" found in ' + configuration)
- else:
- if configuration_format_version == (200 if is_for_maven else 100):
- raise Exception(
- 'No "version" expected in ' + configuration)
- # Disallow prerelease, as older R8 versions cannot parse it causing hard to
- # understand errors.
- check_basic_semver_version(version, 'in ' + configuration, allowPrerelease = False)
- return (name, version)
+ name = 'desugar_jdk_libs_configuration'
+ with open(configuration, 'r') as f:
+ configuration_json = json.loads(f.read())
+ configuration_format_version = \
+ configuration_json.get('configuration_format_version')
+ if (not configuration_format_version):
+ raise Exception('No "configuration_format_version" found in ' +
+ configuration)
+ if (configuration_format_version != 3 and
+ configuration_format_version != 5 and
+ configuration_format_version != (200 if is_for_maven else 100)):
+ raise Exception(
+ 'Unsupported "configuration_format_version" "%s" found in %s' %
+ (configuration_format_version, configuration))
+ version = configuration_json.get('version')
+ if not version:
+ if configuration_format_version == (200 if is_for_maven else 100):
+ identifier = configuration_json.get('identifier')
+ if not identifier:
+ raise Exception('No "identifier" found in ' + configuration)
+ identifier_split = identifier.split(':')
+ if (len(identifier_split) != 3):
+ raise Exception('Invalid "identifier" found in ' +
+ configuration)
+ if (identifier_split[0] != 'com.tools.android'):
+ raise Exception('Invalid "identifier" found in ' +
+ configuration)
+ if not identifier_split[1].startswith(
+ 'desugar_jdk_libs_configuration'):
+ raise Exception('Invalid "identifier" found in ' +
+ configuration)
+ name = identifier_split[1]
+ version = identifier_split[2]
+ else:
+ raise Exception('No "version" found in ' + configuration)
+ else:
+ if configuration_format_version == (200 if is_for_maven else 100):
+ raise Exception('No "version" expected in ' + configuration)
+ # Disallow prerelease, as older R8 versions cannot parse it causing hard to
+ # understand errors.
+ check_basic_semver_version(version,
+ 'in ' + configuration,
+ allowPrerelease=False)
+ return (name, version)
+
class SemanticVersion:
- def __init__(self, major, minor, patch, prerelease):
- self.major = major
- self.minor = minor
- self.patch = patch
- self.prerelease = prerelease
- # Build metadata currently not suppported
- def larger_than(self, other):
- if self.prerelease or other.prerelease:
- raise Exception("Comparison with prerelease not implemented")
- if self.major > other.major:
- return True
- if self.major == other.major and self.minor > other.minor:
- return True
- if self.patch:
- return (self.major == other.major
- and self.minor == other.minor
- and self.patch > other.patch)
- else:
- return False
+ def __init__(self, major, minor, patch, prerelease):
+ self.major = major
+ self.minor = minor
+ self.patch = patch
+ self.prerelease = prerelease
+ # Build metadata currently not suppported
+
+ def larger_than(self, other):
+ if self.prerelease or other.prerelease:
+ raise Exception("Comparison with prerelease not implemented")
+ if self.major > other.major:
+ return True
+ if self.major == other.major and self.minor > other.minor:
+ return True
+ if self.patch:
+ return (self.major == other.major and self.minor == other.minor and
+ self.patch > other.patch)
+ else:
+ return False
# Check that the passed string is formatted as a basic semver version (x.y.z or x.y.z-prerelease
# depending on the value of allowPrerelease).
# See https://semver.org/. The regexp parts used are not all complient with what is suggested
# on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string.
-def check_basic_semver_version(version, error_context = '', components = 3, allowPrerelease = False):
+def check_basic_semver_version(version,
+ error_context='',
+ components=3,
+ allowPrerelease=False):
regexp = '^'
for x in range(components):
- regexp += '([0-9]+)'
- if x < components - 1:
- regexp += '\\.'
+ regexp += '([0-9]+)'
+ if x < components - 1:
+ regexp += '\\.'
if allowPrerelease:
- # This part is from
- # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
- regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?'
+ # This part is from
+ # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+ regexp += r'(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?'
regexp += '$'
reg = re.compile(regexp)
match = reg.match(version)
if not match:
- raise Exception("Invalid version '"
- + version
- + "'"
- + (' ' + error_context) if len(error_context) > 0 else '')
+ raise Exception("Invalid version '" + version + "'" +
+ (' ' + error_context) if len(error_context) > 0 else '')
if components == 2:
- return SemanticVersion(int(match.group(1)), int(match.group(2)), None, None)
+ return SemanticVersion(int(match.group(1)), int(match.group(2)), None,
+ None)
elif components == 3 and not allowPrerelease:
- return SemanticVersion(
- int(match.group(1)), int(match.group(2)), int(match.group(3)), None)
+ return SemanticVersion(int(match.group(1)), int(match.group(2)),
+ int(match.group(3)), None)
elif components == 3 and allowPrerelease:
- return SemanticVersion(
- int(match.group(1)), int(match.group(2)), int(match.group(3)), match.group('prerelease'))
+ return SemanticVersion(int(match.group(1)), int(match.group(2)),
+ int(match.group(3)), match.group('prerelease'))
else:
- raise Exception('Argument "components" must be 2 or 3')
+ raise Exception('Argument "components" must be 2 or 3')
diff --git a/tools/utils_aosp.py b/tools/utils_aosp.py
index 216e43d..e8b0ffb 100644
--- a/tools/utils_aosp.py
+++ b/tools/utils_aosp.py
@@ -3,7 +3,6 @@
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
-
from os.path import join
from subprocess import check_call
@@ -17,19 +16,22 @@
DEFAULT_ROOT = join(utils.REPO_ROOT, 'build', 'aosp')
+
def add_root_argument(parser):
- parser.add_argument('--aosp-root',
- help='Root of the AOSP checkout. ' +
- 'Defaults to ' + DEFAULT_ROOT +'.',
- default=DEFAULT_ROOT)
+ parser.add_argument('--aosp-root',
+ help='Root of the AOSP checkout. ' + 'Defaults to ' +
+ DEFAULT_ROOT + '.',
+ default=DEFAULT_ROOT)
+
def add_common_arguments(parser):
- add_root_argument(parser)
- parser.add_argument('--lunch',
- help='Build menu. ' +
- 'Defaults to ' + DEFAULT_LUNCH + '.',
- default=DEFAULT_LUNCH)
+ add_root_argument(parser)
+ parser.add_argument('--lunch',
+ help='Build menu. ' + 'Defaults to ' + DEFAULT_LUNCH +
+ '.',
+ default=DEFAULT_LUNCH)
+
def run_through_aosp_helper(lunch, args, cwd):
- args[0:0] = [AOSP_HELPER_SH, lunch]
- check_call(args, cwd = cwd)
+ args[0:0] = [AOSP_HELPER_SH, lunch]
+ check_call(args, cwd=cwd)
diff --git a/tools/youtube_data.py b/tools/youtube_data.py
index 3631691..37385c8 100644
--- a/tools/youtube_data.py
+++ b/tools/youtube_data.py
@@ -15,53 +15,65 @@
LATEST_VERSION = '17.19'
VERSIONS = {
- '17.19': {
- 'deploy' : {
- 'sanitize_libraries': False,
- 'inputs': ['%s_deploy.jar' % V17_19_PREFIX],
- 'libraries' : [
- os.path.join(
- V17_19_BASE,
- 'legacy_YouTubeRelease_combined_library_jars_filtered.jar')],
- 'pgconf': [
- '%s_proguard.config' % V17_19_PREFIX,
- '%s_proguard_extra.config' % V17_19_PREFIX,
- '%s/proguardsettings/YouTubeRelease_proguard.config' % utils.THIRD_PARTY,
- utils.IGNORE_WARNINGS_RULES],
- 'min-api' : ANDROID_M_API,
- 'system-properties': [
- # TODO(b/235169948): Reenable -checkenumunboxed.
- # '-Dcom.android.tools.r8.experimental.enablecheckenumunboxed=1',
- '-Dcom.android.tools.r8.experimental.enableconvertchecknotnull=1'],
- 'android_java8_libs': {
- 'config': '%s/desugar_jdk_libs/full_desugar_jdk_libs.json' % V17_19_BASE,
- # Intentionally not adding desugar_jdk_libs_configuration.jar since it
- # is part of jdk_libs_to_desugar.jar in YouTube 17.19.
- 'program': ['%s/desugar_jdk_libs/jdk_libs_to_desugar.jar' % V17_19_BASE],
- 'library': '%s/android_jar/lib-v33/android.jar' % utils.THIRD_PARTY,
- 'pgconf': [
- '%s/desugar_jdk_libs/base.pgcfg' % V17_19_BASE,
- '%s/desugar_jdk_libs/minify_desugar_jdk_libs.pgcfg' % V17_19_BASE
- ]
- }
+ '17.19': {
+ 'deploy': {
+ 'sanitize_libraries': False,
+ 'inputs': ['%s_deploy.jar' % V17_19_PREFIX],
+ 'libraries': [
+ os.path.join(
+ V17_19_BASE,
+ 'legacy_YouTubeRelease_combined_library_jars_filtered.jar')
+ ],
+ 'pgconf': [
+ '%s_proguard.config' % V17_19_PREFIX,
+ '%s_proguard_extra.config' % V17_19_PREFIX,
+ '%s/proguardsettings/YouTubeRelease_proguard.config' %
+ utils.THIRD_PARTY, utils.IGNORE_WARNINGS_RULES
+ ],
+ 'min-api': ANDROID_M_API,
+ 'system-properties': [
+ # TODO(b/235169948): Reenable -checkenumunboxed.
+ # '-Dcom.android.tools.r8.experimental.enablecheckenumunboxed=1',
+ '-Dcom.android.tools.r8.experimental.enableconvertchecknotnull=1'
+ ],
+ 'android_java8_libs': {
+ 'config':
+ '%s/desugar_jdk_libs/full_desugar_jdk_libs.json' %
+ V17_19_BASE,
+ # Intentionally not adding desugar_jdk_libs_configuration.jar since it
+ # is part of jdk_libs_to_desugar.jar in YouTube 17.19.
+ 'program': [
+ '%s/desugar_jdk_libs/jdk_libs_to_desugar.jar' % V17_19_BASE
+ ],
+ 'library':
+ '%s/android_jar/lib-v33/android.jar' % utils.THIRD_PARTY,
+ 'pgconf': [
+ '%s/desugar_jdk_libs/base.pgcfg' % V17_19_BASE,
+ '%s/desugar_jdk_libs/minify_desugar_jdk_libs.pgcfg' %
+ V17_19_BASE
+ ]
+ }
+ },
},
- },
}
+
def GetLatestVersion():
- return LATEST_VERSION
+ return LATEST_VERSION
+
def GetName():
- return 'youtube'
+ return 'youtube'
+
def GetMemoryData(version):
- assert version == '16.20'
- return {
- 'find-xmx-min': 3150,
- 'find-xmx-max': 3300,
- 'find-xmx-range': 64,
- 'oom-threshold': 3100,
- # TODO(b/143431825): Youtube can OOM randomly in memory configurations
- # that should work.
- 'skip-find-xmx-max': True,
- }
+ assert version == '16.20'
+ return {
+ 'find-xmx-min': 3150,
+ 'find-xmx-max': 3300,
+ 'find-xmx-range': 64,
+ 'oom-threshold': 3100,
+ # TODO(b/143431825): Youtube can OOM randomly in memory configurations
+ # that should work.
+ 'skip-find-xmx-max': True,
+ }
diff --git a/tools/zip_utils.py b/tools/zip_utils.py
index 219b443..21077be 100644
--- a/tools/zip_utils.py
+++ b/tools/zip_utils.py
@@ -10,27 +10,32 @@
import utils
+
def add_file_to_zip(file, destination, zip_file):
- with zipfile.ZipFile(zip_file, 'a') as zip:
- zip.write(file, destination)
+ with zipfile.ZipFile(zip_file, 'a') as zip:
+ zip.write(file, destination)
+
def extract_all_that_matches(zip_file, destination, predicate):
- with zipfile.ZipFile(zip_file) as zip:
- names_to_extract = [name for name in zip.namelist() if predicate(name)]
- zip.extractall(path=destination, members=names_to_extract)
- return names_to_extract
+ with zipfile.ZipFile(zip_file) as zip:
+ names_to_extract = [name for name in zip.namelist() if predicate(name)]
+ zip.extractall(path=destination, members=names_to_extract)
+ return names_to_extract
+
def extract_member(zip_file, member, destination):
- with zipfile.ZipFile(zip_file) as zip:
- with utils.TempDir() as temp:
- zip.extract(member, path=temp)
- shutil.move(os.path.join(temp, member), destination)
+ with zipfile.ZipFile(zip_file) as zip:
+ with utils.TempDir() as temp:
+ zip.extract(member, path=temp)
+ shutil.move(os.path.join(temp, member), destination)
+
def get_names_that_matches(zip_file, predicate):
- with zipfile.ZipFile(zip_file) as zip:
- return [name for name in zip.namelist() if predicate(name)]
+ with zipfile.ZipFile(zip_file) as zip:
+ return [name for name in zip.namelist() if predicate(name)]
+
def remove_files_from_zip(files, zip_file):
- assert os.path.exists(zip_file)
- cmd = ['zip', '-d', zip_file] + files
- subprocess.run(cmd)
+ assert os.path.exists(zip_file)
+ cmd = ['zip', '-d', zip_file] + files
+ subprocess.run(cmd)
