Add spdx-gradle-plugin to generate the R8 SBOM
This is using the plugins extension point to provide the download location
from the origin.json which is present in the downloaded dependencies in
third_party/dependencies and third_party/dependencies_new.
The spdx-gradle-plugin currently needs some patches to
work with the R8 project. These changes are in a GitHub fork intended
for upstream merging.
For now the plugin is built from that fork, which is given version
0.2.0-r8-patch01.
To support depending on this locally built plugin the
create_local_maven_with_dependencies.py script has been updated to also
be able to pull from a local repository until the official plugin version
can be used.
An additional third_party/dependencies_plugin has been added for Gradle
plugin dependencies.
The steps to build the plugin:
Pull the spdx-gradle-plugin fork https://github.com/spdx/spdx-gradle-plugin
and build it locally from 567d00cec5f3244ba657189ca1a4d08a823ce1b3 (branch
r8-patch01).
Build:
./gradlew -Dmaven.repo.local=/tmp/spdx-gradle-plugin publishToMavenLocal
Update third_party/dependencies_plugin:
tools/create_local_maven_with_dependencies.py \
--no-upload \
--studio <studio-path> \
--plugin-deps \
--include-maven-local /tmp/spdx-gradle-plugin
Bug: b/280466318
Change-Id: I980ab44afded4704ed43565cb5f19abd1314b16ediff --git a/.gitignore b/.gitignore
index 2fcc852..062fe33 100644
--- a/.gitignore
+++ b/.gitignore
@@ -93,6 +93,8 @@
third_party/dependencies.tar.gz
third_party/dependencies_new/
third_party/dependencies_new.tar.gz
+third_party/dependencies_plugin/
+third_party/dependencies_plugin.tar.gz
third_party/desugar/desugar_*.tar.gz
third_party/desugar/desugar_*/
third_party/examples
diff --git a/d8_r8/main/build.gradle.kts b/d8_r8/main/build.gradle.kts
index 7ce6cc7..47b5da0 100644
--- a/d8_r8/main/build.gradle.kts
+++ b/d8_r8/main/build.gradle.kts
@@ -2,14 +2,25 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import java.net.URI
+import java.nio.file.Path
+import java.nio.file.Paths
+import kotlin.io.path.exists
+import java.nio.file.Files.readString
import net.ltgt.gradle.errorprone.errorprone
+import org.gradle.api.artifacts.ModuleVersionIdentifier
import org.gradle.api.artifacts.component.ModuleComponentIdentifier
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.spdx.sbom.gradle.SpdxSbomTask
+import org.spdx.sbom.gradle.extensions.DefaultSpdxSbomTaskExtension
+
+import com.google.gson.Gson
plugins {
`kotlin-dsl`
id("dependencies-plugin")
id("net.ltgt.errorprone") version "3.0.1"
+ id("org.spdx.sbom") version "0.2.0-r8-patch01"
}
java {
@@ -35,6 +46,33 @@
errorprone(Deps.errorprone)
}
+if (project.hasProperty("spdxVersion")) {
+ project.version = project.property("spdxVersion")
+}
+
+spdxSbom {
+ targets {
+ create("r8") {
+ // Use of both compileClasspath and runtimeClasspath due to how the
+ // dependencies jar is built and dependencies above therefore use
+ // compileOnly for actual runtime dependencies.
+ configurations.set(listOf("compileClasspath", "runtimeClasspath"))
+ scm {
+ uri.set("https://r8.googlesource.com/r8/")
+ if (project.hasProperty("spdxRevision")) {
+ revision.set(project.property("spdxRevision").toString())
+ }
+ }
+ document {
+ name.set("R8 Compiler Suite")
+ namespace.set("https://r8.googlesource.com/r8/-" + project.version + ".jar")
+ creator.set("Organization: Google LLC")
+ packageSupplier.set("Organization: Google LLC")
+ }
+ }
+ }
+}
+
val keepAnnoJarTask = projectTask("keepanno", "jar")
val resourceShrinkerJarTask = projectTask("resourceshrinker", "jar")
val resourceShrinkerDepsTask = projectTask("resourceshrinker", "depsJar")
@@ -61,6 +99,50 @@
dependsOn(gradle.includedBuild("shared").task(":downloadDeps"))
}
+ withType<SpdxSbomTask> {
+ taskExtension.set(object : DefaultSpdxSbomTaskExtension() {
+ override fun mapRepoUri(input: URI?, moduleId: ModuleVersionIdentifier): URI? {
+
+ // Locate the file origin.json with URL for download location.
+ fun getOriginJson() : java.nio.file.Path {
+ var repositoryDir =
+ moduleId.group.replace('.', '/') + "/" + moduleId.name + "/" + moduleId.version
+ val path : Path =
+ Paths.get("third_party", "dependencies", repositoryDir, "origin.json");
+ val path_new : Path =
+ Paths.get("third_party", "dependencies_new", repositoryDir, "origin.json");
+ return if (path.exists()) path else path_new
+ }
+
+ // Simple data model of the content of origin.json generated by the tool to download
+ // and create a local repository. E.g.:
+ /*
+ {
+ "artifacts": [
+ {
+ "file": "org/ow2/asm/asm/9.5/asm-9.5.pom",
+ "repo": "https://repo1.maven.org/maven2/",
+ "artifact": "org.ow2.asm:asm:pom:9.5"
+ },
+ {
+ "file": "org/ow2/asm/asm/9.5/asm-9.5.jar",
+ "repo": "https://repo1.maven.org/maven2/",
+ "artifact": "org.ow2.asm:asm:jar:9.5"
+ }
+ ]
+ }
+ */
+ data class Artifact(val file: String, val repo: String, val artifact: String)
+ data class Artifacts(val artifacts: List<Artifact>)
+
+ // Read origin.json.
+ val json = readString(getOriginJson());
+ val artifacts = Gson().fromJson(json, Artifacts::class.java);
+ return URI.create(artifacts.artifacts.get(0).repo)
+ }
+ })
+ }
+
val consolidatedLicense by registering {
dependsOn(gradle.includedBuild("shared").task(":downloadDeps"))
val root = getRoot()
diff --git a/d8_r8/main/settings.gradle.kts b/d8_r8/main/settings.gradle.kts
index 4e1d975..d4f7ab8 100644
--- a/d8_r8/main/settings.gradle.kts
+++ b/d8_r8/main/settings.gradle.kts
@@ -5,7 +5,7 @@
pluginManagement {
repositories {
maven {
- url = uri("file:../../third_party/dependencies")
+ url = uri("file:../../third_party/dependencies_plugin")
}
maven {
url = uri("file:../../third_party/dependencies_new")
diff --git a/d8_r8/settings.gradle.kts b/d8_r8/settings.gradle.kts
index ac9216a..ad7577f 100644
--- a/d8_r8/settings.gradle.kts
+++ b/d8_r8/settings.gradle.kts
@@ -51,7 +51,7 @@
process.waitFor()
if (process.exitValue() != 0) {
throw GradleException(
- "Bootstrapping dependencies_new download failed:\n"
+ "Bootstrapping ${outputDir} download failed:\n"
+ "${String(process.getErrorStream().readAllBytes(),
java.nio.charset.StandardCharsets.UTF_8)}\n"
+ String(process.getInputStream().readAllBytes(),
@@ -68,6 +68,7 @@
val thirdParty = getRepoRoot().resolve("third_party")
downloadFromGoogleStorage(thirdParty.resolve("dependencies"))
downloadFromGoogleStorage(thirdParty.resolve("dependencies_new"))
+downloadFromGoogleStorage(thirdParty.resolve("dependencies_plugin"))
pluginManagement {
repositories {
diff --git a/third_party/dependencies_plugin.tar.gz.sha1 b/third_party/dependencies_plugin.tar.gz.sha1
new file mode 100644
index 0000000..08fb793
--- /dev/null
+++ b/third_party/dependencies_plugin.tar.gz.sha1
@@ -0,0 +1 @@
+77f5f4042c4340df908bb39cdb40a67009195cac
\ No newline at end of file
diff --git a/tools/archive.py b/tools/archive.py
index 84c0da3..1632231 100755
--- a/tools/archive.py
+++ b/tools/archive.py
@@ -222,6 +222,12 @@
default_pom_file = os.path.join(temp, 'r8.pom')
create_maven_release.write_default_r8_pom_file(default_pom_file,
version)
+ gradle.RunGradle([
+ ':main:spdxSbom',
+ '-PspdxVersion=' + version,
+ '-PspdxRevision=' + GetGitHash()
+ ])
+
for_archiving = [
utils.R8_JAR, utils.R8LIB_JAR, utils.R8LIB_JAR + '.map',
utils.R8LIB_JAR + '_map.zip', utils.R8_FULL_EXCLUDE_DEPS_JAR,
@@ -237,7 +243,9 @@
utils.R8RETRACE_JAR + '_map.zip', utils.R8RETRACE_EXCLUDE_DEPS_JAR,
utils.R8RETRACE_EXCLUDE_DEPS_JAR + '.map',
utils.R8RETRACE_EXCLUDE_DEPS_JAR + '_map.zip',
- utils.KEEPANNO_ANNOTATIONS_JAR, utils.GENERATED_LICENSE
+ utils.KEEPANNO_ANNOTATIONS_JAR,
+ utils.GENERATED_LICENSE,
+ 'd8_r8/main/build/spdx/r8.spdx.json'
]
for file in for_archiving:
file_name = os.path.basename(file)
diff --git a/tools/create_local_maven_with_dependencies.py b/tools/create_local_maven_with_dependencies.py
index 4ae9d25..e5d8885 100755
--- a/tools/create_local_maven_with_dependencies.py
+++ b/tools/create_local_maven_with_dependencies.py
@@ -11,13 +11,13 @@
import utils
-# The local_maven_repository_generator orderes the repositories by name, so
+# The local_maven_repository_generator orders the repositories by name, so
# prefix with X- to control the order, as many dependencies are present
-# in several repositories.
+# in several repositories. Save A- for additional local repository.
REPOSITORIES = [
- 'A-Google=https://maven.google.com/',
- 'B-Maven Central=https://repo1.maven.org/maven2/',
- "C-Gradle Plugins=https://plugins.gradle.org/m2/",
+ 'B-Google=https://maven.google.com/',
+ 'C-Maven Central=https://repo1.maven.org/maven2/',
+ "D-Gradle Plugins=https://plugins.gradle.org/m2/",
]
ANDRDID_SUPPORT_VERSION = '25.4.0'
@@ -97,6 +97,11 @@
version=PROTOBUF_VERSION),
]
+PLUGIN_DEPENDENCIES = [
+ 'org.spdx.sbom:org.spdx.sbom.gradle.plugin:0.2.0-r8-patch01',
+ # See https://github.com/FasterXML/jackson-core/issues/999.
+ 'ch.randelshofer:fastdoubleparser:0.8.0',
+]
def dependencies_tar(dependencies_path):
return os.path.join(os.path.dirname(dependencies_path),
@@ -143,32 +148,89 @@
required=True,
help='Path to a studio-main checkout (to get the tool '
'//tools/base/bazel:local_maven_repository_generator_cli)')
+ result.add_argument(
+ '--plugin-deps',
+ '--plugin_deps',
+ default=False,
+ action='store_true',
+ help='Build repository Gradle plugin dependncies')
+ result.add_argument(
+ '--include-maven-local',
+ '--include_maven_local',
+ metavar=('<path>'),
+ default=False,
+ help='Path to maven local repository to include as dependency source')
+ result.add_argument(
+ '--no-upload',
+ '--no_upload',
+ default=False,
+ action='store_true',
+ help="Don't upload to Google CLoud Storage")
return result.parse_args()
+def set_utime(path):
+ os.utime(path, (0, 0))
+ for root, dirs, files in os.walk(path):
+ for f in files:
+ os.utime(os.path.join(root, f), (0, 0))
+ for d in dirs:
+ os.utime(os.path.join(root, d), (0, 0))
+
def main():
args = parse_options()
- dependencies_path = os.path.join(utils.THIRD_PARTY, 'dependencies')
- print("Downloading to " + dependencies_path)
- remove_local_maven_repository(dependencies_path)
- create_local_maven_repository(args, dependencies_path, REPOSITORIES,
- BUILD_DEPENDENCIES + TEST_DEPENDENCIES)
+ repositories = REPOSITORIES
+ if args.include_maven_local:
+ # Add the local repository as the first for it to take precedence.
+ # Use A- prefix as current local_maven_repository_generator orderes by name.
+ repositories = ['A-Local=file://%s' % args.include_maven_local] + REPOSITORIES
- dependencies_new_path = os.path.join(utils.THIRD_PARTY, 'dependencies_new')
- print("Downloading to " + dependencies_new_path)
- remove_local_maven_repository(dependencies_new_path)
- create_local_maven_repository(args, dependencies_new_path, REPOSITORIES,
- NEW_DEPENDENCIES)
+ dependencies = []
- print("Uploading to Google Cloud Storage:")
- with utils.ChangedWorkingDirectory(utils.THIRD_PARTY):
- for dependency in ['dependencies', 'dependencies_new']:
- cmd = [
- 'upload_to_google_storage.py', '-a', '--bucket', 'r8-deps',
- dependency
- ]
- subprocess.check_call(cmd)
+ if args.plugin_deps:
+ dependencies_plugin_path = os.path.join(utils.THIRD_PARTY, 'dependencies_plugin')
+ remove_local_maven_repository(dependencies_plugin_path)
+ print("Downloading to " + dependencies_plugin_path)
+ create_local_maven_repository(
+ args, dependencies_plugin_path, repositories, PLUGIN_DEPENDENCIES)
+ set_utime(dependencies_plugin_path)
+ dependencies.append('dependencies_plugin')
+ else:
+ dependencies_path = os.path.join(utils.THIRD_PARTY, 'dependencies')
+ remove_local_maven_repository(dependencies_path)
+ print("Downloading to " + dependencies_path)
+ create_local_maven_repository(
+ args, dependencies_path, repositories, BUILD_DEPENDENCIES + TEST_DEPENDENCIES)
+ set_utime(dependencies_path)
+ dependencies.append('dependencies')
+ dependencies_new_path = os.path.join(utils.THIRD_PARTY, 'dependencies_new')
+ remove_local_maven_repository(dependencies_new_path)
+ print("Downloading to " + dependencies_new_path)
+ create_local_maven_repository(
+ args, dependencies_new_path, repositories, NEW_DEPENDENCIES)
+ set_utime(dependencies_new_path)
+ dependencies.append('dependencies_new')
+
+ upload_cmds = []
+ for dependency in dependencies:
+ upload_cmds.append([
+ 'upload_to_google_storage.py',
+ '-a',
+ '--bucket',
+ 'r8-deps',
+ dependency])
+
+ if not args.no_upload:
+ print("Uploading to Google Cloud Storage:")
+ with utils.ChangedWorkingDirectory(utils.THIRD_PARTY):
+ for cmd in upload_cmds:
+ subprocess.check_call(cmd)
+ else:
+ print("Not uploading to Google Cloud Storage. "
+ + "Run the following commands in %s to do so manually" % utils.THIRD_PARTY)
+ for cmd in upload_cmds:
+ print(" ".join(cmd))
if __name__ == '__main__':
