src/resourceshrinker/java/com/android/build/shrinker/ResourceShrinkerImpl.kt

/*
 * Copyright (C) 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.build.shrinker

import com.android.SdkConstants.DOT_9PNG
import com.android.SdkConstants.DOT_PNG
import com.android.SdkConstants.DOT_XML
import com.android.aapt.Resources
import com.android.build.shrinker.DummyContent.TINY_9PNG
import com.android.build.shrinker.DummyContent.TINY_9PNG_CRC
import com.android.build.shrinker.DummyContent.TINY_BINARY_XML
import com.android.build.shrinker.DummyContent.TINY_BINARY_XML_CRC
import com.android.build.shrinker.DummyContent.TINY_PNG
import com.android.build.shrinker.DummyContent.TINY_PNG_CRC
import com.android.build.shrinker.DummyContent.TINY_PROTO_XML
import com.android.build.shrinker.DummyContent.TINY_PROTO_XML_CRC
import com.android.build.shrinker.gatherer.ResourcesGatherer
import com.android.build.shrinker.graph.ResourcesGraphBuilder
import com.android.build.shrinker.obfuscation.ObfuscationMappingsRecorder
import com.android.build.shrinker.usages.ResourceUsageRecorder
import com.android.ide.common.resources.findUnusedResources
import com.android.ide.common.resources.usage.ResourceStore
import com.android.ide.common.resources.usage.ResourceUsageModel.Resource
import com.android.resources.FolderTypeRelationship
import com.android.resources.ResourceFolderType
import com.android.resources.ResourceType
import com.google.common.io.ByteStreams
import com.google.common.io.Files
import java.io.BufferedOutputStream
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.io.InputStream
import java.util.jar.JarEntry
import java.util.jar.JarOutputStream
import java.util.zip.CRC32
import java.util.zip.ZipEntry
import java.util.zip.ZipFile

/**
 * Unit that analyzes all resources (after resource merging, compilation and code shrinking has
 * been completed) and figures out which resources are unused, and replaces them with dummy content
 * inside zip archive file.
 *
 * Resource shrinker implementation that allows to customize:
 * <ul>
 *     <li>application resource gatherer (from R files, resource tables, etc);
 *     <li>recorder for mappings from obfuscated class/methods to original class/methods;
 *     <li>sources from which resource usages are recorded (Dex files, compiled JVM classes,
 *         AndroidManifests, etc);
 *     <li>resources graph builder that connects resources dependent on each other (analyzing
 *         raw resources content in XML, HTML, CSS, JS, analyzing resource content in proto
 *         compiled format);
 * </ul>
 */
class ResourceShrinkerImpl(
    private val resourcesGatherers: List<ResourcesGatherer>,
    private val obfuscationMappingsRecorder: ObfuscationMappingsRecorder?,
    private val usageRecorders: List<ResourceUsageRecorder>,
    private val graphBuilders: List<ResourcesGraphBuilder>,
    private val debugReporter: ShrinkerDebugReporter,
    val supportMultipackages: Boolean,
    private val usePreciseShrinking: Boolean
) : ResourceShrinker {
    val model = ResourceShrinkerModel(debugReporter, supportMultipackages)
    private lateinit var unused: List<Resource>

    override fun analyze() {
        resourcesGatherers.forEach { it.gatherResourceValues(model) }
        obfuscationMappingsRecorder?.recordObfuscationMappings(model)
        usageRecorders.forEach { it.recordUsages(model) }
        graphBuilders.forEach { it.buildGraph(model) }

        model.resourceStore.processToolsAttributes()
        model.keepPossiblyReferencedResources()

        debugReporter.debug { model.resourceStore.dumpResourceModel() }

        unused = findUnusedResources(model.resourceStore.resources) { roots ->
            debugReporter.debug { "The root reachable resources are:" }
            debugReporter.debug { roots.joinToString("\n", transform = { " $it" }) }
        }
        debugReporter.debug { "Unused resources are: " }
        debugReporter.debug { unused.joinToString("\n", transform = { " $it" })}

    }

    override fun close() {
        debugReporter.close()
    }

    override fun getUnusedResourceCount(): Int {
        return unused.size
    }

    override fun rewriteResourcesInApkFormat(
        source: File,
        dest: File,
        format: LinkedResourcesFormat
    ) {
        rewriteResourceZip(source, dest, ApkArchiveFormat(model.resourceStore, format))
    }

    override fun rewriteResourcesInBundleFormat(
        source: File,
        dest: File,
        moduleNameToPackageNameMap: Map<String, String>
    ) {
        rewriteResourceZip(
            source,
            dest,
            BundleArchiveFormat(model.resourceStore, moduleNameToPackageNameMap)
        )
    }

    private fun rewriteResourceZip(source: File, dest: File, format: ArchiveFormat) {
        if (dest.exists() && !dest.delete()) {
            throw IOException("Could not delete $dest")
        }
        JarOutputStream(BufferedOutputStream(FileOutputStream(dest))).use { zos ->
            ZipFile(source).use { zip ->
                // Rather than using Deflater.DEFAULT_COMPRESSION we use 9 here,  since that seems
                // to match the compressed sizes we observe in source .ap_ files encountered by the
                // resource shrinker:
                zos.setLevel(9)
                zip.entries().asSequence().forEach {
                    if (format.fileIsNotReachable(it)) {
                        // If we don't use precise shrinking we don't remove the files, see:
                        // https://b.corp.google.com/issues/37010152
                        if (!usePreciseShrinking) {
                            replaceWithDummyEntry(zos, it, format.resourcesFormat)
                        }
                    } else if (it.name.endsWith("resources.pb") && usePreciseShrinking) {
                            removeResourceUnusedTableEntries(zip.getInputStream(it), zos, it)
                    } else {
                        copyToOutput(zip.getInputStream(it), zos, it)
                    }
                }
            }
        }
        // If net negative, copy original back. This is unusual, but can happen
        // in some circumstances, such as the one described in
        // https://plus.google.com/+SaidTahsinDane/posts/X9sTSwoVUhB
        // "Removed unused resources: Binary resource data reduced from 588KB to 595KB: Removed -1%"
        // Guard against that, and worst case, just use the original.
        val before = source.length()
        val after = dest.length()
        if (after > before) {
            debugReporter.info {
                "Resource shrinking did not work (grew from $before to $after); using original " +
                "instead"
            }
            Files.copy(source, dest)
        }
    }

    private fun removeResourceUnusedTableEntries(zis: InputStream,
                                                 zos: JarOutputStream,
                                                 srcEntry: ZipEntry) {
        val resourceIdsToRemove = unused
            .filterNot { it.type == ResourceType.ID }
            .map { resource -> resource.value }
        val shrunkenResourceTable = Resources.ResourceTable.parseFrom(zis)
                .nullOutEntriesWithIds(resourceIdsToRemove)
        val bytes = shrunkenResourceTable.toByteArray()
        val outEntry = JarEntry(srcEntry.name)
        if (srcEntry.time != -1L) {
            outEntry.time = srcEntry.time
        }
        if (srcEntry.method == JarEntry.STORED) {
            outEntry.method = JarEntry.STORED
            outEntry.size = bytes.size.toLong()
            val crc = CRC32()
            crc.update(bytes, 0, bytes.size)
            outEntry.crc = crc.getValue()
        }
        zos.putNextEntry(outEntry)
        zos.write(bytes)
        zos.closeEntry()
    }

    /** Replaces the given entry with a minimal valid file of that type.  */
    private fun replaceWithDummyEntry(
        zos: JarOutputStream,
        entry: ZipEntry,
        format: LinkedResourcesFormat
    ) {
        // Create a new entry so that the compressed len is recomputed.
        val name = entry.name
        val (bytes, crc) = when {
            // DOT_9PNG (.9.png) must be always before DOT_PNG (.png)
            name.endsWith(DOT_9PNG) -> TINY_9PNG to TINY_9PNG_CRC
            name.endsWith(DOT_PNG) -> TINY_PNG to TINY_PNG_CRC
            name.endsWith(DOT_XML) && format == LinkedResourcesFormat.BINARY ->
                TINY_BINARY_XML to TINY_BINARY_XML_CRC
            name.endsWith(DOT_XML) && format == LinkedResourcesFormat.PROTO ->
                TINY_PROTO_XML to TINY_PROTO_XML_CRC
            else -> ByteArray(0) to 0L
        }

        val outEntry = JarEntry(name)
        if (entry.time != -1L) {
            outEntry.time = entry.time
        }
        if (entry.method == JarEntry.STORED) {
            outEntry.method = JarEntry.STORED
            outEntry.size = bytes.size.toLong()
            outEntry.crc = crc
        }
        zos.putNextEntry(outEntry)
        zos.write(bytes)
        zos.closeEntry()
        debugReporter.info {
            "Skipped unused resource $name: ${entry.size} bytes (replaced with small dummy file " +
            "of size ${bytes.size} bytes)"
        }
    }

    private fun copyToOutput(zis: InputStream, zos: JarOutputStream, entry: ZipEntry) {
        // We can't just compress all files; files that are not compressed in the source .ap_ file
        // must be left uncompressed here, since for example RAW files need to remain uncompressed
        // in the APK such that they can be mmap'ed at runtime.
        // Preserve the STORED method of the input entry.
        val outEntry = when (entry.method) {
            JarEntry.STORED -> JarEntry(entry)
            else -> JarEntry(entry.name)
        }
        if (entry.time != -1L) {
            outEntry.time = entry.time
        }
        zos.putNextEntry(outEntry)
        if (!entry.isDirectory) {
            zis.transferTo(zos);
        }
        zos.closeEntry()
    }
}

private interface ArchiveFormat {
    val resourcesFormat: LinkedResourcesFormat
    fun fileIsNotReachable(entry: ZipEntry): Boolean
}

private class ApkArchiveFormat(
    private val store: ResourceStore,
    override val resourcesFormat: LinkedResourcesFormat
) : ArchiveFormat {

    override fun fileIsNotReachable(entry: ZipEntry): Boolean {
        if (entry.isDirectory || !entry.name.startsWith("res/")) {
            return false
        }
        val (_, folder, name) = entry.name.split('/', limit = 3)
        return !store.isJarPathReachable(folder, name)
    }
}

private class BundleArchiveFormat(
    private val store: ResourceStore,
    private val moduleNameToPackageName: Map<String, String>
) : ArchiveFormat {

    override val resourcesFormat = LinkedResourcesFormat.PROTO

    override fun fileIsNotReachable(entry: ZipEntry): Boolean {
        val module = entry.name.substringBefore('/')
        val packageName = moduleNameToPackageName[module]
        if (entry.isDirectory || packageName == null || !entry.name.startsWith("$module/res/")) {
            return false
        }
        val (_, _, folder, name) = entry.name.split('/', limit = 4)
        return !store.isJarPathReachable(folder, name)
    }
}

private fun ResourceStore.isJarPathReachable(
    folder: String,
    name: String
): Boolean {
    val folderType = ResourceFolderType.getFolderType(folder) ?: return true
    val resourceName = name.substringBefore('.')
    // Bundle format has a restriction: in case the same resource is duplicated in multiple modules
    // its content should be the same in all of them. This restriction means that we can't replace
    // resource with dummy content if its duplicate is used in some module.
    return FolderTypeRelationship.getRelatedResourceTypes(folderType)
        .filterNot { it == ResourceType.ID }
        .flatMap { getResources(it, resourceName) }
        .any { it.isReachable }
}

fun ResourceStore.isJarPathReachable(path: String) : Boolean {
    val (_, folder, name) = path.split('/', limit = 3)
    return isJarPathReachable(folder, name);
}

fun ResourceStore.getResourcesFor(path: String): List<Resource> {
    val (_, folder, name) = path.split('/', limit = 3)
    val folderType = ResourceFolderType.getFolderType(folder) ?: return emptyList()
    val resourceName = name.substringBefore('.')
    return FolderTypeRelationship.getRelatedResourceTypes(folderType)
        .filterNot { it == ResourceType.ID }
        .flatMap { getResources(it, resourceName) }
        .toList()
}