tools/run_on_app.py

#!/usr/bin/env python3
# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

from __future__ import print_function
from glob import glob
import argparse
import copy
import os
import shutil
import sys
import time

import archive
import gradle
import nest_data
from sanitize_libraries import SanitizeLibraries, SanitizeLibrariesInPgconf
import thread_utils
from thread_utils import print_thread
import toolhelper
import update_prebuilds_in_android
import utils
import youtube_data
import chrome_data

TYPES = ['dex', 'deploy', 'proguarded']
APPS = ['nest', 'youtube', 'chrome']
COMPILERS = ['d8', 'r8']
COMPILER_BUILDS = ['full', 'lib']

# We use this magic exit code to signal that the program OOM'ed
OOM_EXIT_CODE = 42
# According to Popen.returncode doc:
# A negative value -N indicates that the child was terminated by signal N.
TIMEOUT_KILL_CODE = -9

# Log file names
FIND_MIN_XMX_FILE = 'find_min_xmx_results'
FIND_MIN_XMX_DIR = 'find_min_xmx'


def ParseOptions(argv):
    result = argparse.ArgumentParser()
    result.add_argument('--compiler',
                        help='The compiler to use',
                        choices=COMPILERS)
    result.add_argument('--compiler-build',
                        help='Compiler build to use',
                        choices=COMPILER_BUILDS,
                        default='lib')
    result.add_argument('--no-fail-fast',
                        help='Whether run_on_app.py should report all failures '
                        'and not just the first one',
                        default=False,
                        action='store_true')
    result.add_argument('--hash', help='The version of D8/R8 to use')
    result.add_argument('--app', help='What app to run on', choices=APPS)
    result.add_argument('--run-all',
                        help='Compile all possible combinations',
                        default=False,
                        action='store_true')
    result.add_argument('--expect-oom',
                        help='Expect that compilation will fail with an OOM',
                        default=False,
                        action='store_true')
    result.add_argument('--type',
                        help='Default for R8: deploy, for D8: proguarded',
                        choices=TYPES)
    result.add_argument('--out',
                        help='Where to place the output',
                        default=utils.BUILD)
    result.add_argument('--no-build',
                        help='Run without building first',
                        default=False,
                        action='store_true')
    result.add_argument('--max-memory',
                        help='The maximum memory in MB to run with',
                        type=int)
    result.add_argument('--find-min-xmx',
                        help='Find the minimum amount of memory we can run in',
                        default=False,
                        action='store_true')
    result.add_argument('--find-min-xmx-min-memory',
                        help='Setting the minimum memory baseline to run in',
                        type=int)
    result.add_argument('--find-min-xmx-max-memory',
                        help='Setting the maximum memory baseline to run in',
                        type=int)
    result.add_argument('--find-min-xmx-range-size',
                        help='Setting the size of the acceptable memory range',
                        type=int,
                        default=32)
    result.add_argument('--find-min-xmx-archive',
                        help='Archive find-min-xmx results on GCS',
                        default=False,
                        action='store_true')
    result.add_argument('--no-extra-pgconf',
                        '--no_extra_pgconf',
                        help='Build without the following extra rules: ' +
                        '-printconfiguration, -printmapping, -printseeds, ' +
                        '-printusage',
                        default=False,
                        action='store_true')
    result.add_argument('--timeout',
                        type=int,
                        default=0,
                        help='Set timeout instead of waiting for OOM.')
    result.add_argument('--ignore-java-version',
                        help='Do not check java version',
                        default=False,
                        action='store_true')
    result.add_argument(
        '--no-libraries',
        help='Do not pass in libraries, even if they exist in conf',
        default=False,
        action='store_true')
    result.add_argument(
        '--disable-assertions',
        '--disable_assertions',
        '-da',
        help='Disable Java assertions when running the compiler '
        '(default enabled)',
        default=False,
        action='store_true')
    result.add_argument('--debug-agent',
                        help='Run with debug agent.',
                        default=False,
                        action='store_true')
    result.add_argument('--version', help='The version of the app to run')
    result.add_argument('-k', help='Override the default ProGuard keep rules')
    result.add_argument('--compiler-flags',
                        help='Additional option(s) for the compiler. ' +
                        'If passing several options use a quoted string.')
    result.add_argument('--r8-flags',
                        help='Additional option(s) for the compiler. ' +
                        'Same as --compiler-flags, keeping it for backward'
                        ' compatibility. ' +
                        'If passing several options use a quoted string.')
    result.add_argument('--track-memory-to-file',
                        help='Track how much memory the jvm is using while ' +
                        ' compiling. Output to the specified file.')
    result.add_argument('--profile',
                        help='Profile R8 run.',
                        default=False,
                        action='store_true')
    result.add_argument(
        '--dump-args-file',
        help='Dump a file with the arguments for the specified ' +
        'configuration. For use as a @<file> argument to perform ' + 'the run.')
    result.add_argument('--print-runtimeraw',
                        metavar='BENCHMARKNAME',
                        help='Print the line \'<BENCHMARKNAME>(RunTimeRaw):' +
                        ' <elapsed> ms\' at the end where <elapsed> is' +
                        ' the elapsed time in milliseconds.')
    result.add_argument('--print-memoryuse',
                        metavar='BENCHMARKNAME',
                        help='Print the line \'<BENCHMARKNAME>(MemoryUse):' +
                        ' <mem>\' at the end where <mem> is the peak' +
                        ' peak resident set size (VmHWM) in bytes.')
    result.add_argument('--print-dexsegments',
                        metavar='BENCHMARKNAME',
                        help='Print the sizes of individual dex segments as ' +
                        '\'<BENCHMARKNAME>-<segment>(CodeSize): <bytes>\'')
    result.add_argument(
        '--track-time-in-memory',
        help='Plot the times taken from memory starting point to '
        'end-point with defined memory increment',
        default=False,
        action='store_true')
    result.add_argument('--track-time-in-memory-max',
                        help='Setting the maximum memory baseline to run in',
                        type=int)
    result.add_argument('--track-time-in-memory-min',
                        help='Setting the minimum memory baseline to run in',
                        type=int)
    result.add_argument('--track-time-in-memory-increment',
                        help='Setting the increment',
                        type=int,
                        default=32)
    result.add_argument('--print-times',
                        help='Include timing',
                        default=False,
                        action='store_true')
    result.add_argument('--cpu-list',
                        help='Run under \'taskset\' with these CPUs. See '
                        'the \'taskset\' -c option for the format')
    result.add_argument('--quiet',
                        help='Disable compiler logging',
                        default=False,
                        action='store_true')
    result.add_argument('--workers',
                        help='Number of workers to use',
                        default=1,
                        type=int)
    (options, args) = result.parse_known_args(argv)
    assert not options.hash or options.no_build, (
        'Argument --no-build is required when using --hash')
    assert not options.hash or options.compiler_build == 'full', (
        'Compiler build lib not yet supported with --hash')
    return (options, args)


# Most apps have -printmapping, -printseeds, -printusage and
# -printconfiguration in the Proguard configuration. However we don't
# want to write these files in the locations specified.
# Instead generate an auxiliary Proguard configuration placing these
# output files together with the dex output.
def GenerateAdditionalProguardConfiguration(temp, outdir):
    name = "output.config"
    with open(os.path.join(temp, name), 'w') as f:
        f.write('-printmapping ' + os.path.join(outdir, 'proguard.map') + "\n")
        f.write('-printseeds ' + os.path.join(outdir, 'proguard.seeds') + "\n")
        f.write('-printusage ' + os.path.join(outdir, 'proguard.usage') + "\n")
        f.write('-printconfiguration ' +
                os.path.join(outdir, 'proguard.config') + "\n")
        return os.path.abspath(f.name)


# Please add bug number for disabled permutations and please explicitly
# do Bug: #BUG in the commit message of disabling to ensure re-enabling
DISABLED_PERMUTATIONS = [
    # (app, version, type), e.g., ('gmail', '180826.15', 'deploy')
]


def get_permutations():
    data_providers = {
        'nest': nest_data,
        'youtube': youtube_data,
        'chrome': chrome_data,
    }
    # Check to ensure that we add all variants here.
    assert len(APPS) == len(data_providers)
    for app, data in data_providers.items():
        for version in data.VERSIONS:
            for type in data.VERSIONS[version]:
                if (app, version, type) not in DISABLED_PERMUTATIONS:
                    # Only run with R8 lib to reduce cycle times.
                    for use_r8lib in [True]:
                        yield app, version, type, use_r8lib


def run_all(options, args):
    # Build first so that each job won't.
    if should_build(options):
        gradle.RunGradle([utils.GRADLE_TASK_R8LIB])
        options.no_build = True
    assert not should_build(options)

    # Args will be destroyed
    assert len(args) == 0
    jobs = []
    for name, version, type, use_r8lib in get_permutations():
        compiler = 'r8' if type == 'deploy' else 'd8'
        compiler_build = 'lib' if use_r8lib else 'full'
        fixed_options = copy.copy(options)
        fixed_options.app = name
        fixed_options.version = version
        fixed_options.compiler = compiler
        fixed_options.compiler_build = compiler_build
        fixed_options.type = type
        jobs.append(
            create_job(compiler, compiler_build, name, fixed_options, type,
                       version))
    exit_code = thread_utils.run_in_parallel(
        jobs,
        number_of_workers=options.workers,
        stop_on_first_failure=not options.no_fail_fast)
    exit(exit_code)


def create_job(compiler, compiler_build, name, options, type, version):
    return lambda worker_id: run_job(compiler, compiler_build, name, options,
                                     type, version, worker_id)


def run_job(compiler, compiler_build, name, options, type, version, worker_id):
    print_thread(
        'Executing %s/%s with %s %s %s' %
        (compiler, compiler_build, name, version, type), worker_id)
    if worker_id is not None:
        options.out = os.path.join(options.out, str(worker_id))
        os.makedirs(options.out, exist_ok=True)
    exit_code = run_with_options(options, [], worker_id=worker_id)
    if exit_code:
        print_thread(
            'Failed %s %s %s with %s/%s' %
            (name, version, type, compiler, compiler_build), worker_id)
    return exit_code


def find_min_xmx(options, args):
    # Args will be destroyed
    assert len(args) == 0
    # If we can run in 128 MB then we are good (which we can for small examples
    # or D8 on medium sized examples)
    if options.find_min_xmx_min_memory:
        not_working = options.find_min_xmx_min_memory
    elif options.compiler == 'd8':
        not_working = 128
    else:
        not_working = 1024
    if options.find_min_xmx_max_memory:
        working = options.find_min_xmx_max_memory
    else:
        working = 1024 * 8
    exit_code = 0
    range = int(options.find_min_xmx_range_size)
    while working - not_working > range:
        next_candidate = int(working - ((working - not_working) / 2))
        print('working: %s, non_working: %s, next_candidate: %s' %
              (working, not_working, next_candidate))
        extra_args = ['-Xmx%sM' % next_candidate]
        t0 = time.time()
        exit_code = run_with_options(options, [], extra_args)
        t1 = time.time()
        print('Running took: %s ms' % (1000.0 * (t1 - t0)))
        if exit_code != 0:
            if exit_code not in [OOM_EXIT_CODE, TIMEOUT_KILL_CODE]:
                print('Non OOM/Timeout error executing, exiting')
                return 2
        if exit_code == 0:
            working = next_candidate
        elif exit_code == TIMEOUT_KILL_CODE:
            print('Timeout. Continue to the next candidate.')
            not_working = next_candidate
        else:
            assert exit_code == OOM_EXIT_CODE
            not_working = next_candidate

    assert working - not_working <= range
    found_range = 'Found range: %s - %s' % (not_working, working)
    print(found_range)

    if options.find_min_xmx_archive:
        sha = utils.get_HEAD_sha1()
        (version, _) = get_version_and_data(options)
        destination = os.path.join(utils.R8_TEST_RESULTS_BUCKET,
                                   FIND_MIN_XMX_DIR, sha, options.compiler,
                                   options.compiler_build, options.app, version,
                                   get_type(options))
        gs_destination = 'gs://%s' % destination
        utils.archive_value(FIND_MIN_XMX_FILE, gs_destination,
                            found_range + '\n')

    return 0


def print_min_xmx_ranges_for_hash(hash, compiler, compiler_build):
    app_directory = os.path.join(utils.R8_TEST_RESULTS_BUCKET, FIND_MIN_XMX_DIR,
                                 hash, compiler, compiler_build)
    gs_base = 'gs://%s' % app_directory
    for app in utils.ls_files_on_cloud_storage(gs_base).strip().split('\n'):
        for version in utils.ls_files_on_cloud_storage(app).strip().split('\n'):
            for type in utils.ls_files_on_cloud_storage(version).strip().split(
                    '\n'):
                gs_location = '%s%s' % (type, FIND_MIN_XMX_FILE)
                value = utils.cat_file_on_cloud_storage(gs_location,
                                                        ignore_errors=True)
                print('%s\n' % value)


def track_time_in_memory(options, args):
    # Args will be destroyed
    assert len(args) == 0
    if not options.track_time_in_memory_min:
        raise Exception(
            'You have to specify --track_time_in_memory_min when running with '
            '--track-time-in-memory')
    if not options.track_time_in_memory_max:
        raise Exception(
            'You have to specify --track_time_in_memory_max when running with '
            '--track-time-in-memory')
    if not options.track_time_in_memory_increment:
        raise Exception(
            'You have to specify --track_time_in_memory_increment when running '
            'with --track-time-in-memory')
    current = options.track_time_in_memory_min
    print('Memory (KB)\tTime (ms)')
    with utils.TempDir() as temp:
        stdout = os.path.join(temp, 'stdout')
        stdout_fd = open(stdout, 'w')
        while current <= options.track_time_in_memory_max:
            extra_args = ['-Xmx%sM' % current]
            t0 = time.time()
            exit_code = run_with_options(options, [],
                                         extra_args,
                                         stdout_fd,
                                         quiet=True)
            t1 = time.time()
            total = (1000.0 * (t1 - t0)) if exit_code == 0 else -1
            print('%s\t%s' % (current, total))
            current += options.track_time_in_memory_increment

    return 0


def main(argv):
    (options, args) = ParseOptions(argv)
    if options.expect_oom and not options.max_memory:
        raise Exception(
            'You should only use --expect-oom if also specifying --max-memory')
    if options.expect_oom and options.timeout:
        raise Exception(
            'You should not use --timeout when also specifying --expect-oom')
    if options.find_min_xmx and options.track_time_in_memory:
        raise Exception(
            'You cannot both find the min xmx and track time at the same time')
    if options.run_all:
        return run_all(options, args)
    if options.find_min_xmx:
        return find_min_xmx(options, args)
    if options.track_time_in_memory:
        return track_time_in_memory(options, args)
    exit_code = run_with_options(options, args, quiet=options.quiet)
    if options.expect_oom:
        exit_code = 0 if exit_code == OOM_EXIT_CODE else 1
    return exit_code


def get_version_and_data(options):
    if options.app == 'nest':
        version = options.version or '20180926'
        data = nest_data
    elif options.app == 'youtube':
        version = options.version or youtube_data.LATEST_VERSION
        data = youtube_data
    elif options.app == 'chrome':
        version = options.version or '180917'
        data = chrome_data
    else:
        raise Exception("You need to specify '--app={}'".format('|'.join(APPS)))
    return version, data


def get_type(options):
    if not options.type:
        return 'deploy' if options.compiler == 'r8' else 'proguarded'
    return options.type


def has_injars_and_libraryjars(pgconfs):
    # Check if there are -injars and -libraryjars in the configuration.
    has_injars = False
    has_libraryjars = False
    for pgconf in pgconfs:
        pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
        with open(pgconf) as pgconf_file:
            for line in pgconf_file:
                trimmed = line.strip()
                if trimmed.startswith('-injars'):
                    has_injars = True
                elif trimmed.startswith('-libraryjars'):
                    has_libraryjars = True
                if has_injars and has_libraryjars:
                    return True
    return False


def check_no_injars_and_no_libraryjars(pgconfs):
    # Ensure that there are no -injars or -libraryjars in the configuration.
    for pgconf in pgconfs:
        pgconf_dirname = os.path.abspath(os.path.dirname(pgconf))
        with open(pgconf) as pgconf_file:
            for line in pgconf_file:
                trimmed = line.strip()
                if trimmed.startswith('-injars'):
                    raise Exception("Unexpected -injars found in " + pgconf)
                elif trimmed.startswith('-libraryjars'):
                    raise Exception("Unexpected -libraryjars found in " +
                                    pgconf)


def should_build(options):
    return not options.no_build


def build_desugared_library_dex(options, quiet, temp, android_java8_libs,
                                desugared_lib_pg_conf, inputs, outdir):
    if not inputs:
        raise Exception(
            "If 'android_java8_libs' is specified the inputs must be explicit" +
            "(not defined using '-injars' in Proguard configuration files)")
    if outdir.endswith('.zip') or outdir.endswith('.jar'):
        raise Exception(
            "If 'android_java8_libs' is specified the output must be a directory"
        )

    jar = None
    main = None
    if options.hash:
        jar = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
        main = 'com.android.tools.r8.R8'

    # Determine the l8 tool.
    assert (options.compiler_build in ['full', 'lib'])
    lib_prefix = 'r8lib-' if options.compiler_build == 'lib' else ''
    tool = lib_prefix + 'l8'

    # Prepare out directory.
    android_java8_libs_output = os.path.join(temp, 'android_java8_libs')
    os.makedirs(android_java8_libs_output)

    # Prepare arguments for L8.
    args = [
        '--desugared-lib',
        android_java8_libs['config'],
        '--lib',
        android_java8_libs['library'],
        '--output',
        android_java8_libs_output,
        '--pg-conf',
        desugared_lib_pg_conf,
        '--release',
    ]
    if 'pgconf' in android_java8_libs:
        for pgconf in android_java8_libs['pgconf']:
            args.extend(['--pg-conf', pgconf])
    args.extend(android_java8_libs['program'])

    # Run L8.
    exit_code = toolhelper.run(tool,
                               args,
                               build=should_build(options),
                               disable_assertions=options.disable_assertions,
                               quiet=quiet,
                               jar=jar,
                               main=main)

    # Copy the desugared library DEX to the output.
    dex_file_name = ('classes' +
                     str(len(glob(os.path.join(outdir, '*.dex'))) + 1) + '.dex')
    shutil.copyfile(os.path.join(android_java8_libs_output, 'classes.dex'),
                    os.path.join(outdir, dex_file_name))


def run_with_options(options,
                     args,
                     extra_args=None,
                     stdout=None,
                     quiet=False,
                     worker_id=None):
    if extra_args is None:
        extra_args = []
    app_provided_pg_conf = False
    # todo(121018500): remove when memory is under control
    if not any('-Xmx' in arg for arg in extra_args):
        if options.max_memory:
            extra_args.append('-Xmx%sM' % options.max_memory)
        else:
            extra_args.append('-Xmx8G')
    if not options.ignore_java_version:
        utils.check_java_version()

    if options.print_times:
        extra_args.append('-Dcom.android.tools.r8.printtimes=1')

    if not options.disable_assertions:
        extra_args.append('-Dcom.android.tools.r8.enableTestAssertions=1')

    outdir = options.out
    (version_id, data) = get_version_and_data(options)

    if options.compiler not in COMPILERS:
        raise Exception("You need to specify '--compiler={}'".format(
            '|'.join(COMPILERS)))

    if options.compiler_build not in COMPILER_BUILDS:
        raise Exception("You need to specify '--compiler-build={}'".format(
            '|'.join(COMPILER_BUILDS)))

    if not version_id in data.VERSIONS.keys():
        print('No version {} for application {}'.format(version_id,
                                                        options.app))
        print('Valid versions are {}'.format(data.VERSIONS.keys()))
        return 1

    version = data.VERSIONS[version_id]

    type = get_type(options)

    if type not in version:
        print('No type {} for version {}'.format(type, version))
        print('Valid types are {}'.format(version.keys()))
        return 1
    values = version[type]

    args.extend(['--output', outdir])
    if 'min-api' in values:
        args.extend(['--min-api', values['min-api']])

    if 'main-dex-list' in values:
        args.extend(['--main-dex-list', values['main-dex-list']])

    inputs = values['inputs']
    libraries = values['libraries'] if 'libraries' in values else []

    if options.compiler == 'r8':
        if 'pgconf' in values and not options.k:
            sanitized_lib_path = os.path.join(os.path.abspath(outdir),
                                              'sanitized_lib.jar')
            if has_injars_and_libraryjars(values['pgconf']):
                sanitized_pgconf_path = os.path.join(os.path.abspath(outdir),
                                                     'sanitized.config')
                SanitizeLibrariesInPgconf(sanitized_lib_path,
                                          sanitized_pgconf_path,
                                          values['pgconf'])
                libraries = [sanitized_lib_path]
                args.extend(['--pg-conf', sanitized_pgconf_path])
                inputs = []
            else:
                # -injars without -libraryjars or vice versa is not supported.
                check_no_injars_and_no_libraryjars(values['pgconf'])
                for pgconf in values['pgconf']:
                    args.extend(['--pg-conf', pgconf])
                if 'sanitize_libraries' in values and values[
                        'sanitize_libraries']:
                    SanitizeLibraries(sanitized_lib_path, values['libraries'],
                                      values['inputs'])
                    libraries = [sanitized_lib_path]
            app_provided_pg_conf = True
            if 'pgconf_extra' in values:
                extra_conf = os.path.join(os.path.abspath(outdir),
                                          'pgconf_extra')
                with open(extra_conf, 'w') as extra_f:
                    extra_f.write(values['pgconf_extra'])
                args.extend(['--pg-conf', extra_conf])
        if options.k:
            args.extend(['--pg-conf', options.k])
        if 'maindexrules' in values:
            for rules in values['maindexrules']:
                args.extend(['--main-dex-rules', rules])
        if 'allow-type-errors' in values:
            extra_args.append('-Dcom.android.tools.r8.allowTypeErrors=1')
        extra_args.append(
            '-Dcom.android.tools.r8.disallowClassInlinerGracefulExit=1')
        if 'system-properties' in values:
            for system_property in values['system-properties']:
                extra_args.append(system_property)

    if options.debug_agent:
        if not options.compiler_build == 'full':
            print(
                'WARNING: Running debugging agent on r8lib is questionable...')
        extra_args.append(
            '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
        )

    if not options.no_libraries:
        for lib in libraries:
            args.extend(['--lib', lib])

    if not outdir.endswith('.zip') and not outdir.endswith('.jar') \
        and not os.path.exists(outdir):
        os.makedirs(outdir)

    if options.hash:
        # Download r8-<hash>.jar from
        # https://storage.googleapis.com/r8-releases/raw/<hash>/.
        download_path = archive.GetUploadDestination(options.hash, 'r8.jar',
                                                     True)
        assert utils.file_exists_on_cloud_storage(download_path), (
            'Could not find r8.jar file from provided hash: %s' % options.hash)
        destination = os.path.join(utils.LIBS, 'r8-' + options.hash + '.jar')
        utils.download_file_from_cloud_storage(download_path,
                                               destination,
                                               quiet=quiet)

    # Additional flags for the compiler from the configuration file.
    if 'flags' in values:
        args.extend(values['flags'].split(' '))
    if options.compiler == 'r8':
        if 'r8-flags' in values:
            args.extend(values['r8-flags'].split(' '))

    # Additional flags for the compiler from the command line.
    if options.compiler_flags:
        args.extend(options.compiler_flags.split(' '))
    if options.r8_flags:
        args.extend(options.r8_flags.split(' '))

    # Feature jars.
    features = values['features'] if 'features' in values else []
    for i, feature in enumerate(features, start=1):
        feature_out = os.path.join(outdir, 'feature-%d.zip' % i)
        for feature_jar in feature['inputs']:
            args.extend(['--feature', feature_jar, feature_out])

    args.extend(inputs)

    t0 = None
    if options.dump_args_file:
        with open(options.dump_args_file, 'w') as args_file:
            args_file.writelines([arg + os.linesep for arg in args])
    else:
        with utils.TempDir() as temp:
            if options.print_memoryuse and not options.track_memory_to_file:
                options.track_memory_to_file = os.path.join(
                    temp, utils.MEMORY_USE_TMP_FILE)
            if options.compiler == 'r8' and app_provided_pg_conf:
                # Ensure that output of -printmapping and -printseeds go to the output
                # location and not where the app Proguard configuration places them.
                if outdir.endswith('.zip') or outdir.endswith('.jar'):
                    pg_outdir = os.path.dirname(outdir)
                else:
                    pg_outdir = outdir
                if not options.no_extra_pgconf:
                    additional_pg_conf = GenerateAdditionalProguardConfiguration(
                        temp, os.path.abspath(pg_outdir))
                    args.extend(['--pg-conf', additional_pg_conf])

            android_java8_libs = values.get('android_java8_libs')
            if android_java8_libs:
                desugared_lib_pg_conf = os.path.join(
                    temp, 'desugared-lib-pg-conf.txt')
                args.extend(['--desugared-lib', android_java8_libs['config']])
                args.extend(
                    ['--desugared-lib-pg-conf-output', desugared_lib_pg_conf])

            stdout_path = os.path.join(temp, 'stdout')
            stderr_path = os.path.join(temp, 'stderr')
            with open(stdout_path, 'w') as stdout_workers:
                with open(stderr_path, 'w') as stderr:
                    jar = None
                    main = None
                    if options.compiler_build == 'full':
                        tool = options.compiler
                    else:
                        assert (options.compiler_build == 'lib')
                        tool = 'r8lib-' + options.compiler
                    if options.hash:
                        jar = os.path.join(utils.LIBS,
                                           'r8-' + options.hash + '.jar')
                        main = 'com.android.tools.r8.' + options.compiler.upper()
                    if should_build(options):
                        gradle.RunGradle([
                            utils.GRADLE_TASK_R8LIB
                            if tool.startswith('r8lib') else utils.GRADLE_TASK_R8
                        ])
                    t0 = time.time()
                    exit_code = toolhelper.run(
                        tool,
                        args,
                        build=False,
                        disable_assertions=options.disable_assertions,
                        profile=options.profile,
                        track_memory_file=options.track_memory_to_file,
                        extra_args=extra_args,
                        stdout=stdout if worker_id is None else stdout_workers,
                        stderr=stderr,
                        timeout=options.timeout,
                        quiet=quiet,
                        cmd_prefix=['taskset', '-c', options.cpu_list]
                        if options.cpu_list else [],
                        jar=jar,
                        main=main,
                        worker_id=worker_id)
            if worker_id is not None:
                with open(stdout_path) as stdout:
                    stdout_text = stdout.read()
                    print_thread(stdout_text, worker_id)
            if exit_code != 0:
                with open(stderr_path) as stderr:
                    stderr_text = stderr.read()
                    if not quiet:
                        print_thread(stderr_text, worker_id)
                    if 'java.lang.OutOfMemoryError' in stderr_text:
                        if not quiet:
                            print('Failure was OOM')
                        return OOM_EXIT_CODE
                    return exit_code

            if options.print_memoryuse:
                print('{}(MemoryUse): {}'.format(
                    options.print_memoryuse,
                    utils.grep_memoryuse(options.track_memory_to_file)))

            if android_java8_libs:
                build_desugared_library_dex(options, quiet, temp,
                                            android_java8_libs,
                                            desugared_lib_pg_conf, inputs,
                                            outdir)

    if options.print_runtimeraw:
        print('{}(RunTimeRaw): {} ms'.format(options.print_runtimeraw,
                                             1000.0 * (time.time() - t0)))

    if options.print_dexsegments:
        dex_files = glob(os.path.join(outdir, '*.dex'))
        utils.print_dexsegments(options.print_dexsegments, dex_files)
        print('{}-Total(CodeSize): {}'.format(
            options.print_dexsegments, compute_size_of_dex_files(dex_files)))
    return 0


def compute_size_of_dex_files(dex_files):
    dex_size = 0
    for dex_file in dex_files:
        dex_size += os.path.getsize(dex_file)
    return dex_size


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))