| #!/usr/bin/env python |
| # Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file |
| # for details. All rights reserved. Use of this source code is governed by a |
| # BSD-style license that can be found in the LICENSE file. |
| |
| # Clone and build AOSP, using D8 instead of JACK or DX, |
| # then run the Android-CTS on the emulator and compare results |
| # to a baseline. |
| # |
| # This script uses the repo manifest file 'third_party/aosp_manifest.xml' |
| # which is a snapshot of the aosp repo set. |
| # The manifest file can be updated with the following commands: |
| # |
| # cd build/aosp |
| # repo manifest -o ../../third_party/aosp_manifest.xml -r |
| # |
| # The baseline is a set of `test_result.xml` files in |
| # third_party/android_cts_baseline/jack. The current test considered a success |
| # if all tests pass that consistently pass in the baseline. |
| |
| from __future__ import print_function |
| from glob import glob |
| from itertools import chain |
| from os.path import join |
| from shutil import copy2 |
| from subprocess import check_call, Popen |
| import argparse |
| import os |
| import re |
| import sys |
| import time |
| |
| import gradle |
| import utils |
| |
| CTS_BASELINE_FILES_DIR = join(utils.REPO_ROOT, |
| 'third_party/android_cts_baseline/jack') |
| AOSP_MANIFEST_XML = join(utils.REPO_ROOT, 'third_party', |
| 'aosp_manifest.xml') |
| AOSP_HELPER_SH = join(utils.REPO_ROOT, 'scripts', 'aosp_helper.sh') |
| |
| D8_JAR = join(utils.REPO_ROOT, 'build/libs/d8.jar') |
| COMPATDX_JAR = join(utils.REPO_ROOT, 'build/libs/compatdx.jar') |
| D8LOGGER_JAR = join(utils.REPO_ROOT, 'build/libs/d8logger.jar') |
| |
| AOSP_ROOT = join(utils.REPO_ROOT, 'build/aosp') |
| |
| AOSP_MANIFEST_URL = 'https://android.googlesource.com/platform/manifest' |
| AOSP_PRESET = 'aosp_x86-eng' |
| |
| AOSP_OUT = join(AOSP_ROOT, 'out') |
| OUT_IMG = join(AOSP_ROOT, 'out_img') # output dir for android img build |
| OUT_CTS = join(AOSP_ROOT, 'out_cts') # output dir for CTS build |
| RESULTS_DIR_BASE = join(OUT_CTS, 'host/linux-x86/cts/android-cts/results') |
| CTS_TRADEFED = join(OUT_CTS, |
| 'host/linux-x86/cts/android-cts/tools/cts-tradefed') |
| |
| J_OPTION = '-j8' |
| |
| EXIT_FAILURE = 1 |
| |
| def parse_arguments(): |
| parser = argparse.ArgumentParser( |
| description = 'Download the AOSP source tree, build an Android image' |
| ' and the CTS targets and run CTS with the emulator on the image.') |
| parser.add_argument('--tool', |
| choices = ['jack', 'dx', 'd8'], |
| default = 'd8', |
| help='compiler tool to use') |
| parser.add_argument('--d8log', |
| metavar = 'FILE', |
| help = 'Enable logging d8 (compatdx) calls to the specified file. Works' |
| ' only with --tool=d8') |
| parser.add_argument('--save-result', |
| metavar = 'FILE', |
| help = 'Save final test_result.xml to the specified file.') |
| parser.add_argument('--no-baseline', |
| action = 'store_true', |
| help = "Don't compare results to baseline hence don't return failure if" |
| ' they differ.') |
| parser.add_argument('--clean-dex', |
| action = 'store_true', |
| help = 'Remove AOSP/dex files always, before the build. By default they' |
| " are removed only if '--tool=d8' and they're older then the D8 tool") |
| return parser.parse_args() |
| |
| # return False on error |
| def remove_aosp_out(): |
| if os.path.exists(AOSP_OUT): |
| if os.path.islink(AOSP_OUT): |
| os.remove(AOSP_OUT) |
| else: |
| print("The AOSP out directory ('" + AOSP_OUT + "') is expected" |
| " to be a symlink", file = sys.stderr) |
| return False |
| return True |
| |
| # Return list of fully qualified names of tests passing in |
| # all the files. |
| def consistently_passing_tests_from_test_results(filenames): |
| tree = {} |
| module = None |
| testcase = None |
| # Build a tree with leaves True|False|None for passing, failing and flaky |
| # tests. |
| for f in filenames: |
| for x in utils.read_cts_test_result(f): |
| if type(x) is utils.CtsModule: |
| module = tree.setdefault(x.name, {}) |
| elif type(x) is utils.CtsTestCase: |
| testcase = module.setdefault(x.name, {}) |
| else: |
| outcome = testcase.setdefault(x.name, x.outcome) |
| if outcome is not None and outcome != x.outcome: |
| testcase[x.name] = None |
| |
| result = [] |
| for module_name, module in tree.iteritems(): |
| for test_case_name, test_case in module.iteritems(): |
| result.extend(['{}/{}/{}'.format(module_name, test_case_name, test_name) |
| for test_name, test in test_case.iteritems() |
| if test]) |
| |
| return result |
| |
| def setup_and_clean(tool_is_d8, clean_dex): |
| # Two output dirs, one for the android image and one for cts tests. |
| # The output is compiled with d8 and jack, respectively. |
| utils.makedirs_if_needed(AOSP_ROOT) |
| utils.makedirs_if_needed(OUT_IMG) |
| utils.makedirs_if_needed(OUT_CTS) |
| |
| # remove dex files older than the current d8 tool |
| counter = 0 |
| if tool_is_d8 or clean_dex: |
| if not clean_dex: |
| d8jar_mtime = os.path.getmtime(D8_JAR) |
| dex_files = (chain.from_iterable(glob(join(x[0], '*.dex')) |
| for x in os.walk(OUT_IMG))) |
| for f in dex_files: |
| if clean_dex or os.path.getmtime(f) <= d8jar_mtime: |
| os.remove(f) |
| counter += 1 |
| if counter > 0: |
| print('Removed {} dex files.'.format(counter)) |
| |
| def checkout_aosp(): |
| # checkout AOSP source |
| manifests_dir = join(AOSP_ROOT, '.repo', 'manifests') |
| utils.makedirs_if_needed(manifests_dir) |
| |
| copy2(AOSP_MANIFEST_XML, manifests_dir) |
| check_call(['repo', 'init', '-u', AOSP_MANIFEST_URL, '-m', |
| 'aosp_manifest.xml', '--depth=1'], cwd = AOSP_ROOT) |
| |
| check_call(['repo', 'sync', '-dq', J_OPTION], cwd = AOSP_ROOT) |
| |
| def Main(): |
| args = parse_arguments() |
| |
| if args.d8log and args.tool != 'd8': |
| print("The '--d8log' option works only with '--tool=d8'.", |
| file = sys.stderr) |
| return EXIT_FAILURE |
| |
| assert args.tool in ['jack', 'dx', 'd8'] |
| |
| jack_option = 'ANDROID_COMPILE_WITH_JACK=' \ |
| + ('true' if args.tool == 'jack' else 'false') |
| |
| # DX_ALT_JAR need to be cleared if not set, for 'make' to work properly |
| alt_jar_option = 'DX_ALT_JAR=' |
| if args.tool == 'd8': |
| if args.d8log: |
| alt_jar_option += D8LOGGER_JAR |
| os.environ['D8LOGGER_OUTPUT'] = args.d8log |
| else: |
| alt_jar_option += COMPATDX_JAR |
| |
| gradle.RunGradle(['d8','d8logger', 'compatdx']) |
| |
| setup_and_clean(args.tool == 'd8', args.clean_dex) |
| |
| checkout_aosp() |
| |
| # activate OUT_CTS and build Android CTS |
| # AOSP has no clean way to set the output directory. |
| # In order to do incremental builds we apply the following symlink-based |
| # workaround. |
| # Note: this does not work on windows, but the AOSP |
| # doesn't build, either |
| |
| if not remove_aosp_out(): |
| return EXIT_FAILURE |
| print("-- Building CTS with 'make {} cts'.".format(J_OPTION)) |
| os.symlink(OUT_CTS, AOSP_OUT) |
| check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, 'cts'], |
| cwd = AOSP_ROOT) |
| |
| # activate OUT_IMG and build the Android image |
| if not remove_aosp_out(): |
| return EXIT_FAILURE |
| print("-- Building Android image with 'make {} {} {}'." \ |
| .format(J_OPTION, jack_option, alt_jar_option)) |
| os.symlink(OUT_IMG, AOSP_OUT) |
| check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, jack_option, |
| alt_jar_option], cwd = AOSP_ROOT) |
| |
| emulator_proc = Popen([AOSP_HELPER_SH, AOSP_PRESET, |
| 'emulator', '-partition-size', '4096', '-wipe-data'], cwd = AOSP_ROOT) |
| |
| if emulator_proc.poll() is not None: |
| print("Can't start Android Emulator.", file = sys.stderr) |
| |
| check_call([AOSP_HELPER_SH, AOSP_PRESET, 'run-cts', |
| CTS_TRADEFED, 'run', 'cts'], cwd = AOSP_ROOT) |
| |
| emulator_proc.terminate() |
| time.sleep(6) # aosp_helper waits to be killed in looping 'sleep 5' |
| |
| # find the newest test_result.xml |
| result_dirs = \ |
| [f for f in glob(join(RESULTS_DIR_BASE, '*')) if os.path.isdir(f)] |
| if len(result_dirs) == 0: |
| print("Can't find result directories in ", RESULTS_DIR_BASE) |
| return EXIT_FAILURE |
| result_dirs.sort(key = os.path.getmtime) |
| results_xml = join(result_dirs[-1], 'test_result.xml') |
| |
| # print summaries |
| re_summary = re.compile('<Summary ') |
| |
| summaries = [('Summary from current test results: ', results_xml)] |
| |
| for (title, result_file) in summaries: |
| print(title, result_file) |
| with open(result_file) as f: |
| for line in f: |
| if re_summary.search(line): |
| print(line) |
| break |
| |
| if args.no_baseline: |
| r = 0 |
| else: |
| print('Comparing test results to baseline:\n') |
| |
| passing_tests = consistently_passing_tests_from_test_results([results_xml]) |
| baseline_results = glob(join(CTS_BASELINE_FILES_DIR, '*.xml')) |
| assert len(baseline_results) != 0 |
| |
| passing_tests_in_baseline = \ |
| consistently_passing_tests_from_test_results(baseline_results) |
| |
| missing_or_failing_tests = \ |
| set(passing_tests_in_baseline) - set(passing_tests) |
| |
| num_tests = len(missing_or_failing_tests) |
| if num_tests != 0: |
| if num_tests > 1: |
| text = '{} tests that consistently pass in the baseline' \ |
| ' are missing or failing in the current test:'.format(num_tests) |
| else: |
| text = '1 test that consistently passes in the baseline' \ |
| ' is missing or failing in the current test:' |
| print(text) |
| for t in missing_or_failing_tests: |
| print(t) |
| r = EXIT_FAILURE |
| else: |
| r = 0 |
| |
| if args.save_result: |
| copy2(results_xml, args.save_result) |
| |
| return r |
| |
| if __name__ == '__main__': |
| sys.exit(Main()) |