blob: fa814bc172915cd4f4ea5f2dd05979bb4b18c387 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Clone and build AOSP, using D8 instead of JACK or DX,
# then run the Android-CTS on the emulator and compare results
# to a baseline.
#
# This script uses the repo manifest file 'third_party/aosp_manifest.xml'
# which is a snapshot of the aosp repo set.
# The manifest file can be updated with the following commands:
#
# cd build/aosp
# repo manifest -o ../../third_party/aosp_manifest.xml -r
#
# The baseline is a set of `test_result.xml` files in
# third_party/android_cts_baseline/jack. The current test considered a success
# if all tests pass that consistently pass in the baseline.
from __future__ import print_function
from glob import glob
from itertools import chain
from os.path import join
from shutil import copy2
from subprocess import check_call, Popen
import argparse
import os
import re
import sys
import time
import checkout_aosp
import gradle
import utils
CTS_BASELINE_FILES_DIR = join(utils.REPO_ROOT,
'third_party/android_cts_baseline/jack')
AOSP_MANIFEST_XML = join(utils.REPO_ROOT, 'third_party',
'aosp_manifest.xml')
AOSP_HELPER_SH = join(utils.REPO_ROOT, 'scripts', 'aosp_helper.sh')
AOSP_ROOT = join(utils.REPO_ROOT, 'build/aosp')
AOSP_MANIFEST_URL = 'https://android.googlesource.com/platform/manifest'
AOSP_PRESET = 'aosp_x86-eng'
AOSP_OUT = join(AOSP_ROOT, 'out')
OUT_IMG = join(AOSP_ROOT, 'out_img') # output dir for android img build
OUT_CTS = join(AOSP_ROOT, 'out_cts') # output dir for CTS build
RESULTS_DIR_BASE = join(OUT_CTS, 'host/linux-x86/cts/android-cts/results')
CTS_TRADEFED = join(OUT_CTS,
'host/linux-x86/cts/android-cts/tools/cts-tradefed')
J_DEFAULT = 8
J_OPTION = '-j' + str(J_DEFAULT)
EXIT_FAILURE = 1
def parse_arguments():
parser = argparse.ArgumentParser(
description = 'Download the AOSP source tree, build an Android image'
' and the CTS targets and run CTS with the emulator on the image.')
parser.add_argument('--tool',
choices = ['dx', 'd8'],
default = 'd8',
help='compiler tool to use')
parser.add_argument('--save-result',
metavar = 'FILE',
help = 'Save final test_result.xml to the specified file.')
parser.add_argument('--no-baseline',
action = 'store_true',
help = "Don't compare results to baseline hence don't return failure if"
' they differ.')
parser.add_argument('--clean-dex',
action = 'store_true',
help = 'Remove AOSP/dex files always, before the build. By default they'
" are removed only if '--tool=d8' and they're older then the D8 tool")
return parser.parse_args()
# return False on error
def remove_aosp_out():
if os.path.exists(AOSP_OUT):
if os.path.islink(AOSP_OUT):
os.remove(AOSP_OUT)
else:
print("The AOSP out directory ('" + AOSP_OUT + "') is expected"
" to be a symlink", file = sys.stderr)
return False
return True
# Return list of fully qualified names of tests passing in
# all the files.
def consistently_passing_tests_from_test_results(filenames):
tree = {}
module = None
testcase = None
# Build a tree with leaves True|False|None for passing, failing and flaky
# tests.
for f in filenames:
for x in utils.read_cts_test_result(f):
if type(x) is utils.CtsModule:
module = tree.setdefault(x.name, {})
elif type(x) is utils.CtsTestCase:
testcase = module.setdefault(x.name, {})
else:
outcome = testcase.setdefault(x.name, x.outcome)
if outcome is not None and outcome != x.outcome:
testcase[x.name] = None
result = []
for module_name, module in tree.iteritems():
for test_case_name, test_case in module.iteritems():
result.extend(['{}/{}/{}'.format(module_name, test_case_name, test_name)
for test_name, test in test_case.iteritems()
if test])
return result
def setup_and_clean(tool_is_d8, clean_dex):
# Two output dirs, one for the android image and one for cts tests.
# The output is compiled with d8 and jack, respectively.
utils.makedirs_if_needed(AOSP_ROOT)
utils.makedirs_if_needed(OUT_IMG)
utils.makedirs_if_needed(OUT_CTS)
# remove dex files older than the current d8 tool
counter = 0
if tool_is_d8 or clean_dex:
if not clean_dex:
d8jar_mtime = os.path.getmtime(utils.D8_JAR)
dex_files = (chain.from_iterable(glob(join(x[0], '*.dex'))
for x in os.walk(OUT_IMG)))
for f in dex_files:
if clean_dex or os.path.getmtime(f) <= d8jar_mtime:
os.remove(f)
counter += 1
if counter > 0:
print('Removed {} dex files.'.format(counter))
def Main():
args = parse_arguments()
assert args.tool in ['dx', 'd8']
use_d8 = 'USE_D8=false'
if args.tool == 'd8':
use_d8 = 'USE_D8=true'
gradle.RunGradle(['d8'])
setup_and_clean(args.tool == 'd8', args.clean_dex)
checkout_aosp.checkout_aosp("test", AOSP_MANIFEST_URL, None,
AOSP_MANIFEST_XML, str(J_DEFAULT), True)
# activate OUT_CTS and build Android CTS
# AOSP has no clean way to set the output directory.
# In order to do incremental builds we apply the following symlink-based
# workaround.
# Note: this does not work on windows, but the AOSP
# doesn't build, either
if not remove_aosp_out():
return EXIT_FAILURE
print("-- Building CTS with 'make {} cts'.".format(J_OPTION))
os.symlink(OUT_CTS, AOSP_OUT)
check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, 'cts'],
cwd = AOSP_ROOT)
# activate OUT_IMG and build the Android image
if not remove_aosp_out():
return EXIT_FAILURE
print("-- Building Android image with 'make {} {}'." \
.format(J_OPTION, use_d8))
os.symlink(OUT_IMG, AOSP_OUT)
check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION,
use_d8_option], cwd = AOSP_ROOT)
emulator_proc = Popen([AOSP_HELPER_SH, AOSP_PRESET,
'emulator', '-partition-size', '4096', '-wipe-data'], cwd = AOSP_ROOT)
if emulator_proc.poll() is not None:
print("Can't start Android Emulator.", file = sys.stderr)
check_call([AOSP_HELPER_SH, AOSP_PRESET, 'run-cts',
CTS_TRADEFED, 'run', 'cts'], cwd = AOSP_ROOT)
emulator_proc.terminate()
time.sleep(6) # aosp_helper waits to be killed in looping 'sleep 5'
# find the newest test_result.xml
result_dirs = \
[f for f in glob(join(RESULTS_DIR_BASE, '*')) if os.path.isdir(f)]
if len(result_dirs) == 0:
print("Can't find result directories in ", RESULTS_DIR_BASE)
return EXIT_FAILURE
result_dirs.sort(key = os.path.getmtime)
results_xml = join(result_dirs[-1], 'test_result.xml')
# print summaries
re_summary = re.compile('<Summary ')
summaries = [('Summary from current test results: ', results_xml)]
for (title, result_file) in summaries:
print(title, result_file)
with open(result_file) as f:
for line in f:
if re_summary.search(line):
print(line)
break
if args.no_baseline:
r = 0
else:
print('Comparing test results to baseline:\n')
passing_tests = consistently_passing_tests_from_test_results([results_xml])
baseline_results = glob(join(CTS_BASELINE_FILES_DIR, '*.xml'))
assert len(baseline_results) != 0
passing_tests_in_baseline = \
consistently_passing_tests_from_test_results(baseline_results)
missing_or_failing_tests = \
set(passing_tests_in_baseline) - set(passing_tests)
num_tests = len(missing_or_failing_tests)
if num_tests != 0:
if num_tests > 1:
text = '{} tests that consistently pass in the baseline' \
' are missing or failing in the current test:'.format(num_tests)
else:
text = '1 test that consistently passes in the baseline' \
' is missing or failing in the current test:'
print(text)
for t in missing_or_failing_tests:
print(t)
r = EXIT_FAILURE
else:
r = 0
if args.save_result:
copy2(results_xml, args.save_result)
return r
if __name__ == '__main__':
sys.exit(Main())