Android CTS-related helper script improvements:
- add new options to 'test_android_cts':
+ --save-result saves the test_result.xml to specified file
+ --no-baseline skips baseline operations (and nonzero return on diff)
+ --clean-dex always removes all dex files before build
- add new script 'compare_cts_results' to compare multiple test_result.xml's
Bug:
Change-Id: I7ab98f7f24debde19bef661f5a8d76a2b6526c78
diff --git a/tools/compare_cts_results.py b/tools/compare_cts_results.py
new file mode 100755
index 0000000..95e29d1
--- /dev/null
+++ b/tools/compare_cts_results.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# Compare multiple CTS test_result.xml files
+
+from __future__ import print_function
+from os.path import basename
+import argparse
+import os
+import re
+import sys
+
+class Module:
+ def __init__(self):
+ self.test_cases = {}
+ self.bf_covered_in_file = 0 # bitfield, one bit per file
+
+ def get_test_case_maybe_create(self, test_case_name):
+ return self.test_cases.setdefault(test_case_name, TestCase())
+
+ def set_file_index_present(self, file_idx):
+ self.bf_covered_in_file |= (1 << file_idx)
+
+ def report(self, module_name, files, diff_only):
+ bf_all_files = self.bf_covered_in_file
+ for test_case_name, test_case in self.test_cases.iteritems():
+ if test_case.bf_covered_in_file != bf_all_files:
+ report_missing_thing('test_case', module_name + '/' + test_case_name,
+ test_case.bf_covered_in_file, files)
+ for test_case_name, test_case in self.test_cases.iteritems():
+ test_case.report(module_name, test_case_name, files, diff_only)
+
+class TestCase:
+ def __init__(self):
+ self.tests = {}
+ self.bf_covered_in_file = 0 # bitfield, one bit per file
+
+ def get_test_maybe_create(self, test_name):
+ return self.tests.setdefault(test_name, Test())
+
+ def set_file_index_present(self, file_idx):
+ self.bf_covered_in_file |= (1 << file_idx)
+
+ def report(self, module_name, test_case_name, files, diff_only):
+ bf_all_files = self.bf_covered_in_file
+ for test_name, test in self.tests.iteritems():
+ do_report = test.bf_passing_in_file != bf_all_files
+ if diff_only:
+ do_report = do_report and test.bf_failing_in_file != bf_all_files
+ if do_report:
+ test.report(module_name, test_case_name, test_name, files)
+
+class Test:
+ def __init__(self):
+ self.bf_failing_in_file = 0 # bitfields, one bit per file
+ self.bf_passing_in_file = 0
+
+ def set_file_index_outcome(self, outcome_is_passed, file_idx):
+ bf_value = (1 << file_idx)
+ if outcome_is_passed:
+ self.bf_passing_in_file |= bf_value
+ else:
+ self.bf_failing_in_file |= bf_value
+
+ # Report test's status in all files: pass/fail/missing
+ def report(self, module_name, test_case_name, test_name, files):
+ print('Test: {}/{}/{}:'.format(module_name, test_case_name, test_name))
+ for file_idx, f in enumerate(files):
+ bf_value = 1 << file_idx
+ print('\t- {:20}'.format(basename(f)), end = '')
+ if self.bf_passing_in_file & bf_value:
+ print('PASS')
+ elif self.bf_failing_in_file & bf_value:
+ print(' FAIL')
+ else:
+ print(' -- -- (missing)')
+
+def parse_arguments():
+ parser = argparse.ArgumentParser(
+ description = 'Compare multiple Android CTS test_result.xml files.')
+ parser.add_argument('files', nargs = '+',
+ help = 'List of (possibly renamed) test_result.xml files')
+ parser.add_argument('--diff-only',
+ action = 'store_true',
+ help = "Don't list tests that consistently fail in all result files,"
+ " list only differences.")
+ return parser.parse_args()
+
+# Read CTS test_result.xml from file and merge into result_tree
+def add_to_result_tree(result_tree, file_xml, file_idx):
+ re_module = re.compile('<Module name="([^"]*)"')
+ re_test_case = re.compile('<TestCase name="([^"]*)"')
+ re_test = re.compile('<Test result="(pass|fail)" name="([^"]*)"')
+ module = None
+ test_case = None
+ with open(file_xml) as f:
+ for line in f:
+ m = re_module.search(line)
+ if m:
+ module_name = m.groups()[0]
+ module = result_tree.setdefault(module_name, Module())
+ module.set_file_index_present(file_idx)
+ continue
+
+ m = re_test_case.search(line)
+ if m:
+ test_case_name = m.groups()[0]
+ test_case = module.get_test_case_maybe_create(test_case_name)
+ test_case.set_file_index_present(file_idx)
+ continue
+
+ m = re_test.search(line)
+ if m:
+ outcome = m.groups()[0]
+ test_name = m.groups()[1]
+ assert outcome in ["fail", "pass"]
+
+ v = test_case.get_test_maybe_create(test_name)
+ v.set_file_index_outcome(outcome == 'pass', file_idx)
+
+# main tree_report function
+def tree_report(result_tree, files, diff_only):
+ bf_all_files = (1 << len(files)) - 1
+ for module_name, module in result_tree.iteritems():
+ if module.bf_covered_in_file != bf_all_files:
+ report_missing_thing('module', module_name, module.bf_covered_in_file,
+ files)
+ for module_name, module in result_tree.iteritems():
+ module.report(module_name, files, diff_only)
+
+def report_missing_thing(thing_type, thing_name, bf_covered_in_file, files):
+ print('Missing {}: {}, from:'.format(thing_type, thing_name))
+ for file_idx, f in enumerate(files):
+ if not (bf_covered_in_file & (1 << file_idx)):
+ print('\t- ' + f)
+
+def Main():
+ m = Module()
+ m.get_test_case_maybe_create('qwe')
+
+ args = parse_arguments()
+
+ result_tree = {}
+ for file_idx, f in enumerate(args.files):
+ add_to_result_tree(result_tree, f, file_idx)
+
+ tree_report(result_tree, args.files, args.diff_only)
+
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(Main())
diff --git a/tools/test_android_cts.py b/tools/test_android_cts.py
index 56fca2d..2f02212 100755
--- a/tools/test_android_cts.py
+++ b/tools/test_android_cts.py
@@ -72,6 +72,17 @@
metavar = 'FILE',
help = 'Enable logging d8 (compatdx) calls to the specified file. Works'
' only with --tool=d8')
+ parser.add_argument('--save-result',
+ metavar = 'FILE',
+ help = 'Save final test_result.xml to the specified file.')
+ parser.add_argument('--no-baseline',
+ action = 'store_true',
+ help = "Don't compare results to baseline hence don't return failure if"
+ ' they differ.')
+ parser.add_argument('--clean-dex',
+ action = 'store_true',
+ help = 'Remove AOSP/dex files always, before the build. By default they'
+ " are removed only if '--tool=d8' and they're older then the D8 tool")
return parser.parse_args()
# return False on error
@@ -177,7 +188,7 @@
print()
return differ
-def setup_and_clean():
+def setup_and_clean(tool_is_d8, clean_dex):
# Two output dirs, one for the android image and one for cts tests.
# The output is compiled with d8 and jack, respectively.
utils.makedirs_if_needed(AOSP_ROOT)
@@ -185,12 +196,18 @@
utils.makedirs_if_needed(OUT_CTS)
# remove dex files older than the current d8 tool
- d8jar_mtime = os.path.getmtime(D8_JAR)
- dex_files = (chain.from_iterable(glob(join(x[0], '*.dex'))
- for x in os.walk(OUT_IMG)))
- for f in dex_files:
- if os.path.getmtime(f) <= d8jar_mtime:
- os.remove(f)
+ counter = 0
+ if tool_is_d8 or clean_dex:
+ if not clean_dex:
+ d8jar_mtime = os.path.getmtime(D8_JAR)
+ dex_files = (chain.from_iterable(glob(join(x[0], '*.dex'))
+ for x in os.walk(OUT_IMG)))
+ for f in dex_files:
+ if clean_dex or os.path.getmtime(f) <= d8jar_mtime:
+ os.remove(f)
+ counter += 1
+ if counter > 0:
+ print('Removed {} dex files.'.format(counter))
def checkout_aosp():
# checkout AOSP source
@@ -216,17 +233,18 @@
jack_option = 'ANDROID_COMPILE_WITH_JACK=' \
+ ('true' if args.tool == 'jack' else 'false')
- alt_jar_option = ''
+ # DX_ALT_JAR need to be cleared if not set, for 'make' to work properly
+ alt_jar_option = 'DX_ALT_JAR='
if args.tool == 'd8':
if args.d8log:
- alt_jar_option = 'DX_ALT_JAR=' + D8LOGGER_JAR
+ alt_jar_option += D8LOGGER_JAR
os.environ['D8LOGGER_OUTPUT'] = args.d8log
else:
- alt_jar_option = 'DX_ALT_JAR=' + COMPATDX_JAR
+ alt_jar_option += COMPATDX_JAR
gradle.RunGradle(['d8','d8logger', 'compatdx'])
- setup_and_clean()
+ setup_and_clean(args.tool == 'd8', args.clean_dex)
checkout_aosp()
@@ -239,6 +257,7 @@
if not remove_aosp_out():
return EXIT_FAILURE
+ print("-- Building CTS with 'make {} cts'.".format(J_OPTION))
os.symlink(OUT_CTS, AOSP_OUT)
check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, 'cts'],
cwd = AOSP_ROOT)
@@ -246,6 +265,8 @@
# activate OUT_IMG and build the Android image
if not remove_aosp_out():
return EXIT_FAILURE
+ print("-- Building Android image with 'make {} {} {}'." \
+ .format(J_OPTION, jack_option, alt_jar_option))
os.symlink(OUT_IMG, AOSP_OUT)
check_call([AOSP_HELPER_SH, AOSP_PRESET, 'make', J_OPTION, jack_option,
alt_jar_option], cwd = AOSP_ROOT)
@@ -272,10 +293,12 @@
# print summaries
re_summary = re.compile('<Summary ')
- for (title, result_file) in [
- ('Summary from current test results: ', results_xml),
- ('Summary from baseline: ', CTS_BASELINE)
- ]:
+
+ summaries = [('Summary from current test results: ', results_xml)]
+ if not args.no_baseline:
+ summaries.append(('Summary from baseline: ', CTS_BASELINE))
+
+ for (title, result_file) in summaries:
print(title, result_file)
with open(result_file) as f:
for line in f:
@@ -283,12 +306,20 @@
print(line)
break
- print('Comparing test results to baseline:\n')
+ if args.no_baseline:
+ r = 0
+ else:
+ print('Comparing test results to baseline:\n')
- result_tree = read_test_result_into_tree(results_xml)
- baseline_tree = read_test_result_into_tree(CTS_BASELINE)
+ result_tree = read_test_result_into_tree(results_xml)
+ baseline_tree = read_test_result_into_tree(CTS_BASELINE)
- return EXIT_FAILURE if diff_tree_report(baseline_tree, result_tree) else 0
+ r = EXIT_FAILURE if diff_tree_report(baseline_tree, result_tree) else 0
+
+ if args.save_result:
+ copy2(results_xml, args.save_result)
+
+ return r
if __name__ == '__main__':
sys.exit(Main())
