blob: 0da116dac6c5218ca25fc4335d71fcd67e3d4c75 [file] [log] [blame]
Ian Zernydcb172e2022-02-22 15:36:45 +01001#!/usr/bin/env python3
Tamas Kenez507481b2017-06-06 15:01:33 +02002# Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file
3# for details. All rights reserved. Use of this source code is governed by a
4# BSD-style license that can be found in the LICENSE file.
5
6# Compare multiple CTS test_result.xml files
7
8from __future__ import print_function
9from os.path import basename
10import argparse
11import os
Tamas Kenez507481b2017-06-06 15:01:33 +020012import sys
13
Tamas Kenez82efeb52017-06-12 13:56:22 +020014import utils
15
Tamas Kenez507481b2017-06-06 15:01:33 +020016class Module:
17 def __init__(self):
18 self.test_cases = {}
19 self.bf_covered_in_file = 0 # bitfield, one bit per file
20
21 def get_test_case_maybe_create(self, test_case_name):
22 return self.test_cases.setdefault(test_case_name, TestCase())
23
24 def set_file_index_present(self, file_idx):
25 self.bf_covered_in_file |= (1 << file_idx)
26
27 def report(self, module_name, files, diff_only):
28 bf_all_files = self.bf_covered_in_file
29 for test_case_name, test_case in self.test_cases.iteritems():
30 if test_case.bf_covered_in_file != bf_all_files:
31 report_missing_thing('test_case', module_name + '/' + test_case_name,
32 test_case.bf_covered_in_file, files)
33 for test_case_name, test_case in self.test_cases.iteritems():
34 test_case.report(module_name, test_case_name, files, diff_only)
35
36class TestCase:
37 def __init__(self):
38 self.tests = {}
39 self.bf_covered_in_file = 0 # bitfield, one bit per file
40
41 def get_test_maybe_create(self, test_name):
42 return self.tests.setdefault(test_name, Test())
43
44 def set_file_index_present(self, file_idx):
45 self.bf_covered_in_file |= (1 << file_idx)
46
47 def report(self, module_name, test_case_name, files, diff_only):
48 bf_all_files = self.bf_covered_in_file
49 for test_name, test in self.tests.iteritems():
50 do_report = test.bf_passing_in_file != bf_all_files
51 if diff_only:
52 do_report = do_report and test.bf_failing_in_file != bf_all_files
53 if do_report:
54 test.report(module_name, test_case_name, test_name, files)
55
56class Test:
57 def __init__(self):
58 self.bf_failing_in_file = 0 # bitfields, one bit per file
59 self.bf_passing_in_file = 0
60
61 def set_file_index_outcome(self, outcome_is_passed, file_idx):
62 bf_value = (1 << file_idx)
63 if outcome_is_passed:
64 self.bf_passing_in_file |= bf_value
65 else:
66 self.bf_failing_in_file |= bf_value
67
68 # Report test's status in all files: pass/fail/missing
69 def report(self, module_name, test_case_name, test_name, files):
70 print('Test: {}/{}/{}:'.format(module_name, test_case_name, test_name))
71 for file_idx, f in enumerate(files):
72 bf_value = 1 << file_idx
73 print('\t- {:20}'.format(basename(f)), end = '')
74 if self.bf_passing_in_file & bf_value:
75 print('PASS')
76 elif self.bf_failing_in_file & bf_value:
77 print(' FAIL')
78 else:
79 print(' -- -- (missing)')
80
81def parse_arguments():
82 parser = argparse.ArgumentParser(
83 description = 'Compare multiple Android CTS test_result.xml files.')
84 parser.add_argument('files', nargs = '+',
85 help = 'List of (possibly renamed) test_result.xml files')
86 parser.add_argument('--diff-only',
87 action = 'store_true',
88 help = "Don't list tests that consistently fail in all result files,"
89 " list only differences.")
90 return parser.parse_args()
91
92# Read CTS test_result.xml from file and merge into result_tree
93def add_to_result_tree(result_tree, file_xml, file_idx):
Tamas Kenez507481b2017-06-06 15:01:33 +020094 module = None
95 test_case = None
Tamas Kenez82efeb52017-06-12 13:56:22 +020096 for x in utils.read_cts_test_result(file_xml):
97 if type(x) is utils.CtsModule:
98 module = result_tree.setdefault(x.name, Module())
99 module.set_file_index_present(file_idx)
100 elif type(x) is utils.CtsTestCase:
101 test_case = module.get_test_case_maybe_create(x.name)
102 test_case.set_file_index_present(file_idx)
103 else:
104 assert(type(x) is utils.CtsTest)
105 v = test_case.get_test_maybe_create(x.name)
106 v.set_file_index_outcome(x.outcome, file_idx)
Tamas Kenez507481b2017-06-06 15:01:33 +0200107
108# main tree_report function
109def tree_report(result_tree, files, diff_only):
110 bf_all_files = (1 << len(files)) - 1
111 for module_name, module in result_tree.iteritems():
112 if module.bf_covered_in_file != bf_all_files:
113 report_missing_thing('module', module_name, module.bf_covered_in_file,
114 files)
115 for module_name, module in result_tree.iteritems():
116 module.report(module_name, files, diff_only)
117
118def report_missing_thing(thing_type, thing_name, bf_covered_in_file, files):
119 print('Missing {}: {}, from:'.format(thing_type, thing_name))
120 for file_idx, f in enumerate(files):
121 if not (bf_covered_in_file & (1 << file_idx)):
122 print('\t- ' + f)
123
124def Main():
125 m = Module()
126 m.get_test_case_maybe_create('qwe')
127
128 args = parse_arguments()
129
130 result_tree = {}
131 for file_idx, f in enumerate(args.files):
132 add_to_result_tree(result_tree, f, file_idx)
133
134 tree_report(result_tree, args.files, args.diff_only)
135
136 return 0
137
138if __name__ == '__main__':
139 sys.exit(Main())