Ian Zerny | dcb172e | 2022-02-22 15:36:45 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Tamas Kenez | 507481b | 2017-06-06 15:01:33 +0200 | [diff] [blame] | 2 | # Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file |
| 3 | # for details. All rights reserved. Use of this source code is governed by a |
| 4 | # BSD-style license that can be found in the LICENSE file. |
| 5 | |
| 6 | # Compare multiple CTS test_result.xml files |
| 7 | |
| 8 | from __future__ import print_function |
| 9 | from os.path import basename |
| 10 | import argparse |
| 11 | import os |
Tamas Kenez | 507481b | 2017-06-06 15:01:33 +0200 | [diff] [blame] | 12 | import sys |
| 13 | |
Tamas Kenez | 82efeb5 | 2017-06-12 13:56:22 +0200 | [diff] [blame] | 14 | import utils |
| 15 | |
Tamas Kenez | 507481b | 2017-06-06 15:01:33 +0200 | [diff] [blame] | 16 | class Module: |
| 17 | def __init__(self): |
| 18 | self.test_cases = {} |
| 19 | self.bf_covered_in_file = 0 # bitfield, one bit per file |
| 20 | |
| 21 | def get_test_case_maybe_create(self, test_case_name): |
| 22 | return self.test_cases.setdefault(test_case_name, TestCase()) |
| 23 | |
| 24 | def set_file_index_present(self, file_idx): |
| 25 | self.bf_covered_in_file |= (1 << file_idx) |
| 26 | |
| 27 | def report(self, module_name, files, diff_only): |
| 28 | bf_all_files = self.bf_covered_in_file |
| 29 | for test_case_name, test_case in self.test_cases.iteritems(): |
| 30 | if test_case.bf_covered_in_file != bf_all_files: |
| 31 | report_missing_thing('test_case', module_name + '/' + test_case_name, |
| 32 | test_case.bf_covered_in_file, files) |
| 33 | for test_case_name, test_case in self.test_cases.iteritems(): |
| 34 | test_case.report(module_name, test_case_name, files, diff_only) |
| 35 | |
| 36 | class TestCase: |
| 37 | def __init__(self): |
| 38 | self.tests = {} |
| 39 | self.bf_covered_in_file = 0 # bitfield, one bit per file |
| 40 | |
| 41 | def get_test_maybe_create(self, test_name): |
| 42 | return self.tests.setdefault(test_name, Test()) |
| 43 | |
| 44 | def set_file_index_present(self, file_idx): |
| 45 | self.bf_covered_in_file |= (1 << file_idx) |
| 46 | |
| 47 | def report(self, module_name, test_case_name, files, diff_only): |
| 48 | bf_all_files = self.bf_covered_in_file |
| 49 | for test_name, test in self.tests.iteritems(): |
| 50 | do_report = test.bf_passing_in_file != bf_all_files |
| 51 | if diff_only: |
| 52 | do_report = do_report and test.bf_failing_in_file != bf_all_files |
| 53 | if do_report: |
| 54 | test.report(module_name, test_case_name, test_name, files) |
| 55 | |
| 56 | class Test: |
| 57 | def __init__(self): |
| 58 | self.bf_failing_in_file = 0 # bitfields, one bit per file |
| 59 | self.bf_passing_in_file = 0 |
| 60 | |
| 61 | def set_file_index_outcome(self, outcome_is_passed, file_idx): |
| 62 | bf_value = (1 << file_idx) |
| 63 | if outcome_is_passed: |
| 64 | self.bf_passing_in_file |= bf_value |
| 65 | else: |
| 66 | self.bf_failing_in_file |= bf_value |
| 67 | |
| 68 | # Report test's status in all files: pass/fail/missing |
| 69 | def report(self, module_name, test_case_name, test_name, files): |
| 70 | print('Test: {}/{}/{}:'.format(module_name, test_case_name, test_name)) |
| 71 | for file_idx, f in enumerate(files): |
| 72 | bf_value = 1 << file_idx |
| 73 | print('\t- {:20}'.format(basename(f)), end = '') |
| 74 | if self.bf_passing_in_file & bf_value: |
| 75 | print('PASS') |
| 76 | elif self.bf_failing_in_file & bf_value: |
| 77 | print(' FAIL') |
| 78 | else: |
| 79 | print(' -- -- (missing)') |
| 80 | |
| 81 | def parse_arguments(): |
| 82 | parser = argparse.ArgumentParser( |
| 83 | description = 'Compare multiple Android CTS test_result.xml files.') |
| 84 | parser.add_argument('files', nargs = '+', |
| 85 | help = 'List of (possibly renamed) test_result.xml files') |
| 86 | parser.add_argument('--diff-only', |
| 87 | action = 'store_true', |
| 88 | help = "Don't list tests that consistently fail in all result files," |
| 89 | " list only differences.") |
| 90 | return parser.parse_args() |
| 91 | |
| 92 | # Read CTS test_result.xml from file and merge into result_tree |
| 93 | def add_to_result_tree(result_tree, file_xml, file_idx): |
Tamas Kenez | 507481b | 2017-06-06 15:01:33 +0200 | [diff] [blame] | 94 | module = None |
| 95 | test_case = None |
Tamas Kenez | 82efeb5 | 2017-06-12 13:56:22 +0200 | [diff] [blame] | 96 | for x in utils.read_cts_test_result(file_xml): |
| 97 | if type(x) is utils.CtsModule: |
| 98 | module = result_tree.setdefault(x.name, Module()) |
| 99 | module.set_file_index_present(file_idx) |
| 100 | elif type(x) is utils.CtsTestCase: |
| 101 | test_case = module.get_test_case_maybe_create(x.name) |
| 102 | test_case.set_file_index_present(file_idx) |
| 103 | else: |
| 104 | assert(type(x) is utils.CtsTest) |
| 105 | v = test_case.get_test_maybe_create(x.name) |
| 106 | v.set_file_index_outcome(x.outcome, file_idx) |
Tamas Kenez | 507481b | 2017-06-06 15:01:33 +0200 | [diff] [blame] | 107 | |
| 108 | # main tree_report function |
| 109 | def tree_report(result_tree, files, diff_only): |
| 110 | bf_all_files = (1 << len(files)) - 1 |
| 111 | for module_name, module in result_tree.iteritems(): |
| 112 | if module.bf_covered_in_file != bf_all_files: |
| 113 | report_missing_thing('module', module_name, module.bf_covered_in_file, |
| 114 | files) |
| 115 | for module_name, module in result_tree.iteritems(): |
| 116 | module.report(module_name, files, diff_only) |
| 117 | |
| 118 | def report_missing_thing(thing_type, thing_name, bf_covered_in_file, files): |
| 119 | print('Missing {}: {}, from:'.format(thing_type, thing_name)) |
| 120 | for file_idx, f in enumerate(files): |
| 121 | if not (bf_covered_in_file & (1 << file_idx)): |
| 122 | print('\t- ' + f) |
| 123 | |
| 124 | def Main(): |
| 125 | m = Module() |
| 126 | m.get_test_case_maybe_create('qwe') |
| 127 | |
| 128 | args = parse_arguments() |
| 129 | |
| 130 | result_tree = {} |
| 131 | for file_idx, f in enumerate(args.files): |
| 132 | add_to_result_tree(result_tree, f, file_idx) |
| 133 | |
| 134 | tree_report(result_tree, args.files, args.diff_only) |
| 135 | |
| 136 | return 0 |
| 137 | |
| 138 | if __name__ == '__main__': |
| 139 | sys.exit(Main()) |