diff --git a/doc/newsfragments/2691_changed.fix_match_assertion.rst b/doc/newsfragments/2691_changed.fix_match_assertion.rst new file mode 100644 index 000000000..bb403764b --- /dev/null +++ b/doc/newsfragments/2691_changed.fix_match_assertion.rst @@ -0,0 +1 @@ +Introduced a new parameter ``include_only_expected`` for py:meth:`result.fix.match() ` assertion. It will only compare the tags present in the expected message when the parameter is set to `True`. \ No newline at end of file diff --git a/testplan/common/utils/comparison.py b/testplan/common/utils/comparison.py index a207a08b6..6989866b3 100644 --- a/testplan/common/utils/comparison.py +++ b/testplan/common/utils/comparison.py @@ -4,7 +4,8 @@ import traceback from collections.abc import Mapping, Iterable, Container from itertools import zip_longest -from typing import List, Tuple, Dict, Hashable, Union +from typing import Any, List, Tuple, Dict, Hashable, Union +from typing import Callable as typing_Callable from .reporting import Absent, fmt, NATIVE_TYPES, callable_name @@ -63,7 +64,7 @@ def check_dict_keys(data, has_keys=None, absent_keys=None): return existing_diff, absent_intersection -class Callable: +class Callable: # TODO: This should not be called Callable - needs refactoring """ Some of our assertions can make use of callables that accept a single argument as comparator values. We also provide the helper @@ -187,7 +188,6 @@ def __str__(self): class MetaCallable(Callable): - delimiter = None def __init__(self, *callables): @@ -209,7 +209,6 @@ def __str__(self): class Or(MetaCallable): - delimiter = "or" def __call__(self, value): @@ -220,7 +219,6 @@ def __call__(self, value): class And(MetaCallable): - delimiter = "and" def __call__(self, value): @@ -481,9 +479,10 @@ def _cmp_dicts( lhs: Dict, rhs: Dict, ignore: Container, - only: Container, + include: Container, report_mode: int, value_cmp_func: Union[Callable, None], + include_only_rhs: bool = False, ) -> Tuple[str, List]: """ Compares two dictionaries with optional restriction to keys, @@ -491,9 +490,10 @@ def _cmp_dicts( :param lhs: dictionary to compare :param rhs: dictionary to compare :param ignore: collection of keys to ignore during comparison - :param only: collection of keys to restrict comparison to + :param include: collection of keys to restrict comparison to :param report_mode: report option code :param value_cmp_func: value comparison function + :param include_only_rhs: use the keys present in rhs. :return: pair of match result and comparison result """ @@ -506,8 +506,10 @@ def should_ignore_key(key: Hashable) -> bool: """ if key in ignore: should_ignore = True - elif only is not None: - should_ignore = key not in only + elif include_only_rhs is True: + should_ignore = key not in rhs.keys() + elif include is not None: + should_ignore = key not in include else: should_ignore = False return should_ignore @@ -521,24 +523,26 @@ def should_ignore_key(key: Hashable) -> bool: # enforce ignorance of match results.append( _rec_compare( - lhs_val, - rhs_val, - ignore, - only, - iter_key, - report_mode, + lhs=lhs_val, + rhs=rhs_val, + ignore=ignore, + include=include, + key=iter_key, + report_mode=report_mode, value_cmp_func=None, + include_only_rhs=include_only_rhs, ) ) else: result = _rec_compare( - lhs_val, - rhs_val, - ignore, - only, - iter_key, - report_mode, - value_cmp_func, + lhs=lhs_val, + rhs=rhs_val, + ignore=ignore, + include=include, + key=iter_key, + report_mode=report_mode, + value_cmp_func=value_cmp_func, + include_only_rhs=include_only_rhs, ) # Decide whether to keep or discard the result, depending on the # reporting mode. @@ -559,11 +563,12 @@ def _rec_compare( lhs, rhs, ignore, - only, + include, key, report_mode, value_cmp_func, _regex_adapter=RegexAdapter, + include_only_rhs=False, ): """ Recursive deep comparison implementation @@ -674,13 +679,14 @@ def _rec_compare( for lhs_item, rhs_item in zip_longest(lhs, rhs): # iterate all elems in both iterable non-mapping objects result = _rec_compare( - lhs_item, - rhs_item, - ignore, - only, + lhs=lhs_item, + rhs=rhs_item, + ignore=ignore, + include=include, key=None, report_mode=report_mode, value_cmp_func=value_cmp_func, + include_only_rhs=include_only_rhs, ) match = Match.combine(match, result[1]) @@ -696,7 +702,13 @@ def _rec_compare( ## DICTS if lhs_cat == rhs_cat == Category.DICT: match, results = _cmp_dicts( - lhs, rhs, ignore, only, report_mode, value_cmp_func + lhs=lhs, + rhs=rhs, + ignore=ignore, + include=include, + report_mode=report_mode, + value_cmp_func=value_cmp_func, + include_only_rhs=include_only_rhs, ) lhs_vals, rhs_vals = _partition(results) return _build_res( @@ -724,7 +736,6 @@ def untyped_fixtag(x, y): isinstance(val, float) or isinstance(val, decimal.Decimal) for val in (x, y) ): - x_, y_ = ( val.rstrip("0").rstrip(".") if "." in val else val for val in (x_, y_) @@ -771,39 +782,36 @@ class ReportOptions(enum.Enum): def compare( - lhs, - rhs, - ignore=None, - only=None, + lhs: Dict, + rhs: Dict, + ignore: List[Hashable] = None, + include: List[Hashable] = None, report_mode=ReportOptions.ALL, - value_cmp_func=COMPARE_FUNCTIONS["native_equality"], -): + value_cmp_func: typing_Callable[[Any, Any], bool] = COMPARE_FUNCTIONS[ + "native_equality" + ], + include_only_rhs: bool = False, +) -> Tuple[bool, List[Tuple]]: """ Compare two iterable key, value objects (e.g. dict or dict-like mapping) and return a status and a detailed comparison table, useful for reporting. - Ignore has precedence over only. + Ignore has precedence over include. :param lhs: object compared against rhs - :type lhs: ``dict`` interface (``__contains__`` and ``.items()``) :param rhs: object compared against lhs - :type rhs: ``dict`` interface (``__contains__`` and ``.items()``) :param ignore: list of keys to ignore in the comparison - :type ignore: ``list`` - :param only: list of keys to exclusively consider in the comparison - :type only: ``list`` + :param include: list of keys to exclusively consider in the comparison :param report_mode: Specify which comparisons should be kept and reported. Default option is to report all comparisons but this can be restricted if desired. See ReportOptions enum for more detail. - :type report_mode: ``ReportOptions`` :param value_cmp_func: function to compare values in a dict. Defaults to COMPARE_FUNCTIONS['native_equality']. - :type value_cmp_func: Callable[[Any, Any], bool] + :param include_only_rhs: use the keys present in rhs. :return: Tuple of comparison bool ``(passed: True, failed: False)`` and a description object for the testdb report - :rtype: ``tuple`` of (``bool``, ``list`` of ``tuple``) """ if (lhs is None) and (rhs is None): @@ -834,16 +842,22 @@ def compare( ignore = ignore or [] match, comparisons = _cmp_dicts( - lhs, rhs, ignore, only, report_mode, value_cmp_func + lhs=lhs, + rhs=rhs, + ignore=ignore, + include=include, + report_mode=report_mode, + value_cmp_func=value_cmp_func, + include_only_rhs=include_only_rhs, ) - # For the keys in only not matching anything, + # For the keys in include not matching anything, # we report them as absent in expected and value. - if isinstance(only, list) and only and comparisons is not None: + if isinstance(include, list) and include and comparisons is not None: keys_found = set() for elem in comparisons: keys_found.add(elem[0]) - for key in only: + for key in include: if key not in keys_found: comparisons.append( (key, Match.IGNORED, Absent.descr, Absent.descr) @@ -988,19 +1002,19 @@ class Expected: Input to the "unordered_compare" function. """ - def __init__(self, value, ignore=None, only=None): + def __init__(self, value, ignore=None, include=None): """ :param value: object compared against each actual value in unordered_compare :type value: ``dict``-like interface (__contains__ and .items()) :param ignore: list of keys to ignore in the comparison :type ignore: ``list`` - :param only: list of keys to exclusively consider in the comparison - :type only: ``list`` + :param include: list of keys to exclusively consider in the comparison + :type include: ``list`` """ self.value = value self.ignore = ignore - self.only = only + self.include = include def unordered_compare( @@ -1105,7 +1119,7 @@ def unordered_compare( cmpr.value, msg, ignore=cmpr.ignore, - only=cmpr.only, + include=cmpr.include, value_cmp_func=value_cmp_func, ) for cmpr in proc_cmps diff --git a/testplan/testing/multitest/entries/assertions.py b/testplan/testing/multitest/entries/assertions.py index 532d6b66d..11d9c9f9d 100644 --- a/testplan/testing/multitest/entries/assertions.py +++ b/testplan/testing/multitest/entries/assertions.py @@ -9,6 +9,7 @@ import cmath import collections import decimal +import lxml import numbers import operator import os @@ -17,8 +18,7 @@ import subprocess import sys import tempfile - -import lxml +from typing import Dict, Hashable, List from testplan.common.utils.convert import make_tuple, flatten_dict_comparison from testplan.common.utils import comparison, difflib @@ -1306,19 +1306,21 @@ class DictMatch(Assertion): def __init__( self, - value, - expected, - include_keys=None, - exclude_keys=None, + value: Dict, + expected: Dict, + include_only_expected: bool = False, + include_keys: List[Hashable] = None, + exclude_keys: List[Hashable] = None, report_mode=comparison.ReportOptions.ALL, - description=None, - category=None, - actual_description=None, - expected_description=None, + description: str = None, + category: str = None, + actual_description: str = None, + expected_description: str = None, value_cmp_func=comparison.COMPARE_FUNCTIONS["native_equality"], ): self.value = value self.expected = expected + self.include_only_expected = include_only_expected self.include_keys = include_keys self.exclude_keys = exclude_keys self.actual_description = actual_description @@ -1337,9 +1339,10 @@ def evaluate(self): lhs=self.value, rhs=self.expected, ignore=self.exclude_keys, - only=self.include_keys, + include=self.include_keys, report_mode=self._report_mode, value_cmp_func=self._value_cmp_func, + include_only_rhs=self.include_only_expected, ) self.comparison = flatten_dict_comparison(cmp_result) return passed @@ -1353,15 +1356,16 @@ class FixMatch(DictMatch): def __init__( self, - value, - expected, - include_tags=None, - exclude_tags=None, + value: Dict, + expected: Dict, + include_only_expected: bool = False, + include_tags: List[Hashable] = None, + exclude_tags: List[Hashable] = None, report_mode=comparison.ReportOptions.ALL, - description=None, - category=None, - actual_description=None, - expected_description=None, + description: str = None, + category: str = None, + actual_description: str = None, + expected_description: str = None, ): """ If both FIX messages are typed, we enable strict type checking. @@ -1379,6 +1383,7 @@ def __init__( super(FixMatch, self).__init__( value=value, expected=expected, + include_only_expected=include_only_expected, include_keys=include_tags, exclude_keys=exclude_tags, report_mode=report_mode, diff --git a/testplan/testing/multitest/result.py b/testplan/testing/multitest/result.py index b8e830c47..d7f7b5b70 100644 --- a/testplan/testing/multitest/result.py +++ b/testplan/testing/multitest/result.py @@ -12,7 +12,7 @@ import re import threading from functools import wraps -from typing import Callable, Optional +from typing import Callable, Optional, Dict, Hashable, List, Any import functools from testplan import defaults @@ -938,17 +938,20 @@ def check( @assertion def match( self, - actual, - expected, - description=None, - category=None, - include_keys=None, - exclude_keys=None, + actual: Dict, + expected: Dict, + include_only_expected: bool = False, + description: str = None, + category: str = None, + include_keys: List[Hashable] = None, + exclude_keys: List[Hashable] = None, report_mode=comparison.ReportOptions.ALL, - actual_description=None, - expected_description=None, - value_cmp_func=comparison.COMPARE_FUNCTIONS["native_equality"], - ): + actual_description: str = None, + expected_description: str = None, + value_cmp_func: Callable[ + [Any, Any], bool + ] = comparison.COMPARE_FUNCTIONS["native_equality"], + ) -> assertions.DictMatch: r""" Matches two dictionaries, supports nested data. Custom comparators can be used as values on the ``expected`` dict. @@ -985,40 +988,31 @@ def match( ) :param actual: Original dictionary. - :type actual: ``dict``. :param expected: Comparison dictionary, can contain custom comparators (e.g. regex, lambda functions) - :type expected: ``dict`` + :param include_only_expected: Use the keys present in the expected dictionary. :param include_keys: Keys to exclusively consider in the comparison. - :type include_keys: ``list`` of ``object`` (items must be hashable) :param exclude_keys: Keys to ignore in the comparison. - :type exclude_keys: ``list`` of ``object`` (items must be hashable) :param report_mode: Specify which comparisons should be kept and reported. Default option is to report all comparisons but this can be restricted if desired. See ReportOptions enum for more detail. - :type report_mode: ``testplan.common.utils.comparison.ReportOptions`` :param actual_description: Column header description for original dict. - :type actual_description: ``str`` :param expected_description: Column header description for expected dict. - :type expected_description: ``str`` :param description: Text description for the assertion. - :type description: ``str`` :param category: Custom category that will be used for summarization. - :type category: ``str`` :param value_cmp_func: Function to use to compare values in expected and actual dicts. Defaults to using `operator.eq()`. - :type value_cmp_func: ``Callable[[Any, Any], bool]`` :return: Assertion pass status - :rtype: ``bool`` """ entry = assertions.DictMatch( value=actual, expected=expected, description=description, + include_only_expected=include_only_expected, include_keys=include_keys, exclude_keys=exclude_keys, report_mode=report_mode, @@ -1178,16 +1172,17 @@ def check( @assertion def match( self, - actual, - expected, - description=None, - category=None, - include_tags=None, - exclude_tags=None, + actual: Dict, + expected: Dict, + include_only_expected: bool = False, + description: str = None, + category: str = None, + include_tags: List[Hashable] = None, + exclude_tags: List[Hashable] = None, report_mode=comparison.ReportOptions.ALL, - actual_description=None, - expected_description=None, - ): + actual_description: str = None, + expected_description: str = None, + ) -> assertions.FixMatch: """ Matches two FIX messages, supports repeating groups (nested data). Custom comparators can be used as values on the ``expected`` msg. @@ -1212,38 +1207,30 @@ def match( ) :param actual: Original FIX message. - :type actual: ``dict`` :param expected: Expected FIX message, can include compiled regex patterns or callables for advanced comparison. - :type expected: ``dict`` + :param include_only_expected: Use the tags present in the expected message. :param include_tags: Tags to exclusively consider in the comparison. - :type include_tags: ``list`` of ``object`` (items must be hashable) :param exclude_tags: Keys to ignore in the comparison. - :type exclude_tags: ``list`` of ``object`` (items must be hashable) :param report_mode: Specify which comparisons should be kept and reported. Default option is to report all comparisons but this can be restricted if desired. See ReportOptions enum for more detail. - :type report_mode: ``testplan.common.utils.comparison.ReportOptions`` :param actual_description: Column header description for original msg. - :type actual_description: ``str`` :param expected_description: Column header description for expected msg. - :type expected_description: ``str`` :param description: Text description for the assertion. - :type description: ``str`` :param category: Custom category that will be used for summarization. - :type category: ``str`` :return: Assertion pass status - :rtype: ``bool`` """ entry = assertions.FixMatch( value=actual, expected=expected, description=description, category=category, + include_only_expected=include_only_expected, include_tags=include_tags, exclude_tags=exclude_tags, report_mode=report_mode, diff --git a/tests/unit/testplan/testing/multitest/test_result.py b/tests/unit/testplan/testing/multitest/test_result.py index 6118120a6..03313c1c5 100644 --- a/tests/unit/testplan/testing/multitest/test_result.py +++ b/tests/unit/testplan/testing/multitest/test_result.py @@ -730,6 +730,139 @@ def test_flattened_comparison_result(self, fix_ns): and _700[4] == (None, "ABSENT") # key not found in expected data ) + def test_subset_of_tags_with_include_tags_true(self, fix_ns): + """ + Test the comparison result when the expected FIX message with repeating groups is the subset of the actual + while include_tags set to True. + """ + + expected = { + 35: "D", + 55: 2, + 555: [ + { + 601: "A", + 683: [ + {689: "a"}, + {689: "b"}, + ], + }, + { + 601: "B", + 683: [ + {688: "c", 689: "c"}, + {688: "d"}, + ], + }, + ], + } + + actual = { + 35: "D", + 22: 5, + 55: 2, + 38: 5, + 555: [ + { + 600: "A", + 601: "A", + 683: [ + {688: "a", 689: "a"}, + {688: "b", 689: "b"}, + ], + }, + { + 600: "B", + 601: "B", + 55: 4, + 683: [ + {688: "c", 689: "c"}, + {688: "d", 689: "d"}, + ], + }, + ], + } + + assert fix_ns.match( + actual=actual, + expected=expected, + description="complex fix message comparison", + include_only_expected=True, + ) + assert len(fix_ns.result.entries) == 1 + + # Comparison result is a list of list items in below format: + # [indent, key, result, (act_type, act_value), (exp_type, exp_value)] + + comp_result = fix_ns.result.entries[0].comparison + + _35 = [item for item in comp_result if item[1] == 35][0] + assert _35[0] == 0 and _35[2][0].lower() == comparison.Match.PASS + _22 = [item for item in comp_result if item[1] == 22][0] + assert ( + _22[0] == 0 + and _22[2][0].lower() == comparison.Match.IGNORED + and _22[3][1] == "5" + and _22[4][1] == "ABSENT" + ) + _55_1, _55_2 = [item for item in comp_result if item[1] == 55] + assert _55_1[0] == 0 and _55_1[2][0].lower() == comparison.Match.PASS + assert ( + _55_2[0] == 1 + and _55_2[2][0].lower() == comparison.Match.IGNORED + and _55_2[3][1] == "4" + and _55_2[4][1] == "ABSENT" + ) + _38 = [item for item in comp_result if item[1] == 38][0] + assert ( + _38[0] == 0 + and _38[2][0].lower() == comparison.Match.IGNORED + and _38[3][1] == "5" + and _38[4][1] == "ABSENT" + ) + _555 = [item for item in comp_result if item[1] == 555][0] + assert _555[0] == 0 and _555[2][0].lower() == comparison.Match.PASS + _600 = [item for item in comp_result if item[1] == 600][0] + assert ( + _600[0] == 1 + and _600[2][0].lower() == comparison.Match.IGNORED + and _600[3][1] == "A" + and _600[4][1] == "ABSENT" + ) + _601 = [item for item in comp_result if item[1] == 601][0] + assert _601[0] == 1 and _601[2][0].lower() == comparison.Match.PASS + _683 = [item for item in comp_result if item[1] == 683][0] + assert _683[0] == 1 and _683[2][0].lower() == comparison.Match.PASS + _688_1, _688_2, _688_3, _688_4 = [ + item for item in comp_result if item[1] == 688 + ] + assert ( + _688_1[0] == 2 + and _688_1[2][0].lower() == comparison.Match.IGNORED + and _688_1[3][1] == "a" + and _688_1[4][1] == "ABSENT" + ) + assert ( + _688_2[0] == 2 + and _688_2[2][0].lower() == comparison.Match.IGNORED + and _688_2[3][1] == "b" + and _688_2[4][1] == "ABSENT" + ) + assert _688_3[0] == 2 and _688_3[2][0].lower() == comparison.Match.PASS + assert _688_4[0] == 2 and _688_4[2][0].lower() == comparison.Match.PASS + _689_1, _689_2, _689_3, _689_4 = [ + item for item in comp_result if item[1] == 689 + ] + assert _689_1[0] == 2 and _689_1[2][0].lower() == comparison.Match.PASS + assert _689_2[0] == 2 and _689_2[2][0].lower() == comparison.Match.PASS + assert _689_3[0] == 2 and _689_3[2][0].lower() == comparison.Match.PASS + assert ( + _689_4[0] == 2 + and _689_4[2][0].lower() == comparison.Match.IGNORED + and _689_4[3][1] == "d" + and _689_4[4][1] == "ABSENT" + ) + class TestResultBaseNamespace: """Test assertions and other methods in the base result.* namespace."""