From 0edbf7686b96eb79da38db4a4041823f54224d06 Mon Sep 17 00:00:00 2001 From: Andreas Maier Date: Sun, 12 Nov 2023 21:38:33 +0100 Subject: [PATCH] Support for parsing complex name-value options Signed-off-by: Andreas Maier --- minimum-constraints.txt | 2 +- requirements.txt | 2 + tests/function/test_parse_name_value.py | 606 ++++++++++++++++++++++++ zhmccli/__init__.py | 1 + zhmccli/_parse_name_value.py | 180 +++++++ 5 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 tests/function/test_parse_name_value.py create mode 100644 zhmccli/_parse_name_value.py diff --git a/minimum-constraints.txt b/minimum-constraints.txt index d077c395..d8961a2b 100644 --- a/minimum-constraints.txt +++ b/minimum-constraints.txt @@ -56,6 +56,7 @@ prompt-toolkit==2.0.1 # PyYAML is pulled in by zhmccli, zhmcclient, yamlloader PyYAML==5.3.1 +pyparsing==3.1.0 jsonschema==3.0.1 yamlloader==0.5.5 @@ -208,7 +209,6 @@ pathlib2==2.3.3; python_version < '3.4' and sys_platform != 'win32' # twine 3.0.0 depends on pkginfo>=1.4.2 pkginfo==1.4.2 py==1.11.0 -pyparsing==3.0.7 # tox 4.0.0 started using pyproject-api and requires pyproject-api>=1.2.1 pyproject-api==1.2.1; python_version >= '3.7' # build is using pyproject-hooks diff --git a/requirements.txt b/requirements.txt index c1faceb8..e154d58c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,8 @@ prompt-toolkit>=2.0.1 PyYAML>=5.3.1,!=5.4.0,!=5.4.1; python_version <= '3.11' PyYAML>=5.3.1,!=5.4.0,!=5.4.1,!=6.0.0; python_version >= '3.12' +pyparsing>=3.1.0 + jsonschema>=3.0.1 yamlloader>=0.5.5 diff --git a/tests/function/test_parse_name_value.py b/tests/function/test_parse_name_value.py new file mode 100644 index 00000000..e4564b1e --- /dev/null +++ b/tests/function/test_parse_name_value.py @@ -0,0 +1,606 @@ +""" +Unit test for '_parse_name_value' module. +""" + +from __future__ import absolute_import, print_function + +import re +import pytest + +from zhmccli import NameValueParseError, parse_name_value + + +TESTCASES_PARSE_NAME_VALUE = [ + # Testcases for test_parse_name_value() + # + # Each list item is a testcase with the following tuple items: + # * desc (str) - Testcase description. + # * input_args (list) - Positional arguments for parse_name_value(). + # * input_kwargs (dict) - Keyword arguments for parse_name_value(). + # * exp_result (object) - Expected return value from parse_name_value() if + # expected to succeed, or None if expected to fail. + # * exp_exc_type (exception) - Expected exception type if expected to fail, + # or None if expected to succeed. + # * exp_exc_message (str) - Regexp pattern to match expected exception + # message if expected to fail, or None if expected to succeed. + + ( + r"""Simple null value 'null'.""", + [], + dict(value=r'null'), + None, + None, None + ), + + ( + r"""Simple null value 'null' as positional arg.""", + [r'null'], + dict(), + None, + None, None + ), + + ( + r"""Simple bool 'true'.""", + [], + dict(value=r'true'), + True, + None, None + ), + ( + r"""Simple bool 'false'.""", + [], + dict(value=r'false'), + False, + None, None + ), + + ( + r"""Simple int '42'.""", + [], + dict(value=r'42'), + 42, + None, None + ), + + ( + r"""Simple float '42.0'.""", + [], + dict(value=r'42.0'), + 42.0, + None, None + ), + ( + r"""Simple float '.1'.""", + [], + dict(value=r'.1'), + 0.1, + None, None + ), + ( + r"""Simple float '1.'.""", + [], + dict(value=r'1.'), + 1.0, + None, None + ), + + ( + r"""Simple quoted string '"abc"'.""", + [], + dict(value=r'"abc"'), + 'abc', + None, None + ), + ( + r"""Simple quoted string '"42"'.""", + [], + dict(value=r'"42"'), + '42', + None, None + ), + ( + r"""Simple quoted string '"true"'.""", + [], + dict(value=r'"true"'), + 'true', + None, None + ), + ( + r"""Simple quoted string '"false"'.""", + [], + dict(value=r'"false"'), + 'false', + None, None + ), + ( + r"""Simple quoted string '"null"'.""", + [], + dict(value=r'"null"'), + 'null', + None, None + ), + ( + r"""Simple unquoted string 'abc'.""", + [], + dict(value=r'abc'), + 'abc', + None, None + ), + ( + r"""Simple unquoted string 'a'.""", + [], + dict(value=r'a'), + 'a', + None, None + ), + ( + r"""Simple quoted string with blank '"a b"'.""", + [], + dict(value=r'"a b"'), + 'a b', + None, None + ), + ( + r"""Simple quoted string with escaped newline '"a\nb"'.""", + [], + dict(value=r'"a\nb"'), + 'a\nb', + None, None + ), + ( + r"""Simple quoted string with escaped carriage return '"a\rb"'.""", + [], + dict(value=r'"a\rb"'), + 'a\rb', + None, None + ), + ( + r"""Simple quoted string with escaped tab '"a\tb"'.""", + [], + dict(value=r'"a\tb"'), + 'a\tb', + None, None + ), + ( + r"""Simple quoted string with escaped backslash '"a\\b"'.""", + [], + dict(value=r'"a\\b"'), + 'a\\b', + None, None + ), + ( + r"""Invalid simple unquoted string with blank 'a b'.""", + [], + dict(value=r'a b'), + None, + NameValueParseError, r".*: Expected end of text, found 'b'.*" + ), + ( + r"""Invalid simple unquoted string with newline 'a\nb'.""", + [], + dict(value=r'a\nb'), + None, + NameValueParseError, r".*: Expected end of text, found '\\'.*" + ), + + ( + r"""Empty array.""", + [], + dict(value=r'[]'), + [], + None, None + ), + ( + r"""Array with one null value 'null'.""", + [], + dict(value=r'[null]'), + [None], + None, None + ), + ( + r"""Array with one boolean 'true'.""", + [], + dict(value=r'[true]'), + [True], + None, None + ), + ( + r"""Array with one boolean 'false'.""", + [], + dict(value=r'[false]'), + [False], + None, None + ), + ( + r"""Array with one int '42'.""", + [], + dict(value=r'[42]'), + [42], + None, None + ), + ( + r"""Array with one float '42.0'.""", + [], + dict(value=r'[42.0]'), + [42.0], + None, None + ), + ( + r"""Array with one quoted string '"null"'.""", + [], + dict(value=r'["null"]'), + ['null'], + None, None + ), + ( + r"""Array with one quoted string '"true"'.""", + [], + dict(value=r'["true"]'), + ['true'], + None, None + ), + ( + r"""Array with one quoted string '"false"'.""", + [], + dict(value=r'["false"]'), + ['false'], + None, None + ), + ( + r"""Array with one quoted string '"a"'.""", + [], + dict(value=r'["a"]'), + ['a'], + None, None + ), + ( + r"""Array with one unquoted string 'a'.""", + [], + dict(value=r'[a]'), + ['a'], + None, None + ), + ( + r"""Array with one quoted string '"a]"'.""", + [], + dict(value=r'["a]"]'), + ['a]'], + None, None + ), + ( + r"""Array with one quoted string '"[a"'.""", + [], + dict(value=r'["[a"]'), + ['[a'], + None, None + ), + ( + r"""Array with two int items.""", + [], + dict(value=r'[42,43]'), + [42, 43], + None, None + ), + ( + r"""Invald array with trailing comma after one item.""", + [], + dict(value=r'[42,]'), + None, + NameValueParseError, r".*: , found ','.*" + ), + ( + r"""Invald array with leading comma before one item.""", + [], + dict(value=r'[,42]'), + None, + NameValueParseError, r".*: , found ','.*" + ), + ( + r"""Invald array with two commas between two items.""", + [], + dict(value=r'[42,,43]'), + None, + NameValueParseError, r".*: , found ','.*" + ), + ( + r"""Invalid array with missing closing ']'.""", + [], + dict(value=r'[42'), + None, + NameValueParseError, r".*: , found end of text.*" + ), + ( + r"""Invalid array with missing opening '['.""", + [], + dict(value=r'42]'), + None, + NameValueParseError, r".*: Expected end of text, found ']'.*" + ), + + ( + r"""Array with one nested empty array.""", + [], + dict(value=r'[[]]'), + [[]], + None, None + ), + ( + r"""Array with one nested empty object.""", + [], + dict(value=r'[{}]'), + [dict()], + None, None + ), + ( + r"""Array with one nested array that has one item.""", + [], + dict(value=r'[[42]]'), + [[42]], + None, None + ), + ( + r"""Array with one nested object that has one item.""", + [], + dict(value=r'[{x=42}]'), + [dict(x=42)], + None, None + ), + ( + r"""Array with two nested array items.""", + [], + dict(value=r'[[42,43],[]]'), + [[42, 43], []], + None, None + ), + ( + r"""Array with mixed array items.""", + [], + dict(value=r'[null,[]]'), + [None, []], + None, None + ), + + ( + r"""Empty object.""", + [], + dict(value=r'{}'), + dict(), + None, None + ), + ( + r"""Object with one null item 'null'.""", + [], + dict(value=r'{x=null}'), + dict(x=None), + None, None + ), + ( + r"""Object with one boolean item 'true'.""", + [], + dict(value=r'{x=true}'), + dict(x=True), + None, None + ), + ( + r"""Object with one boolean item 'false'.""", + [], + dict(value=r'{x=false}'), + dict(x=False), + None, None + ), + ( + r"""Object with one int item '42'.""", + [], + dict(value=r'{x=42}'), + dict(x=42), + None, None + ), + ( + r"""Object with one float item '42.0'.""", + [], + dict(value=r'{x=42.0}'), + dict(x=42.0), + None, None + ), + ( + r"""Object with one quoted string item '"null"'.""", + [], + dict(value=r'{x="null"}'), + dict(x='null'), + None, None + ), + ( + r"""Object with one quoted string item '"true"'.""", + [], + dict(value=r'{x="true"}'), + dict(x='true'), + None, None + ), + ( + r"""Object with one quoted string item '"false"'.""", + [], + dict(value=r'{x="false"}'), + dict(x='false'), + None, None + ), + ( + r"""Object with one quoted string item '"a"'.""", + [], + dict(value=r'{x="a"}'), + dict(x='a'), + None, None + ), + ( + r"""Object with one unquoted string item 'a'.""", + [], + dict(value=r'{x=a}'), + dict(x='a'), + None, None + ), + ( + r"""Object with one quoted string item '"a]"'.""", + [], + dict(value=r'{x="a]"}'), + dict(x='a]'), + None, None + ), + ( + r"""Object with one quoted string item '"[a"'.""", + [], + dict(value=r'{x="[a"}'), + dict(x='[a'), + None, None + ), + ( + r"""Object with two int items.""", + [], + dict(value=r'{x=42,y=43}'), + dict(x=42, y=43), + None, None + ), + ( + r"""Object with an item that has a name with hyphen.""", + [], + dict(value=r'{abc-def=42}'), + {'abc-def': 42}, + None, None + ), + ( + r"""Object with an item that has a name with underscore.""", + [], + dict(value=r'{abc_def=42}'), + {'abc_def': 42}, + None, None + ), + ( + r"""Object with an item that has a name with numeric.""", + [], + dict(value=r'{a0=42}'), + {'a0': 42}, + None, None + ), + ( + r"""Object with an item that has a name with leading hyphen.""", + [], + dict(value=r'{-a=42}'), + None, + NameValueParseError, r".*: , found '-'.*" + ), + ( + r"""Object with an item that has a name with leading underscore.""", + [], + dict(value=r'{_a=42}'), + None, + NameValueParseError, r".*: , found '_'.*" + ), + ( + r"""Object with an item that has a name with leading numeric.""", + [], + dict(value=r'{0x=42}'), + None, + NameValueParseError, r".*: , found '0x'.*" + ), + ( + r"""Invald object with an item that has a name with '='""", + [], + dict(value=r'{a=b=42}'), + None, + NameValueParseError, r".*: , found '='.*" + ), + ( + r"""Invald object with trailing comma after one item.""", + [], + dict(value=r'{x=42,}'), + None, + NameValueParseError, r".*: , found ','.*" + ), + ( + r"""Invald object with leading comma before one item.""", + [], + dict(value=r'{x=,42}'), + None, + NameValueParseError, r".*: , found 'x'.*" + ), + ( + r"""Invald object with two commas between two items.""", + [], + dict(value=r'{x=42,,y=43}'), + None, + NameValueParseError, r".*: , found ','.*" + ), + ( + r"""Invalid object with missing closing '}'.""", + [], + dict(value=r'{x=42'), + None, + NameValueParseError, r".*: , found end of text.*" + ), + ( + r"""Invalid object with missing opening '{'.""", + [], + dict(value=r'x=42}'), + None, + NameValueParseError, r".*: Expected end of text, found '='.*" + ), + + ( + r"""Object with nested empty object.""", + [], + dict(value=r'{x={}}'), + dict(x=dict()), + None, None + ), + ( + r"""Object with nested empty array.""", + [], + dict(value=r'{x=[]}'), + dict(x=[]), + None, None + ), + ( + r"""Object with nested object that has one item.""", + [], + dict(value=r'{x={y=42}}'), + dict(x=dict(y=42)), + None, None + ), + ( + r"""Object with nested array that has one item.""", + [], + dict(value=r'{x=[42]}'), + dict(x=[42]), + None, None + ), + +] + + +@pytest.mark.parametrize( + "desc, input_args, input_kwargs, exp_result, exp_exc_type, exp_exc_message", + TESTCASES_PARSE_NAME_VALUE) +def test_parse_name_value( + desc, input_args, input_kwargs, exp_result, exp_exc_type, + exp_exc_message): + # pylint: disable=unused-argument + """ + Test test_parse_name_value() function. + """ + + if exp_exc_type is None: + + # The code to be tested + result = parse_name_value(*input_args, **input_kwargs) + + assert result == exp_result + + else: + + with pytest.raises(exp_exc_type) as exc_info: + + # The code to be tested + parse_name_value(*input_args, **input_kwargs) + + exc_value = exc_info.value + exc_message = str(exc_value) + assert re.match(exp_exc_message, exc_message) diff --git a/zhmccli/__init__.py b/zhmccli/__init__.py index f4186b7a..e39f9995 100644 --- a/zhmccli/__init__.py +++ b/zhmccli/__init__.py @@ -19,6 +19,7 @@ from __future__ import absolute_import from ._version import * # noqa: F401 +from ._parse_name_value import * # noqa: F401 from ._cmd_info import * # noqa: F401 from ._cmd_session import * # noqa: F401 from ._cmd_console import * # noqa: F401 diff --git a/zhmccli/_parse_name_value.py b/zhmccli/_parse_name_value.py new file mode 100644 index 00000000..d1635227 --- /dev/null +++ b/zhmccli/_parse_name_value.py @@ -0,0 +1,180 @@ +""" +Parser for complex name-value strings. +""" + +from __future__ import absolute_import + +import pyparsing +from pyparsing import Forward, Opt, Group, nums, alphas, Literal, Char, Word, \ + CharsNotIn, QuotedString, DelimitedList, Suppress, ParseResults +# from pyparsing import trace_parse_action + +__all__ = ['NameValueParseError', 'parse_name_value'] + + +# pyparsing grammar for complex name-value strings +NULL_VALUE = Literal('null') +BOOL_VALUE = Literal('true') | Literal('false') +INT_VALUE = Word(nums) +FLOAT_VALUE = Word(nums) + Char('.') + Opt(Word(nums)) | Char('.') + \ + Word(nums) +QUOTED_STRING_VALUE = QuotedString(quote_char='"', esc_char='\\') +PLAIN_STRING_VALUE = CharsNotIn(' \\[]{}(),=') +SIMPLE_VALUE = NULL_VALUE | BOOL_VALUE | FLOAT_VALUE | INT_VALUE | \ + QUOTED_STRING_VALUE | PLAIN_STRING_VALUE +COMPLEX_VALUE = Forward() +NAME = Word(alphas, alphas + nums + '-_') +NAMED_VALUE = Group(NAME + Suppress('=') + COMPLEX_VALUE) +ARRAY_VALUE = Suppress('[') + Opt(DelimitedList(COMPLEX_VALUE, delim=',')) + \ + Suppress(']') +OBJECT_VALUE = Suppress('{') + Opt(DelimitedList(NAMED_VALUE, delim=',')) + \ + Suppress('}') +# pylint: disable=pointless-statement +COMPLEX_VALUE << (ARRAY_VALUE | OBJECT_VALUE | SIMPLE_VALUE) + +# pyparsing parse actions + +# For debugging parse actions, there are two possibilities: +# * Add @pyparsing.trace_parse_action decorator to parse action +# * Invoke set_debug(True) on grammar element + + +def as_python_list(toks): + """Return parsed array as Python list""" + return ParseResults.List(toks.as_list()) + + +ARRAY_VALUE.add_parse_action(as_python_list) + + +def as_python_dict(toks): + """Return parsed object as Python dict""" + result = {} + for name, value in toks: + result[name] = value + return result + + +OBJECT_VALUE.add_parse_action(as_python_dict) + + +def as_python_int(toks): + """Return parsed int string as Python int""" + assert len(toks) == 1 + return int(toks[0]) + + +INT_VALUE.add_parse_action(as_python_int) + + +def as_python_float(toks): + """Return parsed float string as Python float""" + float_str = ''.join(toks) + return float(float_str) + + +FLOAT_VALUE.add_parse_action(as_python_float) + + +def as_python_bool(toks): + """Return parsed bool string as Python bool""" + assert len(toks) == 1 + mapping = { + 'true': True, + 'false': False, + } + return mapping[toks[0]] + + +BOOL_VALUE.add_parse_action(as_python_bool) + + +def as_python_null(toks): + """Return parsed null string as Python None""" + assert len(toks) == 1 + mapping = { + 'null': None, + } + # `None` can only be returned by using the ability of parse actions to + # modify their ParseResults argument in place. + toks[0] = mapping[toks[0]] + + +NULL_VALUE.add_parse_action(as_python_null) + + +class NameValueParseError(Exception): + # pylint: disable=too-few-public-methods + """ + Indicates an error when parsing a complex name-value string. + """ + pass + + +def parse_name_value(value): + """ + Parse a complex name-value string and return the Python data object + representing the value. + + The syntax for the complex name-value string is as follows: + + * ``complex-name-value := simple-value | array-value | object-value`` + + * ``simple-value`` is a value of a simple type, as follows: + + - none/null: ``null``. + Will be returned as Python ``None``. + - bool: ``true`` or ``false``. + Will be returned as Python ``True`` or ``False``. + - int: decimal string representaton, parseable by ``int()``. + Will be returned as a Python ``int`` value. + - float: string representaton that is parseable by ``float()``. + Will be returned as a Python ``float`` value. + - string: string value, with character range and representation as for + JSON string values, surrounded by double quotes. The surrounding double + quotes can be omitted if the (escaped) string value does not contain + any of `` \\[]{}(),=``, and if it cannot be interpreted as a value + of one of the other simple types. + Will be returned as an unescaped Python ``str`` value. + + * ``array-value`` is a value of array/list type, specified as a + comma-separated list of ``complex-name-value``, surrounded by + ``[`` and ``]``. + Will be returned as a Python ``list`` value. + + * ``object-value`` is a value of object/dict type, specified as a + comma-separated list of ``name=complex-name-value``, surrounded by + ``{`` and ``}``. ``name`` must begin with an alphabetic character, + and may contaon zero or more alphanumeric characters and ``-_``. + Will be returned as a Python ``dict`` value. + + * Surrounding blanks are stripped from ``complex-name-value``. + + Examples (using the defaults for `item_sep` and `name_value_sep`): + + * ``42`` + * ``abc`` + * ``"ab c"`` + * ``"ab\nc"`` + * ``[abc,42]`` + * ``true`` # boolean true + * ``"true"`` # string "true" + * ``{interface-name=abc,port=42}`` + * ``{description="ab\tc",host-addresses=[10.11.12.13,null]}`` + + Parameters: + + value(str): The complex name-value string to be parsed. + + Returns: + object: Python object representing the complex name-value string. + + Raises: + NameValueParseError: Parsing error. + """ + try: + parse_result = COMPLEX_VALUE.parse_string(value, parse_all=True) + except pyparsing.ParseException as exc: + raise NameValueParseError( + "Cannot parse complex value {!r}: {}".format(value, exc)) + return parse_result[0]