diff --git a/examples/repeated_basic_example b/examples/repeated_basic_example new file mode 100644 index 0000000..848edbf --- /dev/null +++ b/examples/repeated_basic_example @@ -0,0 +1,7 @@ +This is an example to demonstrate the usage of the 'repeated' keyword, which enables one variable to have multiple captures on one line. + + +normaldata1.1 normaldata1.2 key1.1:data1.1, key1.2:data1.2, key1.3:data1.3, normaldata1.3 normaldata1.3 +normaldata2.1 normaldata2.2 key2.1:data2.1, key2.2:data2.2, normaldata2.3 normaldata2.4 +normaldata3.1 normaldata3.2 normaldata3.3 normaldata3.4 +normaldata4.1 normaldata4.2 key4.1:data4.1, key4.2:data4.2, key4.3:data4.3, normaldata4.3 normaldata4.3 diff --git a/examples/repeated_basic_template b/examples/repeated_basic_template new file mode 100644 index 0000000..9af6cb5 --- /dev/null +++ b/examples/repeated_basic_template @@ -0,0 +1,12 @@ +Value normaldata1 (\S+) +Value normaldata2 (\S+) +Value normaldata3 (\S+) +Value normaldata4 (\S+) +Value Repeated keything (\S+) +Value Repeated valuedata (\S+) +Value Repeated unusedRepeated (\S+) +Value List unused (\S+) + + +Start + ^${normaldata1}\s+${normaldata2} (${keything}:${valuedata},? )*${normaldata3}\s+${normaldata4} -> Record diff --git a/setup.py b/setup.py index e9ff894..55938aa 100755 --- a/setup.py +++ b/setup.py @@ -53,5 +53,5 @@ }, include_package_data=True, package_data={'textfsm': ['../testdata/*']}, - install_requires=['six', 'future'], + install_requires=['six', 'future', 'regex'], ) diff --git a/tests/textfsm_test.py b/tests/textfsm_test.py index ea74f00..ed977ab 100755 --- a/tests/textfsm_test.py +++ b/tests/textfsm_test.py @@ -24,10 +24,13 @@ from builtins import str import unittest from io import StringIO - - - import textfsm +from textfsm.parser import TextFSMTemplateError +try: + import regex as regexModule + useRegex = True +except ImportError: + useRegex = False class UnitTestFSM(unittest.TestCase): @@ -802,7 +805,7 @@ def testEnd(self): def testInvalidRegexp(self): - tplt = 'Value boo (.$*)\n\nStart\n ^$boo -> Next\n' + tplt = 'Value boo ([(\S+]))\n\nStart\n ^$boo -> Next\n' self.assertRaises(textfsm.TextFSMTemplateError, textfsm.TextFSM, StringIO(tplt)) @@ -910,6 +913,123 @@ def testTemplateValue(self): t = textfsm.TextFSM(f) self.assertEqual(str(t), buf_result) + def testRepeated(self): + """Repeated option should work ok.""" + tplt = """Value Repeated repeatedKey (\S+) +Value Repeated repeatedValue (\S+) +Value normalData (\S+) +Value normalData2 (\S+) +Value Repeated repeatedUnused (\S+) + +Start + ^${normalData} (${repeatedKey}:${repeatedValue} )*${normalData2} -> Record""" + + data = """ +normal1 key1:value1 key2:value2 key3:value3 normal2 \n +normal1 normal2 """ + + t = textfsm.TextFSM(StringIO(tplt)) + if useRegex is True: + result = t.ParseText(data) + self.assertListEqual( + [[['key1', 'key2', 'key3'], ['value1', 'value2', 'value3'], 'normal1', 'normal2', []], + [[], [], 'normal1', 'normal2', []]], + result) + else: + # test proper failure when falling back on re module + with self.assertRaises(TextFSMTemplateError, + msg="Expected a ModuleNotFoundError when using keyword 'Repeated' without 'regex' module"): + result = t.ParseText(data) + + def testRepeatedList(self): + """Keywords Repeated and List should work together""" + tplt = """Value List,Repeated repeatedKey (\S+) +Value Repeated,List repeatedValue (\S+) +Value Repeated,List repeatedUnused (\S+) + +Start + ^(${repeatedKey}:${repeatedValue} )+ + ^record -> Record""" + + data = """ +key1:value1 key2:value2 key3:value3 \n +key4:value4 key5:value5 key6:value6 \n +record""" + + t = textfsm.TextFSM(StringIO(tplt)) + if useRegex is True: + result = t.ParseText(data) + else: + return + + self.assertListEqual([[[['key1', 'key2', 'key3'], ['key4', 'key5', 'key6']], [['value1', 'value2', 'value3'], + ['value4', 'value5', 'value6']], []]], + result + ) + + def testRepeatedFilldown(self): + """Keywords Repeated and Filldown should work together""" + tplt = """Value Filldown,Repeated repeatedKey (\S+) +Value Repeated,Filldown repeatedValue (\S+) +Value Required otherMatch (\S+) + +Start + ^record -> Record + ^(${repeatedKey}:${repeatedValue} )+ + ^${otherMatch} + """ + + data = """ +key1:value1 key2:value2 key3:value3 \n +key4:value4 key5:value5 key6:value6 \n +foo \n +bar \n +record \n +foobar \n +record""" + + t = textfsm.TextFSM(StringIO(tplt)) + if useRegex is True: + result = t.ParseText(data) + else: + return + + self.assertListEqual([[['key4', 'key5', 'key6'], ['value4', 'value5', 'value6'], 'bar'], [['key4', 'key5', 'key6'], + ['value4', 'value5', 'value6'], 'foobar']], + result + ) + + def testRepeatedFillup(self): + """Keywords Repeated and Fillup should work together""" + tplt = """Value Fillup,Repeated repeatedKey (\S+) +Value Repeated,Fillup repeatedValue (\S+) +Value Required otherMatch (\S+) + +Start + ^record -> Record + ^(${repeatedKey}:${repeatedValue} )+ + ^${otherMatch} + """ + + data = """ +foo \n +bar \n +record \n +foobar \n +key1:value1 key2:value2 key3:value3 \n +record""" + + t = textfsm.TextFSM(StringIO(tplt)) + if useRegex is True: + result = t.ParseText(data) + else: + return + + self.assertListEqual([[['key1', 'key2', 'key3'], ['value1', 'value2', 'value3'], 'bar'], + [['key1', 'key2', 'key3'], ['value1', 'value2', 'value3'], 'foobar']], + result + ) + if __name__ == '__main__': unittest.main() diff --git a/textfsm/copyable_regex_object.py b/textfsm/copyable_regex_object.py index 5ac29da..e23b21f 100755 --- a/textfsm/copyable_regex_object.py +++ b/textfsm/copyable_regex_object.py @@ -16,8 +16,11 @@ """Work around a regression in Python 2.6 that makes RegexObjects uncopyable.""" +try: + import regex as regexModule +except ImportError: + import re as regexModule -import re from builtins import object # pylint: disable=redefined-builtin @@ -26,7 +29,7 @@ class CopyableRegexObject(object): def __init__(self, pattern): self.pattern = pattern - self.regex = re.compile(pattern) + self.regex = regexModule.compile(pattern) def match(self, *args, **kwargs): return self.regex.match(*args, **kwargs) diff --git a/textfsm/parser.py b/textfsm/parser.py index e892420..766d9cc 100755 --- a/textfsm/parser.py +++ b/textfsm/parser.py @@ -32,13 +32,21 @@ import getopt import inspect -import re import string import sys +import six from builtins import object # pylint: disable=redefined-builtin from builtins import str # pylint: disable=redefined-builtin from builtins import zip # pylint: disable=redefined-builtin -import six +import warnings +warnings.simplefilter("always") +try: + import regex as regexModule + useRegex = True +except ImportError: + import re as regexModule + warnings.warn("Could not locate regex module. Defaulting to re. Repeated keyword disabled.", ImportWarning) + useRegex = False class Error(Exception): @@ -125,6 +133,8 @@ def ValidOptions(cls): obj = getattr(cls, obj_name) if inspect.isclass(obj) and issubclass(obj, cls.OptionBase): valid_options.append(obj_name) + if useRegex is not True: + valid_options.remove("Repeated") return valid_options @classmethod @@ -132,6 +142,25 @@ def GetOption(cls, name): """Returns the class of the requested option name.""" return getattr(cls, name) + class Repeated(OptionBase): + """Will use regex module's 'captures' behavior to get all repeated + values instead of just the last value as re would.""" + + def OnAssignVar(self): + if useRegex is not True: + raise TextFSMTemplateError("Cannot use Repeated option without installing the regex module.") + self.value.value = self.value.values_list + + def OnCreateOptions(self): + self.value.value = [] + + def OnClearVar(self): + if 'Filldown' not in self.value.OptionNames(): + self.value.value = [] + + def OnClearAllVar(self): + self.value.value = [] + class Required(OptionBase): """The Value must be non-empty for the row to be recorded.""" @@ -147,6 +176,8 @@ def OnCreateOptions(self): def OnAssignVar(self): self._myvar = self.value.value + if "Repeated" in self.value.OptionNames(): + self._myvar = self.value.values_list def OnClearVar(self): self.value.value = self._myvar @@ -158,6 +189,10 @@ class Fillup(OptionBase): """Like Filldown, but upwards until it finds a non-empty entry.""" def OnAssignVar(self): + # make sure repeated OnAssignVar runs first so value.value is set + for option in self.value.options: + if option.name == "Repeated": + option.OnAssignVar() # If value is set, copy up the results table, until we # see a set item. if self.value.value: @@ -203,7 +238,10 @@ def OnAssignVar(self): if match and match.groupdict(): self._value.append(match.groupdict()) else: - self._value.append(self.value.value) + if "Repeated" in self.value.OptionNames(): + self._value.append(self.value.values_list) + else: + self._value.append(self.value.value) def OnClearVar(self): if 'Filldown' not in self.value.OptionNames(): @@ -247,12 +285,17 @@ def __init__(self, fsm=None, max_name_len=48, options_class=None): self.options = [] self.regex = None self.value = None + self.values_list = None self.fsm = fsm self._options_cls = options_class def AssignVar(self, value): """Assign a value to this Value.""" - self.value = value + try: + self.value = value[-1] + except IndexError: + self.value = "" + self.values_list = value # Call OnAssignVar on options. _ = [option.OnAssignVar() for option in self.options] @@ -314,20 +357,20 @@ def Parse(self, value): "Invalid Value name '%s' or name too long." % self.name) square_brackets = r'[^\]?\[[^]]*\]' - regex_without_brackets = re.sub(square_brackets, '', self.regex) - if (not re.match(r'^\(.*\)$', self.regex) or + regex_without_brackets = regexModule.sub(square_brackets, '', self.regex) + if (not regexModule.match(r'^\(.*\)$', self.regex) or regex_without_brackets.count('(') != regex_without_brackets.count(')')): raise TextFSMTemplateError( "Value '%s' must be contained within a '()' pair." % self.regex) - self.template = re.sub(r'^\(', '(?P<%s>' % self.name, self.regex) + self.template = regexModule.sub(r'^\(', '(?P<%s>' % self.name, self.regex) # Compile and store the regex object only on List-type values for use in # nested matching if any([isinstance(x, TextFSMOptions.List) for x in self.options]): try: - self.compiled_regex = re.compile(self.regex) - except re.error as e: + self.compiled_regex = regexModule.compile(self.regex) + except regexModule.error as e: raise TextFSMTemplateError(str(e)) def _AddOption(self, name): @@ -374,7 +417,7 @@ class CopyableRegexObject(object): def __init__(self, pattern): self.pattern = pattern - self.regex = re.compile(pattern) + self.regex = regexModule.compile(pattern) def match(self, *args, **kwargs): return self.regex.match(*args, **kwargs) @@ -411,7 +454,7 @@ class TextFSMRule(object): line_num: Integer row number of Value. """ # Implicit default is '(regexp) -> Next.NoRecord' - MATCH_ACTION = re.compile(r'(?P.*)(\s->(?P.*))') + MATCH_ACTION = regexModule.compile(r'(?P.*)(\s->(?P.*))') # The structure to the right of the '->'. LINE_OP = ('Continue', 'Next', 'Error') @@ -427,11 +470,11 @@ class TextFSMRule(object): NEWSTATE_RE = r'(?P\w+|\".*\")' # Compound operator (line and record) with optional new state. - ACTION_RE = re.compile(r'\s+%s(\s+%s)?$' % (OPERATOR_RE, NEWSTATE_RE)) + ACTION_RE = regexModule.compile(r'\s+%s(\s+%s)?$' % (OPERATOR_RE, NEWSTATE_RE)) # Record operator with optional new state. - ACTION2_RE = re.compile(r'\s+%s(\s+%s)?$' % (RECORD_OP_RE, NEWSTATE_RE)) + ACTION2_RE = regexModule.compile(r'\s+%s(\s+%s)?$' % (RECORD_OP_RE, NEWSTATE_RE)) # Default operators with optional new state. - ACTION3_RE = re.compile(r'(\s+%s)?$' % (NEWSTATE_RE)) + ACTION3_RE = regexModule.compile(r'(\s+%s)?$' % (NEWSTATE_RE)) def __init__(self, line, line_num=-1, var_map=None): """Initialise a new rule object. @@ -477,7 +520,7 @@ def __init__(self, line, line_num=-1, var_map=None): try: # Work around a regression in Python 2.6 that makes RE Objects uncopyable. self.regex_obj = CopyableRegexObject(self.regex) - except re.error: + except regexModule.error: raise TextFSMTemplateError( "Invalid regular expression: '%s'. Line: %s." % (self.regex, self.line_num)) @@ -521,7 +564,7 @@ def __init__(self, line, line_num=-1, var_map=None): # Check that an error message is present only with the 'Error' operator. if self.line_op != 'Error' and self.new_state: - if not re.match(r'\w+', self.new_state): + if not regexModule.match(r'\w+', self.new_state): raise TextFSMTemplateError( 'Alphanumeric characters only in state names. Line: %s.' % (self.line_num)) @@ -560,8 +603,8 @@ class TextFSM(object): """ # Variable and State name length. MAX_NAME_LEN = 48 - comment_regex = re.compile(r'^\s*#') - state_name_re = re.compile(r'^(\w+)$') + comment_regex = regexModule.compile(r'^\s*#') + state_name_re = regexModule.compile(r'^(\w+)$') _DEFAULT_OPTIONS = TextFSMOptions def __init__(self, template, options_class=_DEFAULT_OPTIONS): @@ -668,7 +711,7 @@ def _AppendRecord(self): self._ClearRecord() def _Parse(self, template): - """Parses template file for FSM structure. + """Parses template file for FSM structuregex. Args: template: Valid template file. @@ -940,8 +983,13 @@ def _CheckLine(self, line): for rule in self._cur_state: matched = self._CheckRule(rule, line) if matched: - for value in matched.groupdict(): - self._AssignVar(matched, value) + if useRegex is True: + for value in matched.capturesdict(): + self._AssignVar(matched, value) + else: + # workaround to fallback on re module if regex not imported + for value in matched.groupdict(): + self._AssignVar(matched, value) if self._Operations(rule, line): # Not a Continue so check for state transition. @@ -977,7 +1025,10 @@ def _AssignVar(self, matched, value): """ _value = self._GetValue(value) if _value is not None: - _value.AssignVar(matched.group(value)) + if useRegex: + _value.AssignVar(matched.captures(value)) + else: + _value.AssignVar([matched.group(value)]) def _Operations(self, rule, line): """Operators on the data record. diff --git a/textfsm/terminal.py b/textfsm/terminal.py index aa455cc..78964d4 100755 --- a/textfsm/terminal.py +++ b/textfsm/terminal.py @@ -163,12 +163,12 @@ def AnsiText(text, command_list=None, reset=True): def StripAnsiText(text): """Strip ANSI/SGR escape sequences from text.""" - return sgr_re.sub('', text) + return sgr_regex.sub('', text) def EncloseAnsiText(text): """Enclose ANSI/SGR escape sequences with ANSI_START and ANSI_END.""" - return sgr_re.sub(lambda x: ANSI_START + x.group(1) + ANSI_END, text) + return sgr_regex.sub(lambda x: ANSI_START + x.group(1) + ANSI_END, text) def TerminalSize(): @@ -199,7 +199,7 @@ def LineWrap(text, omit_sgr=False): def _SplitWithSgr(text_line): """Tokenise the line so that the sgr sequences can be omitted.""" - token_list = sgr_re.split(text_line) + token_list = sgr_regex.split(text_line) text_line_list = [] line_length = 0 for (index, token) in enumerate(token_list): @@ -207,7 +207,7 @@ def _SplitWithSgr(text_line): if token is '': continue - if sgr_re.match(token): + if sgr_regex.match(token): # Add sgr escape sequences without splitting or counting length. text_line_list.append(token) text_line = ''.join(token_list[index +1:])