Changeset 40919

Timestamp:

Oct 8, 2019, 11:42:36 AM (7 years ago)

Author:

fairlamb

Message:

Finished schema parsing tests

Location:

branches/ipp-259_genericise_backups/tools/backups/utils

Files:

• config_parse_helper.py (modified) (6 diffs)
• config_parse_helper_test.py (modified) (1 diff)

branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py

@@ -3 +3 @@
 from backups.utils.errors import ConfigError, ValidationError
 
-# The as configs should be parsed should be pretty uniform. There's a limited
-# amount that can be done with the actual loading.
-
-# Each backup file should be expecting particular lines from a config. So, this
-# means a schema can be made for what each backup expects.
-
 
 class SpecialSchemaProcessing(Enum):
-    """These enums define how a config line should be processed if it is a list
+    """These enums define how processing should be applied to a string to
+separate it out into list.
     """
     commma_separated = 1
@@ -18 +13 @@
 
 class ConfigSchemaLine():
-    """Details the name of key, whether the option is required,
-    and type expected"""
+    """Defines a config line by what the key is, if it is required or optional, 
+the type that the value should be processed as, and any special processing required
+"""
 
     def __init__(self, key: str, required: bool, expected_type, special_processing: SpecialSchemaProcessing=None):
@@ -30 +26 @@
         self.special_processing = special_processing
 
+    def __str__(self):
+        return (f'<{self.__class__.__name__} - '
+                f'Key:{self.key}, '
+                f'Required:{self.required}, '
+                f'Type:{self.expected_type}, '
+                f'Processing: {self.special_processing}'
+                f'>')
+
 
 class ConfigSchema():
-    """Defines the schema to be used"""
+    """Defines a schema to be used. Consists of a single section name and 
+multiple config lines"""
 
     def __init__(self, section_name, options: [ConfigSchemaLine]):
@@ -52 +57 @@
         self.options[line.key] = line
 
+    def key_in_options(self, key: str) -> bool:
+        return key in self.options.keys()
+
+    def get_schema_line(self, key: str) -> ConfigSchemaLine:
+        if not self.key_in_options(key):
+            raise KeyError(f"KeyError: '{key}' is not present in schema")
+        return self.options[key]
+
+    def __str__(self):
+        options_strs = [ o.__str__() for o in self.options]
+        return (f'{self.__class__.__name__} with {options_strs}')
+
 
 def _split_string_to_list(items: str, separator=',') -> [str]:
@@ -60 +77 @@
 
 def parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict:
-    # attempt to parse everything as intended.
+    """For a given config it will extract all values defined by the schema
+and return them as a dictionary of the keys and the assoicated value as the
+type defined by the schema
+"""
     parsed_items = {}
 
     section = schema.section_name
-    for csl in schema.options:
+    for csl_key in schema.options:
+        csl = schema.get_schema_line(csl_key)
+        print(csl)
 
-        if config.has_option(section, csl.key):
-
+        if config.has_option(section, csl_key):
             if csl.expected_type is bool:
                 parsed_items[csl.key] = config.getboolean(section, csl.key)
-            elif csl.expected is int:
+            elif csl.expected_type is int:
                 parsed_items[csl.key] = config.getint(section, csl.key)
             elif csl.expected_type is float:
@@ -79 +100 @@
                 if csl.special_processing == SpecialSchemaProcessing.commma_separated:
                     items = config[section][csl.key]
-                    parsed_items[csl.key] = _split_string_to_list(items, ' ')
-                    # parse comma_separated stuff
+                    splitup = _split_string_to_list(items, ',')
+                    parsed_items[csl.key] = splitup
                 elif csl.special_processing == SpecialSchemaProcessing.space_separated:
                     items = config[section][csl.key]
-                    parsed_items[csl.key] = _split_string_to_list(items, ',')
-                    # space separated processing
+                    splitup = _split_string_to_list(items, ' ')
+                    parsed_items[csl.key] = splitup
                 else:
-                    raise ConfigError(f"ConfigError: Don't know how to process special schema {csl.special_processing}")
+                    raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}")
             else:
-                raise ConfigError(f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}")
+                raise ConfigError(config, f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}")
 
         else:
             if csl.required:
-                raise ConfigError(f"ConfigError: Key '{csl.key}' is required but was not found in config")
+                raise ConfigError(config, f"ConfigError: Key '{csl.key}' is required but was not found in config")
 
     return parsed_items

branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py

@@ -134 +134 @@
 
 
-# class TestParsingConfig():
-
-#     def test_parsing_fails_with_missing_required_key(self):
-#         assert False is True
-
-#     def test_parsing_schema_with_ints(self):
-#         assert False is True
-
-#     def test_parsing_schema_with_floats(self):
-#         assert False is True
-
-#     def test_parsing_schema_with_bools(self):
-#         assert False is True
-
-#     def test_parsing_schema_with_strs(self):
-#         assert False is True
-
-#     def test_parsing_schema_with_lists_comma_separated(self):
-#         assert False is True
-
-#     def test_parsing_schema_with_lists_space_separated(self):
-#         assert False is True
-
-#     def test_parsing_schema_fails_with_invalid_key(self):
-#         assert False is True
-
-#     def test_parsing_schema_fails_with_unknown_type(self):
-#         assert False is True
-
-#     def test_parsing_schema_fails_with_unknown_special_enum(self):
-#         assert False is True
-
-# def parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict:
+class TestParsingConfig():
+
+    def make_simple_config(self, section_name='DEFAULT') -> helper.ConfigParser:
+        config = helper.ConfigParser()
+        config[section_name] = \
+        { 'inty'        : '3'
+        , 'floaty'      : '3.141'
+        , 'mr_bool'     : 'True'
+        , 'stringer'    : "bell"
+        , 'listo_comma' : "i,am,comma,separated"
+        , 'listo_space' : "yo it's time for space!"
+        }
+        return config
+
+    def test_parsing_fails_with_missing_required_key(self):
+        config = helper.ConfigParser()
+        config['DEFAULT'] = { 'not_the_required_key'   : False }
+
+        csl1 = helper.ConfigSchemaLine('Required_key', True, bool)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+
+        with pytest.raises(ConfigError):
+            helper.parse_config(config, schema)
+
+    def test_schema_of_only_optional_values_is_ok(self):
+        csl1 = helper.ConfigSchemaLine('not_required', False, bool)
+        csl2 = helper.ConfigSchemaLine('not_required_either', False, float)
+        schema = helper.ConfigSchema('DEFAULT', [csl1, csl2] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == {}
+
+    def test_parsing_schema_with_ints(self):
+        csl1 = helper.ConfigSchemaLine('inty', False, int)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == { 'inty' : 3}
+
+    def test_parsing_schema_with_floats(self):
+        csl1 = helper.ConfigSchemaLine('floaty', False, float)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == { 'floaty' : 3.141 }
+
+    def test_parsing_schema_with_bools(self):
+        csl1 = helper.ConfigSchemaLine('mr_bool', True, bool)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == { 'mr_bool' : True }
+
+    def test_parsing_schema_with_strs(self):
+        csl1 = helper.ConfigSchemaLine('stringer', True, str)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == { 'stringer' : 'bell' }
+
+    def test_parsing_schema_with_lists_comma_separated(self):
+        csl1 = helper.ConfigSchemaLine('listo_comma', False, list,
+            helper.SpecialSchemaProcessing.commma_separated)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == { 'listo_comma' : [ "i"
+                                           , "am"
+                                           , "comma"
+                                           , "separated"
+                                           ]
+                         }
+
+    def test_parsing_schema_with_lists_space_separated(self):
+        csl1 = helper.ConfigSchemaLine('listo_space', False, list,
+            helper.SpecialSchemaProcessing.space_separated)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        result = helper.parse_config(config, schema)
+        assert result == { 'listo_space' : [ "yo"
+                                           , "it's"
+                                           , "time"
+                                           , "for"
+                                           , "space!"
+                                           ]
+                         }
+
+    def test_parsing_as_different_types_always_ok_for_str_and_list(self):
+        config = self.make_simple_config()
+
+        csl_float = helper.ConfigSchemaLine('floaty', True, float)
+        csl_int   = helper.ConfigSchemaLine('floaty', True, int)
+        csl_bool  = helper.ConfigSchemaLine('floaty', True, bool)
+        csl_str   = helper.ConfigSchemaLine('floaty', True, str)
+        csl_list  = helper.ConfigSchemaLine('floaty', True, list,
+            helper.SpecialSchemaProcessing.commma_separated)
+
+        schema = helper.ConfigSchema('DEFAULT', [csl_float])
+        result = helper.parse_config(config, schema)
+        assert result == { 'floaty' : 3.141 }
+        assert type(result['floaty']) is float
+
+        schema = helper.ConfigSchema('DEFAULT', [csl_int])
+        with pytest.raises(ValueError):
+            helper.parse_config(config, schema)
+
+        schema = helper.ConfigSchema('DEFAULT', [csl_bool])
+        with pytest.raises(ValueError):
+            helper.parse_config(config, schema)
+
+        schema = helper.ConfigSchema('DEFAULT', [csl_str])
+        result = helper.parse_config(config, schema)
+        assert result == { 'floaty' : '3.141' }
+        assert type(result['floaty']) is str
+
+        schema = helper.ConfigSchema('DEFAULT', [csl_list])
+        result = helper.parse_config(config, schema)
+        assert result == { 'floaty' : ['3.141'] }
+        assert type(result['floaty']) is list
+
+    def test_parsing_schema_fails_with_unknown_type(self):
+        csl1 = helper.ConfigSchemaLine('floaty', True, dict)
+        schema = helper.ConfigSchema('DEFAULT', [csl1] )
+        config = self.make_simple_config()
+        with pytest.raises(ConfigError):
+            helper.parse_config(config, schema)
+
+    def test_all_special_processing_options_are_handled(self):
+        config = self.make_simple_config()
+        config['DEFAULT'] = {'enum_checker' : "comma, and spaces"}
+
+        for e in helper.SpecialSchemaProcessing:
+            csl = helper.ConfigSchemaLine('enum_checker', False, list, e)
+            schema = helper.ConfigSchema('DEFAULT', [csl])
+
+            result = helper.parse_config(config, schema)
+            assert type(result['enum_checker']) is list
+
+    def test_parsing_all(self):
+        config = self.make_simple_config('FULL_TEST')
+
+        csl1 = helper.ConfigSchemaLine('inty', False, int)
+        csl2 = helper.ConfigSchemaLine('floaty', False, float)
+        csl3 = helper.ConfigSchemaLine('mr_bool', True, bool)
+        csl4 = helper.ConfigSchemaLine('stringer', True, str)
+        csl5 = helper.ConfigSchemaLine('listo_comma', False, list,
+            helper.SpecialSchemaProcessing.commma_separated)
+        csl6 = helper.ConfigSchemaLine('listo_space', False, list,
+            helper.SpecialSchemaProcessing.space_separated)
+        schema = helper.ConfigSchema('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6])
+
+        result = helper.parse_config(config, schema)
+        assert result == \
+        { 'inty'        : 3
+        , 'floaty'      : 3.141
+        , 'mr_bool'     : True
+        , 'stringer'    : "bell"
+        , 'listo_comma' : ["i", "am", "comma", "separated"]
+        , 'listo_space' : ["yo", "it's", "time", "for", "space!"]
+        }