Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40918) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40919) @@ -3,13 +3,8 @@ from backups.utils.errors import ConfigError, ValidationError -# The as configs should be parsed should be pretty uniform. There's a limited -# amount that can be done with the actual loading. - -# Each backup file should be expecting particular lines from a config. So, this -# means a schema can be made for what each backup expects. - class SpecialSchemaProcessing(Enum): - """These enums define how a config line should be processed if it is a list + """These enums define how processing should be applied to a string to +separate it out into list. """ commma_separated = 1 @@ -18,6 +13,7 @@ class ConfigSchemaLine(): - """Details the name of key, whether the option is required, - and type expected""" + """Defines a config line by what the key is, if it is required or optional, +the type that the value should be processed as, and any special processing required +""" def __init__(self, key: str, required: bool, expected_type, special_processing: SpecialSchemaProcessing=None): @@ -30,7 +26,16 @@ self.special_processing = special_processing + def __str__(self): + return (f'<{self.__class__.__name__} - ' + f'Key:{self.key}, ' + f'Required:{self.required}, ' + f'Type:{self.expected_type}, ' + f'Processing: {self.special_processing}' + f'>') + class ConfigSchema(): - """Defines the schema to be used""" + """Defines a schema to be used. Consists of a single section name and +multiple config lines""" def __init__(self, section_name, options: [ConfigSchemaLine]): @@ -52,4 +57,16 @@ self.options[line.key] = line + def key_in_options(self, key: str) -> bool: + return key in self.options.keys() + + def get_schema_line(self, key: str) -> ConfigSchemaLine: + if not self.key_in_options(key): + raise KeyError(f"KeyError: '{key}' is not present in schema") + return self.options[key] + + def __str__(self): + options_strs = [ o.__str__() for o in self.options] + return (f'{self.__class__.__name__} with {options_strs}') + def _split_string_to_list(items: str, separator=',') -> [str]: @@ -60,15 +77,19 @@ def parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: - # attempt to parse everything as intended. + """For a given config it will extract all values defined by the schema +and return them as a dictionary of the keys and the assoicated value as the +type defined by the schema +""" parsed_items = {} section = schema.section_name - for csl in schema.options: + for csl_key in schema.options: + csl = schema.get_schema_line(csl_key) + print(csl) - if config.has_option(section, csl.key): - + if config.has_option(section, csl_key): if csl.expected_type is bool: parsed_items[csl.key] = config.getboolean(section, csl.key) - elif csl.expected is int: + elif csl.expected_type is int: parsed_items[csl.key] = config.getint(section, csl.key) elif csl.expected_type is float: @@ -79,18 +100,18 @@ if csl.special_processing == SpecialSchemaProcessing.commma_separated: items = config[section][csl.key] - parsed_items[csl.key] = _split_string_to_list(items, ' ') - # parse comma_separated stuff + splitup = _split_string_to_list(items, ',') + parsed_items[csl.key] = splitup elif csl.special_processing == SpecialSchemaProcessing.space_separated: items = config[section][csl.key] - parsed_items[csl.key] = _split_string_to_list(items, ',') - # space separated processing + splitup = _split_string_to_list(items, ' ') + parsed_items[csl.key] = splitup else: - raise ConfigError(f"ConfigError: Don't know how to process special schema {csl.special_processing}") + raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}") else: - raise ConfigError(f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}") + raise ConfigError(config, f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}") else: if csl.required: - raise ConfigError(f"ConfigError: Key '{csl.key}' is required but was not found in config") + raise ConfigError(config, f"ConfigError: Key '{csl.key}' is required but was not found in config") return parsed_items Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py (revision 40918) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py (revision 40919) @@ -134,35 +134,162 @@ -# class TestParsingConfig(): - -# def test_parsing_fails_with_missing_required_key(self): -# assert False is True - -# def test_parsing_schema_with_ints(self): -# assert False is True - -# def test_parsing_schema_with_floats(self): -# assert False is True - -# def test_parsing_schema_with_bools(self): -# assert False is True - -# def test_parsing_schema_with_strs(self): -# assert False is True - -# def test_parsing_schema_with_lists_comma_separated(self): -# assert False is True - -# def test_parsing_schema_with_lists_space_separated(self): -# assert False is True - -# def test_parsing_schema_fails_with_invalid_key(self): -# assert False is True - -# def test_parsing_schema_fails_with_unknown_type(self): -# assert False is True - -# def test_parsing_schema_fails_with_unknown_special_enum(self): -# assert False is True - -# def parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: +class TestParsingConfig(): + + def make_simple_config(self, section_name='DEFAULT') -> helper.ConfigParser: + config = helper.ConfigParser() + config[section_name] = \ + { 'inty' : '3' + , 'floaty' : '3.141' + , 'mr_bool' : 'True' + , 'stringer' : "bell" + , 'listo_comma' : "i,am,comma,separated" + , 'listo_space' : "yo it's time for space!" + } + return config + + def test_parsing_fails_with_missing_required_key(self): + config = helper.ConfigParser() + config['DEFAULT'] = { 'not_the_required_key' : False } + + csl1 = helper.ConfigSchemaLine('Required_key', True, bool) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + + with pytest.raises(ConfigError): + helper.parse_config(config, schema) + + def test_schema_of_only_optional_values_is_ok(self): + csl1 = helper.ConfigSchemaLine('not_required', False, bool) + csl2 = helper.ConfigSchemaLine('not_required_either', False, float) + schema = helper.ConfigSchema('DEFAULT', [csl1, csl2] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == {} + + def test_parsing_schema_with_ints(self): + csl1 = helper.ConfigSchemaLine('inty', False, int) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == { 'inty' : 3} + + def test_parsing_schema_with_floats(self): + csl1 = helper.ConfigSchemaLine('floaty', False, float) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == { 'floaty' : 3.141 } + + def test_parsing_schema_with_bools(self): + csl1 = helper.ConfigSchemaLine('mr_bool', True, bool) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == { 'mr_bool' : True } + + def test_parsing_schema_with_strs(self): + csl1 = helper.ConfigSchemaLine('stringer', True, str) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == { 'stringer' : 'bell' } + + def test_parsing_schema_with_lists_comma_separated(self): + csl1 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == { 'listo_comma' : [ "i" + , "am" + , "comma" + , "separated" + ] + } + + def test_parsing_schema_with_lists_space_separated(self): + csl1 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper.parse_config(config, schema) + assert result == { 'listo_space' : [ "yo" + , "it's" + , "time" + , "for" + , "space!" + ] + } + + def test_parsing_as_different_types_always_ok_for_str_and_list(self): + config = self.make_simple_config() + + csl_float = helper.ConfigSchemaLine('floaty', True, float) + csl_int = helper.ConfigSchemaLine('floaty', True, int) + csl_bool = helper.ConfigSchemaLine('floaty', True, bool) + csl_str = helper.ConfigSchemaLine('floaty', True, str) + csl_list = helper.ConfigSchemaLine('floaty', True, list, + helper.SpecialSchemaProcessing.commma_separated) + + schema = helper.ConfigSchema('DEFAULT', [csl_float]) + result = helper.parse_config(config, schema) + assert result == { 'floaty' : 3.141 } + assert type(result['floaty']) is float + + schema = helper.ConfigSchema('DEFAULT', [csl_int]) + with pytest.raises(ValueError): + helper.parse_config(config, schema) + + schema = helper.ConfigSchema('DEFAULT', [csl_bool]) + with pytest.raises(ValueError): + helper.parse_config(config, schema) + + schema = helper.ConfigSchema('DEFAULT', [csl_str]) + result = helper.parse_config(config, schema) + assert result == { 'floaty' : '3.141' } + assert type(result['floaty']) is str + + schema = helper.ConfigSchema('DEFAULT', [csl_list]) + result = helper.parse_config(config, schema) + assert result == { 'floaty' : ['3.141'] } + assert type(result['floaty']) is list + + def test_parsing_schema_fails_with_unknown_type(self): + csl1 = helper.ConfigSchemaLine('floaty', True, dict) + schema = helper.ConfigSchema('DEFAULT', [csl1] ) + config = self.make_simple_config() + with pytest.raises(ConfigError): + helper.parse_config(config, schema) + + def test_all_special_processing_options_are_handled(self): + config = self.make_simple_config() + config['DEFAULT'] = {'enum_checker' : "comma, and spaces"} + + for e in helper.SpecialSchemaProcessing: + csl = helper.ConfigSchemaLine('enum_checker', False, list, e) + schema = helper.ConfigSchema('DEFAULT', [csl]) + + result = helper.parse_config(config, schema) + assert type(result['enum_checker']) is list + + def test_parsing_all(self): + config = self.make_simple_config('FULL_TEST') + + csl1 = helper.ConfigSchemaLine('inty', False, int) + csl2 = helper.ConfigSchemaLine('floaty', False, float) + csl3 = helper.ConfigSchemaLine('mr_bool', True, bool) + csl4 = helper.ConfigSchemaLine('stringer', True, str) + csl5 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + csl6 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchema('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6]) + + result = helper.parse_config(config, schema) + assert result == \ + { 'inty' : 3 + , 'floaty' : 3.141 + , 'mr_bool' : True + , 'stringer' : "bell" + , 'listo_comma' : ["i", "am", "comma", "separated"] + , 'listo_space' : ["yo", "it's", "time", "for", "space!"] + }