Index: /branches/ipp-259_genericise_backups/tools/backups/backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40949) @@ -10,7 +10,52 @@ import backups.utils.errors as errs import backups.utils.subprocess_utils as sub_utils -from backups.utils.config_parse_helper import ConfigSchemaSection, ConfigSchemaLine +from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config') + +DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ] +) +COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + ] +) +TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) +RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + ] +) + +MYSQLDUMP_SCHEMA_SECTION = ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) + + +def get_backup_schema() -> ConfigSchema: + backup_schema = ConfigSchema(None) + backup_schema.add_section(DEFAULT_SCHEMA_SECTION) + backup_schema.add_section(COPY_SCHEMA_SECTION) + backup_schema.add_section(TAR_SCHEMA_SECTION) + backup_schema.add_section(RSYNC_SCHEMA_SECTION) + backup_schema.add_section(MYSQLDUMP_SCHEMA_SECTION) + return backup_schema @@ -20,10 +65,13 @@ """ - def __init__(self, config_file=None): + def __init__(self, config_file=None, schema: ConfigSchema=None): if config_file is None: raise errs.ValidationError("ValidationError: Backup class must be given a config") - self.load_config(config_file) - - def load_config(self, config_class_or_filepath): + if schema is None: + schema = get_backup_schema() + print(schema) + self.load_config(config_file, schema) + + def load_config(self, config_class_or_filepath, schema): """Accepts a filepath to a config file, or """ @@ -38,88 +86,47 @@ config.read(config_class_or_filepath) - self.read_default_section(config) - if self._has_copy_section(config): - self.read_copy_config_section(config) - if self._has_tar_section(config): - self.read_tar_config_section(config) - if self._has_rsync_section(config): - self.read_rsync_config_section(config) - if self._has_mysqldump_section(config): - self.read_mysqldump_config_section(config) - - def _has_copy_section(self, config_parser: ConfigParser) -> bool: - section = 'COPY' - self.run_copy = config_parser.has_section(section) - return self.run_copy - - def _has_tar_section(self, config_parser: ConfigParser) -> bool: - section = 'TAR' - self.run_tar = config_parser.has_section(section) - return self.run_tar - - def _has_rsync_section(self, config_parser: ConfigParser) -> bool: - section = 'RSYNC' - self.run_rsync = config_parser.has_section(section) - return self.run_rsync - - def _has_mysqldump_section(self, config_parser: ConfigParser) -> bool: - section = 'MYSQLDUMP' - self.run_mysqldump = config_parser.has_section(section) - return self.run_mysqldump - - DEFAULT_SCHEMA = ConfigSchemaSection('DEFAULT', - [ ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('source_machine', True, str) - , ConfigSchemaLine('verbose', False, bool) - , ConfigSchemaLine('make_dirs', False, bool) - ] - ) - - def read_default_section(self, config: ConfigParser, schema: ConfigSchemaSection=DEFAULT_SCHEMA): - self.default_dict = cfg_help.parse_config(config, schema) + self.read_config_and_schema(config, schema) + # self.read_copy_config_section(config, schema) + # self.read_tar_config_section(config, schema) + # self.read_rsync_config_section(config, schema) + # self.read_mysqldump_config_section(config, schema) + + def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): + parsed_config = ParsedConfig(config, schema) + print(parsed_config) + self.default_dict = parsed_config.get_section('DEFAULT') self.hostname_is_valid() self.maybe_make_backup_dirs() - COPY_SCHEMA = ConfigSchemaSection('COPY', - [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - ] - ) - - def read_copy_config_section(self, config: ConfigParser, schema: ConfigSchemaSection=COPY_SCHEMA): - self.copy_dict = cfg_help.parse_config(config, schema) - - TAR_SCHEMA = ConfigSchemaSection('TAR', - [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - , ConfigSchemaLine('target_filename', True, str) - , ConfigSchemaLine('prefix_date', True, bool) - ] - ) - - def read_tar_config_section(self, config: ConfigParser, schema: ConfigSchemaSection=TAR_SCHEMA): - self.tar_dict = cfg_help.parse_config(config, schema) - - RSYNC_SCHEMA = ConfigSchemaSection('RSYNC', - [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - ] - ) - - def read_rsync_config_section(self, config: ConfigParser, schema: ConfigSchemaSection=RSYNC_SCHEMA): - self.rsync_dict = cfg_help.parse_config(config, schema) - - MYSQLDUMP_SCHEMA = ConfigSchemaSection('MYSQLDUMP', - [ ConfigSchemaLine('user', True, str) - , ConfigSchemaLine('password', True, str) - , ConfigSchemaLine('db_name', True, str) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - , ConfigSchemaLine('target_filename', True, str) - , ConfigSchemaLine('prefix_date', True, bool) - ] - ) - - def read_mysqldump_config_section(self, config: ConfigParser, schema: ConfigSchemaSection=MYSQLDUMP_SCHEMA): - self.mysqldump_dict = cfg_help.parse_config(config, schema) + section = 'COPY' + self.should_run_copy = parsed_config.has_section(section) + if self.should_run_copy: + self.copy_dict = parsed_config.get_section(section) + + section = 'TAR' + self.should_run_tar = parsed_config.has_section(section) + if self.should_run_tar: + self.tar_dict = parsed_config.get_section(section) + + section = 'RSYNC' + self.should_run_rsync = parsed_config.has_section(section) + if self.should_run_rsync: + self.rsync_dict = parsed_config.get_section(section) + + section = 'MYSQLDUMP' + self.should_run_mysqldump = parsed_config.has_section(section) + if self.should_run_mysqldump: + self.mysqldump_dict = parsed_config.get_section(section) + + def run_backup_processes(self): + + if self.should_run_copy: + self._run_copy() + if self.should_run_tar: + self._run_tar() + if self.should_run_rsync: + self._run_rsync() + if self.should_run_mysqldump: + self._run_mysqldump() def hostname_is_valid(self) -> bool: @@ -146,12 +153,6 @@ os.makedirs(backup_path) - def run_backup(self): - self._run_copy() - self._run_tar() - self._run_rsync() - self._run_mysqldump() - def _run_copy(self): - if self.run_copy is None or False: + if self.should_run_copy is None or False: return 0 self.target_backup_paths_ok(raise_error=True) @@ -177,5 +178,5 @@ def _run_tar(self): - if self.run_tar is None or False: + if self.should_run_tar is None or False: return 0 @@ -203,5 +204,5 @@ def _run_rsync(self): - if self.run_rsync is None or False: + if self.should_run_rsync is None or False: return 0 Index: /branches/ipp-259_genericise_backups/tools/backups/backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40949) @@ -8,5 +8,5 @@ import backups.utils.errors as errs from backups.backup import Backup - +from backups.utils.config_parse_helper import ConfigSchema TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config') @@ -92,5 +92,7 @@ def test_loading_only_defaults(self, tmpdir): config = make_default_config(tmpdir, ['bck1', 'bck2'], False) - backy = Backup(config_file=config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) assert backy.default_dict['backup_paths'] == \ @@ -117,14 +119,18 @@ def test_nothing_should_be_set_to_run_if_not_present(self, tmpdir): config = make_default_config(tmpdir, ['bck1', 'bck2'], False) - backy = Backup(config_file=config) - - assert backy.run_copy is False - assert backy.run_tar is False - assert backy.run_rsync is False - assert backy.run_mysqldump is False + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.should_run_copy is False + assert backy.should_run_tar is False + assert backy.should_run_rsync is False + assert backy.should_run_mysqldump is False def test_hostname_assertion_fails(self, tmpdir): config = make_default_config(tmpdir, ['bck1', 'bck2']) - backy = Backup(config_file=config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) backy.default_dict['source_machine'] = 'not_a_real_hostname' @@ -133,5 +139,7 @@ def test_hostname_assertion_passes(self, tmpdir): config = make_default_config(tmpdir, ['bck1', 'bck2']) - backy = Backup(config_file=config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) actual_hostname = socket.gethostname() @@ -168,4 +176,6 @@ , 'verbose' : 'True' } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) bck1_path = os.path.join(tmpdir, 'bck1') bck1b_path = os.path.join(tmpdir, 'bck1/bobby') @@ -177,5 +187,5 @@ assert not os.path.exists(bck2_path) # backup dirs not created automatically - Backup(config) + Backup(config, schema) assert not os.path.exists(bck1_path) assert not os.path.exists(bck1b_path) @@ -190,5 +200,5 @@ , 'make_dirs' : 'True' } - backy = Backup(config) + backy = Backup(config, schema) assert backy.default_dict['make_dirs'] is True assert os.path.exists(bck1_path) @@ -204,9 +214,13 @@ source = os.path.join(tmpdir, "and_specific_file.jpeg") config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) # make file to be copied Path(source).touch() - backy = Backup(config) + backy = Backup(config, schema) + assert backy.should_run_copy is True expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') @@ -226,9 +240,12 @@ , 'additional_args' : "-v --preserve -r" } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) os.makedirs(source1) Path(source2).touch() - backy = Backup(config) + backy = Backup(config, schema) expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') @@ -253,7 +270,10 @@ , 'additional_args' : '-r' } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) assert not os.path.exists(source) - backy = Backup(config) + backy = Backup(config, schema) with pytest.raises(errs.ValidationError): backy._run_copy() @@ -271,7 +291,10 @@ Path(source).touch() config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) assert not os.path.exists(backup_does_not_exist) - backy = Backup(config) + backy = Backup(config, schema) with pytest.raises(errs.ValidationError): backy._run_copy() @@ -284,7 +307,10 @@ } os.makedirs(source) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) with pytest.raises(errs.SubprocessError): - backy = Backup(config) + backy = Backup(config, schema) backy._run_copy() @@ -293,5 +319,5 @@ , 'additional_args' : '-r' } - backy = Backup(config) + backy = Backup(config, schema) expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') @@ -320,5 +346,9 @@ } - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) expected_tar1 = os.path.join(tmpdir, 'bck1', 'cool_tar.tar') @@ -344,5 +374,10 @@ } - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_tar is True date = bckup.get_date() @@ -363,9 +398,12 @@ source = os.path.join(tmpdir, "and_specific_file.jpeg") config['RSYNC'] = {'sources' : f"{source}"} - # make file to be copied Path(source).touch() - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') @@ -389,5 +427,9 @@ Path(source2).touch() - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') @@ -413,6 +455,10 @@ } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + assert not os.path.exists(source) - backy = Backup(config) + backy = Backup(config, schema) with pytest.raises(errs.ValidationError): backy._run_rsync() @@ -431,6 +477,10 @@ config['RSYNC'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + assert not os.path.exists(backup_does_not_exist) - backy = Backup(config) + backy = Backup(config, schema) with pytest.raises(errs.ValidationError): backy._run_rsync() @@ -444,5 +494,10 @@ os.makedirs(source) - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') @@ -458,5 +513,5 @@ } - backy = Backup(config) + backy = Backup(config, schema) expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') @@ -491,5 +546,10 @@ } - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_mysqldump is True date = bckup.get_date() @@ -515,5 +575,9 @@ } - backy = Backup(config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) with pytest.raises(errs.SubprocessError): backy._run_mysqldump() Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py (revision 40949) @@ -4,15 +4,30 @@ import os.path as path +import backups.backup as bck import backups.utils.subprocess_utils as sub_utils import backups.utils.errors as errs from backups.backup import Backup +from backups.utils.config_parse_helper import ConfigSchema EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config') + +CONFLUENCE_SCHEMA = bck.get_backup_schema() + + +def make_confluence_schema(): + schema = ConfigSchema(None) + schema.add_section(bck.DEFAULT_SCHEMA_SECTION) + schema.add_section(bck.COPY_SCHEMA_SECTION) + schema.add_section(bck.TAR_SCHEMA_SECTION) + schema.add_section(bck.MYSQLDUMP_SCHEMA_SECTION) + return schema class ConfluenceBackup(Backup): - def __init__(self, config_file=EXPECTED_CONFIG_FILE): - self.load_config(config_file) + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=None): + if schema is None: + schema = make_confluence_schema() + self.load_config(config_file, schema) @staticmethod @@ -25,5 +40,5 @@ def _run_copy(self, verbose=False): - if self.run_copy is None or False: + if self.should_run_copy is None or False: return 0 Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40949) @@ -6,9 +6,10 @@ from pathlib import Path +import backups.backup as bckup import backups.confluence_backup as jbck_mod import backups.utils.errors as errs from backups.confluence_backup import ConfluenceBackup from backups.utils.errors import ValidationError - +from backups.utils.config_parse_helper import ConfigSchema CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') @@ -146,5 +147,10 @@ } - jb = ConfluenceBackup(config_file=config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = ConfluenceBackup(config, schema) assert jb.default_dict == \ Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py (revision 40949) @@ -4,15 +4,28 @@ import os.path as path +import backups.backup as bck import backups.utils.subprocess_utils as sub_utils import backups.utils.errors as errs from backups.backup import Backup +from backups.utils.config_parse_helper import ConfigSchema EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config') +def make_jira_schema(): + schema = ConfigSchema(None) + schema.add_section(bck.DEFAULT_SCHEMA_SECTION) + schema.add_section(bck.COPY_SCHEMA_SECTION) + schema.add_section(bck.TAR_SCHEMA_SECTION) + schema.add_section(bck.MYSQLDUMP_SCHEMA_SECTION) + return schema + + class JiraBackup(Backup): - def __init__(self, config_file=EXPECTED_CONFIG_FILE): - self.load_config(config_file) + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=None): + if schema is None: + schema = make_jira_schema() + self.load_config(config_file, schema) @staticmethod @@ -25,5 +38,5 @@ def _run_copy(self, verbose=False): - if self.run_copy is None or False: + if self.should_run_copy is None or False: return 0 Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40949) @@ -6,9 +6,10 @@ from pathlib import Path +import backups.backup as bckup import backups.jira_backup as jbck_mod import backups.utils.errors as errs from backups.jira_backup import JiraBackup from backups.utils.errors import ValidationError - +from backups.utils.config_parse_helper import ConfigSchema JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') @@ -146,5 +147,10 @@ } - jb = JiraBackup(config_file=config) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = JiraBackup(config, schema) assert jb.default_dict == \ @@ -212,4 +218,5 @@ assert not os.path.exists(expected_backup_2_copy) + assert jb.should_run_copy is True return_value = jb._run_copy() assert return_value == 0 Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40949) @@ -41,12 +41,12 @@ def __init__(self, section_name, options: [ConfigSchemaLine]): self.section_name = section_name + self.options = {} self._process_options(options) def _process_options(self, options): - self.options = {} if options is None: return if type(options) is not list: - raise ValidationError("ValidationError: ConfigSchema options must be '[ConfigSchemaLine]' or 'None'") + raise ValidationError(f"ValidationError: {self.__class__.__name__} options must be '[ConfigSchemaLine]' or 'None'") if len(options) == 0: return @@ -62,10 +62,42 @@ def get_schema_line(self, key: str) -> ConfigSchemaLine: if not self.key_in_options(key): - raise KeyError(f"KeyError: '{key}' is not present in schema") + raise KeyError(f"KeyError: '{key}' is not present in schema section") return self.options[key] def __str__(self): options_strs = [ o.__str__() for o in self.options] - return (f'{self.__class__.__name__} with {options_strs}') + return (f'{self.__class__.__name__} named "{self.section_name}" with {options_strs}') + + +class ConfigSchema(): + + def __init__(self, sections: [ConfigSchemaSection]): + self.sections = {} + self._process_sections(sections) + + def _process_sections(self, sections): + if sections is None: + return + if type(sections) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} sections must be '[ConfigSchemaSection]' or 'None'") + if len(sections) == 0: + return + for sec in sections: + self.add_section(sec) + + def add_section(self, section: ConfigSchemaSection): + self.sections[section.section_name] = section + + def section_exists(self, section_name: str) -> bool: + return section_name in self.sections.keys() + + def get_schema_section(self, section_name: str) -> ConfigSchemaSection: + if not self.section_exists(section_name): + raise KeyError(f"KeyError: '{section_name}' is not present in schema") + return self.sections[section_name] + + def __str__(self): + section_strs = [ o.__str__() for o in self.sections] + return (f'{self.__class__.__name__} with {section_strs}') @@ -76,5 +108,5 @@ -def parse_config(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: +def _parse_config_section(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: """For a given config it will extract all values defined by the schema and return them as a dictionary of the keys and the assoicated value as the @@ -87,4 +119,5 @@ csl = schema.get_schema_line(csl_key) + # Parses each line if config.has_option(section, csl_key): if csl.expected_type is bool: @@ -115,2 +148,39 @@ return parsed_items + + +# def parse_config(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: +# return _parse_config_section(config, schema) + +def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: + parsed_items = {} + print(config) + for c in config: + print(c) + + for css_name in schema.sections: + print(css_name) + css = schema.get_schema_section(css_name) + print(css) + + # if config.has_section(css_name): + parsed_section = _parse_config_section(config, css) + parsed_items[css_name] = parsed_section + # else: + # raise ConfigError(config, f"ConfigError: Expected section '{css_name}' from schema not present in config") + + return parsed_items + + +class ParsedConfig(): + + def __init__(self, config: ConfigParser, schema: ConfigSchema): + self._section_dict = _parse_config(config, schema) + + def has_section(self, section_name): + return section_name in self._section_dict + + def get_section(self, section_name): + if self.has_section(section_name): + return self._section_dict[section_name] + return None Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py (revision 40948) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py (revision 40949) @@ -156,5 +156,5 @@ with pytest.raises(ConfigError): - helper.parse_config(config, schema) + helper._parse_config_section(config, schema) def test_schema_section_with_only_optional_values_is_ok(self): @@ -163,5 +163,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1, csl2] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == {} @@ -170,5 +170,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'inty' : 3} @@ -177,5 +177,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'floaty' : 3.141 } @@ -184,5 +184,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'mr_bool' : True } @@ -191,5 +191,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'stringer' : 'bell' } @@ -199,5 +199,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'listo_comma' : [ "i" , "am" @@ -212,5 +212,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) config = self.make_simple_config() - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'listo_space' : [ "yo" , "it's" @@ -232,5 +232,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl_float]) - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'floaty' : 3.141 } assert type(result['floaty']) is float @@ -238,17 +238,17 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl_int]) with pytest.raises(ValueError): - helper.parse_config(config, schema) + helper._parse_config_section(config, schema) schema = helper.ConfigSchemaSection('DEFAULT', [csl_bool]) with pytest.raises(ValueError): - helper.parse_config(config, schema) + helper._parse_config_section(config, schema) schema = helper.ConfigSchemaSection('DEFAULT', [csl_str]) - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'floaty' : '3.141' } assert type(result['floaty']) is str schema = helper.ConfigSchemaSection('DEFAULT', [csl_list]) - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == { 'floaty' : ['3.141'] } assert type(result['floaty']) is list @@ -259,5 +259,5 @@ config = self.make_simple_config() with pytest.raises(ConfigError): - helper.parse_config(config, schema) + helper._parse_config_section(config, schema) def test_all_special_processing_options_are_handled(self): @@ -269,5 +269,5 @@ schema = helper.ConfigSchemaSection('DEFAULT', [csl]) - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert type(result['enum_checker']) is list @@ -285,5 +285,5 @@ schema = helper.ConfigSchemaSection('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6]) - result = helper.parse_config(config, schema) + result = helper._parse_config_section(config, schema) assert result == \ { 'inty' : 3