Index: /branches/ipp-259_genericise_backups/tools/backups/backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40920) +++ /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40920) @@ -0,0 +1,120 @@ +import configparser +import os.path as path +import socket + +import backups.utils.config_parse_helper as cfg_help +from backups.utils.errors import ConfigError +from configparser import ConfigParser + +DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config') + + +class Backup(): + """Base class for creating backups, contains all the generic versions of +functions for copying, taring etc. + """ + + def __init__(self, config_file=None): + self.config_file = DEFAULT_CONFIG_FILE if config_file is None else config_file + self.load_config(self.config_file) + + def load_config(self, config_class_or_filepath): + """Accepts a filepath to a config file, or """ + + config = ConfigParser() + if config_class_or_filepath is None: + raise ConfigError(config_class_or_filepath, "Neither config settings or filepath of config given") + elif isinstance(config_class_or_filepath, configparser.ConfigParser): + config = config_class_or_filepath + elif path.exists(config_class_or_filepath): + config.read(config_class_or_filepath) + + self._read_default_section(config) + if self._has_copy_section(config): + self._read_copy_config_section(config) + if self._has_tar_section(config): + self._read_tar_config_section(config) + if self._has_rsync_section(config): + self._read_rsync_config_section(config) + if self._has_mysqldump_section(config): + self._read_mysqldump_config_section(config) + + def _has_copy_section(self, config_parser: ConfigParser) -> bool: + section = 'COPY' + self.run_copy = config_parser.has_section(section) + return self.run_copy + + def _has_tar_section(self, config_parser: ConfigParser) -> bool: + section = 'TAR' + self.run_tar = config_parser.has_section(section) + return self.run_tar + + def _has_rsync_section(self, config_parser: ConfigParser) -> bool: + section = 'RSYNC' + self.run_rsync = config_parser.has_section(section) + return self.run_rsync + + def _has_mysqldump_section(self, config_parser: ConfigParser) -> bool: + section = 'MYSQLDUMP' + self.run_mysqldump = config_parser.has_section(section) + return self.run_mysqldump + + DEFAULT_SCHEMA = cfg_help.ConfigSchema('DEFAULT', + [ cfg_help.ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , cfg_help.ConfigSchemaLine('source_machine', True, str) + , cfg_help.ConfigSchemaLine('verbose', False, bool) + ] + ) + + def _read_default_section(self, config: ConfigParser): + self.default_dict = cfg_help.parse_config(config, self.DEFAULT_SCHEMA) + self.hostname_is_valid() + + COPY_SCHEMA = cfg_help.ConfigSchema('COPY', + [ cfg_help.ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + ] + ) + + def _read_copy_config_section(self, config: ConfigParser): + self.copy_dict = cfg_help.parse_config(config, self.COPY_SCHEMA) + + TAR_SCHEMA = cfg_help.ConfigSchema('TAR', + [ cfg_help.ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , cfg_help.ConfigSchemaLine('target_filename', True, str) + , cfg_help.ConfigSchemaLine('prefix_date', True, bool) + ] + ) + + def _read_tar_config_section(self, config: ConfigParser): + self.tar_dict = cfg_help.parse_config(config, self.TAR_SCHEMA) + + RSYNC_SCHEMA = cfg_help.ConfigSchema('RSYNC', + [ cfg_help.ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + ] + ) + + def _read_rsync_config_section(self, config: ConfigParser): + self.rsync_dict = cfg_help.parse_config(config, self.RSYNC_SCHEMA) + + MYSQLDUMP_SCHEMA = cfg_help.ConfigSchema('MYSQLDUMP', + [ cfg_help.ConfigSchemaLine('user', True, str) + , cfg_help.ConfigSchemaLine('password', True, str) + , cfg_help.ConfigSchemaLine('db_name', True, str) + , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , cfg_help.ConfigSchemaLine('target_filename', True, str) + , cfg_help.ConfigSchemaLine('prefix_date', True, bool) + ] + ) + + def _read_mysqldump_config_section(self, config: ConfigParser): + self.mysqldump_dict = cfg_help.parse_config(config, self.MYSQLDUMP_SCHEMA) + + def hostname_is_valid(self) -> bool: + if self.default_dict is None or 'source_machine' not in self.default_dict.keys(): + raise ValueError('No source machine specified') + + actual_hostname = socket.gethostname() + return actual_hostname == self.default_dict['source_machine'] Index: /branches/ipp-259_genericise_backups/tools/backups/backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40920) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40920) @@ -0,0 +1,131 @@ +import os +import pytest +import socket +from configparser import ConfigParser + +from backups.backup import Backup +from backups.utils.errors import ConfigError + + +TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/full_backup_config.config') + + +def make_default_config( tmpdir: str + , backup_targs: [str] = ['bck1', 'bck2'] + , verbose: bool = False + ) -> ConfigParser: + config = ConfigParser() + + backup_paths = ','.join([os.path.join(tmpdir, b) for b in backup_targs]) + source_machine = socket.gethostname() + + config['DEFAULT'] = \ + { + 'backup_paths' : backup_paths, + 'source_machine' : source_machine, + 'verbose' : verbose, + } + return config + + +class TestArguments(object): + + def test_default_config_file_load_default(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # Default + assert backy.default_dict['backup_paths'] == \ + [ "/export/ippops4.0/atlassian_backups" + , "/data/ippops3.0/atlassian_backups" + , "/data/ippops5.0/atlassian_backups" + ] + assert backy.default_dict['source_machine'] == "hostname_of_computer" + assert backy.default_dict['verbose'] is True + + def test_copy_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.copy_dict['sources'] == \ + [ "/some_dir_to_copy_from/" + , "/some_dir/and_specific_file.jpeg" + ] + assert backy.copy_dict['additional_args'] == ['-v'] + + def test_tar_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.tar_dict['sources'] == \ + [ "/this_dir/" + , "/another_dir/" + , "/and_a_couple_files/subdir/file1.txt" + , "/and_a_couple_files/subdir/file2.png" + , "/and_a_couple_files/subdir/file3.pdf" + ] + assert backy.tar_dict['additional_args'] == [ "--verbose" + , "-z" + , "-p" + ] + assert backy.tar_dict['target_filename'] == "cool_tar" + assert backy.tar_dict['prefix_date'] is True + + def test_rsync_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Rsync] + assert backy.rsync_dict['sources'] == [ "/dir/to/sync"] + assert backy.rsync_dict['additional_args'] == \ + [ "--progress" + , "-r" + ] + + def test_mysqldump_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Mysqldump] + assert backy.mysqldump_dict['user'] == "dumper" + assert backy.mysqldump_dict['password'] == "not_a_real_password" + assert backy.mysqldump_dict['db_name'] == "jiradb" + assert backy.mysqldump_dict['additional_args'] == [''] + assert backy.mysqldump_dict['target_filename'] == "itsa_mysqldump" + assert backy.mysqldump_dict['prefix_date'] is True + + def test_loading_only_defaults(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + backy = Backup(config_file=config) + + assert backy.default_dict['backup_paths'] == \ + [ os.path.join(tmpdir, 'bck1') + , os.path.join(tmpdir, 'bck2') + ] + assert backy.default_dict['source_machine'] == socket.gethostname() + assert backy.default_dict['verbose'] is False + + def test_failure_if_no_defaults_provided(self): + config = ConfigParser() + with pytest.raises(ConfigError): + Backup(config_file=config) + + with pytest.raises(ConfigError): + Backup(config_file=None) + + with pytest.raises(ConfigError): + Backup(config_file="/not/a/real/file.config") + + def test_nothing_should_be_set_to_run_if_not_present(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + backy = Backup(config_file=config) + + assert backy.run_copy is False + assert backy.run_tar is False + assert backy.run_rsync is False + assert backy.run_mysqldump is False + + def test_hostname_assertion_fails(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + backy = Backup(config_file=config) + + backy.default_dict['source_machine'] = 'not_a_real_hostname' + assert backy.hostname_is_valid() is False + + def test_hostname_assertion_passes(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + backy = Backup(config_file=config) + + actual_hostname = socket.gethostname() + backy.default_dict['source_machine'] = actual_hostname + assert backy.hostname_is_valid() is True Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40919) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40920) @@ -13,5 +13,5 @@ class ConfigSchemaLine(): - """Defines a config line by what the key is, if it is required or optional, + """Defines a config line by what the key is, if it is required or optional, the type that the value should be processed as, and any special processing required """ @@ -36,5 +36,5 @@ class ConfigSchema(): - """Defines a schema to be used. Consists of a single section name and + """Defines a schema to be used. Consists of a single section name and multiple config lines"""