Index: trunk/backups/README =================================================================== --- trunk/backups/README (revision 40971) +++ trunk/backups/README (revision 40971) @@ -0,0 +1,13 @@ +README + +This module contains files that perform backups of various services used by +the IPP. Each file has it's own backup class, which is inherited from the base +class. To perform a backup there needs to a config that matches the expected +schema. + +Run 'deploy_backups.py' to deploy the relevant scripts and utils to the machines +that the services live on. + +Check a config is in place for each backup. + +If automated backup is desired run the script periodically in a crontab. Index: trunk/backups/backup.py =================================================================== --- trunk/backups/backup.py (revision 40971) +++ trunk/backups/backup.py (revision 40971) @@ -0,0 +1,286 @@ +import configparser +import datetime +import os +import os.path as path +import socket +import subprocess +from configparser import ConfigParser + +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig, SpecialSchemaProcessing + +DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ] +) +COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + ] +) +TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) +RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + ] +) + +MYSQLDUMP_SCHEMA_SECTION = ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) + + +def get_backup_schema() -> ConfigSchema: + backup_schema = ConfigSchema(None) + backup_schema.add_section(DEFAULT_SCHEMA_SECTION) + backup_schema.add_section(COPY_SCHEMA_SECTION) + backup_schema.add_section(TAR_SCHEMA_SECTION) + backup_schema.add_section(RSYNC_SCHEMA_SECTION) + backup_schema.add_section(MYSQLDUMP_SCHEMA_SECTION) + return backup_schema + + +class Backup(): + """Base class for creating backups, contains all the generic versions of +functions for copying, taring etc. + """ + + def __init__(self, config_file=None, schema: ConfigSchema=None): + if config_file is None: + raise errs.ValidationError("ValidationError: Backup class must be given a config") + if schema is None: + schema = get_backup_schema() + self.load_config(config_file, schema) + + def load_config(self, config_class_or_filepath, schema): + """Accepts a filepath to a config file, or """ + + config = ConfigParser() + expected_class = configparser.ConfigParser + if config_class_or_filepath is None: + raise errs.ValidationError("ValidationError: config settings or filepath to config must be given") + elif isinstance(config_class_or_filepath, expected_class): + config = config_class_or_filepath + elif not path.exists(config_class_or_filepath): + raise errs.ValidationError(f"ValidationError: Config filepath does not exist (or incorrect class given, expected: {expected_class})") + config.read(config_class_or_filepath) + + self.read_config_and_schema(config, schema) + # self.read_copy_config_section(config, schema) + # self.read_tar_config_section(config, schema) + # self.read_rsync_config_section(config, schema) + # self.read_mysqldump_config_section(config, schema) + + def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): + parsed_config = ParsedConfig(config, schema) + self.default_dict = parsed_config.get_section('DEFAULT') + self.hostname_is_valid() + self.maybe_make_backup_dirs() + + section = 'COPY' + self.should_run_copy = parsed_config.has_section(section) + if self.should_run_copy: + self.copy_dict = parsed_config.get_section(section) + + section = 'TAR' + self.should_run_tar = parsed_config.has_section(section) + if self.should_run_tar: + self.tar_dict = parsed_config.get_section(section) + + section = 'RSYNC' + self.should_run_rsync = parsed_config.has_section(section) + if self.should_run_rsync: + self.rsync_dict = parsed_config.get_section(section) + + section = 'MYSQLDUMP' + self.should_run_mysqldump = parsed_config.has_section(section) + if self.should_run_mysqldump: + self.mysqldump_dict = parsed_config.get_section(section) + + def run_backup_processes(self): + self._write_begin_log() + if self.should_run_copy: + self._run_copy() + if self.should_run_tar: + self._run_tar() + if self.should_run_rsync: + self._run_rsync() + if self.should_run_mysqldump: + self._run_mysqldump() + self._write_end_log() + + def hostname_is_valid(self) -> bool: + if self.default_dict is None or 'source_machine' not in self.default_dict.keys(): + raise ValueError('No source machine specified') + + actual_hostname = socket.gethostname() + return actual_hostname == self.default_dict['source_machine'] + + def target_backup_paths_ok(self, raise_error=False): + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + if raise_error: + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + return False + return True + + def maybe_make_backup_dirs(self): + if self.default_dict is None or 'make_dirs' not in self.default_dict.keys(): + return + if self.default_dict['make_dirs']: + for backup_path in self.default_dict['backup_paths']: + if not os.path.exists(backup_path): + os.makedirs(backup_path) + sub_utils.chmod_wrapper(['777', backup_path]) + + def _run_copy(self): + if self.should_run_copy is None or False: + return 0 + self.target_backup_paths_ok(raise_error=True) + + for bp in self.default_dict['backup_paths']: + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.copy_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 + + def _run_tar(self): + if self.should_run_tar is None or False: + return 0 + + backup_paths = self.default_dict['backup_paths'] + + filename = self.tar_dict['target_filename'] + if 'prefix_date' in self.tar_dict.keys() and self.tar_dict['prefix_date']: + filename = f"{get_date()}_{filename}" + filename = path.join(backup_paths[0], filename) # may need some tar suffix magic + + add_args = [] if 'additional_args' not in self.tar_dict.keys() else self.tar_dict['additional_args'] + all_args = add_args + [filename] + self.tar_dict['sources'] + + try: + sub_utils.tar_wrapper(all_args) + sub_utils.chmod_wrapper(['774', filename]) + # copy to backups + if len(backup_paths) == 1: + return 0 + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([ filename, backup_paths[d]]) + except errs.SubprocessError as e: + raise errs.SubprocessError() from e + return 0 + + def _run_rsync(self): + if self.should_run_rsync is None or False: + return 0 + + self.target_backup_paths_ok(raise_error=True) + backup_paths = self.default_dict['backup_paths'] + for bp in backup_paths: + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.rsync_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.rsync_dict.keys(): + args = args + self.rsync_dict['additional_args'] + sub_utils.local_rsync_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: rsync\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 + + def _run_mysqldump(self): + + backup_paths = self.default_dict['backup_paths'] + + mysqldump_args = [ 'mysqldump' + , '-u', self.mysqldump_dict['user'] + , f"-p{self.mysqldump_dict['password']}" + , '--databases', self.mysqldump_dict['db_name'] + ] + if 'additional_args' in self.mysqldump_dict.keys(): + mysqldump_args = mysqldump_args + self.mysqldump_dict['additional_args'] + + dump_filename = self.mysqldump_dict['target_filename'] + if 'prefix_date' in self.mysqldump_dict.keys() and self.mysqldump_dict['prefix_date']: + dump_filename = f"{get_date()}_{dump_filename}" + dump_filename = path.join(backup_paths[0], dump_filename) + + if path.exists(dump_filename): + raise errs.ValidationError(f"ValidationError: mysqldump already exists {dump_filename}") + + with open(dump_filename, "w+") as df: + mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) + dump_result = mysqldump_process.communicate() + + # Get the piped std err. See if it contains the word error + stdo, stde = dump_result + if stde is not None: + decoded_stderr = stde.decode() + if decoded_stderr.casefold().find('error'.casefold()) > -1: + raise errs.SubprocessError(f"SubprocessError: Errors detected in mysqldump:\n{decoded_stderr}") + + # copy the dump to other locations + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([dump_filename, backup_paths[d]]) + return 0 + + def _write_begin_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if path.exists(log_file): + os.remove(log_file) + with open(log_file, "w+") as lf: + lf.write(f'Run started at: {datetime.date.today().strftime("%Y_%b_%d_%H:%M")}\n') + + def _write_end_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if not path.exists(log_file): + return + with open(log_file, "a") as lf: + lf.write(f'Last successful backup completed at : {datetime.date.today().strftime("%Y_%b_%d_%H:%M")}\n') + + +def get_date(custom_date=None, custom_format="%Y_%m_%d"): + # Format is currently: 2019-10-08 + if custom_date is not None: + return custom_date.strftime(custom_format) + return datetime.date.today().strftime(custom_format) Index: trunk/backups/backup_test.py =================================================================== --- trunk/backups/backup_test.py (revision 40971) +++ trunk/backups/backup_test.py (revision 40971) @@ -0,0 +1,729 @@ +import os +import pytest +import stat +import socket +from configparser import ConfigParser +from pathlib import Path + +import backup as bckup +import utils.errors as errs +from backup import Backup +from utils.config_parse_helper import ConfigSchema + +TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config') + + +def make_default_config( tmpdir: str + , backup_targs: [str] = ['bck1', 'bck2'] + , verbose: bool = False + ) -> ConfigParser: + config = ConfigParser() + + # make the paths: + for b in backup_targs: + os.makedirs(os.path.join(tmpdir, b)) + + # Get the correct formats for the config + backup_paths = ','.join([os.path.join(tmpdir, b) for b in backup_targs]) + source_machine = socket.gethostname() + config['DEFAULT'] = \ + { 'backup_paths' : backup_paths + , 'source_machine' : source_machine + , 'verbose' : verbose + } + return config + + +class TestArguments(object): + + def test_default_config_file_load_default(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # Default + assert backy.default_dict['backup_paths'] == \ + [ "/export/ippops4.0/atlassian_backups" + , "/data/ippops3.0/atlassian_backups" + , "/data/ippops5.0/atlassian_backups" + ] + assert backy.default_dict['source_machine'] == "hostname_of_computer" + assert backy.default_dict['verbose'] is True + + def test_copy_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.copy_dict['sources'] == \ + [ "/some_dir_to_copy_from/" + , "/some_dir/and_specific_file.jpeg" + ] + assert backy.copy_dict['additional_args'] == ['-v'] + + def test_tar_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.tar_dict['sources'] == \ + [ "/this_dir/" + , "/another_dir/" + , "/and_a_couple_files/subdir/file1.txt" + , "/and_a_couple_files/subdir/file2.png" + , "/and_a_couple_files/subdir/file3.pdf" + ] + assert backy.tar_dict['additional_args'] == [ "--verbose" + , "-z" + , "-p" + ] + assert backy.tar_dict['target_filename'] == "cool_tar" + assert backy.tar_dict['prefix_date'] is True + + def test_rsync_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Rsync] + assert backy.rsync_dict['sources'] == [ "/dir/to/sync"] + assert backy.rsync_dict['additional_args'] == \ + [ "--progress" + , "-r" + ] + + def test_mysqldump_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Mysqldump] + assert backy.mysqldump_dict['user'] == "dumper" + assert backy.mysqldump_dict['password'] == "not_a_real_password" + assert backy.mysqldump_dict['db_name'] == "jiradb" + assert backy.mysqldump_dict['additional_args'] == [''] + assert backy.mysqldump_dict['target_filename'] == "itsa_mysqldump" + assert backy.mysqldump_dict['prefix_date'] is True + + def test_loading_only_defaults(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.default_dict['backup_paths'] == \ + [ os.path.join(tmpdir, 'bck1') + , os.path.join(tmpdir, 'bck2') + ] + assert backy.default_dict['source_machine'] == socket.gethostname() + assert backy.default_dict['verbose'] is False + + def test_failure_if_no_defaults_provided(self): + config = ConfigParser() + with pytest.raises(errs.ConfigError) as e_empty: + Backup(config_file=config) + assert "is required but was not found in config" in str(e_empty) + + with pytest.raises(errs.ValidationError) as e_none: + Backup(config_file=None) + assert "ValidationError: Backup class must be given a config" in str(e_none) + + with pytest.raises(errs.ValidationError) as e_nofile: + Backup(config_file="/not/a/real/file.config") + assert "ValidationError: Config filepath does not exist" in str(e_nofile) + + def test_nothing_should_be_set_to_run_if_not_present(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.should_run_copy is False + assert backy.should_run_tar is False + assert backy.should_run_rsync is False + assert backy.should_run_mysqldump is False + + def test_hostname_assertion_fails(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + backy.default_dict['source_machine'] = 'not_a_real_hostname' + assert backy.hostname_is_valid() is False + + def test_hostname_assertion_passes(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + actual_hostname = socket.gethostname() + backy.default_dict['source_machine'] = actual_hostname + assert backy.hostname_is_valid() is True + + +class TestFilepathVerifications(object): + + def test_default_target_paths_do_not_exist_in_testing(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.target_backup_paths_ok() is False + + def test_raising_an_error_if_a_backup_path_does_not_exist(self): + with pytest.raises(errs.ValidationError) as e: + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + backy.target_backup_paths_ok(raise_error=True) + assert "ValidationError: target path does not exist" in str(e) + + def test_an_incorrect_path_then_correct_path(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + + backy.default_dict['backup_paths'] = ["/not/real/path"] + assert backy.target_backup_paths_ok() is False + + backy.default_dict['backup_paths'] = [tmpdir] + assert backy.target_backup_paths_ok() is True + + def test_backup_paths_created_if_make_dirs_given(self, tmpdir): + config = ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + bck1_path = os.path.join(tmpdir, 'bck1') + bck1b_path = os.path.join(tmpdir, 'bck1/bobby') + bck1bb_path = os.path.join(tmpdir, 'bck1/bobby/backup') + bck2_path = os.path.join(tmpdir, 'bck2') + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + # backup dirs not created automatically + Backup(config, schema) + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + + # make_dirs included in config - when loaded dirs created. + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + , 'make_dirs' : 'True' + } + backy = Backup(config, schema) + assert backy.default_dict['make_dirs'] is True + assert os.path.exists(bck1_path) + assert os.path.exists(bck1b_path) + assert os.path.exists(bck1bb_path) + assert os.path.exists(bck2_path) + + # Check permissions are 777 too + new_stats = os.stat(bck2_path) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestRunningCopy(): + + def test_simple_copy(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + # make file to be copied + Path(source).touch() + + backy = Backup(config, schema) + assert backy.should_run_copy is True + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_copy_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v --preserve -r" + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + os.makedirs(source1) + Path(source2).touch() + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_copy_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_requires_dash_r_arg_for_copying_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['COPY'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + with pytest.raises(errs.SubprocessError): + backy = Backup(config, schema) + backy._run_copy() + + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + +class TestRunningTar(): + + def test_simply_tar(self, tmpdir): + config = make_default_config(tmpdir, ['bck1']) + source = os.path.join(tmpdir, "folder_to_tar") + os.makedirs(source) + # put a file in it + Path(os.path.join(source, "a_file.txt")) + + config['TAR'] = \ + { 'sources' : f"{source}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_tar1 = os.path.join(tmpdir, 'bck1', 'cool_tar.tar') + print(expected_tar1) + assert not os.path.exists(expected_tar1) + backy._run_tar() + assert os.path.exists(expected_tar1) + + def test_tar_with_multiple_sources_and_date(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "folder_to_tar") + source2 = os.path.join(tmpdir, "another_folder_to_tar") + os.makedirs(source1) + os.makedirs(source2) + # put a file in it + Path(os.path.join(source1, "a_file.txt")) + Path(os.path.join(source2, "other.file")) + + config['TAR'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'True' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_tar is True + + date = bckup.get_date() + + expected_tar1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_tar.tar') + expected_tar2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_tar.tar') + assert not os.path.exists(expected_tar1) + assert not os.path.exists(expected_tar2) + backy._run_tar() + assert os.path.exists(expected_tar1) + assert os.path.exists(expected_tar2) + + +class TestRunningRsync(): + + def test_simple_rsync(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = {'sources' : f"{source}"} + # make file to be copied + Path(source).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_rsync_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source1) + Path(source2).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_rsync_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['RSYNC'] = {'sources' : f"{source}"} + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_requires_dash_r_arg_for_copying_only_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['RSYNC'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + backy = Backup(config, schema) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + def test_rsync_with_delete_after_options(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source_dir = os.path.join(tmpdir, "some_dir") + source_file1 = os.path.join(source_dir, "file1") + source_file2 = os.path.join(source_dir, "file2") + source_file3 = os.path.join(source_dir, "file3") + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source_dir) + Path(source_file1).touch() + Path(source_file2).touch() + Path(source_file3).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file1') + expected2dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file2') + expected3dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file3') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + backy._run_rsync() + print(expected1dir) + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + + # Delete two files, create a new one. + os.remove(source_file2) + os.remove(source_file3) + source_file4 = os.path.join(source_dir, "file4") + Path(source_file4).touch() + expected4dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file4') + + # Rsync will add 4, and 2 and 3 should still exist + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + # with delete after option removed files are cleaned up + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : '-v -z -r --delete-after' + } + backy = Backup(config, schema) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + +@pytest.mark.skip(reason="Requires a fake database to be setup") +class TestMysqlBackup(object): + """ The mySQL dump tests require the machine on which they are +running to have a mysql database running with a user called +'dumper' and a password set to 'not_a_real_password'. + +In production, a proper password should be set on the server and in the config +file. DO NOT USE THE ABOVE IN PRODUCTION + +To change the password: +UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; +""" + + def test_mysql_dump_is_ok(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_mysqldump is True + + date = bckup.get_date() + + expected_dump1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_dump.sql') + expected_dump2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_dump.sql') + assert not os.path.exists(expected_dump1) + assert not os.path.exists(expected_dump2) + result = backy._run_mysqldump() + assert os.path.exists(expected_dump1) + assert os.path.exists(expected_dump2) + + assert result == 0 + + def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'incorrect_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + with pytest.raises(errs.SubprocessError): + backy._run_mysqldump() + + +# class TestRunningAll(object): + +# def test_running_copy_tar_and_rsyc(self): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + + +class TestSimpleLogWriting(object): + + def test_log_writing_success(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + backy.run_backup_processes() + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + # Read file + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + def test_log_writes_only_attempted_if_failure_encountered(self, tmpdir): + + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + # fails due to missing source file in copy + with pytest.raises(errs.ValidationError): + backy.run_backup_processes() + + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] Index: trunk/backups/confluence_backup.py =================================================================== --- trunk/backups/confluence_backup.py (revision 40971) +++ trunk/backups/confluence_backup.py (revision 40971) @@ -0,0 +1,99 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config') + +CONFLUENCE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class ConfluenceBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_confluence_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + confluence_format = "%Y_%m_%d" + if custom_date is not None: + return custom_date.strftime(confluence_format) + return datetime.date.today().strftime(confluence_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For confluence backup there should only be one source: + source = self.copy_dict['sources'][0] + + confluence_wildcard_search = f"{source}/*{ConfluenceBackup.get_confluence_backup_date_format()}*" + confluence_xml_filepaths = glob.glob(confluence_wildcard_search) + if len(confluence_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No confluence backups found of form: {confluence_wildcard_search}") + + for jxf in confluence_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + cb = ConfluenceBackup(config, schema) + cb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/backups/confluence_backup_test.py =================================================================== --- trunk/backups/confluence_backup_test.py (revision 40971) +++ trunk/backups/confluence_backup_test.py (revision 40971) @@ -0,0 +1,288 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backup as bckup +import confluence_backup as cbck_mod + +from confluence_backup import ConfluenceBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'True' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_confluence_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +ConfluenceBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + confluence_backup_path = os.path.join(tmpdir, "confluence_backup_path") + confluence_attachments_path = os.path.join(tmpdir, "confluence_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(confluence_backup_path) + os.makedirs(confluence_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(confluence_backup_path) + assert os.path.exists(confluence_attachments_path) + + # Create backup files that will be copied + confluence_backup_name = f"confluence-xml-backup-{ConfluenceBackup.get_confluence_backup_date_format()}.zip" + confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name) + Path(confluence_backup_filepath).touch() + assert os.path.exists(confluence_backup_filepath) + + # Create some files that will be tar'd + confluence_attachment_1_name = "attachment_1.zip" + confluence_attachment_2_name = "attachment_2.txt" + confluence_attachment_1_filepath = os.path.join(confluence_attachments_path, confluence_attachment_1_name) + confluence_attachment_2_filepath = os.path.join(confluence_attachments_path, confluence_attachment_2_name) + Path(confluence_attachment_1_filepath).touch() + Path(confluence_attachment_2_filepath).touch() + assert os.path.exists(confluence_attachment_1_filepath) + assert os.path.exists(confluence_attachment_2_filepath) + + +class TestArguments(object): + + def assert_defaults(self, jb: ConfluenceBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/confluence_backups' + , '/data/ippops3.0/backups/confluence_backups' + , '/data/ippops5.0/backups/confluence_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/backups'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/attachments'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = cbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + ConfluenceBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = ConfluenceBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + ConfluenceBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + cbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + conf_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_conf_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'conf.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, conf_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + cbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_confluence_backup_date_format(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + # return_value = cb._copy_confluence_backup() + with pytest.raises(ValidationError) as e: + cb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_confluence_copy(self, tmpdir): + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + cb = ConfluenceBackup(config_file=config) + + today = ConfluenceBackup.get_confluence_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"confluence-xml-backup-{today}.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + return_value = cb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_confluence_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + jb = ConfluenceBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + confluence_tar_name = f"{today}_confluence.tar.gz" + expected_confluence_host_tar = os.path.join(tmpdir, "host_backup_path", confluence_tar_name) + expected_confluence_bak1_tar = os.path.join(tmpdir, "backup_1_path", confluence_tar_name) + expected_confluence_bak2_tar = os.path.join(tmpdir, "backup_2_path", confluence_tar_name) + assert not os.path.exists(expected_confluence_host_tar) + assert not os.path.exists(expected_confluence_bak1_tar) + assert not os.path.exists(expected_confluence_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_confluence_host_tar) + assert os.path.exists(expected_confluence_bak1_tar) + assert os.path.exists(expected_confluence_bak2_tar) Index: trunk/backups/deploy_backups.py =================================================================== --- trunk/backups/deploy_backups.py (revision 40971) +++ trunk/backups/deploy_backups.py (revision 40971) @@ -0,0 +1,63 @@ +import argparse +import sys + +import utils.subprocess_utils as sub_utils + +DEFAULT_DEPLOYMENT = \ + { "jira_backup.py" : "ippops4" + , "confluence_backup.py" : "ippops4" + , "svn_backup.py" : "ippops4" + , "mailman_backup.py" : "ippops3" + , "ippops_homedir_backup.py" : "ippops3" + , "website_backup.py" : "ippops3" + } + +ADDITIONAL_FILES = \ + [ './utils' + , 'backup.py' + ] + +PREFIX = "/export/" +SUFFIX = ".0/backups/" + + +def deploy(user=None): + + for file in DEFAULT_DEPLOYMENT: + default_machine = DEFAULT_DEPLOYMENT[file] + + chosen_machine = input(f"Enter machine to copy {file} to [default={default_machine}; ENTER to accept]:") + chosen_machine.strip('\n').strip(' ') + if chosen_machine == '': + chosen_machine = default_machine + + copy_file_and_utils(file, chosen_machine, user) + + +def copy_file_and_utils(file, machine, user): + # copy all the utils across too + user_at = '' if user is None else f'{user}@' + target = f"{user_at}{machine}.ifa.hawaii.edu:{PREFIX}{machine}{SUFFIX}" + args = ['rsync', '-a', '--progress', file, ] + ADDITIONAL_FILES + [target] + print(f"Performing: {' '.join(args)}") + sub_utils.simple_unix_wrapper(args) + + +def main(): + args = parse_args(sys.argv[1:]) + args_dict = vars(args) + deploy(**args_dict) + + +def parse_args(args): + parser = argparse.ArgumentParser( + description='Performs deploys backup files to desired locations') + parser.add_argument('--user' + , nargs='?' + , help='specify the user to transfer files as' + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: trunk/backups/ippops_homedir_backup.py =================================================================== --- trunk/backups/ippops_homedir_backup.py (revision 40971) +++ trunk/backups/ippops_homedir_backup.py (revision 40971) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config') + +IPPOPS_HOMEDIR_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class IPPOpsHomedirBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + ohdb = IPPOpsHomedirBackup(config, schema) + ohdb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/backups/ippops_homedir_backup_test.py =================================================================== --- trunk/backups/ippops_homedir_backup_test.py (revision 40971) +++ trunk/backups/ippops_homedir_backup_test.py (revision 40971) @@ -0,0 +1,133 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import ippops_homedir_backup as ihbck_mod +from ippops_homedir_backup import IPPOpsHomedirBackup +from utils.config_parse_helper import ConfigSchema + +IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/ippops_homedir_test.config') + + +class TestArguments(object): + + def assert_defaults(self, ihb: IPPOpsHomedirBackup): + + assert ihb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/homedir_backups' + , '/data/ippops4.0/backups/homedir_backups' + , '/data/ippops5.0/backups/homedir_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert ihb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/home/panstarrs'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG) + self.assert_defaults(ihb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + ihbck_mod.main(config, schema) + assert True is True + + +class TestIppopsHomedirRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'home')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "home") + subdir1 = os.path.join(homedir, "some_user") + subdir2 = os.path.join(homedir, "some_other_user") + file1 = os.path.join(subdir1, "file_in_subdir1") + file2 = os.path.join(subdir2, "file_in_subdir2") + os.makedirs(homedir) + os.makedirs(subdir1) + os.makedirs(subdir2) + Path(file1).touch() + Path(file2).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + ihb = IPPOpsHomedirBackup(config, schema) + + expected_sub1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user') + expected_sub1_file1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user') + expected_sub2_file2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user/file_in_subdir2') + expected_sub1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user') + expected_sub1_file1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user') + expected_sub2_file2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user/file_in_subdir2') + assert not os.path.exists(expected_sub1_backup1) + assert not os.path.exists(expected_sub1_file1_backup1) + assert not os.path.exists(expected_sub2_backup1) + assert not os.path.exists(expected_sub2_file2_backup1) + assert not os.path.exists(expected_sub1_backup2) + assert not os.path.exists(expected_sub1_file1_backup2) + assert not os.path.exists(expected_sub2_backup2) + assert not os.path.exists(expected_sub2_file2_backup2) + ihb._run_rsync() + assert os.path.exists(expected_sub1_backup1) + assert os.path.exists(expected_sub1_file1_backup1) + assert os.path.exists(expected_sub2_backup1) + assert os.path.exists(expected_sub2_file2_backup1) + assert os.path.exists(expected_sub1_backup2) + assert os.path.exists(expected_sub1_file1_backup2) + assert os.path.exists(expected_sub2_backup2) + assert os.path.exists(expected_sub2_file2_backup2) Index: trunk/backups/jira_backup.py =================================================================== --- trunk/backups/jira_backup.py (revision 40971) +++ trunk/backups/jira_backup.py (revision 40971) @@ -0,0 +1,99 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config') + +JIRA_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class JiraBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_jira_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + jira_format = "%Y-%b-%d" + if custom_date is not None: + return custom_date.strftime(jira_format) + return datetime.date.today().strftime(jira_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For jira backup there should only be one source: + source = self.copy_dict['sources'][0] + + jira_wildcard_search = f"{source}/{JiraBackup.get_jira_backup_date_format()}*" + jira_xml_filepaths = glob.glob(jira_wildcard_search) + if len(jira_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No jira backups found of form: {jira_wildcard_search}") + + for jxf in jira_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + jb = JiraBackup(config, schema) + jb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/backups/jira_backup_test.py =================================================================== --- trunk/backups/jira_backup_test.py (revision 40971) +++ trunk/backups/jira_backup_test.py (revision 40971) @@ -0,0 +1,289 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backups.backup as bckup +import jira_backup as jbck_mod +from jira_backup import JiraBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'True' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_jira_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +JiraBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + jira_backup_path = os.path.join(tmpdir, "jira_backup_path") + jira_attachments_path = os.path.join(tmpdir, "jira_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(jira_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(jira_attachments_path) + + # Create backup files that will be copied + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + # Create some files that will be tar'd + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + +class TestArguments(object): + + def assert_defaults(self, jb: JiraBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/jira_backups' + , '/data/ippops3.0/backups/jira_backups' + , '/data/ippops5.0/backups/jira_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/export'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/data'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = jbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + JiraBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = JiraBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + JiraBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + jbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, jira_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_jira_backup_date_format(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + # return_value = jb._copy_jira_backup() + with pytest.raises(ValidationError) as e: + jb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_jira_copy(self, tmpdir): + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = JiraBackup.get_jira_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_jira_test_file.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_jira_test_file.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_jira_test_file.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + assert jb.should_run_copy is True + return_value = jb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_jira_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + jira_tar_name = f"{today}_jira.tar.gz" + expected_jira_host_tar = os.path.join(tmpdir, "host_backup_path", jira_tar_name) + expected_jira_bak1_tar = os.path.join(tmpdir, "backup_1_path", jira_tar_name) + expected_jira_bak2_tar = os.path.join(tmpdir, "backup_2_path", jira_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) Index: trunk/backups/mailman_backup.py =================================================================== --- trunk/backups/mailman_backup.py (revision 40971) +++ trunk/backups/mailman_backup.py (revision 40971) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config') + +MAILMAN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class MailmanBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + mb = MailmanBackup(config, schema) + mb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/backups/mailman_backup_test.py =================================================================== --- trunk/backups/mailman_backup_test.py (revision 40971) +++ trunk/backups/mailman_backup_test.py (revision 40971) @@ -0,0 +1,119 @@ +import configparser +import os +from pathlib import Path + +import mailman_backup as mailbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from mailman_backup import MailmanBackup +from utils.config_parse_helper import ConfigSchema + + +MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/mailman_test.config') + + +class TestArguments(object): + + def assert_defaults(self, mb: MailmanBackup): + + assert mb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/mailman_backups' + , '/data/ippops4.0/backups/mailman_backups' + , '/data/ippops5.0/backups/mailman_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert mb.rsync_dict == \ + { 'sources' : ['/var/lib/mailman'] + , 'additional_args' : ['-a', '--delete-after', '--copy-links'] + } + + def test_load_default_config_file(self): + mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG) + self.assert_defaults(mb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + mailbck_mod.main(config, schema) + assert True is True + + +class TestMailmanRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'mailman')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + mailman_root = os.path.join(tmpdir, "mailman") + subdir = os.path.join(mailman_root, "some_dir") + file = os.path.join(subdir, "file_in_subdir") + os.makedirs(mailman_root) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + mb = MailmanBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir') + expected1file = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir') + expected2file = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + mb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) Index: trunk/backups/svn_backup.py =================================================================== --- trunk/backups/svn_backup.py (revision 40971) +++ trunk/backups/svn_backup.py (revision 40971) @@ -0,0 +1,45 @@ +import os.path as path + +from utils.config_parse_helper import SpecialSchemaProcessing +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config') + +SVN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class SvnBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + sb = SvnBackup(config, schema) + sb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/backups/svn_backup_test.py =================================================================== --- trunk/backups/svn_backup_test.py (revision 40971) +++ trunk/backups/svn_backup_test.py (revision 40971) @@ -0,0 +1,193 @@ +import configparser +import os +from pathlib import Path + +import svn_backup as svnbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, COPY_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from svn_backup import SvnBackup +from utils.config_parse_helper import ConfigSchema + +SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config') + + +class TestArguments(object): + + def assert_defaults(self, sb: SvnBackup): + + assert sb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/svn_backups' + , '/data/ippops3.0/backups/svn_backups' + , '/data/ippops5.0/backups/svn_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert sb.copy_dict == \ + { 'sources' : ['/etc/apache2/mods-enabled/dav_svn.conf'] + , 'additional_args' : ['-v'] + } + assert sb.rsync_dict == \ + { 'sources' : ['/export/ippops4.0/svnroot'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + sb = SvnBackup(config_file=SVN_TEST_CONFIG) + self.assert_defaults(sb) + + +class TestSvnCopy(object): + """Simple inherit of base class Backup""" + + # Setup config + def test_copy(self, tmpdir): + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'apache2/mods-enabled/dav_svn.conf')}" + , 'additional_args' : "-v" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + apache_folder = os.path.join(tmpdir, "apache2") + subdir = os.path.join(apache_folder, "mods-enabled") + file = os.path.join(subdir, "dav_svn.conf") + os.makedirs(apache_folder) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(COPY_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'dav_svn.conf') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'dav_svn.conf') + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_file_bck2) + sb._run_copy() + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_file_bck2) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + svn_conf_file = "dav_svn.conf" + config['COPY'] = \ + { 'sources' : f'{copy_source}/{svn_conf_file}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, svn_conf_file)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + svnbck_mod.main(config, schema) + assert True is True + + +class TestSvnRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'svnroot')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + svnroot = os.path.join(tmpdir, "svnroot") + subdir = os.path.join(svnroot, "some_dir") + file1 = os.path.join(svnroot, "file_at_root_level") + file2 = os.path.join(subdir, "file_in_subdir") + os.makedirs(svnroot) + os.makedirs(subdir) + Path(file1).touch() + Path(file2).touch() + + # File that will be removed because of --delete-after + os.makedirs(os.path.join(backup_1_path, "svnroot")) + file_that_already_existed = os.path.join(backup_1_path, "svnroot", "file_to_disappear") + Path(file_that_already_existed).touch() + assert os.path.exists(file_that_already_existed) + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir') + expected1file1 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'file_at_root_level') + expected1file2 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir') + expected2file1 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'file_at_root_level') + expected2file2 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file1) + assert not os.path.exists(expected1file2) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file1) + assert not os.path.exists(expected2file2) + assert os.path.exists(file_that_already_existed) + sb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file1) + assert os.path.exists(expected1file2) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file1) + assert os.path.exists(expected2file2) + # file that does not exist in the source, is removed from the target + assert not os.path.exists(file_that_already_existed) Index: trunk/backups/testing/backup_test.config =================================================================== --- trunk/backups/testing/backup_test.config (revision 40971) +++ trunk/backups/testing/backup_test.config (revision 40971) @@ -0,0 +1,34 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = hostname_of_computer +verbose = True + +[COPY] +sources = /some_dir_to_copy_from/, + /some_dir/and_specific_file.jpeg +additional_args = -v + +[TAR] +sources = /this_dir/, + /another_dir/, + /and_a_couple_files/subdir/file1.txt, + /and_a_couple_files/subdir/file2.png, + /and_a_couple_files/subdir/file3.pdf +additional_args = --verbose -z -p +target_filename = cool_tar +prefix_date = True + + +[RSYNC] +sources = /dir/to/sync +additional_args = --progress -r + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +additional_args = +target_filename = itsa_mysqldump +prefix_date = True Index: trunk/backups/testing/confluence_test.config =================================================================== --- trunk/backups/testing/confluence_test.config (revision 40971) +++ trunk/backups/testing/confluence_test.config (revision 40971) @@ -0,0 +1,24 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/confluence_backups, + /data/ippops3.0/backups/confluence_backups, + /data/ippops5.0/backups/confluence_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/confluence/backups +additional_args = -v +conf_prefix = confluence-xml-backup- + +[TAR] +sources = /var/atlassian/application-data/confluence/attachments +additional_args = --verbose -z -p -cf +target_filename = confluence.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = confluencedb +target_filename = confluence_mysqldump.sql +prefix_date = True Index: trunk/backups/testing/ippops_homedir_test.config =================================================================== --- trunk/backups/testing/ippops_homedir_test.config (revision 40971) +++ trunk/backups/testing/ippops_homedir_test.config (revision 40971) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/homedir_backups, + /data/ippops4.0/backups/homedir_backups, + /data/ippops5.0/backups/homedir_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/home/panstarrs +additional_args = -a --delete-after Index: trunk/backups/testing/jira_test.config =================================================================== --- trunk/backups/testing/jira_test.config (revision 40971) +++ trunk/backups/testing/jira_test.config (revision 40971) @@ -0,0 +1,23 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/jira_backups, + /data/ippops3.0/backups/jira_backups, + /data/ippops5.0/backups/jira_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/jira/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/jira/data +additional_args = --verbose -z -p -cf +target_filename = jira.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +target_filename = jira_mysqldump.sql +prefix_date = True Index: trunk/backups/testing/mailman_test.config =================================================================== --- trunk/backups/testing/mailman_test.config (revision 40971) +++ trunk/backups/testing/mailman_test.config (revision 40971) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/mailman_backups, + /data/ippops4.0/backups/mailman_backups, + /data/ippops5.0/backups/mailman_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /var/lib/mailman +additional_args = -a --delete-after --copy-links Index: trunk/backups/testing/svn_test.config =================================================================== --- trunk/backups/testing/svn_test.config (revision 40971) +++ trunk/backups/testing/svn_test.config (revision 40971) @@ -0,0 +1,14 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/svn_backups, + /data/ippops3.0/backups/svn_backups, + /data/ippops5.0/backups/svn_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /etc/apache2/mods-enabled/dav_svn.conf +additional_args = -v + +[RSYNC] +sources = /export/ippops4.0/svnroot +additional_args = -a --delete-after Index: trunk/backups/testing/website_test.config =================================================================== --- trunk/backups/testing/website_test.config (revision 40971) +++ trunk/backups/testing/website_test.config (revision 40971) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/website_backups, + /data/ippops4.0/backups/website_backups, + /data/ippops5.0/backups/website_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/www +additional_args = -a --delete-after Index: trunk/backups/utils/config_parse_helper.py =================================================================== --- trunk/backups/utils/config_parse_helper.py (revision 40971) +++ trunk/backups/utils/config_parse_helper.py (revision 40971) @@ -0,0 +1,180 @@ +from enum import Enum +from configparser import ConfigParser + +from utils.errors import ConfigError, ValidationError + + +class SpecialSchemaProcessing(Enum): + """These enums define how processing should be applied to a string to +separate it out into list. + """ + commma_separated = 1 + space_separated = 2 + + +class ConfigSchemaLine(): + """Defines a config line by what the key is, if it is required or optional, +the type that the value should be processed as, and any special processing required +""" + + def __init__(self, key: str, required: bool, expected_type, special_processing: SpecialSchemaProcessing=None): + self.key = key + self.required = required + self.expected_type = expected_type + if expected_type is list: + if type(special_processing) is not SpecialSchemaProcessing: + raise ValidationError("ValidationError: special_processing must be defined if implementing a list in config") + self.special_processing = special_processing + + def __str__(self): + return (f'<{self.__class__.__name__} - ' + f'Key:{self.key}, ' + f'Required:{self.required}, ' + f'Type:{self.expected_type}, ' + f'Processing: {self.special_processing}' + f'>') + + +class ConfigSchemaSection(): + """Defines a schema to be used. Consists of a single section name and +multiple config lines""" + + def __init__(self, section_name, options: [ConfigSchemaLine]): + self.section_name = section_name + self.options = {} + self._process_options(options) + + def _process_options(self, options): + if options is None: + return + if type(options) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} options must be '[ConfigSchemaLine]' or 'None'") + if len(options) == 0: + return + for o in options: + self.add_line(o) + + def add_line(self, line: ConfigSchemaLine): + self.options[line.key] = line + + def key_in_options(self, key: str) -> bool: + return key in self.options.keys() + + def get_schema_line(self, key: str) -> ConfigSchemaLine: + if not self.key_in_options(key): + raise KeyError(f"KeyError: '{key}' is not present in schema section") + return self.options[key] + + def __str__(self): + options_strs = [ o.__str__() for o in self.options] + return (f'{self.__class__.__name__} named "{self.section_name}" with {options_strs}') + + +class ConfigSchema(): + + def __init__(self, sections: [ConfigSchemaSection]): + self.sections = {} + self._process_sections(sections) + + def _process_sections(self, sections): + if sections is None: + return + if type(sections) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} sections must be '[ConfigSchemaSection]' or 'None'") + if len(sections) == 0: + return + for sec in sections: + self.add_section(sec) + + def add_section(self, section: ConfigSchemaSection): + self.sections[section.section_name] = section + + def section_exists(self, section_name: str) -> bool: + return section_name in self.sections.keys() + + def get_schema_section(self, section_name: str) -> ConfigSchemaSection: + if not self.section_exists(section_name): + raise KeyError(f"KeyError: '{section_name}' is not present in schema") + return self.sections[section_name] + + def __str__(self): + section_strs = [ o.__str__() for o in self.sections] + return (f'{self.__class__.__name__} with {section_strs}') + + +def _split_string_to_list(items: str, separator=',') -> [str]: + split_items = items.split(separator) + processed_items = [ i.strip(' ').strip('\n') for i in split_items] + return processed_items + + +def _parse_config_section(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: + """For a given config it will extract all values defined by the schema +and return them as a dictionary of the keys and the assoicated value as the +type defined by the schema +""" + parsed_items = {} + + section = schema.section_name + for csl_key in schema.options: + csl = schema.get_schema_line(csl_key) + + # Parses each line + if config.has_option(section, csl_key): + if csl.expected_type is bool: + parsed_items[csl.key] = config.getboolean(section, csl.key) + elif csl.expected_type is int: + parsed_items[csl.key] = config.getint(section, csl.key) + elif csl.expected_type is float: + parsed_items[csl.key] = config.getfloat(section, csl.key) + elif csl.expected_type is str: + parsed_items[csl.key] = config[section][csl.key] + elif csl.expected_type is list: + if csl.special_processing == SpecialSchemaProcessing.commma_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ',') + parsed_items[csl.key] = splitup + elif csl.special_processing == SpecialSchemaProcessing.space_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ' ') + parsed_items[csl.key] = splitup + else: + + raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}") + else: + raise ConfigError(config, f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}") + + else: + if csl.required: + raise ConfigError(config, f"ConfigError: Key '{csl.key}' is required but was not found in config") + + return parsed_items + + +def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: + parsed_items = {} + + for css_name in schema.sections: + css = schema.get_schema_section(css_name) + + # if config.has_section(css_name): + parsed_section = _parse_config_section(config, css) + parsed_items[css_name] = parsed_section + # else: + # raise ConfigError(config, f"ConfigError: Expected section '{css_name}' from schema not present in config") + + return parsed_items + + +class ParsedConfig(): + + def __init__(self, config: ConfigParser, schema: ConfigSchema): + self._section_dict = _parse_config(config, schema) + + def has_section(self, section_name): + return section_name in self._section_dict + + def get_section(self, section_name): + if self.has_section(section_name): + return self._section_dict[section_name] + return None Index: trunk/backups/utils/config_parse_helper_test.py =================================================================== --- trunk/backups/utils/config_parse_helper_test.py (revision 40971) +++ trunk/backups/utils/config_parse_helper_test.py (revision 40971) @@ -0,0 +1,295 @@ +import pytest + +import utils.config_parse_helper as helper +from utils.errors import ConfigError, ValidationError + + +class TestSpecialSchemaProcessingEnum(): + + def test_all_enums(self): + cs = helper.SpecialSchemaProcessing.commma_separated + assert cs.name == "commma_separated" + assert cs.value == 1 + + ss = helper.SpecialSchemaProcessing.space_separated + assert ss.name == "space_separated" + assert ss.value == 2 + + assert len(list(helper.SpecialSchemaProcessing)) == 2 + + +class TestConfigSchemaLine(): + + def test_initialising_simple_schema_lines(self): + csl = helper.ConfigSchemaLine('key1', True, bool) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is bool + assert csl.special_processing is None + + def test_initialising_list_in_schema(self): + csl = helper.ConfigSchemaLine('key1', True, list, helper.SpecialSchemaProcessing.commma_separated) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is list + assert csl.special_processing is helper.SpecialSchemaProcessing.commma_separated + + def test_init_list_without_special_processing_fails(self): + not_a_valid_enum = 3 + with pytest.raises(ValidationError): + helper.ConfigSchemaLine('key1', True, list, not_a_valid_enum) + + +class TestConfigSchemaSection(): + + def test_initalising_schema(self): + csl1 = helper.ConfigSchemaLine(key='is_cool', required=False, expected_type=bool) + csl2 = helper.ConfigSchemaLine('fav_word', True, str) + csl3 = helper.ConfigSchemaLine('some_float', False, float) + + schema = helper.ConfigSchemaSection( 'section_name', [csl1, csl2, csl3] ) + assert schema.options == { 'is_cool' : csl1 + , 'fav_word' : csl2 + , 'some_float' : csl3 + } + + def test_empty_schema_section_ok(self): + helper.ConfigSchemaSection("ok_with_none", None) + helper.ConfigSchemaSection("ok_with_empty", []) + + def test_schema_section_fails_if_options_are_not_a_list(self): + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("fails_if_not_a_list", True) + + csl = helper.ConfigSchemaLine('key1', True, bool) + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("even_fails_on_a_single_csl", csl) + + def test_adding_to_schema(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl1 = helper.ConfigSchemaLine('key1', True, bool) + csl2 = helper.ConfigSchemaLine('key2', False, int) + csl3 = helper.ConfigSchemaLine('key3', True, str) + config.add_line(csl1) + assert config.options == { 'key1': csl1 } + + config.add_line(csl2) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + } + + config.add_line(csl3) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + , 'key3': csl3 + } + + def test_adding_a_duplicate_key_to_schema_replaces_previous(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl_ori = helper.ConfigSchemaLine('key', True, bool) + csl_alt = helper.ConfigSchemaLine('key', False, int) + + config.add_line(csl_ori) + assert config.options == { 'key': csl_ori } + + config.add_line(csl_alt) + assert config.options == { 'key': csl_alt } + + +class TestStringSplitting(): + + def test_split_comma_separated_list(self): + before_parse = "/hello/sir/, /glorius/filename,/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected + + def test_removal_of_newlines_too(self): + before_parse = "\n/hello/sir/, /glorius/filename\n,\n/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected +# def _split_string_to_list(items: str, separator=',') -> [str]: + + def test_space_separated_split(self): + before_parse = "--verbose --progress -a -f -z" + expected = [ "--verbose" + , "--progress" + , "-a" + , "-f" + , "-z" + ] + result = helper._split_string_to_list(before_parse, ' ') + assert result == expected + + +class TestParsingConfig(): + + def make_simple_config(self, section_name='DEFAULT') -> helper.ConfigParser: + config = helper.ConfigParser() + config[section_name] = \ + { 'inty' : '3' + , 'floaty' : '3.141' + , 'mr_bool' : 'True' + , 'stringer' : "bell" + , 'listo_comma' : "i,am,comma,separated" + , 'listo_space' : "yo it's time for space!" + } + return config + + def test_parsing_fails_with_missing_required_key(self): + config = helper.ConfigParser() + config['DEFAULT'] = { 'not_the_required_key' : False } + + csl1 = helper.ConfigSchemaLine('Required_key', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_schema_section_with_only_optional_values_is_ok(self): + csl1 = helper.ConfigSchemaLine('not_required', False, bool) + csl2 = helper.ConfigSchemaLine('not_required_either', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1, csl2] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == {} + + def test_parsing_schema_section_with_ints(self): + csl1 = helper.ConfigSchemaLine('inty', False, int) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'inty' : 3} + + def test_parsing_schema_section_with_floats(self): + csl1 = helper.ConfigSchemaLine('floaty', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + + def test_parsing_schema_section_with_bools(self): + csl1 = helper.ConfigSchemaLine('mr_bool', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'mr_bool' : True } + + def test_parsing_schema_section_with_strs(self): + csl1 = helper.ConfigSchemaLine('stringer', True, str) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'stringer' : 'bell' } + + def test_parsing_schema_section_with_lists_comma_separated(self): + csl1 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_comma' : [ "i" + , "am" + , "comma" + , "separated" + ] + } + + def test_parsing_schema_section_with_lists_space_separated(self): + csl1 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_space' : [ "yo" + , "it's" + , "time" + , "for" + , "space!" + ] + } + + def test_parsing_as_different_types_always_ok_for_str_and_list(self): + config = self.make_simple_config() + + csl_float = helper.ConfigSchemaLine('floaty', True, float) + csl_int = helper.ConfigSchemaLine('floaty', True, int) + csl_bool = helper.ConfigSchemaLine('floaty', True, bool) + csl_str = helper.ConfigSchemaLine('floaty', True, str) + csl_list = helper.ConfigSchemaLine('floaty', True, list, + helper.SpecialSchemaProcessing.commma_separated) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_float]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + assert type(result['floaty']) is float + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_int]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_bool]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_str]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : '3.141' } + assert type(result['floaty']) is str + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_list]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : ['3.141'] } + assert type(result['floaty']) is list + + def test_parsing_schema_line_fails_with_unknown_type(self): + csl1 = helper.ConfigSchemaLine('floaty', True, dict) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_all_special_processing_options_are_handled(self): + config = self.make_simple_config() + config['DEFAULT'] = {'enum_checker' : "comma, and spaces"} + + for e in helper.SpecialSchemaProcessing: + csl = helper.ConfigSchemaLine('enum_checker', False, list, e) + schema = helper.ConfigSchemaSection('DEFAULT', [csl]) + + result = helper._parse_config_section(config, schema) + assert type(result['enum_checker']) is list + + def test_parsing_all(self): + config = self.make_simple_config('FULL_TEST') + + csl1 = helper.ConfigSchemaLine('inty', False, int) + csl2 = helper.ConfigSchemaLine('floaty', False, float) + csl3 = helper.ConfigSchemaLine('mr_bool', True, bool) + csl4 = helper.ConfigSchemaLine('stringer', True, str) + csl5 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + csl6 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6]) + + result = helper._parse_config_section(config, schema) + assert result == \ + { 'inty' : 3 + , 'floaty' : 3.141 + , 'mr_bool' : True + , 'stringer' : "bell" + , 'listo_comma' : ["i", "am", "comma", "separated"] + , 'listo_space' : ["yo", "it's", "time", "for", "space!"] + } Index: trunk/backups/utils/errors.py =================================================================== --- trunk/backups/utils/errors.py (revision 40971) +++ trunk/backups/utils/errors.py (revision 40971) @@ -0,0 +1,16 @@ + + +class ConfigError(Exception): + """Error in loading the config file""" + + def __init__(self, config_item, message): + self.config_item = config_item + self.message = message + + +class ValidationError(Exception): + """Validation errors in utils functions. Raised when arguments are not valid""" + + +class SubprocessError(Exception): + """Errors in Popen stderr subprocess in subprocess_utils""" Index: trunk/backups/utils/generic_utils.py =================================================================== --- trunk/backups/utils/generic_utils.py (revision 40971) +++ trunk/backups/utils/generic_utils.py (revision 40971) @@ -0,0 +1,124 @@ +import glob +import os.path as path +import shutil +from distutils.dir_util import copy_tree + +from .subprocess_utils import simple_unix_wrapper +from .errors import ValidationError + + +def copy_to_dir(source_file: str, destination_path: str, verbose: bool=False): + """ Copies a single file into the destination directory. If a directory is + given as a source then all the contents of the directory will be copied (the + directory itself will not be). +""" + + if source_file is None or destination_path is None: + raise ValidationError("Error: source for copy cannot be None type") + if destination_path is None: + raise ValidationError("Error: copy destination cannot be None type") + if not path.exists(source_file): + raise ValidationError(f"Error: Copy Failure: source does not exist: {source_file}") + if not path.exists(destination_path) or not path.isdir(destination_path): + print("Error: Copy Failure: destination path does not exist: ", destination_path) + raise ValidationError(f"Error: Copy Failure: destination path does not exist: {destination_path}") + + try: + if verbose: + print(f"Copying: {source_file} to {destination_path}") + if path.isdir(source_file): + copy_tree(source_file, destination_path) + else: + shutil.copy2(source_file, destination_path) + except IOError as e: + msg = f"Error: copy failure.\n copying: {source_file}\n to: {destination_path}" + raise IOError(msg) from e + return 0 + + +def _list_checks(sources: [str], targets: [str]): + if sources is None or targets is None: + raise ValidationError("Error: source for copy cannot be None type") + if targets is None: + raise ValidationError("Error: copy destination cannot be None type") + if type(sources) is not list or len(sources) == 0: + raise ValidationError("Error: sources must be given as a list with at least one entry") + if type(targets) is not list or len(targets) == 0: + raise ValidationError("Error: targets must be given as a list with at least one entry") + + +def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False): + _list_checks() + + for i in range(0, len(source_paths)): + + if not path.exists(destination_paths[i]): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", destination_paths[i]) + continue + if not path.isdir(destination_paths[i]): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i]) + continue + + result = copy_to_dir(source_paths[i], destination_paths[i], verbose) + if result != 0: + return result + + +def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False): + """ Copies the sources to the destination paths. + +This is a generic function that just wraps shutil.copy2""" + _list_checks(source_paths, destination_paths) + + for dpath in destination_paths: + if not path.exists(dpath): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", dpath) + continue + if not path.isdir(dpath): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", ) + continue + + for spath in source_paths: + result = copy_to_dir(spath, dpath, verbose) + if result != 0: + return result + + return 0 + + +def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False): + """ Moves all files from inside source paths to the destination dits. +It's important to note that it moves index to index +i.e. source[0] -> dest[0]; source[1] -> dest[1] + +This is actually a pretty generic function, should be moved into a utils class. +""" + + _list_checks(source_dirs, destination_dirs) + + if len(source_dirs) != len(destination_dirs): + raise ValidationError("Error: source and target paths are not equal; aborting file movement") + + for i in range(0, len(source_dirs)): + # Identify files in path, move each one: + wildcard_search = "{0}/*".format(source_dirs[i]) + filepaths = glob.glob(wildcard_search) + if len(filepaths) == 0: + print("Warning: directory is empty: ", source_dirs[i]) + continue + try: + for fp in filepaths: + if verbose: + print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) + command_and_args = ["mv", fp, destination_dirs[i]] + # shutil.move(fp, destination_dirs[i]) + simple_unix_wrapper(command_and_args) + except IOError as e: + print("IOError copying: ", e.filename, e.filename2) + return 2 + + return 0 Index: trunk/backups/utils/generic_utils_test.py =================================================================== --- trunk/backups/utils/generic_utils_test.py (revision 40971) +++ trunk/backups/utils/generic_utils_test.py (revision 40971) @@ -0,0 +1,224 @@ +import os +import pytest + +from .generic_utils import * +from .errors import ValidationError +from pathlib import Path + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyingToDir(object): + + def test_copy_fails_with_none_args(self, tmpdir): + with pytest.raises(ValidationError): + copy_to_dir(None, tmpdir) + + with pytest.raises(ValidationError): + real_file = make_a_real_file(tmpdir, "file.txt") + copy_to_dir(real_file, None) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(ValidationError): + copy_to_dir("not a real file", tmpdir) + + def test_copy_fails_if_destination_is_not_real(self, tmpdir): + with pytest.raises(ValidationError): + real_file = make_a_real_file(tmpdir, "real_file.txt") + copy_to_dir(real_file, "not a real directory") + + def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir): + + real_file = make_a_real_file(tmpdir, "not_a_directory.txt") + not_a_directory = real_file + assert os.path.exists(not_a_directory) + assert not os.path.isdir(not_a_directory) + + with pytest.raises(ValidationError): + copy_to_dir(real_file, not_a_directory) + + def test_copy_to_dir(self, tmpdir): + + # Setup source file and target dir + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + file_name = "fun_file.txt" + file_to_copy = os.path.join(tmpdir, file_name) + Path(file_to_copy).touch() + assert os.path.exists(file_to_copy) + + # copied file does not exist yet + expected_copy = os.path.join(tmpdir, "destination", file_name) + assert not os.path.exists(expected_copy) + + # run copy function and confirm copy exists (and original) + result = copy_to_dir(file_to_copy, destination_dir) + assert result == 0 + assert os.path.exists(file_to_copy) + assert os.path.exists(expected_copy) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + filepath_a = make_a_real_file(subdir, filename_a) + filepath_b = make_a_real_file(subsubdir, filename_b) + + print(filepath_a) + print(filepath_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + print(expected_filepath_a) + print(expected_filepath_b) + print(os.path.exists(expected_filepath_a)) + if os.path.exists(expected_filepath_a): + print("apple") + else: + print("banana") + print(os.path.exists(expected_filepath_b)) + + os.path.commonpath + + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # Provide only the original tmpdir to the copying function + copy_to_dir(subdir, destination_dir, True) + + # assert that the subdirs and files all exist + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + +class TestMultiCopy(object): + + def test_copy_fails_if_args_are_none(self, tmpdir): + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpdir, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, tmpdir) + + def test_copy_requires_args_to_be_in_list_format(self, tmpdir): + + not_a_list = "not a list" + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) + + tmpfile = make_a_real_file(tmpdir, "a_file.txt") + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) + + def test_copy_is_fine_with_single_items(self, tmpdir): + + tmpfile = make_a_real_file(tmpdir, "single_file.txt") + destination = os.path.join(tmpdir, "destination") + os.makedirs(destination) + expected_file = os.path.join(tmpdir, "destination", "single_file.txt") + assert not os.path.exists(expected_file) + + result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) + assert result == 0 + assert os.path.exists(expected_file) + + def test_copy_multiple_items_to_multiple_dirs(self, tmpdir): + + file_a = make_a_real_file(tmpdir, "single_file.txt") + file_b = make_a_real_file(tmpdir, "single_file.txt") + file_c = make_a_real_file(tmpdir, "single_file.txt") + source_files = [ file_a + , file_b + , file_c + ] + + destination_1 = os.path.join(tmpdir, "dest_1") + destination_2 = os.path.join(tmpdir, "dest_2") + destination_3 = os.path.join(tmpdir, "dest_3") + dest_paths = [ destination_1 + , destination_2 + , destination_3 + ] + os.makedirs(destination_1) + os.makedirs(destination_2) + os.makedirs(destination_3) + + expected_files = [] + for dp in dest_paths: + for sf in source_files: + expected_files.append( os.path.join(dp, os.path.basename(sf))) + assert not os.path.exists(expected_files[-1]) + assert len(expected_files) == 9 + + result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths) + assert result == 0 + for ef in expected_files: + assert os.path.exists(ef) + + def test_copying_is_recursive(self, tmpdir): + + # make some subdirs, with sub-subdirs, and files within them + subdir_a = os.path.join(tmpdir, "subdir_a") + subsubdir_aa = os.path.join(subdir_a, "subsubdir_aa") + subsubdir_ab = os.path.join(subdir_a, "subsubdir_ab") + subdir_b = os.path.join(tmpdir, "subdir_b") + subsubdir_ba = os.path.join(subdir_b, "subsubdir_ba") + subsubdir_bb = os.path.join(subdir_b, "subsubdir_bb") + os.makedirs(subsubdir_aa) + os.makedirs(subsubdir_ab) + os.makedirs(subsubdir_ba) + os.makedirs(subsubdir_bb) + make_a_real_file(subdir_a, "file_a.txt") + make_a_real_file(subsubdir_aa, "file_aa.txt") + make_a_real_file(subsubdir_ab, "file_ab.txt") + make_a_real_file(subdir_b, "file_b.txt") + make_a_real_file(subsubdir_ba, "file_ba.txt") + make_a_real_file(subsubdir_bb, "file_bb.txt") + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir_a", "file_a.txt") + expected_filepath_aa = os.path.join(destination_dir, "subdir_a", "subsubdir_aa", "file_aa.txt") + expected_filepath_ab = os.path.join(destination_dir, "subdir_a", "subsubdir_ab", "file_ab.txt") + expected_filepath_b = os.path.join(destination_dir, "subdir_b", "file_b.txt") + expected_filepath_ba = os.path.join(destination_dir, "subdir_b", "subsubdir_ba", "file_ba.txt") + expected_filepath_bb = os.path.join(destination_dir, "subdir_b", "subsubdir_bb", "file_bb.txt") + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_aa) + assert not os.path.exists(expected_filepath_ab) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_ba) + assert not os.path.exists(expected_filepath_bb) + + # Provide only the original tmpdir to the copying function + copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) + + # assert that the subdirs and files all exist + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_aa) + assert os.path.exists(expected_filepath_ab) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_ba) + assert os.path.exists(expected_filepath_bb) Index: trunk/backups/utils/subprocess_utils.py =================================================================== --- trunk/backups/utils/subprocess_utils.py (revision 40971) +++ trunk/backups/utils/subprocess_utils.py (revision 40971) @@ -0,0 +1,56 @@ +import subprocess + +# Plans are to flesh this out a bit more to make some smarter copying etc +from .errors import ValidationError, SubprocessError + + +def _args_check(args, function_name): + if args is None: + raise ValidationError(f"Error: no arguments provided in {function_name}") + if type(args) != list or len(args) == 0: + raise ValidationError(f"Error: no arguments provided in {function_name}") + return 0 + + +def cp_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {cp_wrapper.__name__}") + command_and_args = ["cp"] + args + return simple_unix_wrapper(command_and_args) + + +def tar_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {tar_wrapper.__name__}") + command_and_args = ["tar"] + args + return simple_unix_wrapper(command_and_args) + + +def chmod_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {chmod_wrapper.__name__}") + command_and_args = ["chmod"] + args + return simple_unix_wrapper(command_and_args) + + +def local_rsync_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}") + command_and_args = ["rsync"] + args + return simple_unix_wrapper(command_and_args) + + +def simple_unix_wrapper(command_and_args: [str]): + """ This should provide the program name and all arguments in a list. + +Only a single command should be called. + """ + _args_check(command_and_args, simple_unix_wrapper.__name__) + + try: + subprocess_exitcode = subprocess.check_call(command_and_args) + except (subprocess.CalledProcessError) as cpe: + raise SubprocessError(f"{type(cpe)} in python simple_unix_wrapper") from cpe + except (ValidationError) as ve: + raise ValidationError(f"{type(ve)} in python simple_unix_wrapper") from ve + return subprocess_exitcode Index: trunk/backups/utils/subprocess_utils_test.py =================================================================== --- trunk/backups/utils/subprocess_utils_test.py (revision 40971) +++ trunk/backups/utils/subprocess_utils_test.py (revision 40971) @@ -0,0 +1,240 @@ +import backups.utils.subprocess_utils as utils +import os +import pytest +import stat + +from pathlib import Path +from .errors import SubprocessError, ValidationError + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyWrapper(object): + + def test_copy_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.cp_wrapper(None) + + def test_copy_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.cp_wrapper(["not a real file", "not a valid copy_location"]) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.cp_wrapper(["not a real file", tmpdir]) + + def test_simple_copy(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.cp_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_copy_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run copy function and confirm copy exists (and original) + result = utils.cp_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # fails without -r arg: + with pytest.raises(SubprocessError): + utils.cp_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.cp_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + + +class TestTarWrapper(object): + + def test_tar_raise_validation_error_with_none_args(self): + with pytest.raises(ValidationError): + utils.tar_wrapper(None) + + def test_tar_returns_safely_with_too_few_args(self): + with pytest.raises(SubprocessError): + utils.tar_wrapper(["only one arg"]) + + def test_tar_return_error_if_invalid_args_given(self, tmpdir): + tar_filename = os.path.join(tmpdir, "tar_file.tar") + fake_file = os.path.join(tmpdir, "fake_file.txt") + assert not os.path.exists(fake_file) + with pytest.raises(SubprocessError): + utils.tar_wrapper(["-cf", tar_filename, fake_file]) + + def test_simple_tar_of_files(self, tmpdir): + file_a = make_a_real_file(tmpdir, "file_a.txt") + file_b = make_a_real_file(tmpdir, "file_b.txt") + + expected_file = os.path.join(tmpdir, "output.tar") + assert not os.path.exists(expected_file) + + args = ["-cf", expected_file, file_a, file_b] + utils.tar_wrapper(args) + assert os.path.exists(expected_file) + + +class TestChmodWrapper(object): + + def test_chmod_gives_validation_error_when_no_args(self): + with pytest.raises(utils.ValidationError): + utils.chmod_wrapper(None) + + def test_chmod_wrapper_raises_error_with_invalid_args(self): + with pytest.raises(SubprocessError): + utils.chmod_wrapper(["only one arg"]) + + def test_chmod_fails_if_no_permissions_given(self, tmpdir): + real_file = make_a_real_file(tmpdir, 'mr_file') + + with pytest.raises(SubprocessError): + utils.chmod_wrapper([real_file]) + + with pytest.raises(SubprocessError): + utils.chmod_wrapper(["dumb_arg", real_file]) + + def test_can_chmod_file(self, tmpdir): + real_file = make_a_real_file(tmpdir, 'mr_file') + + # check initial permissions + org_stats = os.stat(real_file) + # Default should be: rw-rw-r-- + assert bool(org_stats.st_mode & stat.S_IRUSR) is True + assert bool(org_stats.st_mode & stat.S_IWUSR) is True + assert bool(org_stats.st_mode & stat.S_IXUSR) is False + assert bool(org_stats.st_mode & stat.S_IRGRP) is True + assert bool(org_stats.st_mode & stat.S_IWGRP) is True + assert bool(org_stats.st_mode & stat.S_IXGRP) is False + assert bool(org_stats.st_mode & stat.S_IROTH) is True + assert bool(org_stats.st_mode & stat.S_IWOTH) is False + assert bool(org_stats.st_mode & stat.S_IXOTH) is False + + # change them + utils.chmod_wrapper(["777", real_file]) + + # check they changed + new_stats = os.stat(real_file) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestLocalRsyncWrapper(object): + """This is essentially a copy of the 'copy' tests above. + """ + + def test_rsync_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.local_rsync_wrapper(None) + + def test_rsync_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"]) + + def test_rsync_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", tmpdir]) + + def test_simple_rsync(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_rsync_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run rsync function and confirm rsync exists (and original) + result = utils.local_rsync_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_rsyncing_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # silently passes without -r, just skips over directories + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.local_rsync_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # requires -r to copy directories (and contents) + utils.local_rsync_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) Index: trunk/backups/website_backup.py =================================================================== --- trunk/backups/website_backup.py (revision 40971) +++ trunk/backups/website_backup.py (revision 40971) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config') + +WEBSITE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class WebsiteBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + wb = WebsiteBackup(config, schema) + wb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/backups/website_backup_test.py =================================================================== --- trunk/backups/website_backup_test.py (revision 40971) +++ trunk/backups/website_backup_test.py (revision 40971) @@ -0,0 +1,116 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import website_backup as webbck_mod +from website_backup import WebsiteBackup +from utils.config_parse_helper import ConfigSchema + +WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/website_test.config') + + +class TestArguments(object): + + def assert_defaults(self, wb: WebsiteBackup): + + assert wb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/website_backups' + , '/data/ippops4.0/backups/website_backups' + , '/data/ippops5.0/backups/website_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert wb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/www'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG) + self.assert_defaults(wb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + webbck_mod.main(config, schema) + assert True is True + + +class TestWebsiteRsync(object): + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'www')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "www") + subdir = os.path.join(homedir, "subdir") + file = os.path.join(subdir, "website_file") + os.makedirs(homedir) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + wb = WebsiteBackup(config, schema) + + expected_subdir_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir') + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir/website_file') + expected_subdir_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir/website_file') + assert not os.path.exists(expected_subdir_bck1) + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_subdir_bck2) + assert not os.path.exists(expected_file_bck2) + wb._run_rsync() + assert os.path.exists(expected_subdir_bck1) + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_subdir_bck2) + assert os.path.exists(expected_file_bck2)