Index: trunk/tools/backups/README =================================================================== --- trunk/tools/backups/README (revision 40967) +++ trunk/tools/backups/README (revision 40967) @@ -0,0 +1,13 @@ +README + +This module contains files that perform backups of various services used by +the IPP. Each file has it's own backup class, which is inherited from the base +class. To perform a backup there needs to a config that matches the expected +schema. + +Run 'deploy_backups.py' to deploy the relevant scripts and utils to the machines +that the services live on. + +Check a config is in place for each backup. + +If automated backup is desired run the script periodically in a crontab. Index: trunk/tools/backups/atlassian_backups.config =================================================================== --- trunk/tools/backups/atlassian_backups.config (revision 40936) +++ (revision ) @@ -1,10 +1,0 @@ -[atlassian_backups] -host_backup_path = /export/ippops4.0/atlassian_backups -backup_1_path = /data/ippops3.0/atlassian_backups -backup_2_path = /data/ippops5.0/atlassian_backups -jira_backup_path = /var/atlassian/application-data/jira/export -conf_backup_path = /var/atlassian/application-data/confluence/backups -jira_attachments_path = /var/atlassian/application-data/jira/data -conf_attachments_path = /var/atlassian/application-data/confluence/attachments -mysqldump_password = not_a_real_password -verbose = False Index: trunk/tools/backups/backup.py =================================================================== --- trunk/tools/backups/backup.py (revision 40967) +++ trunk/tools/backups/backup.py (revision 40967) @@ -0,0 +1,286 @@ +import configparser +import datetime +import os +import os.path as path +import socket +import subprocess +from configparser import ConfigParser + +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig, SpecialSchemaProcessing + +DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ] +) +COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + ] +) +TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) +RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + ] +) + +MYSQLDUMP_SCHEMA_SECTION = ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) + + +def get_backup_schema() -> ConfigSchema: + backup_schema = ConfigSchema(None) + backup_schema.add_section(DEFAULT_SCHEMA_SECTION) + backup_schema.add_section(COPY_SCHEMA_SECTION) + backup_schema.add_section(TAR_SCHEMA_SECTION) + backup_schema.add_section(RSYNC_SCHEMA_SECTION) + backup_schema.add_section(MYSQLDUMP_SCHEMA_SECTION) + return backup_schema + + +class Backup(): + """Base class for creating backups, contains all the generic versions of +functions for copying, taring etc. + """ + + def __init__(self, config_file=None, schema: ConfigSchema=None): + if config_file is None: + raise errs.ValidationError("ValidationError: Backup class must be given a config") + if schema is None: + schema = get_backup_schema() + self.load_config(config_file, schema) + + def load_config(self, config_class_or_filepath, schema): + """Accepts a filepath to a config file, or """ + + config = ConfigParser() + expected_class = configparser.ConfigParser + if config_class_or_filepath is None: + raise errs.ValidationError("ValidationError: config settings or filepath to config must be given") + elif isinstance(config_class_or_filepath, expected_class): + config = config_class_or_filepath + elif not path.exists(config_class_or_filepath): + raise errs.ValidationError(f"ValidationError: Config filepath does not exist (or incorrect class given, expected: {expected_class})") + config.read(config_class_or_filepath) + + self.read_config_and_schema(config, schema) + # self.read_copy_config_section(config, schema) + # self.read_tar_config_section(config, schema) + # self.read_rsync_config_section(config, schema) + # self.read_mysqldump_config_section(config, schema) + + def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): + parsed_config = ParsedConfig(config, schema) + self.default_dict = parsed_config.get_section('DEFAULT') + self.hostname_is_valid() + self.maybe_make_backup_dirs() + + section = 'COPY' + self.should_run_copy = parsed_config.has_section(section) + if self.should_run_copy: + self.copy_dict = parsed_config.get_section(section) + + section = 'TAR' + self.should_run_tar = parsed_config.has_section(section) + if self.should_run_tar: + self.tar_dict = parsed_config.get_section(section) + + section = 'RSYNC' + self.should_run_rsync = parsed_config.has_section(section) + if self.should_run_rsync: + self.rsync_dict = parsed_config.get_section(section) + + section = 'MYSQLDUMP' + self.should_run_mysqldump = parsed_config.has_section(section) + if self.should_run_mysqldump: + self.mysqldump_dict = parsed_config.get_section(section) + + def run_backup_processes(self): + self._write_begin_log() + if self.should_run_copy: + self._run_copy() + if self.should_run_tar: + self._run_tar() + if self.should_run_rsync: + self._run_rsync() + if self.should_run_mysqldump: + self._run_mysqldump() + self._write_end_log() + + def hostname_is_valid(self) -> bool: + if self.default_dict is None or 'source_machine' not in self.default_dict.keys(): + raise ValueError('No source machine specified') + + actual_hostname = socket.gethostname() + return actual_hostname == self.default_dict['source_machine'] + + def target_backup_paths_ok(self, raise_error=False): + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + if raise_error: + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + return False + return True + + def maybe_make_backup_dirs(self): + if self.default_dict is None or 'make_dirs' not in self.default_dict.keys(): + return + if self.default_dict['make_dirs']: + for backup_path in self.default_dict['backup_paths']: + if not os.path.exists(backup_path): + os.makedirs(backup_path) + sub_utils.chmod_wrapper(['777', backup_path]) + + def _run_copy(self): + if self.should_run_copy is None or False: + return 0 + self.target_backup_paths_ok(raise_error=True) + + for bp in self.default_dict['backup_paths']: + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.copy_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 + + def _run_tar(self): + if self.should_run_tar is None or False: + return 0 + + backup_paths = self.default_dict['backup_paths'] + + filename = self.tar_dict['target_filename'] + if 'prefix_date' in self.tar_dict.keys() and self.tar_dict['prefix_date']: + filename = f"{get_date()}_{filename}" + filename = path.join(backup_paths[0], filename) # may need some tar suffix magic + + add_args = [] if 'additional_args' not in self.tar_dict.keys() else self.tar_dict['additional_args'] + all_args = add_args + [filename] + self.tar_dict['sources'] + + try: + sub_utils.tar_wrapper(all_args) + sub_utils.chmod_wrapper(['774', filename]) + # copy to backups + if len(backup_paths) == 1: + return 0 + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([ filename, backup_paths[d]]) + except errs.SubprocessError as e: + raise errs.SubprocessError() from e + return 0 + + def _run_rsync(self): + if self.should_run_rsync is None or False: + return 0 + + self.target_backup_paths_ok(raise_error=True) + backup_paths = self.default_dict['backup_paths'] + for bp in backup_paths: + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.rsync_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.rsync_dict.keys(): + args = args + self.rsync_dict['additional_args'] + sub_utils.local_rsync_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: rsync\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 + + def _run_mysqldump(self): + + backup_paths = self.default_dict['backup_paths'] + + mysqldump_args = [ 'mysqldump' + , '-u', self.mysqldump_dict['user'] + , f"-p{self.mysqldump_dict['password']}" + , '--databases', self.mysqldump_dict['db_name'] + ] + if 'additional_args' in self.mysqldump_dict.keys(): + mysqldump_args = mysqldump_args + self.mysqldump_dict['additional_args'] + + dump_filename = self.mysqldump_dict['target_filename'] + if 'prefix_date' in self.mysqldump_dict.keys() and self.mysqldump_dict['prefix_date']: + dump_filename = f"{get_date()}_{dump_filename}" + dump_filename = path.join(backup_paths[0], dump_filename) + + if path.exists(dump_filename): + raise errs.ValidationError(f"ValidationError: mysqldump already exists {dump_filename}") + + with open(dump_filename, "w+") as df: + mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) + dump_result = mysqldump_process.communicate() + + # Get the piped std err. See if it contains the word error + stdo, stde = dump_result + if stde is not None: + decoded_stderr = stde.decode() + if decoded_stderr.casefold().find('error'.casefold()) > -1: + raise errs.SubprocessError(f"SubprocessError: Errors detected in mysqldump:\n{decoded_stderr}") + + # copy the dump to other locations + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([dump_filename, backup_paths[d]]) + return 0 + + def _write_begin_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if path.exists(log_file): + os.remove(log_file) + with open(log_file, "w+") as lf: + lf.write(f'Run started at: {datetime.date.today().strftime("%Y_%b_%d_%H:%M")}\n') + + def _write_end_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if not path.exists(log_file): + return + with open(log_file, "a") as lf: + lf.write(f'Last successful backup completed at : {datetime.date.today().strftime("%Y_%b_%d_%H:%M")}\n') + + +def get_date(custom_date=None, custom_format="%Y_%m_%d"): + # Format is currently: 2019-10-08 + if custom_date is not None: + return custom_date.strftime(custom_format) + return datetime.date.today().strftime(custom_format) Index: trunk/tools/backups/backup_atlassian_applications.py =================================================================== --- trunk/tools/backups/backup_atlassian_applications.py (revision 40936) +++ (revision ) @@ -1,315 +1,0 @@ -import argparse -import configparser -import datetime -import glob -import os -import os.path as path -import shutil -import subprocess -import sys - -import backups.utils.subprocess_utils as sub_utils -from backups.utils.errors import ConfigError, SubprocessError - -DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config') - - -def _move_dir_contents_to_dir(source_dirs: [str], destination_dirs: [str], verbose: bool=False): - """ Moves all files from inside source paths to the destination dits. -It's important to note that it moves index to index -i.e. source[0] -> dest[0]; source[1] -> dest[1] - -This is actually a pretty generic function, should be moved into a utils class. -""" - - if source_dirs is None or destination_dirs is None: - print("Error: You need to supply a source and destination") - return 2 - if type(source_dirs) != list or type(destination_dirs) != list: - print("Error: source_dirs and destination_dirs must be type: list") - print(" type {0} and {1} given".format(type(source_dirs), type(destination_dirs))) - return 2 - if len(source_dirs) != len(destination_dirs): - print("Error: source and target paths are not equal; aborting file movement") - return 2 - - for i in range(0, len(source_dirs)): - # Identify files in path, move each one: - wildcard_search = "{0}/*".format(source_dirs[i]) - filepaths = glob.glob(wildcard_search) - if len(filepaths) == 0: - print("Warning: directory is empty: ", source_dirs[i]) - continue - try: - for fp in filepaths: - if verbose: - print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) - command_and_args = ["mv", fp, destination_dirs[i]] - # shutil.move(fp, destination_dirs[i]) - sub_utils.simple_unix_wrapper(command_and_args) - except IOError as e: - print("IOError copying: ", e.filename, e.filename2) - return 2 - - return 0 - - -class AtlassianBackups(): - - def __init__(self, config_file=None): - self.config_file = DEFAULT_CONFIG_FILE if config_file is None else config_file - self.load_config(self.config_file) - - def load_config(self, config_class_or_filepath): - """Accepts a filepath to a config file, or """ - - config = configparser.ConfigParser() - if config_class_or_filepath is None: - raise ConfigError(config_class_or_filepath, "Neither config settings or filepath of config given") - elif isinstance(config_class_or_filepath, configparser.ConfigParser): - config = config_class_or_filepath - elif path.exists(config_class_or_filepath): - config.read(config_class_or_filepath) - - def _get_str_key_from_section(config, section, key): - if config.has_option(section, key): - return config[section][key] - raise ConfigError(config, "ConfigError: KeyError: no key found for {0}".format(key)) - - section = 'atlassian_backups' - self.host_backup_path = _get_str_key_from_section(config, section, 'host_backup_path') - self.backup_1_path = _get_str_key_from_section(config, section, 'backup_1_path') - self.backup_2_path = _get_str_key_from_section(config, section, 'backup_2_path') - self.jira_backup_path = _get_str_key_from_section(config, section, 'jira_backup_path') - self.conf_backup_path = _get_str_key_from_section(config, section, 'conf_backup_path') - self.jira_attachments_path = _get_str_key_from_section(config, section, 'jira_attachments_path') - self.conf_attachments_path = _get_str_key_from_section(config, section, 'conf_attachments_path') - self.verbose = config.getboolean('atlassian_backups', 'verbose') - - self.mysqldump_password = _get_str_key_from_section(config, section, 'mysqldump_password') - - def target_backup_paths_ok(self) -> bool: - copy_paths = [ self.host_backup_path - , self.backup_1_path - , self.backup_2_path - ] - for cp in copy_paths: - if not path.exists(cp): - return False - self._make_destination_subdirs_if_needed(cp) - - return True - - def _make_destination_subdirs_if_needed(self, copy_path): - subdirs = [ path.join(copy_path, "latest") - , path.join(copy_path, "old") - , path.join(copy_path, "tmp") - ] - for sd in subdirs: - if not path.exists(sd): - if self.verbose: - print("creating dir: ", sd) - os.makedirs(sd) - - def target_paths(self, subdir): - return [ path.join(self.host_backup_path, subdir) - , path.join(self.backup_1_path, subdir) - , path.join(self.backup_2_path, subdir) - ] - - def perform_atlassian_backups(self): - - # Check all paths are ok first: - if not self.target_backup_paths_ok(): - return 1 - - latest_paths = self.target_paths('latest') - tmp_paths = self.target_paths('tmp') - old_paths = self.target_paths('old') - - # Move latest to tmp (and delete if move ok) - _move_dir_contents_to_dir(latest_paths, tmp_paths, self.verbose) - - # copy jira - jira_return = self._copy_jira_backup(self.verbose) - - # copy confluence - conf_return = self._copy_confluence_backup(self.verbose) - - # tarfiles: - tar_return = self._tar_attachments() - - # mysqldump - mysqldump_return = self.make_mysql_dump(self.mysqldump_password) - - # If successful: move tmp to old - if jira_return == 0 and conf_return == 0 and tar_return == 0 and mysqldump_return == 0: - _move_dir_contents_to_dir(tmp_paths, old_paths, self.verbose) - else: - # If any failed: delete latest, move tmp back - _move_dir_contents_to_dir(tmp_paths, latest_paths, self.verbose) - return 5 - - # TODO:clean-up old files (to save space) - - return 0 - - def get_jira_backup_date_format(self, custom_date=None): - # Format is currently: 2019-Aug-13--0325.zip - jira_format = "%Y-%b-%d" - if custom_date is not None: - return custom_date.strftime(jira_format) - return datetime.date.today().strftime(jira_format) - - def _copy_jira_backup(self, verbose=False): - if not self.target_backup_paths_ok(): - return 2 - - jira_wildcard_search = "{0}/{1}*".format(self.jira_backup_path, self.get_jira_backup_date_format()) - jira_xml_filepaths = glob.glob(jira_wildcard_search) - if len(jira_xml_filepaths) == 0: - print("Copy Failure: No jira backups found at: ", jira_wildcard_search) - return 2 - target_paths = self.target_paths('latest') - for jfile in jira_xml_filepaths: - for tpath in target_paths: - if not path.exists(tpath): - print("Copy Failure: target path does not exist:", tpath) - return 2 - try: - if verbose: - print("Copying: {0} to {1}".format(jfile, tpath)) - shutil.copy2(jfile, tpath) - sub_utils.chmod_wrapper(['774', os.path.join(tpath, os.path.basename(jfile))]) - except IOError as e: - print("Error: copying jira backup.\n copying: {0}\n to: {1}".format(e.filename, e.filename2)) - print(e) - return 2 - return 0 - - def get_confluence_backup_date_format(self, custom_date=None): - # Format is: confluence-xml-backup-2019_08_14.zip - conf_format = "%Y_%m_%d" - if custom_date is not None: - return custom_date.strftime(conf_format) - return datetime.date.today().strftime(conf_format) - - def _copy_confluence_backup(self, verbose=False): - if not self.target_backup_paths_ok(): - print("Copy Failure: target path failure") - return 2 - - conf_wildcard_search = "{0}/*{1}*.zip".format(self.conf_backup_path, self.get_confluence_backup_date_format()) - conf_xml_filepaths = glob.glob(conf_wildcard_search) - if len(conf_xml_filepaths) == 0: - print("Copy Failure: No confluence backups found at: ", conf_wildcard_search) - return 2 - target_paths = self.target_paths('latest') - for cfile in conf_xml_filepaths: - for tpath in target_paths: - try: - sub_utils.cp_wrapper([cfile, tpath]) - sub_utils.chmod_wrapper(['774', os.path.join(tpath, os.path.basename(cfile))]) - except Exception as e: - print(e) - return 2 - return 0 - - def _tar_attachments(self): - # tar jira - latest_paths = self.target_paths("latest") - if not self.target_backup_paths_ok(): - return 1 - - jira_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_jira_attachments.tar" - jira_tar_filepath = path.join(latest_paths[0], jira_tar_name) - jira_tar_args = ["-C", "/", "-cf", jira_tar_filepath, self.jira_attachments_path.strip('/')] - sub_utils.tar_wrapper(jira_tar_args) - sub_utils.chmod_wrapper(['774', jira_tar_filepath]) - # copy to backups - sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[1]]) - sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[2]]) - - # tar confluence - confluence_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_confluence_attachments.tar" - confluence_tar_filepath = path.join(latest_paths[0], confluence_tar_name) - confluence_tar_args = ["-C", "/", "-cf", confluence_tar_filepath, self.conf_attachments_path.strip('/')] - sub_utils.tar_wrapper(confluence_tar_args) - sub_utils.chmod_wrapper(['774', confluence_tar_filepath]) - # copy to backups - sub_utils.cp_wrapper([ confluence_tar_filepath, latest_paths[1]]) - sub_utils.cp_wrapper([ confluence_tar_filepath, latest_paths[2]]) - return 0 - - def make_mysql_dump(self, dump_password, dump_name=None): - # do this as the dumper user - - latest_paths = self.target_paths("latest") - if not self.target_backup_paths_ok(): - return 1 - - # make sure target file does not exist (or it will fail) - if dump_name is None: - date = datetime.date.today().strftime("%Y_%m_%d") - dump_filepath = path.join(latest_paths[0], '{0}_mysql_dump.sql'.format(date)) - else: - dump_filepath = path.join(latest_paths[0], dump_name) - if path.exists(dump_filepath): - print("Warning: SQL dump already exists: ", dump_filepath) - return 2 - - # Setup the commands: - mysqldump_args = [ 'mysqldump', '-u', 'dumper', '-p{0}'.format(dump_password) - , '--databases', 'jiradb', 'confluencedb' - ] - with open(dump_filepath, "w+") as df: - mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) - dump_result = mysqldump_process.communicate() - - # Get the piped std err. See it contains the word error - stdo, stde = dump_result - if stde is not None: - decoded_stderr = stde.decode() - if decoded_stderr.casefold().find('error'.casefold()) > -1: - raise SubprocessError() - - # move the dump file to the correct locations - if not path.exists(dump_filepath): - return 2 - cp_1 = sub_utils.cp_wrapper([dump_filepath, latest_paths[1]]) - cp_2 = sub_utils.cp_wrapper([dump_filepath, latest_paths[2]]) - if cp_1 != 0: - return cp_1 - if cp_2 != 0: - return cp_2 - return 0 - - def tidy_backup_folders(self): - return 0 - - def run(self): - return self.perform_atlassian_backups() - - -def main(): - args = parse_args(sys.argv[1:]) - args_dict = vars(args) - ab = AtlassianBackups(**args_dict) - return ab.run() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of atlassian applications\nYou should run this on the machine hosting the atlassian applications') - - parser.add_argument('--config_file', - nargs='?', - help='contains config for all paths and mysqldump details', - default=DEFAULT_CONFIG_FILE) - - return parser.parse_args() - - -if __name__ == "__main__": - main() Index: trunk/tools/backups/backup_atlassian_applications_test.py =================================================================== --- trunk/tools/backups/backup_atlassian_applications_test.py (revision 40936) +++ (revision ) @@ -1,438 +1,0 @@ -import backups.backup_atlassian_applications as baa -import configparser -import datetime -import os -import pytest - -from pathlib import Path - -import backups.full_atlassian_backup_test as fab_test -import backups.utils.subprocess_utils as sub_utils - -from backups.backup_atlassian_applications import AtlassianBackups - -TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), './testing/test.config') - - -def make_config(tmpdir) -> configparser.ConfigParser: - config = configparser.ConfigParser() - config['atlassian_backups'] = \ - { - 'host_backup_path' : os.path.join(tmpdir, 'host_backup_path'), - 'backup_1_path' : os.path.join(tmpdir, 'backup_1_path'), - 'backup_2_path' : os.path.join(tmpdir, 'backup_2_path'), - 'jira_backup_path' : os.path.join(tmpdir, 'jira_backup_path'), - 'conf_backup_path' : os.path.join(tmpdir, 'conf_backup_path'), - 'jira_attachments_path' : os.path.join(tmpdir, 'jira_attachments_path'), - 'conf_attachments_path' : os.path.join(tmpdir, 'conf_attachments_path'), - 'mysqldump_password' : 'not_a_real_password' - } - - -class TestArguments(object): - - def test_load_from_config_file(self): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" - assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" - assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" - assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" - assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" - assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" - assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" - assert ab.mysqldump_password == "not_a_real_password" - assert ab.verbose is True - - def test_default_config_loads_with_no_args(self): - default_config = baa.DEFAULT_CONFIG_FILE - assert os.path.exists(default_config) - ab = AtlassianBackups() - assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" - assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" - assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" - assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" - assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" - assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" - assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" - assert ab.mysqldump_password == "not_a_real_password" - - def test_default_arguments(self): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" - assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" - assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" - assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" - assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" - assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" - assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" - assert ab.mysqldump_password == "not_a_real_password" - - def test_providing_config_directly(self): - config = configparser.ConfigParser() - config.add_section("atlassian_backups") - config.set('atlassian_backups', 'host_backup_path', 'custom_config1') - config.set('atlassian_backups', 'backup_1_path', 'custom_config2') - config.set('atlassian_backups', 'backup_2_path', 'custom_config3') - config.set('atlassian_backups', 'jira_backup_path', 'custom_config4') - config.set('atlassian_backups', 'conf_backup_path', 'custom_config5') - config.set('atlassian_backups', 'jira_attachments_path', 'custom_config6') - config.set('atlassian_backups', 'conf_attachments_path', 'custom_config7') - config.set('atlassian_backups', 'mysqldump_password', 'custom_config8') - config.set('atlassian_backups', 'verbose', 'False') - - ab = AtlassianBackups(config_file=config) - - assert ab.host_backup_path == 'custom_config1' - assert ab.backup_1_path == 'custom_config2' - assert ab.backup_2_path == 'custom_config3' - assert ab.jira_backup_path == 'custom_config4' - assert ab.conf_backup_path == 'custom_config5' - assert ab.jira_attachments_path == 'custom_config6' - assert ab.conf_attachments_path == 'custom_config7' - assert ab.mysqldump_password == 'custom_config8' - assert ab.verbose is False - - def test_nonexistant_config_file_raises_error(self): - with pytest.raises(baa.ConfigError): - AtlassianBackups("not a real file path") - - def test_default_config_loads_from_main_call(self): - try: - baa.main() - except Exception: - pytest.fail("An error was raised when it should not") - - -class TestFilepathVerifications(object): - - def test_default_target_paths_do_not_exists_in_testing(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - assert ab.target_backup_paths_ok() is False - - def test_an_incorrect_path_then_correct_path(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - assert os.path.exists(ab.jira_backup_path) is False - ab.jira_backup_path = tmpdir - assert os.path.exists(ab.jira_backup_path) is True - - def test_each_path_individually(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - assert os.path.exists(ab.host_backup_path) is False - ab.host_backup_path = tmpdir - assert os.path.exists(ab.host_backup_path) is True - assert ab.target_backup_paths_ok() is False - - assert os.path.exists(ab.backup_1_path) is False - ab.backup_1_path = tmpdir - assert os.path.exists(ab.backup_1_path) is True - assert ab.target_backup_paths_ok() is False - - assert os.path.exists(ab.backup_2_path) is False - ab.backup_2_path = tmpdir - assert os.path.exists(ab.backup_2_path) is True - # Once all backup paths are set, they are ok - assert ab.target_backup_paths_ok() is True - - assert os.path.exists(ab.jira_backup_path) is False - ab.jira_backup_path = tmpdir - assert os.path.exists(ab.jira_backup_path) is True - - assert os.path.exists(ab.conf_backup_path) is False - ab.conf_backup_path = tmpdir - assert os.path.exists(ab.conf_backup_path) is True - - assert os.path.exists(ab.jira_attachments_path) is False - ab.jira_attachments_path = tmpdir - assert os.path.exists(ab.jira_attachments_path) is True - - assert os.path.exists(ab.conf_attachments_path) is False - ab.conf_attachments_path = tmpdir - assert os.path.exists(ab.conf_attachments_path) is True - - def test_backup_subdirs_are_created_if_missing(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab.jira_path = tmpdir - ab.conf_path = tmpdir - - backup_paths = [ os.path.join(tmpdir, 'host') - , os.path.join(tmpdir, 'backup_1') - , os.path.join(tmpdir, 'backup_2') - ] - - for bp in backup_paths: - - # make the parent dir - os.makedirs(bp) - ab.bp = bp - assert os.path.exists(bp) - - # Expected subdirs - expected_latest = os.path.join(bp, 'latest') - expected_tmp = os.path.join(bp, 'tmp') - expected_old = os.path.join(bp, 'old') - assert not os.path.exists(expected_latest) - assert not os.path.exists(expected_tmp) - assert not os.path.exists(expected_old) - - # all_paths_ok will call subdir creation - ab._make_destination_subdirs_if_needed(bp) - assert os.path.exists(expected_latest) - assert os.path.exists(expected_tmp) - assert os.path.exists(expected_old) - - -class TestMovingBackups(object): - - def test_target_paths_with_defaults_function(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - default = ab.target_paths('default_paths') - - assert default[0] == "/export/ippops4.0/atlassian_backups/default_paths" - assert default[1] == "/data/ippops3.0/atlassian_backups/default_paths" - assert default[2] == "/data/ippops5.0/atlassian_backups/default_paths" - - def test_target_paths_function_after_changing_paths(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - ab.host_backup_path = "/different_path/for_host" - ab.backup_1_path = "/bk/one" - ab.backup_2_path = "/bk/two" - - another_subdir = ab.target_paths('a_different_subdir') - - assert another_subdir[0] == "/different_path/for_host/a_different_subdir" - assert another_subdir[1] == "/bk/one/a_different_subdir" - assert another_subdir[2] == "/bk/two/a_different_subdir" - - def test_move_aborts_if_source_and_destination_path_lists_are_different_lengths(self): - source_paths = [ "/Just/one/path" ] - destination_paths = [ "/Two", "/paths"] - - return_is_non_zero = baa._move_dir_contents_to_dir(source_paths, destination_paths) - assert return_is_non_zero != 0 - - def test_moving_old_backups(self, tmpdir): - # Need to set up X source_dirs, and X target_subdirs - tmpdir_host = os.path.join(tmpdir, "host") - tmpdir_bak1 = os.path.join(tmpdir, "bak1") - tmpdir_bak2 = os.path.join(tmpdir, "bak2") - os.makedirs(tmpdir_host) - os.makedirs(tmpdir_bak1) - os.makedirs(tmpdir_bak2) - - # Setup some subdirectories - source_paths = [ os.path.join(tmpdir_host, 'source') - , os.path.join(tmpdir_bak1, 'source') - , os.path.join(tmpdir_bak2, 'source') - ] - destination_paths = [ os.path.join(tmpdir_host, 'destination') - , os.path.join(tmpdir_bak1, 'destination') - , os.path.join(tmpdir_bak2, 'destination') - ] - for i in range(0, len(source_paths)): - os.makedirs(source_paths[i]) - os.makedirs(destination_paths[i]) - - # Put some files in the source directories - source_host_file1 = os.path.join(source_paths[0], "host_file1.zip") - source_bak1_file1 = os.path.join(source_paths[1], "bak1_file1.txt") - source_bak1_file2 = os.path.join(source_paths[1], "bak1_file2.txt") - source_bak1_file3 = os.path.join(source_paths[1], "bak1_file3.txt") - source_bak2_file1 = os.path.join(source_paths[2], "bak2_file1.lol") - source_bak2_file2 = os.path.join(source_paths[2], "bak2_file2.lol") - Path(source_host_file1).touch() - Path(source_bak1_file1).touch() - Path(source_bak1_file2).touch() - Path(source_bak1_file3).touch() - Path(source_bak2_file1).touch() - Path(source_bak2_file2).touch() - assert os.path.exists(source_host_file1) - assert os.path.exists(source_bak1_file1) - assert os.path.exists(source_bak1_file2) - assert os.path.exists(source_bak1_file3) - assert os.path.exists(source_bak2_file1) - assert os.path.exists(source_bak2_file2) - - # Assert files do NOT exist in target dirs yet - destination_host_file1 = os.path.join(destination_paths[0], "host_file1.zip") - destination_bak1_file1 = os.path.join(destination_paths[1], "bak1_file1.txt") - destination_bak1_file2 = os.path.join(destination_paths[1], "bak1_file2.txt") - destination_bak1_file3 = os.path.join(destination_paths[1], "bak1_file3.txt") - destination_bak2_file1 = os.path.join(destination_paths[2], "bak2_file1.lol") - destination_bak2_file2 = os.path.join(destination_paths[2], "bak2_file2.lol") - assert not os.path.exists(destination_host_file1) - assert not os.path.exists(destination_bak1_file1) - assert not os.path.exists(destination_bak1_file2) - assert not os.path.exists(destination_bak1_file3) - assert not os.path.exists(destination_bak2_file1) - assert not os.path.exists(destination_bak2_file2) - - # run the move - return_result = baa._move_dir_contents_to_dir(source_paths, destination_paths) - assert return_result == 0 - - # check that source files are no longer there, but the target ones do exist - assert not os.path.exists(source_host_file1) - assert not os.path.exists(source_bak1_file1) - assert not os.path.exists(source_bak1_file2) - assert not os.path.exists(source_bak1_file3) - assert not os.path.exists(source_bak2_file1) - assert not os.path.exists(source_bak2_file2) - assert os.path.exists(destination_host_file1) - assert os.path.exists(destination_bak1_file1) - assert os.path.exists(destination_bak1_file2) - assert os.path.exists(destination_bak1_file3) - assert os.path.exists(destination_bak2_file1) - assert os.path.exists(destination_bak2_file2) - - -class TestCopyingBackups(object): - - def test_get_jira_backup_date_format(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - today = datetime.date.today() - expected_format = today.strftime("%Y-%b-%d") - assert ab.get_jira_backup_date_format() == expected_format - - def test_copying_fails_if_paths_not_present(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - return_value = ab._copy_jira_backup() - assert return_value != 0 - - return_value = ab._copy_confluence_backup() - assert return_value != 0 - - def test_jira_copy(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - return_value = ab._copy_jira_backup() - assert return_value == 0 - - def test_get_confluence_backup_date_format(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - today = datetime.date.today() - expected_format = today.strftime("%Y_%m_%d") - assert ab.get_confluence_backup_date_format() == expected_format - - def test_confluence_copy(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - return_value = ab._copy_confluence_backup() - assert return_value == 0 - - def test_full_copy(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - result = ab.perform_atlassian_backups() - - assert result == 0 - - def test_atlsssian_copying_fails_when_paths_do_not_exist(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - fake_path = "/not/a/real/path" - ab.jira_backup_path = fake_path - ab.conf_backup_path = fake_path - ab.host_backup_path = fake_path - ab.backup_1_path = fake_path - ab.backup_2_path = fake_path - assert not os.path.exists(ab.jira_backup_path) - assert not os.path.exists(ab.conf_backup_path) - assert not os.path.exists(ab.host_backup_path) - assert not os.path.exists(ab.backup_1_path) - assert not os.path.exists(ab.backup_2_path) - - result = ab.perform_atlassian_backups() - assert result == 1 - - -class TestTaringOfAttachments(object): - - def test_attachments_tar(self, tmpdir): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - today = datetime.date.today().strftime("%Y_%m_%d") - jira_tar_name = today + "_jira_attachments.tar" - conf_tar_name = today + "_confluence_attachments.tar" - expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name) - expected_conf_host_tar = os.path.join(ab.host_backup_path, "latest", conf_tar_name) - expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name) - expected_conf_bak1_tar = os.path.join(ab.backup_1_path, "latest", conf_tar_name) - expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name) - expected_conf_bak2_tar = os.path.join(ab.backup_2_path, "latest", conf_tar_name) - assert not os.path.exists(expected_jira_host_tar) - assert not os.path.exists(expected_conf_host_tar) - assert not os.path.exists(expected_jira_bak1_tar) - assert not os.path.exists(expected_conf_bak1_tar) - assert not os.path.exists(expected_jira_bak2_tar) - assert not os.path.exists(expected_conf_bak2_tar) - - tar_result = ab._tar_attachments() - assert tar_result == 0 - - assert os.path.exists(expected_jira_host_tar) - assert os.path.exists(expected_conf_host_tar) - assert os.path.exists(expected_jira_bak1_tar) - assert os.path.exists(expected_conf_bak1_tar) - assert os.path.exists(expected_jira_bak2_tar) - assert os.path.exists(expected_conf_bak2_tar) - - -@pytest.mark.skip(reason="Requires a fake database to be setup") -class TestMysqlBackup(object): - """ The mySQL dump tests require the machine on which they are -running to have a mysql database running with a user called -'dumper' and a password set to 'not_a_real_password'. - -In production, a proper password should be set on the server and in the config -file. - -To change the password: -UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; -""" - - dumper_password = "not_a_real_password" - - def test_mysql_dump_is_ok(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - date = datetime.date.today().strftime("%Y_%m_%d") - expected_dump = os.path.join(ab.host_backup_path, "latest", '{0}_mysql_dump.sql'.format(date)) - print(expected_dump) - assert not os.path.exists(expected_dump) - - mysqldump_result = ab.make_mysql_dump(self.dumper_password) - assert mysqldump_result == 0 - - assert os.path.exists(expected_dump) - - def test_mysql_dump_is_ok_to_custom_file(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - expected_dump = os.path.join(ab.host_backup_path, "latest", "humptydumpy.sql") - print(expected_dump) - assert not os.path.exists(expected_dump) - - mysqldump_result = ab.make_mysql_dump(self.dumper_password, "humptydumpy.sql") - assert mysqldump_result == 0 - - assert os.path.exists(expected_dump) - - def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - with pytest.raises(sub_utils.SubprocessError): - ab.make_mysql_dump("Not the correct password") Index: trunk/tools/backups/backup_test.py =================================================================== --- trunk/tools/backups/backup_test.py (revision 40967) +++ trunk/tools/backups/backup_test.py (revision 40967) @@ -0,0 +1,729 @@ +import os +import pytest +import stat +import socket +from configparser import ConfigParser +from pathlib import Path + +import backup as bckup +import utils.errors as errs +from backup import Backup +from utils.config_parse_helper import ConfigSchema + +TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config') + + +def make_default_config( tmpdir: str + , backup_targs: [str] = ['bck1', 'bck2'] + , verbose: bool = False + ) -> ConfigParser: + config = ConfigParser() + + # make the paths: + for b in backup_targs: + os.makedirs(os.path.join(tmpdir, b)) + + # Get the correct formats for the config + backup_paths = ','.join([os.path.join(tmpdir, b) for b in backup_targs]) + source_machine = socket.gethostname() + config['DEFAULT'] = \ + { 'backup_paths' : backup_paths + , 'source_machine' : source_machine + , 'verbose' : verbose + } + return config + + +class TestArguments(object): + + def test_default_config_file_load_default(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # Default + assert backy.default_dict['backup_paths'] == \ + [ "/export/ippops4.0/atlassian_backups" + , "/data/ippops3.0/atlassian_backups" + , "/data/ippops5.0/atlassian_backups" + ] + assert backy.default_dict['source_machine'] == "hostname_of_computer" + assert backy.default_dict['verbose'] is True + + def test_copy_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.copy_dict['sources'] == \ + [ "/some_dir_to_copy_from/" + , "/some_dir/and_specific_file.jpeg" + ] + assert backy.copy_dict['additional_args'] == ['-v'] + + def test_tar_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.tar_dict['sources'] == \ + [ "/this_dir/" + , "/another_dir/" + , "/and_a_couple_files/subdir/file1.txt" + , "/and_a_couple_files/subdir/file2.png" + , "/and_a_couple_files/subdir/file3.pdf" + ] + assert backy.tar_dict['additional_args'] == [ "--verbose" + , "-z" + , "-p" + ] + assert backy.tar_dict['target_filename'] == "cool_tar" + assert backy.tar_dict['prefix_date'] is True + + def test_rsync_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Rsync] + assert backy.rsync_dict['sources'] == [ "/dir/to/sync"] + assert backy.rsync_dict['additional_args'] == \ + [ "--progress" + , "-r" + ] + + def test_mysqldump_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Mysqldump] + assert backy.mysqldump_dict['user'] == "dumper" + assert backy.mysqldump_dict['password'] == "not_a_real_password" + assert backy.mysqldump_dict['db_name'] == "jiradb" + assert backy.mysqldump_dict['additional_args'] == [''] + assert backy.mysqldump_dict['target_filename'] == "itsa_mysqldump" + assert backy.mysqldump_dict['prefix_date'] is True + + def test_loading_only_defaults(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.default_dict['backup_paths'] == \ + [ os.path.join(tmpdir, 'bck1') + , os.path.join(tmpdir, 'bck2') + ] + assert backy.default_dict['source_machine'] == socket.gethostname() + assert backy.default_dict['verbose'] is False + + def test_failure_if_no_defaults_provided(self): + config = ConfigParser() + with pytest.raises(errs.ConfigError) as e_empty: + Backup(config_file=config) + assert "is required but was not found in config" in str(e_empty) + + with pytest.raises(errs.ValidationError) as e_none: + Backup(config_file=None) + assert "ValidationError: Backup class must be given a config" in str(e_none) + + with pytest.raises(errs.ValidationError) as e_nofile: + Backup(config_file="/not/a/real/file.config") + assert "ValidationError: Config filepath does not exist" in str(e_nofile) + + def test_nothing_should_be_set_to_run_if_not_present(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.should_run_copy is False + assert backy.should_run_tar is False + assert backy.should_run_rsync is False + assert backy.should_run_mysqldump is False + + def test_hostname_assertion_fails(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + backy.default_dict['source_machine'] = 'not_a_real_hostname' + assert backy.hostname_is_valid() is False + + def test_hostname_assertion_passes(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + actual_hostname = socket.gethostname() + backy.default_dict['source_machine'] = actual_hostname + assert backy.hostname_is_valid() is True + + +class TestFilepathVerifications(object): + + def test_default_target_paths_do_not_exist_in_testing(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.target_backup_paths_ok() is False + + def test_raising_an_error_if_a_backup_path_does_not_exist(self): + with pytest.raises(errs.ValidationError) as e: + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + backy.target_backup_paths_ok(raise_error=True) + assert "ValidationError: target path does not exist" in str(e) + + def test_an_incorrect_path_then_correct_path(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + + backy.default_dict['backup_paths'] = ["/not/real/path"] + assert backy.target_backup_paths_ok() is False + + backy.default_dict['backup_paths'] = [tmpdir] + assert backy.target_backup_paths_ok() is True + + def test_backup_paths_created_if_make_dirs_given(self, tmpdir): + config = ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + bck1_path = os.path.join(tmpdir, 'bck1') + bck1b_path = os.path.join(tmpdir, 'bck1/bobby') + bck1bb_path = os.path.join(tmpdir, 'bck1/bobby/backup') + bck2_path = os.path.join(tmpdir, 'bck2') + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + # backup dirs not created automatically + Backup(config, schema) + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + + # make_dirs included in config - when loaded dirs created. + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + , 'make_dirs' : 'True' + } + backy = Backup(config, schema) + assert backy.default_dict['make_dirs'] is True + assert os.path.exists(bck1_path) + assert os.path.exists(bck1b_path) + assert os.path.exists(bck1bb_path) + assert os.path.exists(bck2_path) + + # Check permissions are 777 too + new_stats = os.stat(bck2_path) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestRunningCopy(): + + def test_simple_copy(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + # make file to be copied + Path(source).touch() + + backy = Backup(config, schema) + assert backy.should_run_copy is True + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_copy_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v --preserve -r" + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + os.makedirs(source1) + Path(source2).touch() + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_copy_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_requires_dash_r_arg_for_copying_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['COPY'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + with pytest.raises(errs.SubprocessError): + backy = Backup(config, schema) + backy._run_copy() + + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + +class TestRunningTar(): + + def test_simply_tar(self, tmpdir): + config = make_default_config(tmpdir, ['bck1']) + source = os.path.join(tmpdir, "folder_to_tar") + os.makedirs(source) + # put a file in it + Path(os.path.join(source, "a_file.txt")) + + config['TAR'] = \ + { 'sources' : f"{source}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_tar1 = os.path.join(tmpdir, 'bck1', 'cool_tar.tar') + print(expected_tar1) + assert not os.path.exists(expected_tar1) + backy._run_tar() + assert os.path.exists(expected_tar1) + + def test_tar_with_multiple_sources_and_date(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "folder_to_tar") + source2 = os.path.join(tmpdir, "another_folder_to_tar") + os.makedirs(source1) + os.makedirs(source2) + # put a file in it + Path(os.path.join(source1, "a_file.txt")) + Path(os.path.join(source2, "other.file")) + + config['TAR'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'True' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_tar is True + + date = bckup.get_date() + + expected_tar1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_tar.tar') + expected_tar2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_tar.tar') + assert not os.path.exists(expected_tar1) + assert not os.path.exists(expected_tar2) + backy._run_tar() + assert os.path.exists(expected_tar1) + assert os.path.exists(expected_tar2) + + +class TestRunningRsync(): + + def test_simple_rsync(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = {'sources' : f"{source}"} + # make file to be copied + Path(source).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_rsync_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source1) + Path(source2).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_rsync_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['RSYNC'] = {'sources' : f"{source}"} + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_requires_dash_r_arg_for_copying_only_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['RSYNC'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + backy = Backup(config, schema) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + def test_rsync_with_delete_after_options(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source_dir = os.path.join(tmpdir, "some_dir") + source_file1 = os.path.join(source_dir, "file1") + source_file2 = os.path.join(source_dir, "file2") + source_file3 = os.path.join(source_dir, "file3") + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source_dir) + Path(source_file1).touch() + Path(source_file2).touch() + Path(source_file3).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file1') + expected2dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file2') + expected3dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file3') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + backy._run_rsync() + print(expected1dir) + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + + # Delete two files, create a new one. + os.remove(source_file2) + os.remove(source_file3) + source_file4 = os.path.join(source_dir, "file4") + Path(source_file4).touch() + expected4dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file4') + + # Rsync will add 4, and 2 and 3 should still exist + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + # with delete after option removed files are cleaned up + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : '-v -z -r --delete-after' + } + backy = Backup(config, schema) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + +@pytest.mark.skip(reason="Requires a fake database to be setup") +class TestMysqlBackup(object): + """ The mySQL dump tests require the machine on which they are +running to have a mysql database running with a user called +'dumper' and a password set to 'not_a_real_password'. + +In production, a proper password should be set on the server and in the config +file. DO NOT USE THE ABOVE IN PRODUCTION + +To change the password: +UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; +""" + + def test_mysql_dump_is_ok(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_mysqldump is True + + date = bckup.get_date() + + expected_dump1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_dump.sql') + expected_dump2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_dump.sql') + assert not os.path.exists(expected_dump1) + assert not os.path.exists(expected_dump2) + result = backy._run_mysqldump() + assert os.path.exists(expected_dump1) + assert os.path.exists(expected_dump2) + + assert result == 0 + + def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'incorrect_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + with pytest.raises(errs.SubprocessError): + backy._run_mysqldump() + + +# class TestRunningAll(object): + +# def test_running_copy_tar_and_rsyc(self): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + + +class TestSimpleLogWriting(object): + + def test_log_writing_success(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + backy.run_backup_processes() + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + # Read file + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + def test_log_writes_only_attempted_if_failure_encountered(self, tmpdir): + + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + # fails due to missing source file in copy + with pytest.raises(errs.ValidationError): + backy.run_backup_processes() + + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] Index: trunk/tools/backups/confluence_backup.py =================================================================== --- trunk/tools/backups/confluence_backup.py (revision 40967) +++ trunk/tools/backups/confluence_backup.py (revision 40967) @@ -0,0 +1,99 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config') + +CONFLUENCE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class ConfluenceBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_confluence_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + confluence_format = "%Y_%m_%d" + if custom_date is not None: + return custom_date.strftime(confluence_format) + return datetime.date.today().strftime(confluence_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For confluence backup there should only be one source: + source = self.copy_dict['sources'][0] + + confluence_wildcard_search = f"{source}/*{ConfluenceBackup.get_confluence_backup_date_format()}*" + confluence_xml_filepaths = glob.glob(confluence_wildcard_search) + if len(confluence_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No confluence backups found of form: {confluence_wildcard_search}") + + for jxf in confluence_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + cb = ConfluenceBackup(config, schema) + cb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/confluence_backup_test.py =================================================================== --- trunk/tools/backups/confluence_backup_test.py (revision 40967) +++ trunk/tools/backups/confluence_backup_test.py (revision 40967) @@ -0,0 +1,288 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backup as bckup +import confluence_backup as cbck_mod + +from confluence_backup import ConfluenceBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'True' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_confluence_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +ConfluenceBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + confluence_backup_path = os.path.join(tmpdir, "confluence_backup_path") + confluence_attachments_path = os.path.join(tmpdir, "confluence_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(confluence_backup_path) + os.makedirs(confluence_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(confluence_backup_path) + assert os.path.exists(confluence_attachments_path) + + # Create backup files that will be copied + confluence_backup_name = f"confluence-xml-backup-{ConfluenceBackup.get_confluence_backup_date_format()}.zip" + confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name) + Path(confluence_backup_filepath).touch() + assert os.path.exists(confluence_backup_filepath) + + # Create some files that will be tar'd + confluence_attachment_1_name = "attachment_1.zip" + confluence_attachment_2_name = "attachment_2.txt" + confluence_attachment_1_filepath = os.path.join(confluence_attachments_path, confluence_attachment_1_name) + confluence_attachment_2_filepath = os.path.join(confluence_attachments_path, confluence_attachment_2_name) + Path(confluence_attachment_1_filepath).touch() + Path(confluence_attachment_2_filepath).touch() + assert os.path.exists(confluence_attachment_1_filepath) + assert os.path.exists(confluence_attachment_2_filepath) + + +class TestArguments(object): + + def assert_defaults(self, jb: ConfluenceBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/confluence_backups' + , '/data/ippops3.0/backups/confluence_backups' + , '/data/ippops5.0/backups/confluence_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/backups'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/attachments'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = cbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + ConfluenceBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = ConfluenceBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + ConfluenceBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + cbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + conf_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_conf_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'conf.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, conf_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + cbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_confluence_backup_date_format(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + # return_value = cb._copy_confluence_backup() + with pytest.raises(ValidationError) as e: + cb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_confluence_copy(self, tmpdir): + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + cb = ConfluenceBackup(config_file=config) + + today = ConfluenceBackup.get_confluence_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"confluence-xml-backup-{today}.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + return_value = cb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_confluence_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + jb = ConfluenceBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + confluence_tar_name = f"{today}_confluence.tar.gz" + expected_confluence_host_tar = os.path.join(tmpdir, "host_backup_path", confluence_tar_name) + expected_confluence_bak1_tar = os.path.join(tmpdir, "backup_1_path", confluence_tar_name) + expected_confluence_bak2_tar = os.path.join(tmpdir, "backup_2_path", confluence_tar_name) + assert not os.path.exists(expected_confluence_host_tar) + assert not os.path.exists(expected_confluence_bak1_tar) + assert not os.path.exists(expected_confluence_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_confluence_host_tar) + assert os.path.exists(expected_confluence_bak1_tar) + assert os.path.exists(expected_confluence_bak2_tar) Index: trunk/tools/backups/deploy_backups.py =================================================================== --- trunk/tools/backups/deploy_backups.py (revision 40967) +++ trunk/tools/backups/deploy_backups.py (revision 40967) @@ -0,0 +1,63 @@ +import argparse +import sys + +import utils.subprocess_utils as sub_utils + +DEFAULT_DEPLOYMENT = \ + { "jira_backup.py" : "ippops4" + , "confluence_backup.py" : "ippops4" + , "svn_backup.py" : "ippops4" + , "mailman_backup.py" : "ippops3" + , "ippops_homedir_backup.py" : "ippops3" + , "website_backup.py" : "ippops3" + } + +ADDITIONAL_FILES = \ + [ './utils' + , 'backup.py' + ] + +PREFIX = "/export/" +SUFFIX = ".0/backups/" + + +def deploy(user=None): + + for file in DEFAULT_DEPLOYMENT: + default_machine = DEFAULT_DEPLOYMENT[file] + + chosen_machine = input(f"Enter machine to copy {file} to [default={default_machine}; ENTER to accept]:") + chosen_machine.strip('\n').strip(' ') + if chosen_machine == '': + chosen_machine = default_machine + + copy_file_and_utils(file, chosen_machine, user) + + +def copy_file_and_utils(file, machine, user): + # copy all the utils across too + user_at = '' if user is None else f'{user}@' + target = f"{user_at}{machine}.ifa.hawaii.edu:{PREFIX}{machine}{SUFFIX}" + args = ['rsync', '-a', '--progress', file, ] + ADDITIONAL_FILES + [target] + print(f"Performing: {' '.join(args)}") + sub_utils.simple_unix_wrapper(args) + + +def main(): + args = parse_args(sys.argv[1:]) + args_dict = vars(args) + deploy(**args_dict) + + +def parse_args(args): + parser = argparse.ArgumentParser( + description='Performs deploys backup files to desired locations') + parser.add_argument('--user' + , nargs='?' + , help='specify the user to transfer files as' + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/fixture_test.py =================================================================== --- trunk/tools/backups/fixture_test.py (revision 40936) +++ (revision ) @@ -1,37 +1,0 @@ -import pytest - - -@pytest.fixture -def mr_fixture(): - - ab = {} - - def _add_to_dict(): - return ab.update({"hiya": 12}) - - def _add_more_to_dict(): - return ab.update({"bonjour": 13}) - - def _full_creation(): - _add_to_dict() - _add_more_to_dict() - print(ab) - return ab - - yield _full_creation() - - def _full_removal(): - ab = {} - print(ab) - - _full_removal() - - -class TestFixtures(object): - - def test_loading_fixture(self, mr_fixture): - ab = mr_fixture - print(mr_fixture) - type(mr_fixture) - assert ab["hiya"] == 12 - assert ab["bonjour"] == 13 Index: trunk/tools/backups/full_atlassian_backup_test.py =================================================================== --- trunk/tools/backups/full_atlassian_backup_test.py (revision 40936) +++ (revision ) @@ -1,146 +1,0 @@ -import datetime -import os -import backup_atlassian_applications as baa - -from pathlib import Path - -from backup_atlassian_applications import AtlassianBackups - -def create_all_paths_and_folders(ab_class: AtlassianBackups, tmpdir) -> AtlassianBackups: - """This creates all paths and files needed for thorough testing of the -atlassian backup class. The class returned will have all the paths set too. -Assertions are made that the files exist and they are set on the class object. - - The directory strucutre is as follows: - - tmpdir/host_backup/latest/2019-09-01_jira_prev_latest_file.zip - /2019_09_01_conf_prev_latest_file.zip - /backup_1/old/2019-08-13_jira_old_file.zip - /2019_08_13_conf_old_file.zip - /backup_2/latest/2019-09-01_jira_prev_latest_file.zip - /2019_09_01_conf_prev_latest_file.zip - /old/2019-08-13_jira_old_file.zip - /2019_08_13_conf_old_file.zip - /tmp - - # Directories where the original files come from (date will be todays) - tmpdir/jira_home/zip_in_here/2019-11-23_jira_test_file.zip - /attachments/attachment_1.zip - /attachments/attachment_2.txt - tmpdir/conf_home/zip_in_here/2019_11_23_conf_test_file.zip - /attachments/attachment_1.zip - /attachments/attachment_2.txt - -""" - - # Create all main paths required - host_backup_path = os.path.join(tmpdir, "host_backup") - backup_1_path = os.path.join(tmpdir, "backup_1") - backup_2_path = os.path.join(tmpdir, "backup_2") - jira_backup_path = os.path.join(tmpdir, "jira_home", "zip_in_here") - conf_backup_path = os.path.join(tmpdir, "conf_home", "zip_in_here") - jira_attachments_path = os.path.join(tmpdir, "jira_home", "attachments") - conf_attachments_path = os.path.join(tmpdir, "conf_home", "attachments") - os.makedirs(host_backup_path) - os.makedirs(backup_1_path) - os.makedirs(backup_2_path) - os.makedirs(jira_backup_path) - os.makedirs(conf_backup_path) - os.makedirs(jira_attachments_path) - os.makedirs(conf_attachments_path) - assert os.path.exists(host_backup_path) - assert os.path.exists(backup_1_path) - assert os.path.exists(backup_2_path) - assert os.path.exists(jira_backup_path) - assert os.path.exists(conf_backup_path) - assert os.path.exists(jira_attachments_path) - assert os.path.exists(conf_attachments_path) - - # And the some of the subdirs (only one of the backups has all of them) - os.makedirs(os.path.join(host_backup_path, "latest")) - os.makedirs(os.path.join(backup_1_path, "old")) - os.makedirs(os.path.join(backup_2_path, "old")) - os.makedirs(os.path.join(backup_2_path, "tmp")) - os.makedirs(os.path.join(backup_2_path, "latest")) - assert os.path.exists(os.path.join(host_backup_path, "latest")) - assert os.path.exists(os.path.join(backup_1_path, "old")) - assert os.path.exists(os.path.join(backup_2_path, "old")) - assert os.path.exists(os.path.join(backup_2_path, "tmp")) - assert os.path.exists(os.path.join(backup_2_path, "latest")) - - # set them on the class - ab_class.host_backup_path = host_backup_path - ab_class.backup_1_path = backup_1_path - ab_class.backup_2_path = backup_2_path - ab_class.jira_backup_path = jira_backup_path - ab_class.conf_backup_path = conf_backup_path - ab_class.jira_attachments_path = jira_attachments_path - ab_class.conf_attachments_path = conf_attachments_path - - # Create backup files - jira_backup_name = ab_class.get_jira_backup_date_format() + "_jira_test_file.zip" - jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) - Path(jira_backup_filepath).touch() - assert os.path.exists(jira_backup_filepath) - - conf_backup_name = ab_class.get_confluence_backup_date_format() + "_conf_test_file.zip" - conf_backup_filepath = os.path.join(conf_backup_path, conf_backup_name) - Path(conf_backup_filepath).touch() - assert os.path.exists(conf_backup_filepath) - - # Create some files that are attchments - jira_attachment_1_name = "attachment_1.zip" - jira_attachment_2_name = "attachment_2.txt" - jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) - jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) - Path(jira_attachment_1_filepath).touch() - Path(jira_attachment_2_filepath).touch() - assert os.path.exists(jira_attachment_1_filepath) - assert os.path.exists(jira_attachment_2_filepath) - - conf_attachment_1_name = "attachment_1.zip" - conf_attachment_2_name = "attachment_2.txt" - conf_attachment_1_filepath = os.path.join(conf_attachments_path, conf_attachment_1_name) - conf_attachment_2_filepath = os.path.join(conf_attachments_path, conf_attachment_2_name) - Path(conf_attachment_1_filepath).touch() - Path(conf_attachment_2_filepath).touch() - assert os.path.exists(conf_attachment_1_filepath) - assert os.path.exists(conf_attachment_2_filepath) - - # Have some old and latest versions in the backup dirs: - old_date = datetime.date(2019, 8, 13) - latest_date = datetime.date(2019, 9, 1) - jira_old_backup_name = ab_class.get_jira_backup_date_format(old_date) + "_jira_old_file.zip" - jira_latest_backup_name = ab_class.get_jira_backup_date_format(latest_date) + "_jira_prev_latest_file.zip" - conf_old_backup_name = ab_class.get_confluence_backup_date_format(old_date) + "_conf_old_file.zip" - conf_latest_backup_name = ab_class.get_confluence_backup_date_format(latest_date) + "_conf_prev_latest_file.zip" - - def assert_not_create_assert_exists(root_path, subdir, filename): - filepath = os.path.join(root_path, subdir, filename) - assert not os.path.exists(filepath) - Path(filepath).touch() - assert os.path.exists(filepath) - - assert_not_create_assert_exists(host_backup_path, "latest", jira_latest_backup_name) - assert_not_create_assert_exists(host_backup_path, "latest", conf_latest_backup_name) - assert_not_create_assert_exists(backup_1_path, "old", jira_old_backup_name) - assert_not_create_assert_exists(backup_1_path, "old", conf_old_backup_name) - assert_not_create_assert_exists(backup_2_path, "old", jira_old_backup_name) - assert_not_create_assert_exists(backup_2_path, "old", conf_old_backup_name) - assert_not_create_assert_exists(backup_2_path, "latest", jira_latest_backup_name) - assert_not_create_assert_exists(backup_2_path, "latest", conf_latest_backup_name) - - return ab_class - - -class TestEntireProcess(object): - - def test_entire_process(self, tmpdir): - test_config_file = os.path.join(os.path.dirname(__file__), 'testing', 'test.config') - - ab = AtlassianBackups(config_file=test_config_file) - ab = create_all_paths_and_folders(ab, tmpdir) - - result = ab.perform_atlassian_backups() - - assert result == 0 Index: trunk/tools/backups/ippops_homedir_backup.py =================================================================== --- trunk/tools/backups/ippops_homedir_backup.py (revision 40967) +++ trunk/tools/backups/ippops_homedir_backup.py (revision 40967) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config') + +IPPOPS_HOMEDIR_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class IPPOpsHomedirBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + ohdb = IPPOpsHomedirBackup(config, schema) + ohdb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/ippops_homedir_backup_test.py =================================================================== --- trunk/tools/backups/ippops_homedir_backup_test.py (revision 40967) +++ trunk/tools/backups/ippops_homedir_backup_test.py (revision 40967) @@ -0,0 +1,133 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import ippops_homedir_backup as ihbck_mod +from ippops_homedir_backup import IPPOpsHomedirBackup +from utils.config_parse_helper import ConfigSchema + +IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/ippops_homedir_test.config') + + +class TestArguments(object): + + def assert_defaults(self, ihb: IPPOpsHomedirBackup): + + assert ihb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/homedir_backups' + , '/data/ippops4.0/backups/homedir_backups' + , '/data/ippops5.0/backups/homedir_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert ihb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/home/panstarrs'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG) + self.assert_defaults(ihb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + ihbck_mod.main(config, schema) + assert True is True + + +class TestIppopsHomedirRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'home')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "home") + subdir1 = os.path.join(homedir, "some_user") + subdir2 = os.path.join(homedir, "some_other_user") + file1 = os.path.join(subdir1, "file_in_subdir1") + file2 = os.path.join(subdir2, "file_in_subdir2") + os.makedirs(homedir) + os.makedirs(subdir1) + os.makedirs(subdir2) + Path(file1).touch() + Path(file2).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + ihb = IPPOpsHomedirBackup(config, schema) + + expected_sub1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user') + expected_sub1_file1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user') + expected_sub2_file2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user/file_in_subdir2') + expected_sub1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user') + expected_sub1_file1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user') + expected_sub2_file2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user/file_in_subdir2') + assert not os.path.exists(expected_sub1_backup1) + assert not os.path.exists(expected_sub1_file1_backup1) + assert not os.path.exists(expected_sub2_backup1) + assert not os.path.exists(expected_sub2_file2_backup1) + assert not os.path.exists(expected_sub1_backup2) + assert not os.path.exists(expected_sub1_file1_backup2) + assert not os.path.exists(expected_sub2_backup2) + assert not os.path.exists(expected_sub2_file2_backup2) + ihb._run_rsync() + assert os.path.exists(expected_sub1_backup1) + assert os.path.exists(expected_sub1_file1_backup1) + assert os.path.exists(expected_sub2_backup1) + assert os.path.exists(expected_sub2_file2_backup1) + assert os.path.exists(expected_sub1_backup2) + assert os.path.exists(expected_sub1_file1_backup2) + assert os.path.exists(expected_sub2_backup2) + assert os.path.exists(expected_sub2_file2_backup2) Index: trunk/tools/backups/jira_backup.py =================================================================== --- trunk/tools/backups/jira_backup.py (revision 40967) +++ trunk/tools/backups/jira_backup.py (revision 40967) @@ -0,0 +1,99 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config') + +JIRA_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class JiraBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_jira_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + jira_format = "%Y-%b-%d" + if custom_date is not None: + return custom_date.strftime(jira_format) + return datetime.date.today().strftime(jira_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For jira backup there should only be one source: + source = self.copy_dict['sources'][0] + + jira_wildcard_search = f"{source}/{JiraBackup.get_jira_backup_date_format()}*" + jira_xml_filepaths = glob.glob(jira_wildcard_search) + if len(jira_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No jira backups found of form: {jira_wildcard_search}") + + for jxf in jira_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + jb = JiraBackup(config, schema) + jb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/jira_backup_test.py =================================================================== --- trunk/tools/backups/jira_backup_test.py (revision 40967) +++ trunk/tools/backups/jira_backup_test.py (revision 40967) @@ -0,0 +1,289 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backups.backup as bckup +import jira_backup as jbck_mod +from jira_backup import JiraBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'True' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_jira_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +JiraBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + jira_backup_path = os.path.join(tmpdir, "jira_backup_path") + jira_attachments_path = os.path.join(tmpdir, "jira_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(jira_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(jira_attachments_path) + + # Create backup files that will be copied + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + # Create some files that will be tar'd + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + +class TestArguments(object): + + def assert_defaults(self, jb: JiraBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/jira_backups' + , '/data/ippops3.0/backups/jira_backups' + , '/data/ippops5.0/backups/jira_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/export'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/data'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = jbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + JiraBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = JiraBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + JiraBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + jbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, jira_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_jira_backup_date_format(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + # return_value = jb._copy_jira_backup() + with pytest.raises(ValidationError) as e: + jb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_jira_copy(self, tmpdir): + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = JiraBackup.get_jira_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_jira_test_file.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_jira_test_file.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_jira_test_file.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + assert jb.should_run_copy is True + return_value = jb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_jira_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + jira_tar_name = f"{today}_jira.tar.gz" + expected_jira_host_tar = os.path.join(tmpdir, "host_backup_path", jira_tar_name) + expected_jira_bak1_tar = os.path.join(tmpdir, "backup_1_path", jira_tar_name) + expected_jira_bak2_tar = os.path.join(tmpdir, "backup_2_path", jira_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) Index: trunk/tools/backups/mailman_backup.py =================================================================== --- trunk/tools/backups/mailman_backup.py (revision 40967) +++ trunk/tools/backups/mailman_backup.py (revision 40967) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config') + +MAILMAN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class MailmanBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + mb = MailmanBackup(config, schema) + mb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/mailman_backup_test.py =================================================================== --- trunk/tools/backups/mailman_backup_test.py (revision 40967) +++ trunk/tools/backups/mailman_backup_test.py (revision 40967) @@ -0,0 +1,119 @@ +import configparser +import os +from pathlib import Path + +import mailman_backup as mailbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from mailman_backup import MailmanBackup +from utils.config_parse_helper import ConfigSchema + + +MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/mailman_test.config') + + +class TestArguments(object): + + def assert_defaults(self, mb: MailmanBackup): + + assert mb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/mailman_backups' + , '/data/ippops4.0/backups/mailman_backups' + , '/data/ippops5.0/backups/mailman_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert mb.rsync_dict == \ + { 'sources' : ['/var/lib/mailman'] + , 'additional_args' : ['-a', '--delete-after', '--copy-links'] + } + + def test_load_default_config_file(self): + mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG) + self.assert_defaults(mb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + mailbck_mod.main(config, schema) + assert True is True + + +class TestMailmanRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'mailman')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + mailman_root = os.path.join(tmpdir, "mailman") + subdir = os.path.join(mailman_root, "some_dir") + file = os.path.join(subdir, "file_in_subdir") + os.makedirs(mailman_root) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + mb = MailmanBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir') + expected1file = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir') + expected2file = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + mb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) Index: trunk/tools/backups/svn_backup.py =================================================================== --- trunk/tools/backups/svn_backup.py (revision 40967) +++ trunk/tools/backups/svn_backup.py (revision 40967) @@ -0,0 +1,45 @@ +import os.path as path + +from utils.config_parse_helper import SpecialSchemaProcessing +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config') + +SVN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class SvnBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + sb = SvnBackup(config, schema) + sb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/svn_backup_test.py =================================================================== --- trunk/tools/backups/svn_backup_test.py (revision 40967) +++ trunk/tools/backups/svn_backup_test.py (revision 40967) @@ -0,0 +1,193 @@ +import configparser +import os +from pathlib import Path + +import svn_backup as svnbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, COPY_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from svn_backup import SvnBackup +from utils.config_parse_helper import ConfigSchema + +SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config') + + +class TestArguments(object): + + def assert_defaults(self, sb: SvnBackup): + + assert sb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/svn_backups' + , '/data/ippops3.0/backups/svn_backups' + , '/data/ippops5.0/backups/svn_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert sb.copy_dict == \ + { 'sources' : ['/etc/apache2/mods-enabled/dav_svn.conf'] + , 'additional_args' : ['-v'] + } + assert sb.rsync_dict == \ + { 'sources' : ['/export/ippops4.0/svnroot'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + sb = SvnBackup(config_file=SVN_TEST_CONFIG) + self.assert_defaults(sb) + + +class TestSvnCopy(object): + """Simple inherit of base class Backup""" + + # Setup config + def test_copy(self, tmpdir): + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'apache2/mods-enabled/dav_svn.conf')}" + , 'additional_args' : "-v" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + apache_folder = os.path.join(tmpdir, "apache2") + subdir = os.path.join(apache_folder, "mods-enabled") + file = os.path.join(subdir, "dav_svn.conf") + os.makedirs(apache_folder) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(COPY_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'dav_svn.conf') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'dav_svn.conf') + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_file_bck2) + sb._run_copy() + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_file_bck2) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + svn_conf_file = "dav_svn.conf" + config['COPY'] = \ + { 'sources' : f'{copy_source}/{svn_conf_file}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, svn_conf_file)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + svnbck_mod.main(config, schema) + assert True is True + + +class TestSvnRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'svnroot')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + svnroot = os.path.join(tmpdir, "svnroot") + subdir = os.path.join(svnroot, "some_dir") + file1 = os.path.join(svnroot, "file_at_root_level") + file2 = os.path.join(subdir, "file_in_subdir") + os.makedirs(svnroot) + os.makedirs(subdir) + Path(file1).touch() + Path(file2).touch() + + # File that will be removed because of --delete-after + os.makedirs(os.path.join(backup_1_path, "svnroot")) + file_that_already_existed = os.path.join(backup_1_path, "svnroot", "file_to_disappear") + Path(file_that_already_existed).touch() + assert os.path.exists(file_that_already_existed) + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir') + expected1file1 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'file_at_root_level') + expected1file2 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir') + expected2file1 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'file_at_root_level') + expected2file2 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file1) + assert not os.path.exists(expected1file2) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file1) + assert not os.path.exists(expected2file2) + assert os.path.exists(file_that_already_existed) + sb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file1) + assert os.path.exists(expected1file2) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file1) + assert os.path.exists(expected2file2) + # file that does not exist in the source, is removed from the target + assert not os.path.exists(file_that_already_existed) Index: trunk/tools/backups/testing/backup_test.config =================================================================== --- trunk/tools/backups/testing/backup_test.config (revision 40967) +++ trunk/tools/backups/testing/backup_test.config (revision 40967) @@ -0,0 +1,34 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = hostname_of_computer +verbose = True + +[COPY] +sources = /some_dir_to_copy_from/, + /some_dir/and_specific_file.jpeg +additional_args = -v + +[TAR] +sources = /this_dir/, + /another_dir/, + /and_a_couple_files/subdir/file1.txt, + /and_a_couple_files/subdir/file2.png, + /and_a_couple_files/subdir/file3.pdf +additional_args = --verbose -z -p +target_filename = cool_tar +prefix_date = True + + +[RSYNC] +sources = /dir/to/sync +additional_args = --progress -r + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +additional_args = +target_filename = itsa_mysqldump +prefix_date = True Index: trunk/tools/backups/testing/confluence_test.config =================================================================== --- trunk/tools/backups/testing/confluence_test.config (revision 40967) +++ trunk/tools/backups/testing/confluence_test.config (revision 40967) @@ -0,0 +1,24 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/confluence_backups, + /data/ippops3.0/backups/confluence_backups, + /data/ippops5.0/backups/confluence_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/confluence/backups +additional_args = -v +conf_prefix = confluence-xml-backup- + +[TAR] +sources = /var/atlassian/application-data/confluence/attachments +additional_args = --verbose -z -p -cf +target_filename = confluence.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = confluencedb +target_filename = confluence_mysqldump.sql +prefix_date = True Index: trunk/tools/backups/testing/ippops_homedir_test.config =================================================================== --- trunk/tools/backups/testing/ippops_homedir_test.config (revision 40967) +++ trunk/tools/backups/testing/ippops_homedir_test.config (revision 40967) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/homedir_backups, + /data/ippops4.0/backups/homedir_backups, + /data/ippops5.0/backups/homedir_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/home/panstarrs +additional_args = -a --delete-after Index: trunk/tools/backups/testing/jira_test.config =================================================================== --- trunk/tools/backups/testing/jira_test.config (revision 40967) +++ trunk/tools/backups/testing/jira_test.config (revision 40967) @@ -0,0 +1,23 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/jira_backups, + /data/ippops3.0/backups/jira_backups, + /data/ippops5.0/backups/jira_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/jira/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/jira/data +additional_args = --verbose -z -p -cf +target_filename = jira.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +target_filename = jira_mysqldump.sql +prefix_date = True Index: trunk/tools/backups/testing/mailman_test.config =================================================================== --- trunk/tools/backups/testing/mailman_test.config (revision 40967) +++ trunk/tools/backups/testing/mailman_test.config (revision 40967) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/mailman_backups, + /data/ippops4.0/backups/mailman_backups, + /data/ippops5.0/backups/mailman_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /var/lib/mailman +additional_args = -a --delete-after --copy-links Index: trunk/tools/backups/testing/svn_test.config =================================================================== --- trunk/tools/backups/testing/svn_test.config (revision 40967) +++ trunk/tools/backups/testing/svn_test.config (revision 40967) @@ -0,0 +1,14 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/svn_backups, + /data/ippops3.0/backups/svn_backups, + /data/ippops5.0/backups/svn_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /etc/apache2/mods-enabled/dav_svn.conf +additional_args = -v + +[RSYNC] +sources = /export/ippops4.0/svnroot +additional_args = -a --delete-after Index: trunk/tools/backups/testing/test.config =================================================================== --- trunk/tools/backups/testing/test.config (revision 40936) +++ (revision ) @@ -1,10 +1,0 @@ -[atlassian_backups] -host_backup_path = /export/ippops4.0/atlassian_backups -backup_1_path = /data/ippops3.0/atlassian_backups -backup_2_path = /data/ippops5.0/atlassian_backups -jira_backup_path = /var/atlassian/application-data/jira/export -conf_backup_path = /var/atlassian/application-data/confluence/backups -jira_attachments_path = /var/atlassian/application-data/jira/data -conf_attachments_path = /var/atlassian/application-data/confluence/attachments -mysqldump_password = not_a_real_password -verbose = True Index: trunk/tools/backups/testing/website_test.config =================================================================== --- trunk/tools/backups/testing/website_test.config (revision 40967) +++ trunk/tools/backups/testing/website_test.config (revision 40967) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/website_backups, + /data/ippops4.0/backups/website_backups, + /data/ippops5.0/backups/website_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/www +additional_args = -a --delete-after Index: trunk/tools/backups/utils/config_parse_helper.py =================================================================== --- trunk/tools/backups/utils/config_parse_helper.py (revision 40967) +++ trunk/tools/backups/utils/config_parse_helper.py (revision 40967) @@ -0,0 +1,180 @@ +from enum import Enum +from configparser import ConfigParser + +from utils.errors import ConfigError, ValidationError + + +class SpecialSchemaProcessing(Enum): + """These enums define how processing should be applied to a string to +separate it out into list. + """ + commma_separated = 1 + space_separated = 2 + + +class ConfigSchemaLine(): + """Defines a config line by what the key is, if it is required or optional, +the type that the value should be processed as, and any special processing required +""" + + def __init__(self, key: str, required: bool, expected_type, special_processing: SpecialSchemaProcessing=None): + self.key = key + self.required = required + self.expected_type = expected_type + if expected_type is list: + if type(special_processing) is not SpecialSchemaProcessing: + raise ValidationError("ValidationError: special_processing must be defined if implementing a list in config") + self.special_processing = special_processing + + def __str__(self): + return (f'<{self.__class__.__name__} - ' + f'Key:{self.key}, ' + f'Required:{self.required}, ' + f'Type:{self.expected_type}, ' + f'Processing: {self.special_processing}' + f'>') + + +class ConfigSchemaSection(): + """Defines a schema to be used. Consists of a single section name and +multiple config lines""" + + def __init__(self, section_name, options: [ConfigSchemaLine]): + self.section_name = section_name + self.options = {} + self._process_options(options) + + def _process_options(self, options): + if options is None: + return + if type(options) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} options must be '[ConfigSchemaLine]' or 'None'") + if len(options) == 0: + return + for o in options: + self.add_line(o) + + def add_line(self, line: ConfigSchemaLine): + self.options[line.key] = line + + def key_in_options(self, key: str) -> bool: + return key in self.options.keys() + + def get_schema_line(self, key: str) -> ConfigSchemaLine: + if not self.key_in_options(key): + raise KeyError(f"KeyError: '{key}' is not present in schema section") + return self.options[key] + + def __str__(self): + options_strs = [ o.__str__() for o in self.options] + return (f'{self.__class__.__name__} named "{self.section_name}" with {options_strs}') + + +class ConfigSchema(): + + def __init__(self, sections: [ConfigSchemaSection]): + self.sections = {} + self._process_sections(sections) + + def _process_sections(self, sections): + if sections is None: + return + if type(sections) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} sections must be '[ConfigSchemaSection]' or 'None'") + if len(sections) == 0: + return + for sec in sections: + self.add_section(sec) + + def add_section(self, section: ConfigSchemaSection): + self.sections[section.section_name] = section + + def section_exists(self, section_name: str) -> bool: + return section_name in self.sections.keys() + + def get_schema_section(self, section_name: str) -> ConfigSchemaSection: + if not self.section_exists(section_name): + raise KeyError(f"KeyError: '{section_name}' is not present in schema") + return self.sections[section_name] + + def __str__(self): + section_strs = [ o.__str__() for o in self.sections] + return (f'{self.__class__.__name__} with {section_strs}') + + +def _split_string_to_list(items: str, separator=',') -> [str]: + split_items = items.split(separator) + processed_items = [ i.strip(' ').strip('\n') for i in split_items] + return processed_items + + +def _parse_config_section(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: + """For a given config it will extract all values defined by the schema +and return them as a dictionary of the keys and the assoicated value as the +type defined by the schema +""" + parsed_items = {} + + section = schema.section_name + for csl_key in schema.options: + csl = schema.get_schema_line(csl_key) + + # Parses each line + if config.has_option(section, csl_key): + if csl.expected_type is bool: + parsed_items[csl.key] = config.getboolean(section, csl.key) + elif csl.expected_type is int: + parsed_items[csl.key] = config.getint(section, csl.key) + elif csl.expected_type is float: + parsed_items[csl.key] = config.getfloat(section, csl.key) + elif csl.expected_type is str: + parsed_items[csl.key] = config[section][csl.key] + elif csl.expected_type is list: + if csl.special_processing == SpecialSchemaProcessing.commma_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ',') + parsed_items[csl.key] = splitup + elif csl.special_processing == SpecialSchemaProcessing.space_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ' ') + parsed_items[csl.key] = splitup + else: + + raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}") + else: + raise ConfigError(config, f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}") + + else: + if csl.required: + raise ConfigError(config, f"ConfigError: Key '{csl.key}' is required but was not found in config") + + return parsed_items + + +def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: + parsed_items = {} + + for css_name in schema.sections: + css = schema.get_schema_section(css_name) + + # if config.has_section(css_name): + parsed_section = _parse_config_section(config, css) + parsed_items[css_name] = parsed_section + # else: + # raise ConfigError(config, f"ConfigError: Expected section '{css_name}' from schema not present in config") + + return parsed_items + + +class ParsedConfig(): + + def __init__(self, config: ConfigParser, schema: ConfigSchema): + self._section_dict = _parse_config(config, schema) + + def has_section(self, section_name): + return section_name in self._section_dict + + def get_section(self, section_name): + if self.has_section(section_name): + return self._section_dict[section_name] + return None Index: trunk/tools/backups/utils/config_parse_helper_test.py =================================================================== --- trunk/tools/backups/utils/config_parse_helper_test.py (revision 40967) +++ trunk/tools/backups/utils/config_parse_helper_test.py (revision 40967) @@ -0,0 +1,295 @@ +import pytest + +import utils.config_parse_helper as helper +from utils.errors import ConfigError, ValidationError + + +class TestSpecialSchemaProcessingEnum(): + + def test_all_enums(self): + cs = helper.SpecialSchemaProcessing.commma_separated + assert cs.name == "commma_separated" + assert cs.value == 1 + + ss = helper.SpecialSchemaProcessing.space_separated + assert ss.name == "space_separated" + assert ss.value == 2 + + assert len(list(helper.SpecialSchemaProcessing)) == 2 + + +class TestConfigSchemaLine(): + + def test_initialising_simple_schema_lines(self): + csl = helper.ConfigSchemaLine('key1', True, bool) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is bool + assert csl.special_processing is None + + def test_initialising_list_in_schema(self): + csl = helper.ConfigSchemaLine('key1', True, list, helper.SpecialSchemaProcessing.commma_separated) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is list + assert csl.special_processing is helper.SpecialSchemaProcessing.commma_separated + + def test_init_list_without_special_processing_fails(self): + not_a_valid_enum = 3 + with pytest.raises(ValidationError): + helper.ConfigSchemaLine('key1', True, list, not_a_valid_enum) + + +class TestConfigSchemaSection(): + + def test_initalising_schema(self): + csl1 = helper.ConfigSchemaLine(key='is_cool', required=False, expected_type=bool) + csl2 = helper.ConfigSchemaLine('fav_word', True, str) + csl3 = helper.ConfigSchemaLine('some_float', False, float) + + schema = helper.ConfigSchemaSection( 'section_name', [csl1, csl2, csl3] ) + assert schema.options == { 'is_cool' : csl1 + , 'fav_word' : csl2 + , 'some_float' : csl3 + } + + def test_empty_schema_section_ok(self): + helper.ConfigSchemaSection("ok_with_none", None) + helper.ConfigSchemaSection("ok_with_empty", []) + + def test_schema_section_fails_if_options_are_not_a_list(self): + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("fails_if_not_a_list", True) + + csl = helper.ConfigSchemaLine('key1', True, bool) + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("even_fails_on_a_single_csl", csl) + + def test_adding_to_schema(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl1 = helper.ConfigSchemaLine('key1', True, bool) + csl2 = helper.ConfigSchemaLine('key2', False, int) + csl3 = helper.ConfigSchemaLine('key3', True, str) + config.add_line(csl1) + assert config.options == { 'key1': csl1 } + + config.add_line(csl2) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + } + + config.add_line(csl3) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + , 'key3': csl3 + } + + def test_adding_a_duplicate_key_to_schema_replaces_previous(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl_ori = helper.ConfigSchemaLine('key', True, bool) + csl_alt = helper.ConfigSchemaLine('key', False, int) + + config.add_line(csl_ori) + assert config.options == { 'key': csl_ori } + + config.add_line(csl_alt) + assert config.options == { 'key': csl_alt } + + +class TestStringSplitting(): + + def test_split_comma_separated_list(self): + before_parse = "/hello/sir/, /glorius/filename,/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected + + def test_removal_of_newlines_too(self): + before_parse = "\n/hello/sir/, /glorius/filename\n,\n/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected +# def _split_string_to_list(items: str, separator=',') -> [str]: + + def test_space_separated_split(self): + before_parse = "--verbose --progress -a -f -z" + expected = [ "--verbose" + , "--progress" + , "-a" + , "-f" + , "-z" + ] + result = helper._split_string_to_list(before_parse, ' ') + assert result == expected + + +class TestParsingConfig(): + + def make_simple_config(self, section_name='DEFAULT') -> helper.ConfigParser: + config = helper.ConfigParser() + config[section_name] = \ + { 'inty' : '3' + , 'floaty' : '3.141' + , 'mr_bool' : 'True' + , 'stringer' : "bell" + , 'listo_comma' : "i,am,comma,separated" + , 'listo_space' : "yo it's time for space!" + } + return config + + def test_parsing_fails_with_missing_required_key(self): + config = helper.ConfigParser() + config['DEFAULT'] = { 'not_the_required_key' : False } + + csl1 = helper.ConfigSchemaLine('Required_key', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_schema_section_with_only_optional_values_is_ok(self): + csl1 = helper.ConfigSchemaLine('not_required', False, bool) + csl2 = helper.ConfigSchemaLine('not_required_either', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1, csl2] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == {} + + def test_parsing_schema_section_with_ints(self): + csl1 = helper.ConfigSchemaLine('inty', False, int) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'inty' : 3} + + def test_parsing_schema_section_with_floats(self): + csl1 = helper.ConfigSchemaLine('floaty', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + + def test_parsing_schema_section_with_bools(self): + csl1 = helper.ConfigSchemaLine('mr_bool', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'mr_bool' : True } + + def test_parsing_schema_section_with_strs(self): + csl1 = helper.ConfigSchemaLine('stringer', True, str) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'stringer' : 'bell' } + + def test_parsing_schema_section_with_lists_comma_separated(self): + csl1 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_comma' : [ "i" + , "am" + , "comma" + , "separated" + ] + } + + def test_parsing_schema_section_with_lists_space_separated(self): + csl1 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_space' : [ "yo" + , "it's" + , "time" + , "for" + , "space!" + ] + } + + def test_parsing_as_different_types_always_ok_for_str_and_list(self): + config = self.make_simple_config() + + csl_float = helper.ConfigSchemaLine('floaty', True, float) + csl_int = helper.ConfigSchemaLine('floaty', True, int) + csl_bool = helper.ConfigSchemaLine('floaty', True, bool) + csl_str = helper.ConfigSchemaLine('floaty', True, str) + csl_list = helper.ConfigSchemaLine('floaty', True, list, + helper.SpecialSchemaProcessing.commma_separated) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_float]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + assert type(result['floaty']) is float + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_int]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_bool]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_str]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : '3.141' } + assert type(result['floaty']) is str + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_list]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : ['3.141'] } + assert type(result['floaty']) is list + + def test_parsing_schema_line_fails_with_unknown_type(self): + csl1 = helper.ConfigSchemaLine('floaty', True, dict) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_all_special_processing_options_are_handled(self): + config = self.make_simple_config() + config['DEFAULT'] = {'enum_checker' : "comma, and spaces"} + + for e in helper.SpecialSchemaProcessing: + csl = helper.ConfigSchemaLine('enum_checker', False, list, e) + schema = helper.ConfigSchemaSection('DEFAULT', [csl]) + + result = helper._parse_config_section(config, schema) + assert type(result['enum_checker']) is list + + def test_parsing_all(self): + config = self.make_simple_config('FULL_TEST') + + csl1 = helper.ConfigSchemaLine('inty', False, int) + csl2 = helper.ConfigSchemaLine('floaty', False, float) + csl3 = helper.ConfigSchemaLine('mr_bool', True, bool) + csl4 = helper.ConfigSchemaLine('stringer', True, str) + csl5 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + csl6 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6]) + + result = helper._parse_config_section(config, schema) + assert result == \ + { 'inty' : 3 + , 'floaty' : 3.141 + , 'mr_bool' : True + , 'stringer' : "bell" + , 'listo_comma' : ["i", "am", "comma", "separated"] + , 'listo_space' : ["yo", "it's", "time", "for", "space!"] + } Index: trunk/tools/backups/utils/generic_utils.py =================================================================== --- trunk/tools/backups/utils/generic_utils.py (revision 40936) +++ trunk/tools/backups/utils/generic_utils.py (revision 40967) @@ -1,7 +1,9 @@ +import glob import os.path as path import shutil +from distutils.dir_util import copy_tree -from distutils.dir_util import copy_tree -from backups.utils.errors import ValidationError +from .subprocess_utils import simple_unix_wrapper +from .errors import ValidationError @@ -35,18 +37,39 @@ +def _list_checks(sources: [str], targets: [str]): + if sources is None or targets is None: + raise ValidationError("Error: source for copy cannot be None type") + if targets is None: + raise ValidationError("Error: copy destination cannot be None type") + if type(sources) is not list or len(sources) == 0: + raise ValidationError("Error: sources must be given as a list with at least one entry") + if type(targets) is not list or len(targets) == 0: + raise ValidationError("Error: targets must be given as a list with at least one entry") + + +def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False): + _list_checks() + + for i in range(0, len(source_paths)): + + if not path.exists(destination_paths[i]): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", destination_paths[i]) + continue + if not path.isdir(destination_paths[i]): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i]) + continue + + result = copy_to_dir(source_paths[i], destination_paths[i], verbose) + if result != 0: + return result + + def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False): """ Copies the sources to the destination paths. This is a generic function that just wraps shutil.copy2""" - - if source_paths is None: - raise ValidationError("Error: source for copy cannot be None type") - if destination_paths is None: - raise ValidationError("Error: copy destination cannot be None type") - - if type(source_paths) is not list or len(source_paths) == 0: - raise ValidationError("Error: sources must be given as a list with at least one entry") - if type(destination_paths) is not list or len(destination_paths) == 0: - raise ValidationError("Error: destinations must be given as a list with at least one entry") + _list_checks(source_paths, destination_paths) for dpath in destination_paths: @@ -66,2 +89,36 @@ return 0 + + +def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False): + """ Moves all files from inside source paths to the destination dits. +It's important to note that it moves index to index +i.e. source[0] -> dest[0]; source[1] -> dest[1] + +This is actually a pretty generic function, should be moved into a utils class. +""" + + _list_checks(source_dirs, destination_dirs) + + if len(source_dirs) != len(destination_dirs): + raise ValidationError("Error: source and target paths are not equal; aborting file movement") + + for i in range(0, len(source_dirs)): + # Identify files in path, move each one: + wildcard_search = "{0}/*".format(source_dirs[i]) + filepaths = glob.glob(wildcard_search) + if len(filepaths) == 0: + print("Warning: directory is empty: ", source_dirs[i]) + continue + try: + for fp in filepaths: + if verbose: + print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) + command_and_args = ["mv", fp, destination_dirs[i]] + # shutil.move(fp, destination_dirs[i]) + simple_unix_wrapper(command_and_args) + except IOError as e: + print("IOError copying: ", e.filename, e.filename2) + return 2 + + return 0 Index: trunk/tools/backups/utils/generic_utils_test.py =================================================================== --- trunk/tools/backups/utils/generic_utils_test.py (revision 40936) +++ trunk/tools/backups/utils/generic_utils_test.py (revision 40967) @@ -1,7 +1,7 @@ -import backups.utils.generic_utils as utils import os import pytest -from backups.utils.errors import ValidationError +from .generic_utils import * +from .errors import ValidationError from pathlib import Path @@ -19,18 +19,18 @@ def test_copy_fails_with_none_args(self, tmpdir): with pytest.raises(ValidationError): - utils.copy_to_dir(None, tmpdir) + copy_to_dir(None, tmpdir) with pytest.raises(ValidationError): real_file = make_a_real_file(tmpdir, "file.txt") - utils.copy_to_dir(real_file, None) + copy_to_dir(real_file, None) def test_copy_fails_if_source_does_not_exist(self, tmpdir): with pytest.raises(ValidationError): - utils.copy_to_dir("not a real file", tmpdir) + copy_to_dir("not a real file", tmpdir) def test_copy_fails_if_destination_is_not_real(self, tmpdir): with pytest.raises(ValidationError): real_file = make_a_real_file(tmpdir, "real_file.txt") - utils.copy_to_dir(real_file, "not a real directory") + copy_to_dir(real_file, "not a real directory") def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir): @@ -42,5 +42,5 @@ with pytest.raises(ValidationError): - utils.copy_to_dir(real_file, not_a_directory) + copy_to_dir(real_file, not_a_directory) def test_copy_to_dir(self, tmpdir): @@ -60,5 +60,5 @@ # run copy function and confirm copy exists (and original) - result = utils.copy_to_dir(file_to_copy, destination_dir) + result = copy_to_dir(file_to_copy, destination_dir) assert result == 0 assert os.path.exists(file_to_copy) @@ -98,5 +98,5 @@ # Provide only the original tmpdir to the copying function - utils.copy_to_dir(subdir, destination_dir, True) + copy_to_dir(subdir, destination_dir, True) # assert that the subdirs and files all exist @@ -110,11 +110,11 @@ with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(None, None) - - with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(tmpdir, None) - - with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(None, tmpdir) + copy_multiple_items_to_multiple_dirs(None, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpdir, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, tmpdir) def test_copy_requires_args_to_be_in_list_format(self, tmpdir): @@ -122,9 +122,9 @@ not_a_list = "not a list" with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) + copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) tmpfile = make_a_real_file(tmpdir, "a_file.txt") with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) + copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) def test_copy_is_fine_with_single_items(self, tmpdir): @@ -136,5 +136,5 @@ assert not os.path.exists(expected_file) - result = utils.copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) + result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) assert result == 0 assert os.path.exists(expected_file) @@ -168,5 +168,5 @@ assert len(expected_files) == 9 - result = utils.copy_multiple_items_to_multiple_dirs(source_files, dest_paths) + result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths) assert result == 0 for ef in expected_files: @@ -212,5 +212,5 @@ # Provide only the original tmpdir to the copying function - utils.copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) + copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) # assert that the subdirs and files all exist Index: trunk/tools/backups/utils/subprocess_utils.py =================================================================== --- trunk/tools/backups/utils/subprocess_utils.py (revision 40936) +++ trunk/tools/backups/utils/subprocess_utils.py (revision 40967) @@ -2,5 +2,5 @@ # Plans are to flesh this out a bit more to make some smarter copying etc -from backups.utils.errors import ValidationError, SubprocessError +from .errors import ValidationError, SubprocessError @@ -34,4 +34,11 @@ +def local_rsync_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}") + command_and_args = ["rsync"] + args + return simple_unix_wrapper(command_and_args) + + def simple_unix_wrapper(command_and_args: [str]): """ This should provide the program name and all arguments in a list. @@ -42,8 +49,8 @@ try: - subprocess.check_call(command_and_args) + subprocess_exitcode = subprocess.check_call(command_and_args) except (subprocess.CalledProcessError) as cpe: raise SubprocessError(f"{type(cpe)} in python simple_unix_wrapper") from cpe except (ValidationError) as ve: raise ValidationError(f"{type(ve)} in python simple_unix_wrapper") from ve - return 0 + return subprocess_exitcode Index: trunk/tools/backups/utils/subprocess_utils_test.py =================================================================== --- trunk/tools/backups/utils/subprocess_utils_test.py (revision 40936) +++ trunk/tools/backups/utils/subprocess_utils_test.py (revision 40967) @@ -5,5 +5,5 @@ from pathlib import Path -from backups.utils.errors import SubprocessError, ValidationError +from .errors import SubprocessError, ValidationError @@ -164,2 +164,77 @@ assert bool(new_stats.st_mode & stat.S_IWOTH) is True assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestLocalRsyncWrapper(object): + """This is essentially a copy of the 'copy' tests above. + """ + + def test_rsync_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.local_rsync_wrapper(None) + + def test_rsync_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"]) + + def test_rsync_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", tmpdir]) + + def test_simple_rsync(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_rsync_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run rsync function and confirm rsync exists (and original) + result = utils.local_rsync_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_rsyncing_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # silently passes without -r, just skips over directories + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.local_rsync_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # requires -r to copy directories (and contents) + utils.local_rsync_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) Index: trunk/tools/backups/website_backup.py =================================================================== --- trunk/tools/backups/website_backup.py (revision 40967) +++ trunk/tools/backups/website_backup.py (revision 40967) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config') + +WEBSITE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class WebsiteBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + wb = WebsiteBackup(config, schema) + wb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: trunk/tools/backups/website_backup_test.py =================================================================== --- trunk/tools/backups/website_backup_test.py (revision 40967) +++ trunk/tools/backups/website_backup_test.py (revision 40967) @@ -0,0 +1,116 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import website_backup as webbck_mod +from website_backup import WebsiteBackup +from utils.config_parse_helper import ConfigSchema + +WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/website_test.config') + + +class TestArguments(object): + + def assert_defaults(self, wb: WebsiteBackup): + + assert wb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/website_backups' + , '/data/ippops4.0/backups/website_backups' + , '/data/ippops5.0/backups/website_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert wb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/www'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG) + self.assert_defaults(wb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + webbck_mod.main(config, schema) + assert True is True + + +class TestWebsiteRsync(object): + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'www')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "www") + subdir = os.path.join(homedir, "subdir") + file = os.path.join(subdir, "website_file") + os.makedirs(homedir) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + wb = WebsiteBackup(config, schema) + + expected_subdir_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir') + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir/website_file') + expected_subdir_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir/website_file') + assert not os.path.exists(expected_subdir_bck1) + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_subdir_bck2) + assert not os.path.exists(expected_file_bck2) + wb._run_rsync() + assert os.path.exists(expected_subdir_bck1) + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_subdir_bck2) + assert os.path.exists(expected_file_bck2)