Index: /branches/ipp-259_genericise_backups/tools/backups/backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40936) +++ /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40937) @@ -1,12 +1,14 @@ import configparser import datetime +import os import os.path as path import socket import subprocess +from configparser import ConfigParser import backups.utils.config_parse_helper as cfg_help import backups.utils.errors as errs import backups.utils.subprocess_utils as sub_utils -from configparser import ConfigParser +from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaLine DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config') @@ -19,6 +21,7 @@ def __init__(self, config_file=None): - self.config_file = DEFAULT_CONFIG_FILE if config_file is None else config_file - self.load_config(self.config_file) + if config_file is None: + raise errs.ValidationError("ValidationError: Backup class must be given a config") + self.load_config(config_file) def load_config(self, config_class_or_filepath): @@ -33,13 +36,13 @@ config.read(config_class_or_filepath) - self._read_default_section(config) + self.read_default_section(config) if self._has_copy_section(config): - self._read_copy_config_section(config) + self.read_copy_config_section(config) if self._has_tar_section(config): - self._read_tar_config_section(config) + self.read_tar_config_section(config) if self._has_rsync_section(config): - self._read_rsync_config_section(config) + self.read_rsync_config_section(config) if self._has_mysqldump_section(config): - self._read_mysqldump_config_section(config) + self.read_mysqldump_config_section(config) def _has_copy_section(self, config_parser: ConfigParser) -> bool: @@ -63,56 +66,58 @@ return self.run_mysqldump - DEFAULT_SCHEMA = cfg_help.ConfigSchema('DEFAULT', - [ cfg_help.ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , cfg_help.ConfigSchemaLine('source_machine', True, str) - , cfg_help.ConfigSchemaLine('verbose', False, bool) - ] - ) - - def _read_default_section(self, config: ConfigParser): - self.default_dict = cfg_help.parse_config(config, self.DEFAULT_SCHEMA) + DEFAULT_SCHEMA = ConfigSchema('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ] + ) + + def read_default_section(self, config: ConfigParser, schema: ConfigSchema=DEFAULT_SCHEMA): + self.default_dict = cfg_help.parse_config(config, schema) self.hostname_is_valid() - - COPY_SCHEMA = cfg_help.ConfigSchema('COPY', - [ cfg_help.ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - ] - ) - - def _read_copy_config_section(self, config: ConfigParser): - self.copy_dict = cfg_help.parse_config(config, self.COPY_SCHEMA) - - TAR_SCHEMA = cfg_help.ConfigSchema('TAR', - [ cfg_help.ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - , cfg_help.ConfigSchemaLine('target_filename', True, str) - , cfg_help.ConfigSchemaLine('prefix_date', True, bool) - ] - ) - - def _read_tar_config_section(self, config: ConfigParser): - self.tar_dict = cfg_help.parse_config(config, self.TAR_SCHEMA) - - RSYNC_SCHEMA = cfg_help.ConfigSchema('RSYNC', - [ cfg_help.ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - ] - ) - - def _read_rsync_config_section(self, config: ConfigParser): - self.rsync_dict = cfg_help.parse_config(config, self.RSYNC_SCHEMA) - - MYSQLDUMP_SCHEMA = cfg_help.ConfigSchema('MYSQLDUMP', - [ cfg_help.ConfigSchemaLine('user', True, str) - , cfg_help.ConfigSchemaLine('password', True, str) - , cfg_help.ConfigSchemaLine('db_name', True, str) - , cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - , cfg_help.ConfigSchemaLine('target_filename', True, str) - , cfg_help.ConfigSchemaLine('prefix_date', True, bool) - ] - ) - - def _read_mysqldump_config_section(self, config: ConfigParser): - self.mysqldump_dict = cfg_help.parse_config(config, self.MYSQLDUMP_SCHEMA) + self.maybe_make_backup_dirs() + + COPY_SCHEMA = ConfigSchema('COPY', + [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + ] + ) + + def read_copy_config_section(self, config: ConfigParser, schema: ConfigSchema=COPY_SCHEMA): + self.copy_dict = cfg_help.parse_config(config, schema) + + TAR_SCHEMA = ConfigSchema('TAR', + [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] + ) + + def read_tar_config_section(self, config: ConfigParser, schema: ConfigSchema=TAR_SCHEMA): + self.tar_dict = cfg_help.parse_config(config, schema) + + RSYNC_SCHEMA = ConfigSchema('RSYNC', + [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + ] + ) + + def read_rsync_config_section(self, config: ConfigParser, schema: ConfigSchema=RSYNC_SCHEMA): + self.rsync_dict = cfg_help.parse_config(config, schema) + + MYSQLDUMP_SCHEMA = ConfigSchema('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] + ) + + def read_mysqldump_config_section(self, config: ConfigParser, schema: ConfigSchema=MYSQLDUMP_SCHEMA): + self.mysqldump_dict = cfg_help.parse_config(config, schema) def hostname_is_valid(self) -> bool: @@ -122,4 +127,20 @@ actual_hostname = socket.gethostname() return actual_hostname == self.default_dict['source_machine'] + + def target_backup_paths_ok(self, raise_error=False): + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + if raise_error: + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + return False + return True + + def maybe_make_backup_dirs(self): + if self.default_dict is None or 'make_dirs' not in self.default_dict.keys(): + return + if self.default_dict['make_dirs']: + for backup_path in self.default_dict['backup_paths']: + if not os.path.exists(backup_path): + os.makedirs(backup_path) def run_backup(self): @@ -132,8 +153,9 @@ if self.run_copy is None or False: return 0 + self.target_backup_paths_ok(raise_error=True) for bp in self.default_dict['backup_paths']: - if not path.exists(bp): - raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") for s in self.copy_dict['sources']: @@ -143,5 +165,4 @@ args = [s, bp] if 'additional_args' in self.copy_dict.keys(): - print("hiya") args = args + self.copy_dict['additional_args'] sub_utils.cp_wrapper(args) @@ -168,5 +189,4 @@ try: - print(all_args) sub_utils.tar_wrapper(all_args) sub_utils.chmod_wrapper(['774', filename]) @@ -184,8 +204,9 @@ return 0 + self.target_backup_paths_ok(raise_error=True) backup_paths = self.default_dict['backup_paths'] for bp in backup_paths: - if not path.exists(bp): - raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") for s in self.rsync_dict['sources']: @@ -195,7 +216,5 @@ args = [s, bp] if 'additional_args' in self.rsync_dict.keys(): - print("hiya") args = args + self.rsync_dict['additional_args'] - print(args) sub_utils.local_rsync_wrapper(args) except errs.SubprocessError as e: @@ -240,7 +259,8 @@ for d in range(1, len(backup_paths)): sub_utils.cp_wrapper([dump_filename, backup_paths[d]]) - - -def get_date(custom_date=None, custom_format="%Y-%b-%d"): + return 0 + + +def get_date(custom_date=None, custom_format="%Y_%m_%d"): # Format is currently: 2019-10-08 if custom_date is not None: Index: anches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications.py (revision 40936) +++ (revision ) @@ -1,290 +1,0 @@ -import argparse -import configparser -import datetime -import glob -import os -import os.path as path -import shutil -import subprocess -import sys - -import backups.utils.subprocess_utils as sub_utils -import backups.utils.generic_utils as gen_utils -from backups.utils.errors import ConfigError, SubprocessError - -DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config') - - -class AtlassianBackups(): - - def __init__(self, config_file=None): - self.config_file = DEFAULT_CONFIG_FILE if config_file is None else config_file - self.load_config(self.config_file) - - def load_config(self, config_class_or_filepath): - """Accepts a filepath to a config file, or """ - - config = configparser.ConfigParser() - if config_class_or_filepath is None: - raise ConfigError(config_class_or_filepath, "Neither config settings or filepath of config given") - elif isinstance(config_class_or_filepath, configparser.ConfigParser): - config = config_class_or_filepath - elif path.exists(config_class_or_filepath): - config.read(config_class_or_filepath) - - def _get_str_key_from_section(config, section, key): - if config.has_option(section, key): - return config[section][key] - raise ConfigError(config, "ConfigError: KeyError: no key found for {0}".format(key)) - - section = 'atlassian_backups' - self.host_backup_path = _get_str_key_from_section(config, section, 'host_backup_path') - self.backup_1_path = _get_str_key_from_section(config, section, 'backup_1_path') - self.backup_2_path = _get_str_key_from_section(config, section, 'backup_2_path') - self.jira_backup_path = _get_str_key_from_section(config, section, 'jira_backup_path') - self.conf_backup_path = _get_str_key_from_section(config, section, 'conf_backup_path') - self.jira_attachments_path = _get_str_key_from_section(config, section, 'jira_attachments_path') - self.conf_attachments_path = _get_str_key_from_section(config, section, 'conf_attachments_path') - self.verbose = config.getboolean('atlassian_backups', 'verbose') - - self.mysqldump_password = _get_str_key_from_section(config, section, 'mysqldump_password') - - def target_backup_paths_ok(self) -> bool: - copy_paths = [ self.host_backup_path - , self.backup_1_path - , self.backup_2_path - ] - for cp in copy_paths: - if not path.exists(cp): - return False - self._make_destination_subdirs_if_needed(cp) - - return True - - def _make_destination_subdirs_if_needed(self, copy_path): - subdirs = [ path.join(copy_path, "latest") - , path.join(copy_path, "old") - , path.join(copy_path, "tmp") - ] - for sd in subdirs: - if not path.exists(sd): - if self.verbose: - print("creating dir: ", sd) - os.makedirs(sd) - - def target_paths(self, subdir): - return [ path.join(self.host_backup_path, subdir) - , path.join(self.backup_1_path, subdir) - , path.join(self.backup_2_path, subdir) - ] - - def perform_backups(self): - - # Check all paths are ok first: - if not self.target_backup_paths_ok(): - return 1 - - latest_paths = self.target_paths('latest') - tmp_paths = self.target_paths('tmp') - old_paths = self.target_paths('old') - - # Move latest to tmp (and delete if move ok) - gen_utils.move_dirs(latest_paths, tmp_paths, self.verbose) - - return_status = self._jira_backups() - return_status = return_status + self._conf_backups() - return_status = return_status + self._db_dump() - - # If successful: move tmp to old - failures = [ rs for rs in return_status if rs != 0 ] - if len(failures) == 0: - gen_utils.move_dirs(tmp_paths, old_paths, self.verbose) - else: - # If any failed: delete latest, move tmp back - gen_utils.move_dirs(tmp_paths, latest_paths, self.verbose) - return 5 - - # TODO:clean-up old files (to save space) - - return 0 - - def _jira_backups(self): - return_status = [] - return_status.append(self._copy_jira_backup(self.verbose)) - return_status.append(self._tar_jira_attachments()) - return return_status - - def _conf_backups(self): - return_status = [] - return_status.append(self._copy_confluence_backup(self.verbose)) - return_status.append(self._tar_conf_attachments()) - return return_status - - def _db_dump(self): - return [self.make_mysql_dump(self.mysqldump_password)] - - def get_jira_backup_date_format(self, custom_date=None): - # Format is currently: 2019-Aug-13--0325.zip - jira_format = "%Y-%b-%d" - if custom_date is not None: - return custom_date.strftime(jira_format) - return datetime.date.today().strftime(jira_format) - - def _copy_jira_backup(self, verbose=False): - if not self.target_backup_paths_ok(): - return 2 - - jira_wildcard_search = "{0}/{1}*".format(self.jira_backup_path, self.get_jira_backup_date_format()) - jira_xml_filepaths = glob.glob(jira_wildcard_search) - if len(jira_xml_filepaths) == 0: - print("Copy Failure: No jira backups found at: ", jira_wildcard_search) - return 2 - target_paths = self.target_paths('latest') - for jfile in jira_xml_filepaths: - for tpath in target_paths: - if not path.exists(tpath): - print("Copy Failure: target path does not exist:", tpath) - return 2 - try: - if verbose: - print("Copying: {0} to {1}".format(jfile, tpath)) - shutil.copy2(jfile, tpath) - sub_utils.chmod_wrapper(['774', os.path.join(tpath, os.path.basename(jfile))]) - except IOError as e: - print("Error: copying jira backup.\n copying: {0}\n to: {1}".format(e.filename, e.filename2)) - print(e) - return 2 - return 0 - - def get_confluence_backup_date_format(self, custom_date=None): - # Format is: confluence-xml-backup-2019_08_14.zip - conf_format = "%Y_%m_%d" - if custom_date is not None: - return custom_date.strftime(conf_format) - return datetime.date.today().strftime(conf_format) - - def _copy_confluence_backup(self, verbose=False): - if not self.target_backup_paths_ok(): - print("Copy Failure: target path failure") - return 2 - - conf_wildcard_search = "{0}/*{1}*.zip".format(self.conf_backup_path, self.get_confluence_backup_date_format()) - conf_xml_filepaths = glob.glob(conf_wildcard_search) - if len(conf_xml_filepaths) == 0: - print("Copy Failure: No confluence backups found at: ", conf_wildcard_search) - return 2 - target_paths = self.target_paths('latest') - for cfile in conf_xml_filepaths: - for tpath in target_paths: - try: - sub_utils.cp_wrapper([cfile, tpath]) - sub_utils.chmod_wrapper(['774', os.path.join(tpath, os.path.basename(cfile))]) - except Exception as e: - print(e) - return 2 - return 0 - - def _tar_jira_attachments(self): - # tar jira - latest_paths = self.target_paths("latest") - if not self.target_backup_paths_ok(): - return 1 - - jira_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_jira_attachments.tar" - jira_tar_filepath = path.join(latest_paths[0], jira_tar_name) - jira_tar_args = ["-C", "/", "-cf", jira_tar_filepath, self.jira_attachments_path.strip('/')] - sub_utils.tar_wrapper(jira_tar_args) - sub_utils.chmod_wrapper(['774', jira_tar_filepath]) - # copy to backups - sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[1]]) - sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[2]]) - return 0 - - def _tar_conf_attachments(self): - latest_paths = self.target_paths("latest") - if not self.target_backup_paths_ok(): - return 1 - - # tar confluence - confluence_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_confluence_attachments.tar" - confluence_tar_filepath = path.join(latest_paths[0], confluence_tar_name) - confluence_tar_args = ["-C", "/", "-cf", confluence_tar_filepath, self.conf_attachments_path.strip('/')] - sub_utils.tar_wrapper(confluence_tar_args) - sub_utils.chmod_wrapper(['774', confluence_tar_filepath]) - # copy to backups - sub_utils.cp_wrapper([ confluence_tar_filepath, latest_paths[1]]) - sub_utils.cp_wrapper([ confluence_tar_filepath, latest_paths[2]]) - return 0 - - def make_mysql_dump(self, dump_password, dump_name=None): - # do this as the dumper user - - latest_paths = self.target_paths("latest") - if not self.target_backup_paths_ok(): - return 1 - - # make sure target file does not exist (or it will fail) - if dump_name is None: - date = datetime.date.today().strftime("%Y_%m_%d") - dump_filepath = path.join(latest_paths[0], '{0}_mysql_dump.sql'.format(date)) - else: - dump_filepath = path.join(latest_paths[0], dump_name) - if path.exists(dump_filepath): - print("Warning: SQL dump already exists: ", dump_filepath) - return 2 - - # Setup the commands: - mysqldump_args = [ 'mysqldump', '-u', 'dumper', '-p{0}'.format(dump_password) - , '--databases', 'jiradb', 'confluencedb' - ] - with open(dump_filepath, "w+") as df: - mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) - dump_result = mysqldump_process.communicate() - - # Get the piped std err. See it contains the word error - stdo, stde = dump_result - if stde is not None: - decoded_stderr = stde.decode() - if decoded_stderr.casefold().find('error'.casefold()) > -1: - raise SubprocessError() - - # move the dump file to the correct locations - if not path.exists(dump_filepath): - return 2 - cp_1 = sub_utils.cp_wrapper([dump_filepath, latest_paths[1]]) - cp_2 = sub_utils.cp_wrapper([dump_filepath, latest_paths[2]]) - if cp_1 != 0: - return cp_1 - if cp_2 != 0: - return cp_2 - return 0 - - def tidy_backup_folders(self): - return 0 - - def run(self): - return self.perform_backups() - - -def main(): - args = parse_args(sys.argv[1:]) - args_dict = vars(args) - ab = AtlassianBackups(**args_dict) - return ab.run() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of atlassian applications\nYou should run this on the machine hosting the atlassian applications') - - parser.add_argument('--config_file', - nargs='?', - help='contains config for all paths and mysqldump details', - default=DEFAULT_CONFIG_FILE) - - return parser.parse_args() - - -if __name__ == "__main__": - main() Index: anches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications_test.py (revision 40936) +++ (revision ) @@ -1,451 +1,0 @@ -import backups.backup_atlassian_applications as baa -import configparser -import datetime -import os -import pytest - -from pathlib import Path - -import backups.full_atlassian_backup_test as fab_test -import backups.utils.subprocess_utils as sub_utils -import backups.utils.generic_utils as gen_utils - -from backups.backup_atlassian_applications import AtlassianBackups -from backups.utils.errors import ValidationError - -TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), './testing/test.config') - - -def make_config(tmpdir) -> configparser.ConfigParser: - config = configparser.ConfigParser() - config['atlassian_backups'] = \ - { - 'host_backup_path' : os.path.join(tmpdir, 'host_backup_path'), - 'backup_1_path' : os.path.join(tmpdir, 'backup_1_path'), - 'backup_2_path' : os.path.join(tmpdir, 'backup_2_path'), - 'jira_backup_path' : os.path.join(tmpdir, 'jira_backup_path'), - 'conf_backup_path' : os.path.join(tmpdir, 'conf_backup_path'), - 'jira_attachments_path' : os.path.join(tmpdir, 'jira_attachments_path'), - 'conf_attachments_path' : os.path.join(tmpdir, 'conf_attachments_path'), - 'mysqldump_password' : 'not_a_real_password' - } - - -class TestArguments(object): - - def test_load_from_config_file(self): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" - assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" - assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" - assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" - assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" - assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" - assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" - assert ab.mysqldump_password == "not_a_real_password" - assert ab.verbose is True - - def test_default_config_loads_with_no_args(self): - default_config = baa.DEFAULT_CONFIG_FILE - assert os.path.exists(default_config) - ab = AtlassianBackups() - assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" - assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" - assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" - assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" - assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" - assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" - assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" - assert ab.mysqldump_password == "not_a_real_password" - - def test_default_arguments(self): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" - assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" - assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" - assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" - assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" - assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" - assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" - assert ab.mysqldump_password == "not_a_real_password" - - def test_providing_config_directly(self): - config = configparser.ConfigParser() - config.add_section("atlassian_backups") - config.set('atlassian_backups', 'host_backup_path', 'custom_config1') - config.set('atlassian_backups', 'backup_1_path', 'custom_config2') - config.set('atlassian_backups', 'backup_2_path', 'custom_config3') - config.set('atlassian_backups', 'jira_backup_path', 'custom_config4') - config.set('atlassian_backups', 'conf_backup_path', 'custom_config5') - config.set('atlassian_backups', 'jira_attachments_path', 'custom_config6') - config.set('atlassian_backups', 'conf_attachments_path', 'custom_config7') - config.set('atlassian_backups', 'mysqldump_password', 'custom_config8') - config.set('atlassian_backups', 'verbose', 'False') - - ab = AtlassianBackups(config_file=config) - - assert ab.host_backup_path == 'custom_config1' - assert ab.backup_1_path == 'custom_config2' - assert ab.backup_2_path == 'custom_config3' - assert ab.jira_backup_path == 'custom_config4' - assert ab.conf_backup_path == 'custom_config5' - assert ab.jira_attachments_path == 'custom_config6' - assert ab.conf_attachments_path == 'custom_config7' - assert ab.mysqldump_password == 'custom_config8' - assert ab.verbose is False - - def test_nonexistant_config_file_raises_error(self): - with pytest.raises(baa.ConfigError): - AtlassianBackups("not a real file path") - - def test_default_config_loads_from_main_call(self): - try: - baa.main() - except Exception: - pytest.fail("An error was raised when it should not") - - -class TestFilepathVerifications(object): - - def test_default_target_paths_do_not_exists_in_testing(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - assert ab.target_backup_paths_ok() is False - - def test_an_incorrect_path_then_correct_path(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - assert os.path.exists(ab.jira_backup_path) is False - ab.jira_backup_path = tmpdir - assert os.path.exists(ab.jira_backup_path) is True - - def test_each_path_individually(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - assert os.path.exists(ab.host_backup_path) is False - ab.host_backup_path = tmpdir - assert os.path.exists(ab.host_backup_path) is True - assert ab.target_backup_paths_ok() is False - - assert os.path.exists(ab.backup_1_path) is False - ab.backup_1_path = tmpdir - assert os.path.exists(ab.backup_1_path) is True - assert ab.target_backup_paths_ok() is False - - assert os.path.exists(ab.backup_2_path) is False - ab.backup_2_path = tmpdir - assert os.path.exists(ab.backup_2_path) is True - # Once all backup paths are set, they are ok - assert ab.target_backup_paths_ok() is True - - assert os.path.exists(ab.jira_backup_path) is False - ab.jira_backup_path = tmpdir - assert os.path.exists(ab.jira_backup_path) is True - - assert os.path.exists(ab.conf_backup_path) is False - ab.conf_backup_path = tmpdir - assert os.path.exists(ab.conf_backup_path) is True - - assert os.path.exists(ab.jira_attachments_path) is False - ab.jira_attachments_path = tmpdir - assert os.path.exists(ab.jira_attachments_path) is True - - assert os.path.exists(ab.conf_attachments_path) is False - ab.conf_attachments_path = tmpdir - assert os.path.exists(ab.conf_attachments_path) is True - - def test_backup_subdirs_are_created_if_missing(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab.jira_path = tmpdir - ab.conf_path = tmpdir - - backup_paths = [ os.path.join(tmpdir, 'host') - , os.path.join(tmpdir, 'backup_1') - , os.path.join(tmpdir, 'backup_2') - ] - - for bp in backup_paths: - - # make the parent dir - os.makedirs(bp) - ab.bp = bp - assert os.path.exists(bp) - - # Expected subdirs - expected_latest = os.path.join(bp, 'latest') - expected_tmp = os.path.join(bp, 'tmp') - expected_old = os.path.join(bp, 'old') - assert not os.path.exists(expected_latest) - assert not os.path.exists(expected_tmp) - assert not os.path.exists(expected_old) - - # all_paths_ok will call subdir creation - ab._make_destination_subdirs_if_needed(bp) - assert os.path.exists(expected_latest) - assert os.path.exists(expected_tmp) - assert os.path.exists(expected_old) - - -class TestMovingBackups(object): - - def test_target_paths_with_defaults_function(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - default = ab.target_paths('default_paths') - - assert default[0] == "/export/ippops4.0/atlassian_backups/default_paths" - assert default[1] == "/data/ippops3.0/atlassian_backups/default_paths" - assert default[2] == "/data/ippops5.0/atlassian_backups/default_paths" - - def test_target_paths_function_after_changing_paths(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - - ab.host_backup_path = "/different_path/for_host" - ab.backup_1_path = "/bk/one" - ab.backup_2_path = "/bk/two" - - another_subdir = ab.target_paths('a_different_subdir') - - assert another_subdir[0] == "/different_path/for_host/a_different_subdir" - assert another_subdir[1] == "/bk/one/a_different_subdir" - assert another_subdir[2] == "/bk/two/a_different_subdir" - - def test_move_aborts_if_source_and_destination_path_lists_are_different_lengths(self): - source_paths = [ "/Just/one/path" ] - destination_paths = [ "/Two", "/paths"] - - with pytest.raises(ValidationError): - gen_utils.move_dirs(source_paths, destination_paths) - - def test_moving_old_backups(self, tmpdir): - # Need to set up X source_dirs, and X target_subdirs - tmpdir_host = os.path.join(tmpdir, "host") - tmpdir_bak1 = os.path.join(tmpdir, "bak1") - tmpdir_bak2 = os.path.join(tmpdir, "bak2") - os.makedirs(tmpdir_host) - os.makedirs(tmpdir_bak1) - os.makedirs(tmpdir_bak2) - - # Setup some subdirectories - source_paths = [ os.path.join(tmpdir_host, 'source') - , os.path.join(tmpdir_bak1, 'source') - , os.path.join(tmpdir_bak2, 'source') - ] - destination_paths = [ os.path.join(tmpdir_host, 'destination') - , os.path.join(tmpdir_bak1, 'destination') - , os.path.join(tmpdir_bak2, 'destination') - ] - for i in range(0, len(source_paths)): - os.makedirs(source_paths[i]) - os.makedirs(destination_paths[i]) - - # Put some files in the source directories - source_host_file1 = os.path.join(source_paths[0], "host_file1.zip") - source_bak1_file1 = os.path.join(source_paths[1], "bak1_file1.txt") - source_bak1_file2 = os.path.join(source_paths[1], "bak1_file2.txt") - source_bak1_file3 = os.path.join(source_paths[1], "bak1_file3.txt") - source_bak2_file1 = os.path.join(source_paths[2], "bak2_file1.lol") - source_bak2_file2 = os.path.join(source_paths[2], "bak2_file2.lol") - Path(source_host_file1).touch() - Path(source_bak1_file1).touch() - Path(source_bak1_file2).touch() - Path(source_bak1_file3).touch() - Path(source_bak2_file1).touch() - Path(source_bak2_file2).touch() - assert os.path.exists(source_host_file1) - assert os.path.exists(source_bak1_file1) - assert os.path.exists(source_bak1_file2) - assert os.path.exists(source_bak1_file3) - assert os.path.exists(source_bak2_file1) - assert os.path.exists(source_bak2_file2) - - # Assert files do NOT exist in target dirs yet - destination_host_file1 = os.path.join(destination_paths[0], "host_file1.zip") - destination_bak1_file1 = os.path.join(destination_paths[1], "bak1_file1.txt") - destination_bak1_file2 = os.path.join(destination_paths[1], "bak1_file2.txt") - destination_bak1_file3 = os.path.join(destination_paths[1], "bak1_file3.txt") - destination_bak2_file1 = os.path.join(destination_paths[2], "bak2_file1.lol") - destination_bak2_file2 = os.path.join(destination_paths[2], "bak2_file2.lol") - assert not os.path.exists(destination_host_file1) - assert not os.path.exists(destination_bak1_file1) - assert not os.path.exists(destination_bak1_file2) - assert not os.path.exists(destination_bak1_file3) - assert not os.path.exists(destination_bak2_file1) - assert not os.path.exists(destination_bak2_file2) - - # run the move - return_result = gen_utils.move_dirs(source_paths, destination_paths) - assert return_result == 0 - - # check that source files are no longer there, but the target ones do exist - assert not os.path.exists(source_host_file1) - assert not os.path.exists(source_bak1_file1) - assert not os.path.exists(source_bak1_file2) - assert not os.path.exists(source_bak1_file3) - assert not os.path.exists(source_bak2_file1) - assert not os.path.exists(source_bak2_file2) - assert os.path.exists(destination_host_file1) - assert os.path.exists(destination_bak1_file1) - assert os.path.exists(destination_bak1_file2) - assert os.path.exists(destination_bak1_file3) - assert os.path.exists(destination_bak2_file1) - assert os.path.exists(destination_bak2_file2) - - -class TestCopyingBackups(object): - - def test_get_jira_backup_date_format(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - today = datetime.date.today() - expected_format = today.strftime("%Y-%b-%d") - assert ab.get_jira_backup_date_format() == expected_format - - def test_copying_fails_if_paths_not_present(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - return_value = ab._copy_jira_backup() - assert return_value != 0 - - return_value = ab._copy_confluence_backup() - assert return_value != 0 - - def test_jira_copy(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - return_value = ab._copy_jira_backup() - assert return_value == 0 - - def test_get_confluence_backup_date_format(self): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - today = datetime.date.today() - expected_format = today.strftime("%Y_%m_%d") - assert ab.get_confluence_backup_date_format() == expected_format - - def test_confluence_copy(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - return_value = ab._copy_confluence_backup() - assert return_value == 0 - - def test_full_copy(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - result = ab.perform_backups() - - assert result == 0 - - def test_atlsssian_copying_fails_when_paths_do_not_exist(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - fake_path = "/not/a/real/path" - ab.jira_backup_path = fake_path - ab.conf_backup_path = fake_path - ab.host_backup_path = fake_path - ab.backup_1_path = fake_path - ab.backup_2_path = fake_path - assert not os.path.exists(ab.jira_backup_path) - assert not os.path.exists(ab.conf_backup_path) - assert not os.path.exists(ab.host_backup_path) - assert not os.path.exists(ab.backup_1_path) - assert not os.path.exists(ab.backup_2_path) - - result = ab.perform_backups() - assert result == 1 - - -class TestTaringOfAttachments(object): - - def test_jira_attachments_tar(self, tmpdir): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - today = datetime.date.today().strftime("%Y_%m_%d") - jira_tar_name = today + "_jira_attachments.tar" - expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name) - expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name) - expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name) - assert not os.path.exists(expected_jira_host_tar) - assert not os.path.exists(expected_jira_bak1_tar) - assert not os.path.exists(expected_jira_bak2_tar) - - tar_result = ab._tar_jira_attachments() - assert tar_result == 0 - - assert os.path.exists(expected_jira_host_tar) - assert os.path.exists(expected_jira_bak1_tar) - assert os.path.exists(expected_jira_bak2_tar) - - def test_conf_attachments_tar(self, tmpdir): - - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - today = datetime.date.today().strftime("%Y_%m_%d") - conf_tar_name = today + "_confluence_attachments.tar" - expected_conf_host_tar = os.path.join(ab.host_backup_path, "latest", conf_tar_name) - expected_conf_bak1_tar = os.path.join(ab.backup_1_path, "latest", conf_tar_name) - expected_conf_bak2_tar = os.path.join(ab.backup_2_path, "latest", conf_tar_name) - assert not os.path.exists(expected_conf_host_tar) - assert not os.path.exists(expected_conf_bak1_tar) - assert not os.path.exists(expected_conf_bak2_tar) - - tar_result = ab._tar_conf_attachments() - assert tar_result == 0 - - assert os.path.exists(expected_conf_host_tar) - assert os.path.exists(expected_conf_bak1_tar) - assert os.path.exists(expected_conf_bak2_tar) - - -@pytest.mark.skip(reason="Requires a fake database to be setup") -class TestMysqlBackup(object): - """ The mySQL dump tests require the machine on which they are -running to have a mysql database running with a user called -'dumper' and a password set to 'not_a_real_password'. - -In production, a proper password should be set on the server and in the config -file. - -To change the password: -UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; -""" - - dumper_password = "not_a_real_password" - - def test_mysql_dump_is_ok(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - date = datetime.date.today().strftime("%Y_%m_%d") - expected_dump = os.path.join(ab.host_backup_path, "latest", '{0}_mysql_dump.sql'.format(date)) - print(expected_dump) - assert not os.path.exists(expected_dump) - - mysqldump_result = ab.make_mysql_dump(self.dumper_password) - assert mysqldump_result == 0 - - assert os.path.exists(expected_dump) - - def test_mysql_dump_is_ok_to_custom_file(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - expected_dump = os.path.join(ab.host_backup_path, "latest", "humptydumpy.sql") - print(expected_dump) - assert not os.path.exists(expected_dump) - - mysqldump_result = ab.make_mysql_dump(self.dumper_password, "humptydumpy.sql") - assert mysqldump_result == 0 - - assert os.path.exists(expected_dump) - - def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): - ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) - ab = fab_test.create_all_paths_and_folders(ab, tmpdir) - - with pytest.raises(sub_utils.SubprocessError): - ab.make_mysql_dump("Not the correct password") Index: /branches/ipp-259_genericise_backups/tools/backups/backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40936) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40937) @@ -27,9 +27,8 @@ source_machine = socket.gethostname() config['DEFAULT'] = \ - { - 'backup_paths' : backup_paths, - 'source_machine' : source_machine, - 'verbose' : verbose, - } + { 'backup_paths' : backup_paths + , 'source_machine' : source_machine + , 'verbose' : verbose + } return config @@ -107,5 +106,5 @@ Backup(config_file=config) - with pytest.raises(errs.ConfigError): + with pytest.raises(errs.ValidationError): Backup(config_file=None) @@ -136,4 +135,62 @@ backy.default_dict['source_machine'] = actual_hostname assert backy.hostname_is_valid() is True + + +class TestFilepathVerifications(object): + + def test_default_target_paths_do_not_exist_in_testing(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.target_backup_paths_ok() is False + + def test_raising_an_error_if_a_backup_path_does_not_exist(self): + with pytest.raises(errs.ValidationError) as e: + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + backy.target_backup_paths_ok(raise_error=True) + assert "ValidationError: target path does not exist" in str(e) + + def test_an_incorrect_path_then_correct_path(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + + backy.default_dict['backup_paths'] = ["/not/real/path"] + assert backy.target_backup_paths_ok() is False + + backy.default_dict['backup_paths'] = [tmpdir] + assert backy.target_backup_paths_ok() is True + + def test_backup_paths_created_if_make_dirs_given(self, tmpdir): + config = ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + bck1_path = os.path.join(tmpdir, 'bck1') + bck1b_path = os.path.join(tmpdir, 'bck1/bobby') + bck1bb_path = os.path.join(tmpdir, 'bck1/bobby/backup') + bck2_path = os.path.join(tmpdir, 'bck2') + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + # backup dirs not created automatically + Backup(config) + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + + # make_dirs included in config - when loaded dirs created. + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + , 'make_dirs' : 'True' + } + backy = Backup(config) + assert backy.default_dict['make_dirs'] is True + assert os.path.exists(bck1_path) + assert os.path.exists(bck1b_path) + assert os.path.exists(bck1bb_path) + assert os.path.exists(bck2_path) @@ -265,5 +322,4 @@ assert not os.path.exists(expected_tar1) backy._run_tar() - print(expected_tar1) assert os.path.exists(expected_tar1) @@ -422,6 +478,4 @@ """ - dumper_password = "not_a_real_password" - def test_mysql_dump_is_ok(self, tmpdir): config = make_default_config(tmpdir, ['bck1', 'bck2']) @@ -442,7 +496,9 @@ assert not os.path.exists(expected_dump1) assert not os.path.exists(expected_dump2) - backy._run_mysqldump() + result = backy._run_mysqldump() assert os.path.exists(expected_dump1) assert os.path.exists(expected_dump2) + + assert result == 0 def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.config (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.config (revision 40937) @@ -0,0 +1,23 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/confluence/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/confluence/data +additional_args = --verbose -z -p -cf +target_filename = confluence.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = confluencedb +target_filename = confluence_mysqldump.sql +prefix_date = True Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py (revision 40937) @@ -0,0 +1,77 @@ +import argparse +import datetime +import glob +import os.path as path + +import backups.utils.subprocess_utils as sub_utils +import backups.utils.errors as errs +from backups.backup import Backup + +DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'confluence_backup.config') + + +class ConfluenceBackup(Backup): + + def __init__(self, config_file=DEFAULT_CONFIG_FILE): + self.load_config(config_file) + + @staticmethod + def get_confluence_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + confluence_format = "%Y_%m_%d" + if custom_date is not None: + return custom_date.strftime(confluence_format) + return datetime.date.today().strftime(confluence_format) + + def _run_copy(self, verbose=False): + if self.run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For confluence backup there should only be one source: + source = self.copy_dict['sources'][0] + + confluence_wildcard_search = f"{source}/{ConfluenceBackup.get_confluence_backup_date_format()}*" + confluence_xml_filepaths = glob.glob(confluence_wildcard_search) + if len(confluence_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No confluence backups found of form: {confluence_wildcard_search}") + + for jxf in confluence_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(): + jb = ConfluenceBackup() + jb.run_backup() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of confluence.') + + parser.add_argument('--config_file', + nargs='?', + help='specify the location of a config that matches the ConfigSchema', + default=DEFAULT_CONFIG_FILE) + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40937) @@ -0,0 +1,241 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backups.confluence_backup as jbck_mod +import backups.utils.errors as errs +from backups.confluence_backup import ConfluenceBackup +from backups.utils.errors import ValidationError + + +JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'True' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_confluence_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +ConfluenceBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + confluence_backup_path = os.path.join(tmpdir, "confluence_backup_path") + confluence_attachments_path = os.path.join(tmpdir, "confluence_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(confluence_backup_path) + os.makedirs(confluence_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(confluence_backup_path) + assert os.path.exists(confluence_attachments_path) + + # Create backup files that will be copied + confluence_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_confluence_test_file.zip" + confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name) + Path(confluence_backup_filepath).touch() + assert os.path.exists(confluence_backup_filepath) + + # Create some files that will be tar'd + confluence_attachment_1_name = "attachment_1.zip" + confluence_attachment_2_name = "attachment_2.txt" + confluence_attachment_1_filepath = os.path.join(confluence_attachments_path, confluence_attachment_1_name) + confluence_attachment_2_filepath = os.path.join(confluence_attachments_path, confluence_attachment_2_name) + Path(confluence_attachment_1_filepath).touch() + Path(confluence_attachment_2_filepath).touch() + assert os.path.exists(confluence_attachment_1_filepath) + assert os.path.exists(confluence_attachment_2_filepath) + + +class TestArguments(object): + + def assert_defaults(self, jb: ConfluenceBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/atlassian_backups' + , '/data/ippops3.0/atlassian_backups' + , '/data/ippops5.0/atlassian_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/export'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/data'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = ConfluenceBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_loads_with_no_args(self): + default_config = jbck_mod.DEFAULT_CONFIG_FILE + assert os.path.exists(default_config) + jb = ConfluenceBackup() + self.assert_defaults(jb) + + def test_default_arguments(self): + jb = ConfluenceBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'False' + } + + jb = ConfluenceBackup(config_file=config) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(errs.ConfigError): + ConfluenceBackup("not a real file path") + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(errs.ValidationError) as e: + jbck_mod.main() + assert "ValidationError: target path does not exist:" in str(e) + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_confluence_backup_date_format(self): + jb = ConfluenceBackup(config_file=JIRA_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y_%m_%d") + assert jb.get_confluence_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y_%m_%d") + assert jb.get_confluence_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + jb = ConfluenceBackup(config_file=JIRA_TEST_CONFIG) + # return_value = jb._copy_confluence_backup() + with pytest.raises(ValidationError) as e: + jb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_confluence_copy(self, tmpdir): + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + jb = ConfluenceBackup(config_file=config) + + today = ConfluenceBackup.get_confluence_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_confluence_test_file.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_confluence_test_file.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_confluence_test_file.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + return_value = jb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_confluence_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + jb = ConfluenceBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + confluence_tar_name = f"{today}_confluence.tar.gz" + expected_confluence_host_tar = os.path.join(tmpdir, "host_backup_path", confluence_tar_name) + expected_confluence_bak1_tar = os.path.join(tmpdir, "backup_1_path", confluence_tar_name) + expected_confluence_bak2_tar = os.path.join(tmpdir, "backup_2_path", confluence_tar_name) + assert not os.path.exists(expected_confluence_host_tar) + assert not os.path.exists(expected_confluence_bak1_tar) + assert not os.path.exists(expected_confluence_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_confluence_host_tar) + assert os.path.exists(expected_confluence_bak1_tar) + assert os.path.exists(expected_confluence_bak2_tar) Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup.config (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup.config (revision 40937) @@ -0,0 +1,23 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/jira/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/jira/data +additional_args = --verbose -z -p -cf +target_filename = jira.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +target_filename = jira_mysqldump.sql +prefix_date = True Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py (revision 40937) @@ -0,0 +1,77 @@ +import argparse +import datetime +import glob +import os.path as path + +import backups.utils.subprocess_utils as sub_utils +import backups.utils.errors as errs +from backups.backup import Backup + +DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'jira_backup.config') + + +class JiraBackup(Backup): + + def __init__(self, config_file=DEFAULT_CONFIG_FILE): + self.load_config(config_file) + + @staticmethod + def get_jira_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + jira_format = "%Y-%b-%d" + if custom_date is not None: + return custom_date.strftime(jira_format) + return datetime.date.today().strftime(jira_format) + + def _run_copy(self, verbose=False): + if self.run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For jira backup there should only be one source: + source = self.copy_dict['sources'][0] + + jira_wildcard_search = f"{source}/{JiraBackup.get_jira_backup_date_format()}*" + jira_xml_filepaths = glob.glob(jira_wildcard_search) + if len(jira_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No jira backups found of form: {jira_wildcard_search}") + + for jxf in jira_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(): + jb = JiraBackup() + jb.run_backup() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of jira.') + + parser.add_argument('--config_file', + nargs='?', + help='specify the location of a config that matches the ConfigSchema', + default=DEFAULT_CONFIG_FILE) + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40937) @@ -0,0 +1,241 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backups.jira_backup as jbck_mod +import backups.utils.errors as errs +from backups.jira_backup import JiraBackup +from backups.utils.errors import ValidationError + + +JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'True' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_jira_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +JiraBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + jira_backup_path = os.path.join(tmpdir, "jira_backup_path") + jira_attachments_path = os.path.join(tmpdir, "jira_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(jira_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(jira_attachments_path) + + # Create backup files that will be copied + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + # Create some files that will be tar'd + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + +class TestArguments(object): + + def assert_defaults(self, jb: JiraBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/atlassian_backups' + , '/data/ippops3.0/atlassian_backups' + , '/data/ippops5.0/atlassian_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/export'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/data'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_loads_with_no_args(self): + default_config = jbck_mod.DEFAULT_CONFIG_FILE + assert os.path.exists(default_config) + jb = JiraBackup() + self.assert_defaults(jb) + + def test_default_arguments(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + + jb = JiraBackup(config_file=config) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(errs.ConfigError): + JiraBackup("not a real file path") + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(errs.ValidationError) as e: + jbck_mod.main() + assert "ValidationError: target path does not exist:" in str(e) + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_jira_backup_date_format(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + # return_value = jb._copy_jira_backup() + with pytest.raises(ValidationError) as e: + jb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_jira_copy(self, tmpdir): + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = JiraBackup.get_jira_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_jira_test_file.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_jira_test_file.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_jira_test_file.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + return_value = jb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_jira_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + jira_tar_name = f"{today}_jira.tar.gz" + expected_jira_host_tar = os.path.join(tmpdir, "host_backup_path", jira_tar_name) + expected_jira_bak1_tar = os.path.join(tmpdir, "backup_1_path", jira_tar_name) + expected_jira_bak2_tar = os.path.join(tmpdir, "backup_2_path", jira_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) Index: /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config (revision 40937) @@ -0,0 +1,23 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/confluence/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/confluence/data +additional_args = --verbose -z -p -cf +target_filename = confluence.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = confluencedb +target_filename = confluence_mysqldump.sql +prefix_date = True Index: /branches/ipp-259_genericise_backups/tools/backups/testing/jira_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/jira_test.config (revision 40937) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/jira_test.config (revision 40937) @@ -0,0 +1,23 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/jira/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/jira/data +additional_args = --verbose -z -p -cf +target_filename = jira.tar.gz +prefix_date = True + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +target_filename = jira_mysqldump.sql +prefix_date = True Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40936) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40937) @@ -86,5 +86,4 @@ for csl_key in schema.options: csl = schema.get_schema_line(csl_key) - print(csl) if config.has_option(section, csl_key):