Index: /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications.py (revision 40911) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications.py (revision 40912) @@ -90,24 +90,28 @@ # Move latest to tmp (and delete if move ok) - gen_utils.copy_multiple_items_to_multiple_dirs(latest_paths, tmp_paths, self.verbose) + gen_utils.move_dirs(latest_paths, tmp_paths, self.verbose) + + return_status = [] # copy jira - jira_return = self._copy_jira_backup(self.verbose) + return_status.append(self._copy_jira_backup(self.verbose)) # copy confluence - conf_return = self._copy_confluence_backup(self.verbose) + return_status.append(self._copy_confluence_backup(self.verbose)) # tarfiles: - tar_return = self._tar_attachments() + return_status.append(self._tar_jira_attachments()) + return_status.append(self._tar_conf_attachments()) # mysqldump - mysqldump_return = self.make_mysql_dump(self.mysqldump_password) + return_status.append(self.make_mysql_dump(self.mysqldump_password)) # If successful: move tmp to old - if jira_return == 0 and conf_return == 0 and tar_return == 0 and mysqldump_return == 0: - gen_utils.copy_multiple_items_to_multiple_dirs(tmp_paths, old_paths, self.verbose) + failures = [ rs for rs in return_status if rs != 0 ] + if len(failures) == 0: + gen_utils.move_dirs(tmp_paths, old_paths, self.verbose) else: # If any failed: delete latest, move tmp back - gen_utils.copy_multiple_items_to_multiple_dirs(tmp_paths, latest_paths, self.verbose) + gen_utils.move_dirs(tmp_paths, latest_paths, self.verbose) return 5 @@ -177,5 +181,5 @@ return 0 - def _tar_attachments(self): + def _tar_jira_attachments(self): # tar jira latest_paths = self.target_paths("latest") @@ -191,4 +195,11 @@ sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[1]]) sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[2]]) + return 0 + + + def _tar_conf_attachments(self): + latest_paths = self.target_paths("latest") + if not self.target_backup_paths_ok(): + return 1 # tar confluence Index: /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications_test.py (revision 40911) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications_test.py (revision 40912) @@ -9,6 +9,8 @@ import backups.full_atlassian_backup_test as fab_test import backups.utils.subprocess_utils as sub_utils +import backups.utils.generic_utils as gen_utils from backups.backup_atlassian_applications import AtlassianBackups +from backups.utils.errors import ValidationError TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), './testing/test.config') @@ -215,6 +217,6 @@ destination_paths = [ "/Two", "/paths"] - return_is_non_zero = baa._move_dir_contents_to_dir(source_paths, destination_paths) - assert return_is_non_zero != 0 + with pytest.raises(ValidationError): + gen_utils.move_dirs(source_paths, destination_paths) def test_moving_old_backups(self, tmpdir): @@ -275,5 +277,5 @@ # run the move - return_result = baa._move_dir_contents_to_dir(source_paths, destination_paths) + return_result = gen_utils.move_dirs(source_paths, destination_paths) assert return_result == 0 @@ -357,5 +359,5 @@ class TestTaringOfAttachments(object): - def test_attachments_tar(self, tmpdir): + def test_jira_attachments_tar(self, tmpdir): ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) @@ -364,26 +366,37 @@ today = datetime.date.today().strftime("%Y_%m_%d") jira_tar_name = today + "_jira_attachments.tar" + expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name) + expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name) + expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + + tar_result = ab._tar_jira_attachments() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) + + def test_conf_attachments_tar(self, tmpdir): + + ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE) + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) + + today = datetime.date.today().strftime("%Y_%m_%d") conf_tar_name = today + "_confluence_attachments.tar" - expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name) expected_conf_host_tar = os.path.join(ab.host_backup_path, "latest", conf_tar_name) - expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name) expected_conf_bak1_tar = os.path.join(ab.backup_1_path, "latest", conf_tar_name) - expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name) expected_conf_bak2_tar = os.path.join(ab.backup_2_path, "latest", conf_tar_name) - assert not os.path.exists(expected_jira_host_tar) assert not os.path.exists(expected_conf_host_tar) - assert not os.path.exists(expected_jira_bak1_tar) assert not os.path.exists(expected_conf_bak1_tar) - assert not os.path.exists(expected_jira_bak2_tar) assert not os.path.exists(expected_conf_bak2_tar) - tar_result = ab._tar_attachments() + tar_result = ab._tar_conf_attachments() assert tar_result == 0 - assert os.path.exists(expected_jira_host_tar) assert os.path.exists(expected_conf_host_tar) - assert os.path.exists(expected_jira_bak1_tar) assert os.path.exists(expected_conf_bak1_tar) - assert os.path.exists(expected_jira_bak2_tar) assert os.path.exists(expected_conf_bak2_tar) Index: /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py (revision 40911) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py (revision 40912) @@ -1,6 +1,9 @@ +import glob import os.path as path import shutil from distutils.dir_util import copy_tree + +import backups.utils.subprocess_utils as sub_utils from backups.utils.errors import ValidationError @@ -35,18 +38,39 @@ +def _list_checks(sources: [str], targets: [str]): + if sources is None or targets is None: + raise ValidationError("Error: source for copy cannot be None type") + if targets is None: + raise ValidationError("Error: copy destination cannot be None type") + if type(sources) is not list or len(sources) == 0: + raise ValidationError("Error: sources must be given as a list with at least one entry") + if type(targets) is not list or len(targets) == 0: + raise ValidationError("Error: targets must be given as a list with at least one entry") + + +def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False): + _list_checks() + + for i in range(0, len(source_paths)): + + if not path.exists(destination_paths[i]): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", destination_paths[i]) + continue + if not path.isdir(destination_paths[i]): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i]) + continue + + result = copy_to_dir(source_paths[i], destination_paths[i], verbose) + if result != 0: + return result + + def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False): """ Copies the sources to the destination paths. This is a generic function that just wraps shutil.copy2""" - - if source_paths is None: - raise ValidationError("Error: source for copy cannot be None type") - if destination_paths is None: - raise ValidationError("Error: copy destination cannot be None type") - - if type(source_paths) is not list or len(source_paths) == 0: - raise ValidationError("Error: sources must be given as a list with at least one entry") - if type(destination_paths) is not list or len(destination_paths) == 0: - raise ValidationError("Error: destinations must be given as a list with at least one entry") + _list_checks(source_paths, destination_paths) for dpath in destination_paths: @@ -66,2 +90,36 @@ return 0 + + +def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False): + """ Moves all files from inside source paths to the destination dits. +It's important to note that it moves index to index +i.e. source[0] -> dest[0]; source[1] -> dest[1] + +This is actually a pretty generic function, should be moved into a utils class. +""" + + _list_checks(source_dirs, destination_dirs) + + if len(source_dirs) != len(destination_dirs): + raise ValidationError("Error: source and target paths are not equal; aborting file movement") + + for i in range(0, len(source_dirs)): + # Identify files in path, move each one: + wildcard_search = "{0}/*".format(source_dirs[i]) + filepaths = glob.glob(wildcard_search) + if len(filepaths) == 0: + print("Warning: directory is empty: ", source_dirs[i]) + continue + try: + for fp in filepaths: + if verbose: + print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) + command_and_args = ["mv", fp, destination_dirs[i]] + # shutil.move(fp, destination_dirs[i]) + sub_utils.simple_unix_wrapper(command_and_args) + except IOError as e: + print("IOError copying: ", e.filename, e.filename2) + return 2 + + return 0