Changeset 40912

Timestamp:

Oct 4, 2019, 4:50:00 PM (7 years ago)

Author:

fairlamb

Message:

that function was used - moved it to generic; separated out jira/conf tar

Location:

branches/ipp-259_genericise_backups/tools/backups

Files:

• backup_atlassian_applications.py (modified) (3 diffs)
• backup_atlassian_applications_test.py (modified) (5 diffs)
• utils/generic_utils.py (modified) (3 diffs)

branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications.py

@@ -90 +90 @@
 
         # Move latest to tmp (and delete if move ok)
-        gen_utils.copy_multiple_items_to_multiple_dirs(latest_paths, tmp_paths, self.verbose)
+        gen_utils.move_dirs(latest_paths, tmp_paths, self.verbose)
+
+        return_status = []
 
         # copy jira
-        jira_return = self._copy_jira_backup(self.verbose)
+        return_status.append(self._copy_jira_backup(self.verbose))
 
         # copy confluence
-        conf_return = self._copy_confluence_backup(self.verbose)
+        return_status.append(self._copy_confluence_backup(self.verbose))
 
         # tarfiles:
-        tar_return = self._tar_attachments()
+        return_status.append(self._tar_jira_attachments())
+        return_status.append(self._tar_conf_attachments())
 
         # mysqldump
-        mysqldump_return = self.make_mysql_dump(self.mysqldump_password)
+        return_status.append(self.make_mysql_dump(self.mysqldump_password))
 
         # If successful: move tmp to old
-        if jira_return == 0 and conf_return == 0 and tar_return == 0 and mysqldump_return == 0:
-            gen_utils.copy_multiple_items_to_multiple_dirs(tmp_paths, old_paths, self.verbose)
+        failures = [ rs for rs in return_status if rs != 0 ]
+        if len(failures) == 0:
+            gen_utils.move_dirs(tmp_paths, old_paths, self.verbose)
         else:
             # If any failed: delete latest, move tmp back
-            gen_utils.copy_multiple_items_to_multiple_dirs(tmp_paths, latest_paths, self.verbose)
+            gen_utils.move_dirs(tmp_paths, latest_paths, self.verbose)
             return 5
 
@@ -177 +181 @@
         return 0
 
-    def _tar_attachments(self):
+    def _tar_jira_attachments(self):
         # tar jira
         latest_paths = self.target_paths("latest")
@@ -191 +195 @@
         sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[1]])
         sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[2]])
+        return 0
+
+
+    def _tar_conf_attachments(self):
+        latest_paths = self.target_paths("latest")
+        if not self.target_backup_paths_ok():
+            return 1
 
         # tar confluence

branches/ipp-259_genericise_backups/tools/backups/backup_atlassian_applications_test.py

@@ -9 +9 @@
 import backups.full_atlassian_backup_test as fab_test
 import backups.utils.subprocess_utils as sub_utils
+import backups.utils.generic_utils as gen_utils
 
 from backups.backup_atlassian_applications import AtlassianBackups
+from backups.utils.errors import ValidationError
 
 TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), './testing/test.config')
@@ -215 +217 @@
         destination_paths = [ "/Two", "/paths"]
 
-        return_is_non_zero = baa._move_dir_contents_to_dir(source_paths, destination_paths)
-        assert return_is_non_zero != 0
+        with pytest.raises(ValidationError):
+            gen_utils.move_dirs(source_paths, destination_paths)
 
     def test_moving_old_backups(self, tmpdir):
@@ -275 +277 @@
 
         # run the move
-        return_result = baa._move_dir_contents_to_dir(source_paths, destination_paths)
+        return_result = gen_utils.move_dirs(source_paths, destination_paths)
         assert return_result == 0
 
@@ -357 +359 @@
 class TestTaringOfAttachments(object):
 
-    def test_attachments_tar(self, tmpdir):
+    def test_jira_attachments_tar(self, tmpdir):
 
         ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE)
@@ -364 +366 @@
         today = datetime.date.today().strftime("%Y_%m_%d")
         jira_tar_name = today + "_jira_attachments.tar"
+        expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name)
+        expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name)
+        expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name)
+        assert not os.path.exists(expected_jira_host_tar)
+        assert not os.path.exists(expected_jira_bak1_tar)
+        assert not os.path.exists(expected_jira_bak2_tar)
+
+        tar_result = ab._tar_jira_attachments()
+        assert tar_result == 0
+
+        assert os.path.exists(expected_jira_host_tar)
+        assert os.path.exists(expected_jira_bak1_tar)
+        assert os.path.exists(expected_jira_bak2_tar)
+
+    def test_conf_attachments_tar(self, tmpdir):
+
+        ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE)
+        ab = fab_test.create_all_paths_and_folders(ab, tmpdir)
+
+        today = datetime.date.today().strftime("%Y_%m_%d")
         conf_tar_name = today + "_confluence_attachments.tar"
-        expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name)
         expected_conf_host_tar = os.path.join(ab.host_backup_path, "latest", conf_tar_name)
-        expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name)
         expected_conf_bak1_tar = os.path.join(ab.backup_1_path, "latest", conf_tar_name)
-        expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name)
         expected_conf_bak2_tar = os.path.join(ab.backup_2_path, "latest", conf_tar_name)
-        assert not os.path.exists(expected_jira_host_tar)
         assert not os.path.exists(expected_conf_host_tar)
-        assert not os.path.exists(expected_jira_bak1_tar)
         assert not os.path.exists(expected_conf_bak1_tar)
-        assert not os.path.exists(expected_jira_bak2_tar)
         assert not os.path.exists(expected_conf_bak2_tar)
 
-        tar_result = ab._tar_attachments()
+        tar_result = ab._tar_conf_attachments()
         assert tar_result == 0
 
-        assert os.path.exists(expected_jira_host_tar)
         assert os.path.exists(expected_conf_host_tar)
-        assert os.path.exists(expected_jira_bak1_tar)
         assert os.path.exists(expected_conf_bak1_tar)
-        assert os.path.exists(expected_jira_bak2_tar)
         assert os.path.exists(expected_conf_bak2_tar)
 

branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py

@@ +1 @@
+import glob
 import os.path as path
 import shutil
 
 from distutils.dir_util import copy_tree
+
+import backups.utils.subprocess_utils as sub_utils
 from backups.utils.errors import ValidationError
 
@@ -35 +38 @@
 
 
+def _list_checks(sources: [str], targets: [str]):
+    if sources is None or targets is None:
+        raise ValidationError("Error: source for copy cannot be None type")
+    if targets is None:
+        raise ValidationError("Error: copy destination cannot be None type")
+    if type(sources) is not list or len(sources) == 0:
+        raise ValidationError("Error: sources must be given as a list with at least one entry")
+    if type(targets) is not list or len(targets) == 0:
+        raise ValidationError("Error: targets must be given as a list with at least one entry")
+
+
+def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False):
+    _list_checks()
+
+    for i in range(0, len(source_paths)):
+
+        if not path.exists(destination_paths[i]):
+            if verbose:
+                print("Copy Warning: target path does not exist (skipping): ", destination_paths[i])
+            continue
+        if not path.isdir(destination_paths[i]):
+            if verbose:
+                print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i])
+            continue
+
+        result = copy_to_dir(source_paths[i], destination_paths[i], verbose)
+        if result != 0:
+            return result
+
+
 def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False):
     """ Copies the sources to the destination paths.
 
 This is a generic function that just wraps shutil.copy2"""
-
-    if source_paths is None:
-        raise ValidationError("Error: source for copy cannot be None type")
-    if destination_paths is None:
-        raise ValidationError("Error: copy destination cannot be None type")
-
-    if type(source_paths) is not list or len(source_paths) == 0:
-        raise ValidationError("Error: sources must be given as a list with at least one entry")
-    if type(destination_paths) is not list or len(destination_paths) == 0:
-        raise ValidationError("Error: destinations must be given as a list with at least one entry")
+    _list_checks(source_paths, destination_paths)
 
     for dpath in destination_paths:
@@ -66 +90 @@
 
     return 0
+
+
+def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False):
+    """ Moves all files from inside source paths to the destination dits.
+It's important to note that it moves index to index
+i.e. source[0] -> dest[0]; source[1] -> dest[1]
+
+This is actually a pretty generic function, should be moved into a utils class.
+"""
+
+    _list_checks(source_dirs, destination_dirs)
+
+    if len(source_dirs) != len(destination_dirs):
+        raise ValidationError("Error: source and target paths are not equal; aborting file movement")
+
+    for i in range(0, len(source_dirs)):
+        # Identify files in path, move each one:
+        wildcard_search = "{0}/*".format(source_dirs[i])
+        filepaths = glob.glob(wildcard_search)
+        if len(filepaths) == 0:
+            print("Warning: directory is empty: ", source_dirs[i])
+            continue
+        try:
+            for fp in filepaths:
+                if verbose:
+                    print("Moving: {0} -> {1}".format(fp, destination_dirs[i]))
+                command_and_args = ["mv", fp, destination_dirs[i]]
+                # shutil.move(fp, destination_dirs[i])
+                sub_utils.simple_unix_wrapper(command_and_args)
+        except IOError as e:
+            print("IOError copying: ", e.filename, e.filename2)
+            return 2
+
+    return 0