Changeset 41067 for trunk/backups

Timestamp:

Nov 5, 2019, 3:18:35 PM (7 years ago)

Author:

fairlamb

Message:

merge of branch ipp-350 into trunk

Location:

trunk/backups

Files:

• . (modified) (1 prop)
• backup.py (modified) (3 diffs)
• backup_test.py (modified) (1 diff)
• confluence_backup.py (modified) (1 diff)
• confluence_backup_test.py (modified) (3 diffs)
• jira_backup.py (modified) (1 diff)
• jira_backup_test.py (modified) (3 diffs)
• testing/confluence_test.config (modified) (1 diff)
• testing/jira_test.config (modified) (1 diff)

trunk/backups/backup.py

@@ -33 +33 @@
     [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated)
     , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated)
+    , ConfigSchemaLine('sub-dirs', False, list, SpecialSchemaProcessing.commma_separated)
     ]
 )
@@ -83 +84 @@
 
         self.read_config_and_schema(config, schema)
-        # self.read_copy_config_section(config, schema)
-        # self.read_tar_config_section(config, schema)
-        # self.read_rsync_config_section(config, schema)
-        # self.read_mysqldump_config_section(config, schema)
 
     def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema):
@@ -206 +203 @@
         self.target_backup_paths_ok(raise_error=True)
         backup_paths = self.default_dict['backup_paths']
-        for bp in backup_paths:
-            # if not path.exists(bp):
-            #     raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}")
-
-            for s in self.rsync_dict['sources']:
-                if not path.exists(s):
-                    raise errs.ValidationError(f"ValidationError: target path does not exist: {s}")
+
+        sources = self.rsync_dict['sources']
+        for i in range(0, len(sources)):
+            source = sources[i]
+            if not path.exists(source):
+                    raise errs.ValidationError(f"ValidationError: target path does not exist: {source}")
+
+            for bp in backup_paths:
                 try:
-                    args = [s, bp]
+                    additional_args = []
                     if 'additional_args' in self.rsync_dict.keys():
-                        args = args + self.rsync_dict['additional_args']
+                        additional_args = self.rsync_dict['additional_args']
+
+                    source_and_target_args = [source, bp]
+
+                    # sub-dirs option allows items to be backed up to subdirs within
+                    # the source for better naming
+                    if 'sub-dirs' in self.rsync_dict:
+                        sub_dirs = self.rsync_dict['sub-dirs']
+                        if len(sub_dirs) != len(sources):
+                            raise errs.ValidationError(f"ValidationError: if using " +
+                                "subdirs the number of subdirs must equal the number of sources")
+                        bp_with_subdir = os.path.join(bp, sub_dirs[i])
+                        if not os.path.exists(bp_with_subdir):
+                            os.makedirs(bp_with_subdir)
+                        source_and_target_args = [source, bp_with_subdir]
+
+                    args = source_and_target_args + additional_args
                     sub_utils.local_rsync_wrapper(args)
+
                 except errs.SubprocessError as e:
                     raise errs.SubprocessError( f"Error: rsync\n"
-                                                f"    from: {s}\n"
-                                                f"    to: {bp}\n"
+                                                f"    from: {source_and_target_args[0]}\n"
+                                                f"    to: {source_and_target_args[1]}\n"
                                               ) from e
         return 0

trunk/backups/backup_test.py

@@ -596 +596 @@
         assert os.path.exists(expected4dir)
 
+    def test_rsync_with_subdirs_option(self, tmpdir):
+        config = make_default_config(tmpdir, ['bck1', 'bck2'])
+        source1 = os.path.join(tmpdir, "some_dir")
+        source2 = os.path.join(tmpdir, "some_other_dir")
+        file_in_dir = os.path.join(source1, "and_specific_file.jpeg")
+        file_in_other_dir = os.path.join(source2, "another_file.jpeg")
+        os.makedirs(source1)
+        os.makedirs(source2)
+        Path(file_in_dir).touch()
+        Path(file_in_other_dir).touch()
+
+        config['RSYNC'] = \
+        { 'sources' : f"{source1}, {source2}"
+        , 'additional_args' : "-a --progress -v"
+        , 'sub-dirs' : "subdir_with_better_name1, subdir_with_better_name2"
+        }
+
+        schema = ConfigSchema(None)
+        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
+        schema.add_section(bckup.RSYNC_SCHEMA_SECTION)
+
+        backy = Backup(config, schema)
+
+        expected_bak1_src1_dir  = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir')
+        expected_bak1_src1_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg')
+        expected_bak1_src2_dir  = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir')
+        expected_bak1_src2_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg')
+        expected_bak2_src1_dir  = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir')
+        expected_bak2_src1_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg')
+        expected_bak2_src2_dir  = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir')
+        expected_bak2_src2_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg')
+        assert not os.path.exists(expected_bak1_src1_dir)
+        assert not os.path.exists(expected_bak1_src1_file)
+        assert not os.path.exists(expected_bak1_src2_dir)
+        assert not os.path.exists(expected_bak1_src2_file)
+        assert not os.path.exists(expected_bak2_src1_dir)
+        assert not os.path.exists(expected_bak2_src1_file)
+        assert not os.path.exists(expected_bak2_src2_dir)
+        assert not os.path.exists(expected_bak2_src2_file)
+        backy._run_rsync()
+        assert os.path.exists(expected_bak1_src1_dir)
+        assert os.path.exists(expected_bak1_src1_file)
+        assert os.path.exists(expected_bak1_src2_dir)
+        assert os.path.exists(expected_bak1_src2_file)
+        assert os.path.exists(expected_bak2_src1_dir)
+        assert os.path.exists(expected_bak2_src1_file)
+        assert os.path.exists(expected_bak2_src2_dir)
+        assert os.path.exists(expected_bak2_src2_file)
 
 @pytest.mark.skip(reason="Requires a fake database to be setup")

trunk/backups/confluence_backup.py

@@ -32 +32 @@
         , ConfigSchemaLine('target_filename', True,  str)
         , ConfigSchemaLine('prefix_date',     True,  bool)
+        ])
+    , ConfigSchemaSection('RSYNC',
+        [ ConfigSchemaLine('sources', True, list,
+            cfg_help.SpecialSchemaProcessing.commma_separated)
+        , ConfigSchemaLine('additional_args', False, list,
+            cfg_help.SpecialSchemaProcessing.space_separated)
+        , ConfigSchemaLine('sub-dirs', False, list,
+            cfg_help.SpecialSchemaProcessing.commma_separated)
         ])
     , ConfigSchemaSection('MYSQLDUMP',

trunk/backups/confluence_backup_test.py

@@ -34 +34 @@
         , 'target_filename' : 'confluence.tar.gz'
         , 'prefix_date' : 'True'
+        }
+    config['RSYNC'] = \
+        { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')},"
+                       f"{os.path.join(tmpdir, 'rsync_source_dir2')}")
+        , 'additional_args' : '-a --delete-after --exclude exclude_me_if_you_like'
+        , 'sub-dirs' : 'conf_subdir1, conf_subdir2'
         }
     config['MYSQLDUMP'] = \
@@ -82 +88 @@
     assert os.path.exists(confluence_attachment_2_filepath)
 
+    # Create a few folders to be rsync'd
+    rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1")
+    rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2")
+    file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync")
+    file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync")
+    exluded_file = os.path.join(rsync_source_dir1, "exclude_me_if_you_like")
+    os.makedirs(rsync_source_dir1)
+    os.makedirs(rsync_source_dir2)
+    Path(file_in_rsync_dir1).touch()
+    Path(file_in_rsync_dir2).touch()
+    Path(exluded_file).touch()
+    assert os.path.exists(file_in_rsync_dir1)
+    assert os.path.exists(file_in_rsync_dir2)
+    assert os.path.exists(exluded_file)
+
 
 class TestArguments(object):
@@ -287 +308 @@
         assert os.path.exists(expected_confluence_bak1_tar)
         assert os.path.exists(expected_confluence_bak2_tar)
+
+
+class TestRsyncOfFiles(object):
+
+    def test_confluence_rsync_with_subdirs(self, tmpdir):
+        """This should be similar to the regular Backup version"""
+        create_items_for_full_confluence_backup_test(tmpdir)
+        config = make_config(tmpdir)
+
+        cb = ConfluenceBackup(config_file=config)
+
+        # Expected resulting rsync directories
+        expected_confluence_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "conf_subdir1", "rsync_source_dir1")
+        expected_confluence_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path",    "conf_subdir1", "rsync_source_dir1")
+        expected_confluence_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path",    "conf_subdir1", "rsync_source_dir1")
+        expected_confluence_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "conf_subdir2", "rsync_source_dir2")
+        expected_confluence_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path",    "conf_subdir2", "rsync_source_dir2")
+        expected_confluence_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path",    "conf_subdir2", "rsync_source_dir2")
+        assert not os.path.exists(expected_confluence_host_rsync1)
+        assert not os.path.exists(expected_confluence_bak1_rsync1)
+        assert not os.path.exists(expected_confluence_bak2_rsync1)
+        assert not os.path.exists(expected_confluence_host_rsync2)
+        assert not os.path.exists(expected_confluence_bak1_rsync2)
+        assert not os.path.exists(expected_confluence_bak2_rsync2)
+        # Expected resulting rsync files
+        expected_file_host_dir1 = os.path.join( expected_confluence_host_rsync1, "some_file_to_rsync")
+        expected_file_bak1_dir1 = os.path.join( expected_confluence_bak1_rsync1, "some_file_to_rsync")
+        expected_file_bak2_dir1 = os.path.join( expected_confluence_bak2_rsync1, "some_file_to_rsync")
+        expected_file_host_dir2 = os.path.join( expected_confluence_host_rsync2, "some_other_file_to_rsync")
+        expected_file_bak1_dir2 = os.path.join( expected_confluence_bak1_rsync2, "some_other_file_to_rsync")
+        expected_file_bak2_dir2 = os.path.join( expected_confluence_bak2_rsync2, "some_other_file_to_rsync")
+        assert not os.path.exists(expected_file_host_dir1)
+        assert not os.path.exists(expected_file_bak1_dir1)
+        assert not os.path.exists(expected_file_bak2_dir1)
+        assert not os.path.exists(expected_file_host_dir2)
+        assert not os.path.exists(expected_file_bak1_dir2)
+        assert not os.path.exists(expected_file_bak2_dir2)
+        an_excluded_file = os.path.join(expected_confluence_host_rsync1, "exclude_me_if_you_like")
+        assert not os.path.exists(an_excluded_file)
+
+        rsync_result = cb._run_rsync()
+        assert rsync_result == 0
+
+        # All files Rsync'd
+        assert os.path.exists(expected_confluence_host_rsync1)
+        assert os.path.exists(expected_confluence_bak1_rsync1)
+        assert os.path.exists(expected_confluence_bak2_rsync1)
+        assert os.path.exists(expected_confluence_host_rsync2)
+        assert os.path.exists(expected_confluence_bak1_rsync2)
+        assert os.path.exists(expected_confluence_bak2_rsync2)
+        assert os.path.exists(expected_file_host_dir1)
+        assert os.path.exists(expected_file_bak1_dir1)
+        assert os.path.exists(expected_file_bak2_dir1)
+        assert os.path.exists(expected_file_host_dir2)
+        assert os.path.exists(expected_file_bak1_dir2)
+        assert os.path.exists(expected_file_bak2_dir2)
+        assert not os.path.exists(an_excluded_file)

trunk/backups/jira_backup.py

@@ -32 +32 @@
         , ConfigSchemaLine('target_filename', True,  str)
         , ConfigSchemaLine('prefix_date',     True,  bool)
+        ])
+    , ConfigSchemaSection('RSYNC',
+        [ ConfigSchemaLine('sources', True, list,
+            cfg_help.SpecialSchemaProcessing.commma_separated)
+        , ConfigSchemaLine('additional_args', False, list,
+            cfg_help.SpecialSchemaProcessing.space_separated)
+        , ConfigSchemaLine('sub-dirs', False, list,
+            cfg_help.SpecialSchemaProcessing.commma_separated)
         ])
     , ConfigSchemaSection('MYSQLDUMP',

trunk/backups/jira_backup_test.py

@@ -33 +33 @@
         , 'target_filename' : 'jira.tar.gz'
         , 'prefix_date' : 'True'
+        }
+    config['RSYNC'] = \
+        { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')},"
+                       f"{os.path.join(tmpdir, 'rsync_source_dir2')}")
+        , 'additional_args' : '-a --delete-after'
+        , 'sub-dirs' : 'jira_subdir1, jira_subdir2'
         }
     config['MYSQLDUMP'] = \
@@ -81 +87 @@
     assert os.path.exists(jira_attachment_2_filepath)
 
+    # Create a few folders to be rsync'd
+    rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1")
+    rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2")
+    file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync")
+    file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync")
+    os.makedirs(rsync_source_dir1)
+    os.makedirs(rsync_source_dir2)
+    Path(file_in_rsync_dir1).touch()
+    Path(file_in_rsync_dir2).touch()
+    assert os.path.exists(file_in_rsync_dir1)
+    assert os.path.exists(file_in_rsync_dir2)
+
 
 class TestArguments(object):
@@ -288 +306 @@
         assert os.path.exists(expected_jira_bak1_tar)
         assert os.path.exists(expected_jira_bak2_tar)
+
+
+class TestRsyncOfFiles(object):
+
+    def test_jira_rsync_with_subdirs(self, tmpdir):
+        """This should be similar to the regular Backup version"""
+        create_items_for_full_jira_backup_test(tmpdir)
+        config = make_config(tmpdir)
+
+        jb = JiraBackup(config_file=config)
+
+        # Expected resulting rsync directories
+        expected_jira_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "jira_subdir1", "rsync_source_dir1")
+        expected_jira_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path",    "jira_subdir1", "rsync_source_dir1")
+        expected_jira_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path",    "jira_subdir1", "rsync_source_dir1")
+        expected_jira_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "jira_subdir2", "rsync_source_dir2")
+        expected_jira_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path",    "jira_subdir2", "rsync_source_dir2")
+        expected_jira_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path",    "jira_subdir2", "rsync_source_dir2")
+        assert not os.path.exists(expected_jira_host_rsync1)
+        assert not os.path.exists(expected_jira_bak1_rsync1)
+        assert not os.path.exists(expected_jira_bak2_rsync1)
+        assert not os.path.exists(expected_jira_host_rsync2)
+        assert not os.path.exists(expected_jira_bak1_rsync2)
+        assert not os.path.exists(expected_jira_bak2_rsync2)
+        # Expected resulting rsync files
+        expected_file_host_dir1 = os.path.join( expected_jira_host_rsync1, "some_file_to_rsync")
+        expected_file_bak1_dir1 = os.path.join( expected_jira_bak1_rsync1, "some_file_to_rsync")
+        expected_file_bak2_dir1 = os.path.join( expected_jira_bak2_rsync1, "some_file_to_rsync")
+        expected_file_host_dir2 = os.path.join( expected_jira_host_rsync2, "some_other_file_to_rsync")
+        expected_file_bak1_dir2 = os.path.join( expected_jira_bak1_rsync2, "some_other_file_to_rsync")
+        expected_file_bak2_dir2 = os.path.join( expected_jira_bak2_rsync2, "some_other_file_to_rsync")
+        assert not os.path.exists(expected_file_host_dir1)
+        assert not os.path.exists(expected_file_bak1_dir1)
+        assert not os.path.exists(expected_file_bak2_dir1)
+        assert not os.path.exists(expected_file_host_dir2)
+        assert not os.path.exists(expected_file_bak1_dir2)
+        assert not os.path.exists(expected_file_bak2_dir2)
+
+        rsync_result = jb._run_rsync()
+        assert rsync_result == 0
+
+        # All files Rsync'd
+        assert os.path.exists(expected_jira_host_rsync1)
+        assert os.path.exists(expected_jira_bak1_rsync1)
+        assert os.path.exists(expected_jira_bak2_rsync1)
+        assert os.path.exists(expected_jira_host_rsync2)
+        assert os.path.exists(expected_jira_bak1_rsync2)
+        assert os.path.exists(expected_jira_bak2_rsync2)
+        assert os.path.exists(expected_file_host_dir1)
+        assert os.path.exists(expected_file_bak1_dir1)
+        assert os.path.exists(expected_file_bak2_dir1)
+        assert os.path.exists(expected_file_host_dir2)
+        assert os.path.exists(expected_file_bak1_dir2)
+        assert os.path.exists(expected_file_bak2_dir2)

trunk/backups/testing/confluence_test.config

@@ -17 +17 @@
 prefix_date = True
 
+[RSYNC]
+sources = /opt/atlassian, /var/atlassian
+additional_args = -a --delete-after
+sub-dirs = conf-install, conf-home
+
 [MYSQLDUMP]
 user = dumper

trunk/backups/testing/jira_test.config

@@ -16 +16 @@
 prefix_date = True
 
+[RSYNC]
+sources = /opt/atlassian, /var/atlassian
+additional_args = -a --delete-after
+sub-dirs = jira-install, jira-home
+
 [MYSQLDUMP]
 user = dumper