Changeset 41059

Timestamp:

Nov 5, 2019, 9:17:31 AM (7 years ago)

Author:

fairlamb

Message:

Added rsync to jira/conf schema and tests to boot

Location:

branches/ipp-350_add_to_jira_conf_backups

Files:

• backup.py (modified) (2 diffs)
• confluence_backup.py (modified) (1 diff)
• confluence_backup_test.py (modified) (3 diffs)
• jira_backup.py (modified) (1 diff)
• jira_backup_test.py (modified) (3 diffs)
• testing/confluence_test.config (modified) (1 diff)
• testing/jira_test.config (modified) (1 diff)

branches/ipp-350_add_to_jira_conf_backups/backup.py

@@ -83 +83 @@
 
         self.read_config_and_schema(config, schema)
-        # self.read_copy_config_section(config, schema)
-        # self.read_tar_config_section(config, schema)
-        # self.read_rsync_config_section(config, schema)
-        # self.read_mysqldump_config_section(config, schema)
 
     def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema):
@@ -206 +202 @@
         self.target_backup_paths_ok(raise_error=True)
         backup_paths = self.default_dict['backup_paths']
-        for bp in backup_paths:
-            # if not path.exists(bp):
-            #     raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}")
-
-            for s in self.rsync_dict['sources']:
-                if not path.exists(s):
+
+        sources = self.rsync_dict['sources']
+        for i in range(0, len(sources)):
+            s = sources[i]
+            if not path.exists(s):
                     raise errs.ValidationError(f"ValidationError: target path does not exist: {s}")
+
+            for bp in backup_paths:
                 try:
                     args = [s, bp]
+
                     if 'additional_args' in self.rsync_dict.keys():
                         args = args + self.rsync_dict['additional_args']
                     sub_utils.local_rsync_wrapper(args)
+
                 except errs.SubprocessError as e:
                     raise errs.SubprocessError( f"Error: rsync\n"

branches/ipp-350_add_to_jira_conf_backups/confluence_backup.py

@@ -32 +32 @@
         , ConfigSchemaLine('target_filename', True,  str)
         , ConfigSchemaLine('prefix_date',     True,  bool)
+        ])
+    , ConfigSchemaSection('RSYNC',
+        [ ConfigSchemaLine('sources', True, list,
+            cfg_help.SpecialSchemaProcessing.commma_separated)
+        , ConfigSchemaLine('additional_args', False, list,
+            cfg_help.SpecialSchemaProcessing.space_separated)
         ])
     , ConfigSchemaSection('MYSQLDUMP',

branches/ipp-350_add_to_jira_conf_backups/confluence_backup_test.py

@@ -34 +34 @@
         , 'target_filename' : 'confluence.tar.gz'
         , 'prefix_date' : 'True'
+        }
+    config['RSYNC'] = \
+        { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')},"
+                       f"{os.path.join(tmpdir, 'rsync_source_dir2')}")
+        , 'additional_args' : '-a --delete-after'
         }
     config['MYSQLDUMP'] = \
@@ -82 +87 @@
     assert os.path.exists(confluence_attachment_2_filepath)
 
+    # Create a few folders to be rsync'd
+    rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1")
+    rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2")
+    file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync")
+    file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync")
+    os.makedirs(rsync_source_dir1)
+    os.makedirs(rsync_source_dir2)
+    Path(file_in_rsync_dir1).touch()
+    Path(file_in_rsync_dir2).touch()
+    assert os.path.exists(file_in_rsync_dir1)
+    assert os.path.exists(file_in_rsync_dir2)
+
 
 class TestArguments(object):
@@ -287 +304 @@
         assert os.path.exists(expected_confluence_bak1_tar)
         assert os.path.exists(expected_confluence_bak2_tar)
+
+
+class TestRsyncOfFiles(object):
+
+    def test_confluence_rsync(self, tmpdir):
+        """This should be similar to the regular Backup version"""
+        create_items_for_full_confluence_backup_test(tmpdir)
+        config = make_config(tmpdir)
+
+        cb = ConfluenceBackup(config_file=config)
+
+        # Expected resulting rsync directories
+        expected_confluence_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "rsync_source_dir1")
+        expected_confluence_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path",    "rsync_source_dir1")
+        expected_confluence_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path",    "rsync_source_dir1")
+        expected_confluence_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "rsync_source_dir2")
+        expected_confluence_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path",    "rsync_source_dir2")
+        expected_confluence_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path",    "rsync_source_dir2")
+        assert not os.path.exists(expected_confluence_host_rsync1)
+        assert not os.path.exists(expected_confluence_bak1_rsync1)
+        assert not os.path.exists(expected_confluence_bak2_rsync1)
+        assert not os.path.exists(expected_confluence_host_rsync2)
+        assert not os.path.exists(expected_confluence_bak1_rsync2)
+        assert not os.path.exists(expected_confluence_bak2_rsync2)
+        # Expected resulting rsync files
+        expected_file_host_dir1 = os.path.join( expected_confluence_host_rsync1, "some_file_to_rsync")
+        expected_file_bak1_dir1 = os.path.join( expected_confluence_bak1_rsync1, "some_file_to_rsync")
+        expected_file_bak2_dir1 = os.path.join( expected_confluence_bak2_rsync1, "some_file_to_rsync")
+        expected_file_host_dir2 = os.path.join( expected_confluence_host_rsync2, "some_other_file_to_rsync")
+        expected_file_bak1_dir2 = os.path.join( expected_confluence_bak1_rsync2, "some_other_file_to_rsync")
+        expected_file_bak2_dir2 = os.path.join( expected_confluence_bak2_rsync2, "some_other_file_to_rsync")
+        assert not os.path.exists(expected_file_host_dir1)
+        assert not os.path.exists(expected_file_bak1_dir1)
+        assert not os.path.exists(expected_file_bak2_dir1)
+        assert not os.path.exists(expected_file_host_dir2)
+        assert not os.path.exists(expected_file_bak1_dir2)
+        assert not os.path.exists(expected_file_bak2_dir2)
+
+        rsync_result = cb._run_rsync()
+        assert rsync_result == 0
+
+        # All files Rsync'd
+        assert os.path.exists(expected_confluence_host_rsync1)
+        assert os.path.exists(expected_confluence_bak1_rsync1)
+        assert os.path.exists(expected_confluence_bak2_rsync1)
+        assert os.path.exists(expected_confluence_host_rsync2)
+        assert os.path.exists(expected_confluence_bak1_rsync2)
+        assert os.path.exists(expected_confluence_bak2_rsync2)
+        assert os.path.exists(expected_file_host_dir1)
+        assert os.path.exists(expected_file_bak1_dir1)
+        assert os.path.exists(expected_file_bak2_dir1)
+        assert os.path.exists(expected_file_host_dir2)
+        assert os.path.exists(expected_file_bak1_dir2)
+        assert os.path.exists(expected_file_bak2_dir2)

branches/ipp-350_add_to_jira_conf_backups/jira_backup.py

@@ -32 +32 @@
         , ConfigSchemaLine('target_filename', True,  str)
         , ConfigSchemaLine('prefix_date',     True,  bool)
+        ])
+    , ConfigSchemaSection('RSYNC',
+        [ ConfigSchemaLine('sources', True, list,
+            cfg_help.SpecialSchemaProcessing.commma_separated)
+        , ConfigSchemaLine('additional_args', False, list,
+            cfg_help.SpecialSchemaProcessing.space_separated)
         ])
     , ConfigSchemaSection('MYSQLDUMP',

branches/ipp-350_add_to_jira_conf_backups/jira_backup_test.py

@@ -33 +33 @@
         , 'target_filename' : 'jira.tar.gz'
         , 'prefix_date' : 'True'
+        }
+    config['RSYNC'] = \
+        { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')},"
+                       f"{os.path.join(tmpdir, 'rsync_source_dir2')}")
+        , 'additional_args' : '-a --delete-after'
         }
     config['MYSQLDUMP'] = \
@@ -81 +86 @@
     assert os.path.exists(jira_attachment_2_filepath)
 
+    # Create a few folders to be rsync'd
+    rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1")
+    rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2")
+    file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync")
+    file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync")
+    os.makedirs(rsync_source_dir1)
+    os.makedirs(rsync_source_dir2)
+    Path(file_in_rsync_dir1).touch()
+    Path(file_in_rsync_dir2).touch()
+    assert os.path.exists(file_in_rsync_dir1)
+    assert os.path.exists(file_in_rsync_dir2)
+
 
 class TestArguments(object):
@@ -288 +305 @@
         assert os.path.exists(expected_jira_bak1_tar)
         assert os.path.exists(expected_jira_bak2_tar)
+
+
+class TestRsyncOfFiles(object):
+
+    def test_jira_rsync(self, tmpdir):
+        """This should be similar to the regular Backup version"""
+        create_items_for_full_jira_backup_test(tmpdir)
+        config = make_config(tmpdir)
+
+        jb = JiraBackup(config_file=config)
+
+        # Expected resulting rsync directories
+        expected_jira_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "rsync_source_dir1")
+        expected_jira_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path",    "rsync_source_dir1")
+        expected_jira_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path",    "rsync_source_dir1")
+        expected_jira_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "rsync_source_dir2")
+        expected_jira_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path",    "rsync_source_dir2")
+        expected_jira_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path",    "rsync_source_dir2")
+        assert not os.path.exists(expected_jira_host_rsync1)
+        assert not os.path.exists(expected_jira_bak1_rsync1)
+        assert not os.path.exists(expected_jira_bak2_rsync1)
+        assert not os.path.exists(expected_jira_host_rsync2)
+        assert not os.path.exists(expected_jira_bak1_rsync2)
+        assert not os.path.exists(expected_jira_bak2_rsync2)
+        # Expected resulting rsync files
+        expected_file_host_dir1 = os.path.join( expected_jira_host_rsync1, "some_file_to_rsync")
+        expected_file_bak1_dir1 = os.path.join( expected_jira_bak1_rsync1, "some_file_to_rsync")
+        expected_file_bak2_dir1 = os.path.join( expected_jira_bak2_rsync1, "some_file_to_rsync")
+        expected_file_host_dir2 = os.path.join( expected_jira_host_rsync2, "some_other_file_to_rsync")
+        expected_file_bak1_dir2 = os.path.join( expected_jira_bak1_rsync2, "some_other_file_to_rsync")
+        expected_file_bak2_dir2 = os.path.join( expected_jira_bak2_rsync2, "some_other_file_to_rsync")
+        assert not os.path.exists(expected_file_host_dir1)
+        assert not os.path.exists(expected_file_bak1_dir1)
+        assert not os.path.exists(expected_file_bak2_dir1)
+        assert not os.path.exists(expected_file_host_dir2)
+        assert not os.path.exists(expected_file_bak1_dir2)
+        assert not os.path.exists(expected_file_bak2_dir2)
+
+        rsync_result = jb._run_rsync()
+        assert rsync_result == 0
+
+        # All files Rsync'd
+        assert os.path.exists(expected_jira_host_rsync1)
+        assert os.path.exists(expected_jira_bak1_rsync1)
+        assert os.path.exists(expected_jira_bak2_rsync1)
+        assert os.path.exists(expected_jira_host_rsync2)
+        assert os.path.exists(expected_jira_bak1_rsync2)
+        assert os.path.exists(expected_jira_bak2_rsync2)
+        assert os.path.exists(expected_file_host_dir1)
+        assert os.path.exists(expected_file_bak1_dir1)
+        assert os.path.exists(expected_file_bak2_dir1)
+        assert os.path.exists(expected_file_host_dir2)
+        assert os.path.exists(expected_file_bak1_dir2)
+        assert os.path.exists(expected_file_bak2_dir2)

branches/ipp-350_add_to_jira_conf_backups/testing/confluence_test.config

@@ -17 +17 @@
 prefix_date = True
 
+[RSYNC]
+sources = /opt/atlassian, /var/atlassian
+additional_args = -a --delete-after
+
 [MYSQLDUMP]
 user = dumper

branches/ipp-350_add_to_jira_conf_backups/testing/jira_test.config

@@ -16 +16 @@
 prefix_date = True
 
+[RSYNC]
+sources = /opt/atlassian, /var/atlassian
+additional_args = -a --delete-after
+
 [MYSQLDUMP]
 user = dumper