Changeset 41060

Timestamp:

Nov 5, 2019, 10:34:28 AM (7 years ago)

Author:

fairlamb

Message:

Added a new option to place rsyncs in specific subdirs (for clearer naming)

Location:

branches/ipp-350_add_to_jira_conf_backups

Files:

• backup.py (modified) (2 diffs)
• backup_test.py (modified) (1 diff)

branches/ipp-350_add_to_jira_conf_backups/backup.py

@@ -33 +33 @@
     [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated)
     , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated)
+    , ConfigSchemaLine('sub-dirs', False, list, SpecialSchemaProcessing.commma_separated)
     ]
 )
@@ -205 +206 @@
         sources = self.rsync_dict['sources']
         for i in range(0, len(sources)):
-            s = sources[i]
-            if not path.exists(s):
-                    raise errs.ValidationError(f"ValidationError: target path does not exist: {s}")
+            source = sources[i]
+            if not path.exists(source):
+                    raise errs.ValidationError(f"ValidationError: target path does not exist: {source}")
 
             for bp in backup_paths:
                 try:
-                    args = [s, bp]
-
+                    additional_args = []
                     if 'additional_args' in self.rsync_dict.keys():
-                        args = args + self.rsync_dict['additional_args']
+                        additional_args = self.rsync_dict['additional_args']
+
+                    source_and_target_args = [source, bp]
+
+                    # sub-dirs option allows items to be backed up to subdirs within
+                    # the source for better naming
+                    if 'sub-dirs' in self.rsync_dict:
+                        sub_dirs = self.rsync_dict['sub-dirs']
+                        if len(sub_dirs) != len(sources):
+                            raise errs.ValidationError(f"ValidationError: if using " +
+                                "subdirs the number of subdirs must equal the number of sources")
+                        bp_with_subdir = os.path.join(bp, sub_dirs[i])
+                        if not os.path.exists(bp_with_subdir):
+                            os.makedirs(bp_with_subdir)
+                        source_and_target_args = [source, bp_with_subdir]
+
+                    args = source_and_target_args + additional_args
                     sub_utils.local_rsync_wrapper(args)
 
                 except errs.SubprocessError as e:
                     raise errs.SubprocessError( f"Error: rsync\n"
-                                                f"    from: {s}\n"
-                                                f"    to: {bp}\n"
+                                                f"    from: {source_and_target_args[0]}\n"
+                                                f"    to: {source_and_target_args[1]}\n"
                                               ) from e
         return 0

branches/ipp-350_add_to_jira_conf_backups/backup_test.py

@@ -596 +596 @@
         assert os.path.exists(expected4dir)
 
+    def test_rsync_with_subdirs_option(self, tmpdir):
+        config = make_default_config(tmpdir, ['bck1', 'bck2'])
+        source1 = os.path.join(tmpdir, "some_dir")
+        source2 = os.path.join(tmpdir, "some_other_dir")
+        file_in_dir = os.path.join(source1, "and_specific_file.jpeg")
+        file_in_other_dir = os.path.join(source2, "another_file.jpeg")
+        os.makedirs(source1)
+        os.makedirs(source2)
+        Path(file_in_dir).touch()
+        Path(file_in_other_dir).touch()
+
+        config['RSYNC'] = \
+        { 'sources' : f"{source1}, {source2}"
+        , 'additional_args' : "-a --progress -v"
+        , 'sub-dirs' : "subdir_with_better_name1, subdir_with_better_name2"
+        }
+
+        schema = ConfigSchema(None)
+        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
+        schema.add_section(bckup.RSYNC_SCHEMA_SECTION)
+
+        backy = Backup(config, schema)
+
+        expected_bak1_src1_dir  = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir')
+        expected_bak1_src1_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg')
+        expected_bak1_src2_dir  = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir')
+        expected_bak1_src2_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg')
+        expected_bak2_src1_dir  = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir')
+        expected_bak2_src1_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg')
+        expected_bak2_src2_dir  = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir')
+        expected_bak2_src2_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg')
+        assert not os.path.exists(expected_bak1_src1_dir)
+        assert not os.path.exists(expected_bak1_src1_file)
+        assert not os.path.exists(expected_bak1_src2_dir)
+        assert not os.path.exists(expected_bak1_src2_file)
+        assert not os.path.exists(expected_bak2_src1_dir)
+        assert not os.path.exists(expected_bak2_src1_file)
+        assert not os.path.exists(expected_bak2_src2_dir)
+        assert not os.path.exists(expected_bak2_src2_file)
+        backy._run_rsync()
+        assert os.path.exists(expected_bak1_src1_dir)
+        assert os.path.exists(expected_bak1_src1_file)
+        assert os.path.exists(expected_bak1_src2_dir)
+        assert os.path.exists(expected_bak1_src2_file)
+        assert os.path.exists(expected_bak2_src1_dir)
+        assert os.path.exists(expected_bak2_src1_file)
+        assert os.path.exists(expected_bak2_src2_dir)
+        assert os.path.exists(expected_bak2_src2_file)
 
 @pytest.mark.skip(reason="Requires a fake database to be setup")