Changeset 40932

Timestamp:

Oct 9, 2019, 2:15:34 PM (7 years ago)

Author:

fairlamb

Message:

Added local rsync to subprocess and how to handle it in backups (functionally the same as cp)

Location:

branches/ipp-259_genericise_backups/tools/backups

Files:

• backup.py (modified) (1 diff)
• backup_test.py (modified) (4 diffs)
• utils/subprocess_utils.py (modified) (1 diff)
• utils/subprocess_utils_test.py (modified) (1 diff)

branches/ipp-259_genericise_backups/tools/backups/backup.py

@@ -181 +181 @@
 
     def _run_rsync(self):
-        raise Exception("Not implemented")
+        if self.run_rsync is None or False:
+            return 0
+
+        backup_paths = self.default_dict['backup_paths']
+        for bp in backup_paths:
+            if not path.exists(bp):
+                raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}")
+
+            for s in self.rsync_dict['sources']:
+                if not path.exists(s):
+                    raise errs.ValidationError(f"ValidationError: target path does not exist: {s}")
+                try:
+                    args = [s, bp]
+                    if 'additional_args' in self.rsync_dict.keys():
+                        print("hiya")
+                        args = args + self.rsync_dict['additional_args']
+                        print(args)
+                    sub_utils.local_rsync_wrapper(args)
+                except errs.SubprocessError as e:
+                    raise errs.SubprocessError( f"Error: rsync\n"
+                                                f"    from: {s}\n"
+                                                f"    to: {bp}\n"
+                                              ) from e
+        return 0
 
     def _run_mysqldump(self):
 
-        # cfg_help.ConfigSchemaLine('user', True, str)
-        # cfg_help.ConfigSchemaLine('password', True, str)
-        # cfg_help.ConfigSchemaLine('db_name', True, str)
-        # cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated)
-        # cfg_help.ConfigSchemaLine('target_filename', True, str)
-        # cfg_help.ConfigSchemaLine('prefix_date', True, bool)
-
         backup_paths = self.default_dict['backup_paths']
 
-        # Setup the commands:
         mysqldump_args = [ 'mysqldump'
                          , '-u', self.mysqldump_dict['user']

branches/ipp-259_genericise_backups/tools/backups/backup_test.py

@@ -186 +186 @@
         assert os.path.exists(expected2file)
 
-    def copy_fails_if_source_file_does_not_exist(self, tmpdir):
+    def test_copy_fails_if_source_file_does_not_exist(self, tmpdir):
         config = make_default_config(tmpdir, ['bck1', 'bck2'])
         source = os.path.join(tmpdir, "and_specific_file.jpeg")
@@ -196 +196 @@
         assert not os.path.exists(source)
         backy = Backup(config)
-        with pytest.raises(errs.SubprocessError):
+        with pytest.raises(errs.ValidationError):
             backy._run_copy()
 
-    def copy_fails_if_destination_does_not_exist(self, tmpdir):
+    def test_copy_fails_if_destination_does_not_exist(self, tmpdir):
         config = ConfigParser()
 
@@ -214 +214 @@
         assert not os.path.exists(backup_does_not_exist)
         backy = Backup(config)
-        with pytest.raises(errs.SubprocessError):
+        with pytest.raises(errs.ValidationError):
             backy._run_copy()
 
@@ -298 +298 @@
 
 
-# class TestRunningRsync():
-
-#     def test_simply_rsync(self):
-#         assert True is False
-
-#     def test_rsync_with_additional_args(self):
-#         assert True is False
+class TestRunningRsync():
+
+    def test_simple_rsync(self, tmpdir):
+        config = make_default_config(tmpdir, ['bck1', 'bck2'])
+        source = os.path.join(tmpdir, "and_specific_file.jpeg")
+        config['RSYNC'] = {'sources' : f"{source}"}
+
+        # make file to be copied
+        Path(source).touch()
+
+        backy = Backup(config)
+
+        expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg')
+        expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg')
+        assert not os.path.exists(expected1file)
+        assert not os.path.exists(expected2file)
+        backy._run_rsync()
+        assert os.path.exists(expected1file)
+        assert os.path.exists(expected2file)
+
+    def test_rsync_with_additional_args(self, tmpdir):
+        config = make_default_config(tmpdir, ['bck1', 'bck2'])
+        source1 = os.path.join(tmpdir, "some_dir")
+        source2 = os.path.join(tmpdir, "and_specific_file.jpeg")
+        config['RSYNC'] = \
+        { 'sources' : f"{source1}, {source2}"
+        , 'additional_args' : "-v -z -r --progress"
+        }
+
+        os.makedirs(source1)
+        Path(source2).touch()
+
+        backy = Backup(config)
+
+        expected1dir  = os.path.join(tmpdir, 'bck1', 'some_dir')
+        expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg')
+        expected2dir  = os.path.join(tmpdir, 'bck2', 'some_dir')
+        expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg')
+        assert not os.path.exists(expected1dir)
+        assert not os.path.exists(expected1file)
+        assert not os.path.exists(expected2dir)
+        assert not os.path.exists(expected2file)
+        backy._run_rsync()
+        assert os.path.exists(expected1dir)
+        assert os.path.exists(expected1file)
+        assert os.path.exists(expected2dir)
+        assert os.path.exists(expected2file)
+
+    def test_rsync_fails_if_source_file_does_not_exist(self, tmpdir):
+        config = make_default_config(tmpdir, ['bck1', 'bck2'])
+        source = os.path.join(tmpdir, "and_specific_file.jpeg")
+        config['RSYNC'] = \
+        { 'sources' : f"{source}"
+        , 'additional_args' : '-r'
+        }
+
+        assert not os.path.exists(source)
+        backy = Backup(config)
+        with pytest.raises(errs.ValidationError):
+            backy._run_rsync()
+
+    def test_rsync_fails_if_destination_does_not_exist(self, tmpdir):
+        config = ConfigParser()
+
+        backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist')
+
+        config['DEFAULT'] = \
+        { 'backup_paths' : backup_does_not_exist
+        , 'source_machine': socket.gethostname()
+        }
+        source = os.path.join(tmpdir, "and_specific_file.jpeg")
+        Path(source).touch()
+        config['RSYNC'] = {'sources' : f"{source}"}
+
+        assert not os.path.exists(backup_does_not_exist)
+        backy = Backup(config)
+        with pytest.raises(errs.ValidationError):
+            backy._run_rsync()
+
+    def test_rsync_requires_dash_r_arg_for_copying_dirs(self, tmpdir):
+        config = make_default_config(tmpdir, ['bck1', 'bck2'])
+        source = os.path.join(tmpdir, "some_dir")
+        config['RSYNC'] = \
+        { 'sources' : f"{source}"
+        }
+        os.makedirs(source)
+
+        backy = Backup(config)
+        expected1dir  = os.path.join(tmpdir, 'bck1', 'some_dir')
+        expected2dir  = os.path.join(tmpdir, 'bck2', 'some_dir')
+        assert not os.path.exists(expected1dir)
+        assert not os.path.exists(expected2dir)
+        backy._run_rsync()
+        assert not os.path.exists(expected1dir)
+        assert not os.path.exists(expected2dir)
+
+        config['RSYNC'] = \
+        { 'sources' : f"{source}"
+        , 'additional_args' : '-r'
+        }
+
+        backy = Backup(config)
+        expected1dir  = os.path.join(tmpdir, 'bck1', 'some_dir')
+        expected2dir  = os.path.join(tmpdir, 'bck2', 'some_dir')
+        assert not os.path.exists(expected1dir)
+        assert not os.path.exists(expected2dir)
+        backy._run_rsync()
+        assert os.path.exists(expected1dir)
+        assert os.path.exists(expected2dir)
 
 

branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py

@@ -34 +34 @@
 
 
+def local_rsync_wrapper(args: [str]):
+    if args is None:
+        raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}")
+    command_and_args = ["rsync"] + args
+    return simple_unix_wrapper(command_and_args)
+
+
 def simple_unix_wrapper(command_and_args: [str]):
     """ This should provide the program name and all arguments in a list.

branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py

@@ -164 +164 @@
         assert bool(new_stats.st_mode & stat.S_IWOTH) is True
         assert bool(new_stats.st_mode & stat.S_IXOTH) is True
+
+
+class TestLocalRsyncWrapper(object):
+    """This is essentially a copy of the 'copy' tests above.
+    """
+
+    def test_rsync_raises_validation_error_with_no_args(self):
+        with pytest.raises(ValidationError):
+            utils.local_rsync_wrapper(None)
+
+    def test_rsync_fails_with_invalid_args(self, tmpdir):
+        with pytest.raises(SubprocessError):
+            utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"])
+
+    def test_rsync_fails_if_source_does_not_exist(self, tmpdir):
+        with pytest.raises(SubprocessError):
+            utils.local_rsync_wrapper(["not a real file", tmpdir])
+
+    def test_simple_rsync(self, tmpdir):
+        real_file = make_a_real_file(tmpdir, "real_file.txt")
+        expected_filepath = os.path.join(tmpdir, "copied_file.txt")
+        assert not os.path.isfile(expected_filepath)
+
+        result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath])
+        assert os.path.isfile(expected_filepath)
+        assert result == 0
+
+    def test_simple_rsync_to_directory(self, tmpdir):
+
+        # Setup source file and target dir
+        filename = "real_file.txt"
+        real_file = make_a_real_file(tmpdir, filename)
+
+        destination_dir = os.path.join(tmpdir, "destination")
+        os.makedirs(destination_dir)
+        assert os.path.exists(destination_dir)
+
+        expected_filepath = os.path.join(destination_dir, filename)
+        assert not os.path.exists(expected_filepath)
+
+        # run rsync function and confirm rsync exists (and original)
+        result = utils.local_rsync_wrapper([real_file, destination_dir])
+        assert result == 0
+        assert os.path.isfile(real_file)
+        assert os.path.exists(expected_filepath)
+
+    def test_rsyncing_directories_and_subdirectories(self, tmpdir):
+
+        subdir = os.path.join(tmpdir, "subdir")
+        subsubdir = os.path.join(subdir, "subsubdir")
+        os.makedirs(subsubdir)
+        filename_a = "file_in_subdir.txt"
+        filename_b = "file_in_subsubdir.txt"
+        make_a_real_file(subdir, filename_a)
+        make_a_real_file(subsubdir, filename_b)
+
+        destination_dir = os.path.join(tmpdir, "destination")
+        os.makedirs(destination_dir)
+        expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a)
+        expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b)
+        assert not os.path.exists(expected_filepath_a)
+        assert not os.path.exists(expected_filepath_b)
+
+        # silently passes without -r, just skips over directories
+        assert not os.path.exists(expected_filepath_a)
+        assert not os.path.exists(expected_filepath_b)
+
+        utils.local_rsync_wrapper([subdir, destination_dir])
+        assert not os.path.exists(expected_filepath_a)
+        assert not os.path.exists(expected_filepath_b)
+
+        # requires -r to copy directories (and contents)
+        utils.local_rsync_wrapper(["-r", subdir, destination_dir])
+        assert os.path.exists(expected_filepath_a)
+        assert os.path.exists(expected_filepath_b)