Index: /branches/ipp-259_genericise_backups/tools/backups/backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40931) +++ /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40932) @@ -181,18 +181,33 @@ def _run_rsync(self): - raise Exception("Not implemented") + if self.run_rsync is None or False: + return 0 + + backup_paths = self.default_dict['backup_paths'] + for bp in backup_paths: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.rsync_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.rsync_dict.keys(): + print("hiya") + args = args + self.rsync_dict['additional_args'] + print(args) + sub_utils.local_rsync_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: rsync\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 def _run_mysqldump(self): - # cfg_help.ConfigSchemaLine('user', True, str) - # cfg_help.ConfigSchemaLine('password', True, str) - # cfg_help.ConfigSchemaLine('db_name', True, str) - # cfg_help.ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) - # cfg_help.ConfigSchemaLine('target_filename', True, str) - # cfg_help.ConfigSchemaLine('prefix_date', True, bool) - backup_paths = self.default_dict['backup_paths'] - # Setup the commands: mysqldump_args = [ 'mysqldump' , '-u', self.mysqldump_dict['user'] Index: /branches/ipp-259_genericise_backups/tools/backups/backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40931) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40932) @@ -186,5 +186,5 @@ assert os.path.exists(expected2file) - def copy_fails_if_source_file_does_not_exist(self, tmpdir): + def test_copy_fails_if_source_file_does_not_exist(self, tmpdir): config = make_default_config(tmpdir, ['bck1', 'bck2']) source = os.path.join(tmpdir, "and_specific_file.jpeg") @@ -196,8 +196,8 @@ assert not os.path.exists(source) backy = Backup(config) - with pytest.raises(errs.SubprocessError): + with pytest.raises(errs.ValidationError): backy._run_copy() - def copy_fails_if_destination_does_not_exist(self, tmpdir): + def test_copy_fails_if_destination_does_not_exist(self, tmpdir): config = ConfigParser() @@ -214,5 +214,5 @@ assert not os.path.exists(backup_does_not_exist) backy = Backup(config) - with pytest.raises(errs.SubprocessError): + with pytest.raises(errs.ValidationError): backy._run_copy() @@ -298,11 +298,113 @@ -# class TestRunningRsync(): - -# def test_simply_rsync(self): -# assert True is False - -# def test_rsync_with_additional_args(self): -# assert True is False +class TestRunningRsync(): + + def test_simple_rsync(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = {'sources' : f"{source}"} + + # make file to be copied + Path(source).touch() + + backy = Backup(config) + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_rsync_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source1) + Path(source2).touch() + + backy = Backup(config) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_rsync_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + assert not os.path.exists(source) + backy = Backup(config) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['RSYNC'] = {'sources' : f"{source}"} + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_requires_dash_r_arg_for_copying_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['RSYNC'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + + backy = Backup(config) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + backy = Backup(config) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) Index: /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py (revision 40931) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py (revision 40932) @@ -34,4 +34,11 @@ +def local_rsync_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}") + command_and_args = ["rsync"] + args + return simple_unix_wrapper(command_and_args) + + def simple_unix_wrapper(command_and_args: [str]): """ This should provide the program name and all arguments in a list. Index: /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py (revision 40931) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py (revision 40932) @@ -164,2 +164,77 @@ assert bool(new_stats.st_mode & stat.S_IWOTH) is True assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestLocalRsyncWrapper(object): + """This is essentially a copy of the 'copy' tests above. + """ + + def test_rsync_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.local_rsync_wrapper(None) + + def test_rsync_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"]) + + def test_rsync_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", tmpdir]) + + def test_simple_rsync(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_rsync_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run rsync function and confirm rsync exists (and original) + result = utils.local_rsync_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_rsyncing_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # silently passes without -r, just skips over directories + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.local_rsync_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # requires -r to copy directories (and contents) + utils.local_rsync_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b)