Index: trunk/backups/backup.py =================================================================== --- trunk/backups/backup.py (revision 40973) +++ trunk/backups/backup.py (revision 41067) @@ -33,4 +33,5 @@ [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, SpecialSchemaProcessing.commma_separated) ] ) @@ -83,8 +84,4 @@ self.read_config_and_schema(config, schema) - # self.read_copy_config_section(config, schema) - # self.read_tar_config_section(config, schema) - # self.read_rsync_config_section(config, schema) - # self.read_mysqldump_config_section(config, schema) def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): @@ -206,20 +203,38 @@ self.target_backup_paths_ok(raise_error=True) backup_paths = self.default_dict['backup_paths'] - for bp in backup_paths: - # if not path.exists(bp): - # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") - - for s in self.rsync_dict['sources']: - if not path.exists(s): - raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + + sources = self.rsync_dict['sources'] + for i in range(0, len(sources)): + source = sources[i] + if not path.exists(source): + raise errs.ValidationError(f"ValidationError: target path does not exist: {source}") + + for bp in backup_paths: try: - args = [s, bp] + additional_args = [] if 'additional_args' in self.rsync_dict.keys(): - args = args + self.rsync_dict['additional_args'] + additional_args = self.rsync_dict['additional_args'] + + source_and_target_args = [source, bp] + + # sub-dirs option allows items to be backed up to subdirs within + # the source for better naming + if 'sub-dirs' in self.rsync_dict: + sub_dirs = self.rsync_dict['sub-dirs'] + if len(sub_dirs) != len(sources): + raise errs.ValidationError(f"ValidationError: if using " + + "subdirs the number of subdirs must equal the number of sources") + bp_with_subdir = os.path.join(bp, sub_dirs[i]) + if not os.path.exists(bp_with_subdir): + os.makedirs(bp_with_subdir) + source_and_target_args = [source, bp_with_subdir] + + args = source_and_target_args + additional_args sub_utils.local_rsync_wrapper(args) + except errs.SubprocessError as e: raise errs.SubprocessError( f"Error: rsync\n" - f" from: {s}\n" - f" to: {bp}\n" + f" from: {source_and_target_args[0]}\n" + f" to: {source_and_target_args[1]}\n" ) from e return 0 Index: trunk/backups/backup_test.py =================================================================== --- trunk/backups/backup_test.py (revision 40973) +++ trunk/backups/backup_test.py (revision 41067) @@ -596,4 +596,52 @@ assert os.path.exists(expected4dir) + def test_rsync_with_subdirs_option(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "some_other_dir") + file_in_dir = os.path.join(source1, "and_specific_file.jpeg") + file_in_other_dir = os.path.join(source2, "another_file.jpeg") + os.makedirs(source1) + os.makedirs(source2) + Path(file_in_dir).touch() + Path(file_in_other_dir).touch() + + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-a --progress -v" + , 'sub-dirs' : "subdir_with_better_name1, subdir_with_better_name2" + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_bak1_src1_dir = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir') + expected_bak1_src1_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg') + expected_bak1_src2_dir = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir') + expected_bak1_src2_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg') + expected_bak2_src1_dir = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir') + expected_bak2_src1_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg') + expected_bak2_src2_dir = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir') + expected_bak2_src2_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg') + assert not os.path.exists(expected_bak1_src1_dir) + assert not os.path.exists(expected_bak1_src1_file) + assert not os.path.exists(expected_bak1_src2_dir) + assert not os.path.exists(expected_bak1_src2_file) + assert not os.path.exists(expected_bak2_src1_dir) + assert not os.path.exists(expected_bak2_src1_file) + assert not os.path.exists(expected_bak2_src2_dir) + assert not os.path.exists(expected_bak2_src2_file) + backy._run_rsync() + assert os.path.exists(expected_bak1_src1_dir) + assert os.path.exists(expected_bak1_src1_file) + assert os.path.exists(expected_bak1_src2_dir) + assert os.path.exists(expected_bak1_src2_file) + assert os.path.exists(expected_bak2_src1_dir) + assert os.path.exists(expected_bak2_src1_file) + assert os.path.exists(expected_bak2_src2_dir) + assert os.path.exists(expected_bak2_src2_file) @pytest.mark.skip(reason="Requires a fake database to be setup") Index: trunk/backups/confluence_backup.py =================================================================== --- trunk/backups/confluence_backup.py (revision 40973) +++ trunk/backups/confluence_backup.py (revision 41067) @@ -32,4 +32,12 @@ , ConfigSchemaLine('target_filename', True, str) , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, + cfg_help.SpecialSchemaProcessing.commma_separated) ]) , ConfigSchemaSection('MYSQLDUMP', Index: trunk/backups/confluence_backup_test.py =================================================================== --- trunk/backups/confluence_backup_test.py (revision 40973) +++ trunk/backups/confluence_backup_test.py (revision 41067) @@ -34,4 +34,10 @@ , 'target_filename' : 'confluence.tar.gz' , 'prefix_date' : 'True' + } + config['RSYNC'] = \ + { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')}," + f"{os.path.join(tmpdir, 'rsync_source_dir2')}") + , 'additional_args' : '-a --delete-after --exclude exclude_me_if_you_like' + , 'sub-dirs' : 'conf_subdir1, conf_subdir2' } config['MYSQLDUMP'] = \ @@ -82,4 +88,19 @@ assert os.path.exists(confluence_attachment_2_filepath) + # Create a few folders to be rsync'd + rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1") + rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2") + file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync") + file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync") + exluded_file = os.path.join(rsync_source_dir1, "exclude_me_if_you_like") + os.makedirs(rsync_source_dir1) + os.makedirs(rsync_source_dir2) + Path(file_in_rsync_dir1).touch() + Path(file_in_rsync_dir2).touch() + Path(exluded_file).touch() + assert os.path.exists(file_in_rsync_dir1) + assert os.path.exists(file_in_rsync_dir2) + assert os.path.exists(exluded_file) + class TestArguments(object): @@ -287,2 +308,59 @@ assert os.path.exists(expected_confluence_bak1_tar) assert os.path.exists(expected_confluence_bak2_tar) + + +class TestRsyncOfFiles(object): + + def test_confluence_rsync_with_subdirs(self, tmpdir): + """This should be similar to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + + cb = ConfluenceBackup(config_file=config) + + # Expected resulting rsync directories + expected_confluence_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "conf_subdir2", "rsync_source_dir2") + expected_confluence_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path", "conf_subdir2", "rsync_source_dir2") + expected_confluence_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path", "conf_subdir2", "rsync_source_dir2") + assert not os.path.exists(expected_confluence_host_rsync1) + assert not os.path.exists(expected_confluence_bak1_rsync1) + assert not os.path.exists(expected_confluence_bak2_rsync1) + assert not os.path.exists(expected_confluence_host_rsync2) + assert not os.path.exists(expected_confluence_bak1_rsync2) + assert not os.path.exists(expected_confluence_bak2_rsync2) + # Expected resulting rsync files + expected_file_host_dir1 = os.path.join( expected_confluence_host_rsync1, "some_file_to_rsync") + expected_file_bak1_dir1 = os.path.join( expected_confluence_bak1_rsync1, "some_file_to_rsync") + expected_file_bak2_dir1 = os.path.join( expected_confluence_bak2_rsync1, "some_file_to_rsync") + expected_file_host_dir2 = os.path.join( expected_confluence_host_rsync2, "some_other_file_to_rsync") + expected_file_bak1_dir2 = os.path.join( expected_confluence_bak1_rsync2, "some_other_file_to_rsync") + expected_file_bak2_dir2 = os.path.join( expected_confluence_bak2_rsync2, "some_other_file_to_rsync") + assert not os.path.exists(expected_file_host_dir1) + assert not os.path.exists(expected_file_bak1_dir1) + assert not os.path.exists(expected_file_bak2_dir1) + assert not os.path.exists(expected_file_host_dir2) + assert not os.path.exists(expected_file_bak1_dir2) + assert not os.path.exists(expected_file_bak2_dir2) + an_excluded_file = os.path.join(expected_confluence_host_rsync1, "exclude_me_if_you_like") + assert not os.path.exists(an_excluded_file) + + rsync_result = cb._run_rsync() + assert rsync_result == 0 + + # All files Rsync'd + assert os.path.exists(expected_confluence_host_rsync1) + assert os.path.exists(expected_confluence_bak1_rsync1) + assert os.path.exists(expected_confluence_bak2_rsync1) + assert os.path.exists(expected_confluence_host_rsync2) + assert os.path.exists(expected_confluence_bak1_rsync2) + assert os.path.exists(expected_confluence_bak2_rsync2) + assert os.path.exists(expected_file_host_dir1) + assert os.path.exists(expected_file_bak1_dir1) + assert os.path.exists(expected_file_bak2_dir1) + assert os.path.exists(expected_file_host_dir2) + assert os.path.exists(expected_file_bak1_dir2) + assert os.path.exists(expected_file_bak2_dir2) + assert not os.path.exists(an_excluded_file) Index: trunk/backups/jira_backup.py =================================================================== --- trunk/backups/jira_backup.py (revision 40973) +++ trunk/backups/jira_backup.py (revision 41067) @@ -32,4 +32,12 @@ , ConfigSchemaLine('target_filename', True, str) , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, + cfg_help.SpecialSchemaProcessing.commma_separated) ]) , ConfigSchemaSection('MYSQLDUMP', Index: trunk/backups/jira_backup_test.py =================================================================== --- trunk/backups/jira_backup_test.py (revision 40973) +++ trunk/backups/jira_backup_test.py (revision 41067) @@ -33,4 +33,10 @@ , 'target_filename' : 'jira.tar.gz' , 'prefix_date' : 'True' + } + config['RSYNC'] = \ + { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')}," + f"{os.path.join(tmpdir, 'rsync_source_dir2')}") + , 'additional_args' : '-a --delete-after' + , 'sub-dirs' : 'jira_subdir1, jira_subdir2' } config['MYSQLDUMP'] = \ @@ -81,4 +87,16 @@ assert os.path.exists(jira_attachment_2_filepath) + # Create a few folders to be rsync'd + rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1") + rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2") + file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync") + file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync") + os.makedirs(rsync_source_dir1) + os.makedirs(rsync_source_dir2) + Path(file_in_rsync_dir1).touch() + Path(file_in_rsync_dir2).touch() + assert os.path.exists(file_in_rsync_dir1) + assert os.path.exists(file_in_rsync_dir2) + class TestArguments(object): @@ -288,2 +306,56 @@ assert os.path.exists(expected_jira_bak1_tar) assert os.path.exists(expected_jira_bak2_tar) + + +class TestRsyncOfFiles(object): + + def test_jira_rsync_with_subdirs(self, tmpdir): + """This should be similar to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + + jb = JiraBackup(config_file=config) + + # Expected resulting rsync directories + expected_jira_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "jira_subdir2", "rsync_source_dir2") + expected_jira_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path", "jira_subdir2", "rsync_source_dir2") + expected_jira_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path", "jira_subdir2", "rsync_source_dir2") + assert not os.path.exists(expected_jira_host_rsync1) + assert not os.path.exists(expected_jira_bak1_rsync1) + assert not os.path.exists(expected_jira_bak2_rsync1) + assert not os.path.exists(expected_jira_host_rsync2) + assert not os.path.exists(expected_jira_bak1_rsync2) + assert not os.path.exists(expected_jira_bak2_rsync2) + # Expected resulting rsync files + expected_file_host_dir1 = os.path.join( expected_jira_host_rsync1, "some_file_to_rsync") + expected_file_bak1_dir1 = os.path.join( expected_jira_bak1_rsync1, "some_file_to_rsync") + expected_file_bak2_dir1 = os.path.join( expected_jira_bak2_rsync1, "some_file_to_rsync") + expected_file_host_dir2 = os.path.join( expected_jira_host_rsync2, "some_other_file_to_rsync") + expected_file_bak1_dir2 = os.path.join( expected_jira_bak1_rsync2, "some_other_file_to_rsync") + expected_file_bak2_dir2 = os.path.join( expected_jira_bak2_rsync2, "some_other_file_to_rsync") + assert not os.path.exists(expected_file_host_dir1) + assert not os.path.exists(expected_file_bak1_dir1) + assert not os.path.exists(expected_file_bak2_dir1) + assert not os.path.exists(expected_file_host_dir2) + assert not os.path.exists(expected_file_bak1_dir2) + assert not os.path.exists(expected_file_bak2_dir2) + + rsync_result = jb._run_rsync() + assert rsync_result == 0 + + # All files Rsync'd + assert os.path.exists(expected_jira_host_rsync1) + assert os.path.exists(expected_jira_bak1_rsync1) + assert os.path.exists(expected_jira_bak2_rsync1) + assert os.path.exists(expected_jira_host_rsync2) + assert os.path.exists(expected_jira_bak1_rsync2) + assert os.path.exists(expected_jira_bak2_rsync2) + assert os.path.exists(expected_file_host_dir1) + assert os.path.exists(expected_file_bak1_dir1) + assert os.path.exists(expected_file_bak2_dir1) + assert os.path.exists(expected_file_host_dir2) + assert os.path.exists(expected_file_bak1_dir2) + assert os.path.exists(expected_file_bak2_dir2) Index: trunk/backups/testing/confluence_test.config =================================================================== --- trunk/backups/testing/confluence_test.config (revision 40973) +++ trunk/backups/testing/confluence_test.config (revision 41067) @@ -17,4 +17,9 @@ prefix_date = True +[RSYNC] +sources = /opt/atlassian, /var/atlassian +additional_args = -a --delete-after +sub-dirs = conf-install, conf-home + [MYSQLDUMP] user = dumper Index: trunk/backups/testing/jira_test.config =================================================================== --- trunk/backups/testing/jira_test.config (revision 40973) +++ trunk/backups/testing/jira_test.config (revision 41067) @@ -16,4 +16,9 @@ prefix_date = True +[RSYNC] +sources = /opt/atlassian, /var/atlassian +additional_args = -a --delete-after +sub-dirs = jira-install, jira-home + [MYSQLDUMP] user = dumper