Index: /branches/ipp-259_genericise_backups/tools/backups/backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40951) +++ /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40952) @@ -11,6 +11,4 @@ import backups.utils.subprocess_utils as sub_utils from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig - -DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config') DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', @@ -70,5 +68,4 @@ if schema is None: schema = get_backup_schema() - print(schema) self.load_config(config_file, schema) @@ -94,5 +91,4 @@ def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): parsed_config = ParsedConfig(config, schema) - print(parsed_config) self.default_dict = parsed_config.get_section('DEFAULT') self.hostname_is_valid() Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40951) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40952) @@ -88,7 +88,7 @@ assert jb.default_dict == \ - { 'backup_paths' : [ '/export/ippops4.0/atlassian_backups' - , '/data/ippops3.0/atlassian_backups' - , '/data/ippops5.0/atlassian_backups' + { 'backup_paths' : [ '/export/ippops4.0/backups/confluence_backups' + , '/data/ippops3.0/backups/confluence_backups' + , '/data/ippops5.0/backups/confluence_backups' ] , 'source_machine' : 'ippops4' Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40951) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40952) @@ -88,7 +88,7 @@ assert jb.default_dict == \ - { 'backup_paths' : [ '/export/ippops4.0/atlassian_backups' - , '/data/ippops3.0/atlassian_backups' - , '/data/ippops5.0/atlassian_backups' + { 'backup_paths' : [ '/export/ippops4.0/backups/jira_backups' + , '/data/ippops3.0/backups/jira_backups' + , '/data/ippops5.0/backups/jira_backups' ] , 'source_machine' : 'ippops4' Index: /branches/ipp-259_genericise_backups/tools/backups/svn_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/svn_backup.py (revision 40952) +++ /branches/ipp-259_genericise_backups/tools/backups/svn_backup.py (revision 40952) @@ -0,0 +1,59 @@ +import argparse +import os.path as path + +import backups.utils.config_parse_helper as cfg_help +from backups.backup import Backup +from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config') + +SVN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class SvnBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + self.load_config(config_file, schema) + + +def main(): + sb = SvnBackup() + sb.run_backup() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of svn.') + + parser.add_argument('--config_file', + nargs='?', + help='specify the location of a config that matches the ConfigSchema', + default=EXPECTED_CONFIG_FILE) + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py (revision 40952) +++ /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py (revision 40952) @@ -0,0 +1,105 @@ +import configparser +import os +from pathlib import Path + +import backups.backup as bckup +from backups.svn_backup import SvnBackup +from backups.utils.config_parse_helper import ConfigSchema + +SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config') + + +class TestArguments(object): + + def assert_defaults(self, sb: SvnBackup): + + assert sb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/svn_backups' + , '/data/ippops3.0/backups/svn_backups' + , '/data/ippops5.0/backups/svn_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert sb.copy_dict == \ + { 'sources' : ['/etc/apache2/mods-enabled/dav_svn.conf'] + , 'additional_args' : ['-v'] + } + assert sb.rsync_dict == \ + { 'sources' : ['/svnroot'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + sb = SvnBackup(config_file=SVN_TEST_CONFIG) + self.assert_defaults(sb) + + +class TestSvnRsync(object): + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'svnroot')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + svnroot = os.path.join(tmpdir, "svnroot") + subdir = os.path.join(svnroot, "some_dir") + file1 = os.path.join(svnroot, "file_at_root_level") + file2 = os.path.join(subdir, "file_in_subdir") + os.makedirs(svnroot) + os.makedirs(subdir) + Path(file1).touch() + Path(file2).touch() + + # File that will be removed because of --delete-after + os.makedirs(os.path.join(backup_1_path, "svnroot")) + file_that_already_existed = os.path.join(backup_1_path, "svnroot", "file_to_disappear") + Path(file_that_already_existed).touch() + assert os.path.exists(file_that_already_existed) + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir') + expected1file1 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'file_at_root_level') + expected1file2 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir') + expected2file1 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'file_at_root_level') + expected2file2 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file1) + assert not os.path.exists(expected1file2) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file1) + assert not os.path.exists(expected2file2) + assert os.path.exists(file_that_already_existed) + sb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file1) + assert os.path.exists(expected1file2) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file1) + assert os.path.exists(expected2file2) + # file that does not exist in the source, is removed from the target + assert not os.path.exists(file_that_already_existed) Index: /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config (revision 40951) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config (revision 40952) @@ -1,6 +1,6 @@ [DEFAULT] -backup_paths = /export/ippops4.0/atlassian_backups, - /data/ippops3.0/atlassian_backups, - /data/ippops5.0/atlassian_backups +backup_paths = /export/ippops4.0/backups/confluence_backups, + /data/ippops3.0/backups/confluence_backups, + /data/ippops5.0/backups/confluence_backups source_machine = ippops4 verbose = False Index: /branches/ipp-259_genericise_backups/tools/backups/testing/jira_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/jira_test.config (revision 40951) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/jira_test.config (revision 40952) @@ -1,6 +1,6 @@ [DEFAULT] -backup_paths = /export/ippops4.0/atlassian_backups, - /data/ippops3.0/atlassian_backups, - /data/ippops5.0/atlassian_backups +backup_paths = /export/ippops4.0/backups/jira_backups, + /data/ippops3.0/backups/jira_backups, + /data/ippops5.0/backups/jira_backups source_machine = ippops4 verbose = False Index: /branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config (revision 40952) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config (revision 40952) @@ -0,0 +1,14 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/svn_backups, + /data/ippops3.0/backups/svn_backups, + /data/ippops5.0/backups/svn_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /etc/apache2/mods-enabled/dav_svn.conf +additional_args = -v + +[RSYNC] +sources = /svnroot +additional_args = -a --delete-after Index: anches/ipp-259_genericise_backups/tools/backups/testing/test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/test.config (revision 40951) +++ (revision ) @@ -1,10 +1,0 @@ -[atlassian_backups] -host_backup_path = /export/ippops4.0/atlassian_backups -backup_1_path = /data/ippops3.0/atlassian_backups -backup_2_path = /data/ippops5.0/atlassian_backups -jira_backup_path = /var/atlassian/application-data/jira/export -conf_backup_path = /var/atlassian/application-data/confluence/backups -jira_attachments_path = /var/atlassian/application-data/jira/data -conf_attachments_path = /var/atlassian/application-data/confluence/attachments -mysqldump_password = not_a_real_password -verbose = True