Index: anches/ipp-259_genericise_backups/tools/backups/atlassian_backups.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/atlassian_backups.config (revision 40957) +++ (revision ) @@ -1,10 +1,0 @@ -[atlassian_backups] -host_backup_path = /export/ippops4.0/atlassian_backups -backup_1_path = /data/ippops3.0/atlassian_backups -backup_2_path = /data/ippops5.0/atlassian_backups -jira_backup_path = /var/atlassian/application-data/jira/export -conf_backup_path = /var/atlassian/application-data/confluence/backups -jira_attachments_path = /var/atlassian/application-data/jira/data -conf_attachments_path = /var/atlassian/application-data/confluence/attachments -mysqldump_password = not_a_real_password -verbose = False Index: /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py (revision 40958) @@ -0,0 +1,53 @@ +import argparse +import os.path as path + +import backups.utils.config_parse_helper as cfg_help +from backups.backup import Backup +from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config') + +IPPOPS_HOMEDIR_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class IPPOpsHomedirBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + self.load_config(config_file, schema) + + +def main(): + ohdb = IPPOpsHomedirBackup() + ohdb.run_backup() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of the ippops homedirs.') + + parser.add_argument('--config_file', + nargs='?', + help='specify the location of a config that matches the ConfigSchema', + default=EXPECTED_CONFIG_FILE) + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py (revision 40958) @@ -0,0 +1,102 @@ +import configparser +import os +from pathlib import Path + +import backups.backup as bckup +from backups.ippops_homedir_backup import IPPOpsHomedirBackup +from backups.utils.config_parse_helper import ConfigSchema + +IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/ippops_homedir_test.config') + + +class TestArguments(object): + + def assert_defaults(self, ihb: IPPOpsHomedirBackup): + + assert ihb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/homedir_backups' + , '/data/ippops4.0/backups/homedir_backups' + , '/data/ippops5.0/backups/homedir_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert ihb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/home/panstarrs'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG) + self.assert_defaults(ihb) + + +class TestIppopsHomedirRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'home')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "home") + subdir1 = os.path.join(homedir, "some_user") + subdir2 = os.path.join(homedir, "some_other_user") + file1 = os.path.join(subdir1, "file_in_subdir1") + file2 = os.path.join(subdir2, "file_in_subdir2") + os.makedirs(homedir) + os.makedirs(subdir1) + os.makedirs(subdir2) + Path(file1).touch() + Path(file2).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + ihb = IPPOpsHomedirBackup(config, schema) + + expected_sub1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user') + expected_sub1_file1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user') + expected_sub2_file2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user/file_in_subdir2') + expected_sub1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user') + expected_sub1_file1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user') + expected_sub2_file2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user/file_in_subdir2') + assert not os.path.exists(expected_sub1_backup1) + assert not os.path.exists(expected_sub1_file1_backup1) + assert not os.path.exists(expected_sub2_backup1) + assert not os.path.exists(expected_sub2_file2_backup1) + assert not os.path.exists(expected_sub1_backup2) + assert not os.path.exists(expected_sub1_file1_backup2) + assert not os.path.exists(expected_sub2_backup2) + assert not os.path.exists(expected_sub2_file2_backup2) + ihb._run_rsync() + assert os.path.exists(expected_sub1_backup1) + assert os.path.exists(expected_sub1_file1_backup1) + assert os.path.exists(expected_sub2_backup1) + assert os.path.exists(expected_sub2_file2_backup1) + assert os.path.exists(expected_sub1_backup2) + assert os.path.exists(expected_sub1_file1_backup2) + assert os.path.exists(expected_sub2_backup2) + assert os.path.exists(expected_sub2_file2_backup2) Index: /branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py (revision 40958) @@ -0,0 +1,53 @@ +import argparse +import os.path as path + +import backups.utils.config_parse_helper as cfg_help +from backups.backup import Backup +from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config') + +MAILMAN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class MailmanBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + self.load_config(config_file, schema) + + +def main(): + mb = MailmanBackup() + mb.run_backup() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of mailman.') + + parser.add_argument('--config_file', + nargs='?', + help='specify the location of a config that matches the ConfigSchema', + default=EXPECTED_CONFIG_FILE) + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py (revision 40958) @@ -0,0 +1,86 @@ +import configparser +import os +from pathlib import Path + +import backups.backup as bckup +from backups.mailman_backup import MailmanBackup +from backups.utils.config_parse_helper import ConfigSchema + +MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/mailman_test.config') + + +class TestArguments(object): + + def assert_defaults(self, mb: MailmanBackup): + + assert mb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/mailman_backups' + , '/data/ippops4.0/backups/mailman_backups' + , '/data/ippops5.0/backups/mailman_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert mb.rsync_dict == \ + { 'sources' : ['/var/lib/mailman'] + , 'additional_args' : ['-a', '--delete-after', '--copy-links'] + } + + def test_load_default_config_file(self): + mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG) + self.assert_defaults(mb) + + +class TestMailmanRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'mailman')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + mailman_root = os.path.join(tmpdir, "mailman") + subdir = os.path.join(mailman_root, "some_dir") + file = os.path.join(subdir, "file_in_subdir") + os.makedirs(mailman_root) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + mb = MailmanBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir') + expected1file = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir') + expected2file = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + mb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) Index: /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py (revision 40957) +++ /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py (revision 40958) @@ -36,5 +36,53 @@ +class TestSvnCopy(object): + """Simple inherit of base class Backup""" + + # Setup config + def test_copy(self, tmpdir): + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'apache2/mods-enabled/dav_svn.conf')}" + , 'additional_args' : "-v" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + apache_folder = os.path.join(tmpdir, "apache2") + subdir = os.path.join(apache_folder, "mods-enabled") + file = os.path.join(subdir, "dav_svn.conf") + os.makedirs(apache_folder) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'dav_svn.conf') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'dav_svn.conf') + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_file_bck2) + sb._run_copy() + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_file_bck2) + + class TestSvnRsync(object): + """Simple inherit of base class Backup""" def test_rsync_with_additional_args(self, tmpdir): Index: /branches/ipp-259_genericise_backups/tools/backups/testing/ippops_homedir_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/ippops_homedir_test.config (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/ippops_homedir_test.config (revision 40958) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/homedir_backups, + /data/ippops4.0/backups/homedir_backups, + /data/ippops5.0/backups/homedir_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/home/panstarrs +additional_args = -a --delete-after Index: /branches/ipp-259_genericise_backups/tools/backups/testing/mailman_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/mailman_test.config (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/mailman_test.config (revision 40958) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/mailman_backups, + /data/ippops4.0/backups/mailman_backups, + /data/ippops5.0/backups/mailman_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /var/lib/mailman +additional_args = -a --delete-after --copy-links Index: /branches/ipp-259_genericise_backups/tools/backups/testing/website_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/website_test.config (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/website_test.config (revision 40958) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/website_backups, + /data/ippops4.0/backups/website_backups, + /data/ippops5.0/backups/website_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/www +additional_args = -a --delete-after Index: /branches/ipp-259_genericise_backups/tools/backups/website_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/website_backup.py (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/website_backup.py (revision 40958) @@ -0,0 +1,53 @@ +import argparse +import os.path as path + +import backups.utils.config_parse_helper as cfg_help +from backups.backup import Backup +from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config') + +WEBSITE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class WebsiteBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + self.load_config(config_file, schema) + + +def main(): + wb = WebsiteBackup() + wb.run_backup() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of the panstarrs website.') + + parser.add_argument('--config_file', + nargs='?', + help='specify the location of a config that matches the ConfigSchema', + default=EXPECTED_CONFIG_FILE) + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py (revision 40958) +++ /branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py (revision 40958) @@ -0,0 +1,85 @@ +import configparser +import os +from pathlib import Path + +import backups.backup as bckup +from backups.website_backup import WebsiteBackup +from backups.utils.config_parse_helper import ConfigSchema + +WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/website_test.config') + + +class TestArguments(object): + + def assert_defaults(self, wb: WebsiteBackup): + + assert wb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/website_backups' + , '/data/ippops4.0/backups/website_backups' + , '/data/ippops5.0/backups/website_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert wb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/www'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG) + self.assert_defaults(wb) + + +class TestWebsiteRsync(object): + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'www')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "www") + subdir = os.path.join(homedir, "subdir") + file = os.path.join(subdir, "website_file") + os.makedirs(homedir) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + wb = WebsiteBackup(config, schema) + + expected_subdir_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir') + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir/website_file') + expected_subdir_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir/website_file') + assert not os.path.exists(expected_subdir_bck1) + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_subdir_bck2) + assert not os.path.exists(expected_file_bck2) + wb._run_rsync() + assert os.path.exists(expected_subdir_bck1) + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_subdir_bck2) + assert os.path.exists(expected_file_bck2)