Index: /branches/ipp-259_genericise_backups/tools/backups/backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/backup.py (revision 40962) @@ -7,11 +7,10 @@ from configparser import ConfigParser -import backups.utils.config_parse_helper as cfg_help -import backups.utils.errors as errs -import backups.utils.subprocess_utils as sub_utils -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig, SpecialSchemaProcessing DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', - [ ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) + [ ConfigSchemaLine('backup_paths', True, list, SpecialSchemaProcessing.commma_separated) , ConfigSchemaLine('source_machine', True, str) , ConfigSchemaLine('verbose', False, bool) @@ -20,11 +19,11 @@ ) COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY', - [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) ] ) TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR', - [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) , ConfigSchemaLine('target_filename', True, str) , ConfigSchemaLine('prefix_date', True, bool) @@ -32,6 +31,6 @@ ) RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC', - [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) ] ) @@ -41,5 +40,5 @@ , ConfigSchemaLine('password', True, str) , ConfigSchemaLine('db_name', True, str) - , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) , ConfigSchemaLine('target_filename', True, str) , ConfigSchemaLine('prefix_date', True, bool) @@ -148,4 +147,5 @@ if not os.path.exists(backup_path): os.makedirs(backup_path) + sub_utils.chmod_wrapper(['777', backup_path]) def _run_copy(self): Index: /branches/ipp-259_genericise_backups/tools/backups/backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/backup_test.py (revision 40962) @@ -1,12 +1,13 @@ import os import pytest +import stat import socket from configparser import ConfigParser from pathlib import Path -import backups.backup as bckup -import backups.utils.errors as errs -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +import backup as bckup +import utils.errors as errs +from backup import Backup +from utils.config_parse_helper import ConfigSchema TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config') @@ -207,4 +208,16 @@ assert os.path.exists(bck2_path) + # Check permissions are 777 too + new_stats = os.stat(bck2_path) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + class TestRunningCopy(): Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py (revision 40962) @@ -1,12 +1,11 @@ -import argparse import datetime import glob import os.path as path -import backups.utils.config_parse_helper as cfg_help -import backups.utils.errors as errs -import backups.utils.subprocess_utils as sub_utils -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config') @@ -71,5 +70,5 @@ source = self.copy_dict['sources'][0] - confluence_wildcard_search = f"{source}/{ConfluenceBackup.get_confluence_backup_date_format()}*" + confluence_wildcard_search = f"{source}/*{ConfluenceBackup.get_confluence_backup_date_format()}*" confluence_xml_filepaths = glob.glob(confluence_wildcard_search) if len(confluence_xml_filepaths) == 0: @@ -92,20 +91,7 @@ -def main(): - cb = ConfluenceBackup() - cb.run_backup() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of confluence.') - - parser.add_argument('--config_file', - nargs='?', - help='specify the location of a config that matches the ConfigSchema', - default=EXPECTED_CONFIG_FILE) - - return parser.parse_args() +def main(config=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + cb = ConfluenceBackup(config, schema) + cb.run_backup_processes() Index: /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py (revision 40962) @@ -6,10 +6,10 @@ from pathlib import Path -import backups.backup as bckup -import backups.confluence_backup as jbck_mod -import backups.utils.errors as errs -from backups.confluence_backup import ConfluenceBackup -from backups.utils.errors import ValidationError -from backups.utils.config_parse_helper import ConfigSchema +import backup as bckup +import confluence_backup as cbck_mod + +from confluence_backup import ConfluenceBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') @@ -67,5 +67,5 @@ # Create backup files that will be copied - confluence_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_confluence_test_file.zip" + confluence_backup_name = f"confluence-xml-backup-{ConfluenceBackup.get_confluence_backup_date_format()}.zip" confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name) Path(confluence_backup_filepath).touch() @@ -96,9 +96,9 @@ } assert jb.copy_dict == \ - { 'sources' : ['/var/atlassian/application-data/confluence/export'] + { 'sources' : ['/var/atlassian/application-data/confluence/backups'] , 'additional_args' : ['-v'] } assert jb.tar_dict == \ - { 'sources' : ['/var/atlassian/application-data/confluence/data'] + { 'sources' : ['/var/atlassian/application-data/confluence/attachments'] , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] , 'target_filename' : 'confluence.tar.gz' @@ -118,8 +118,8 @@ def test_default_config_fails_to_load_if_the_file_does_not_exist(self): - default_config = jbck_mod.EXPECTED_CONFIG_FILE + default_config = cbck_mod.EXPECTED_CONFIG_FILE assert not os.path.exists(default_config) - with pytest.raises(errs.ValidationError) as e: + with pytest.raises(ValidationError) as e: ConfluenceBackup() assert "ValidationError: Config filepath does not exist" in str(e) @@ -174,14 +174,52 @@ def test_nonexistant_config_file_raises_error(self): - with pytest.raises(errs.ValidationError) as e: + with pytest.raises(ValidationError) as e: ConfluenceBackup("not a real file path") assert "ValidationError: Config filepath does not exist" in str(e) def test_main_loads_default_and_fails_due_to_missing_paths(self): - with pytest.raises(errs.ValidationError) as e: - jbck_mod.main() + with pytest.raises(ValidationError) as e: + cbck_mod.main() assert "ValidationError: Config filepath does not exist" in str(e) +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + conf_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_conf_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'conf.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, conf_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + cbck_mod.main(config, schema) + assert True is True + + class TestCopyingBackups(object): @@ -189,18 +227,18 @@ def test_get_confluence_backup_date_format(self): - jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) today = datetime.date.today() expected_format = today.strftime("%Y_%m_%d") - assert jb.get_confluence_backup_date_format() == expected_format + assert cb.get_confluence_backup_date_format() == expected_format another_date = datetime.date(1988, 11, 23) expected_format = another_date.strftime("%Y_%m_%d") - assert jb.get_confluence_backup_date_format(another_date) == expected_format + assert cb.get_confluence_backup_date_format(another_date) == expected_format def test_copying_fails_if_paths_not_present(self): - jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) - # return_value = jb._copy_confluence_backup() - with pytest.raises(ValidationError) as e: - jb._run_copy() + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + # return_value = cb._copy_confluence_backup() + with pytest.raises(ValidationError) as e: + cb._run_copy() assert "ValidationError: target path does not exist:" in str(e) @@ -208,15 +246,15 @@ create_items_for_full_confluence_backup_test(tmpdir) config = make_config(tmpdir) - jb = ConfluenceBackup(config_file=config) + cb = ConfluenceBackup(config_file=config) today = ConfluenceBackup.get_confluence_backup_date_format() - expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_confluence_test_file.zip" ) - expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_confluence_test_file.zip" ) - expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_confluence_test_file.zip" ) + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"confluence-xml-backup-{today}.zip" ) assert not os.path.exists(expected_host_backup_copy) assert not os.path.exists(expected_backup_1_copy) assert not os.path.exists(expected_backup_2_copy) - return_value = jb._run_copy() + return_value = cb._run_copy() assert return_value == 0 Index: /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py (revision 40962) @@ -1,8 +1,7 @@ -import argparse import os.path as path -import backups.utils.config_parse_helper as cfg_help -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config') @@ -32,20 +31,7 @@ -def main(): - ohdb = IPPOpsHomedirBackup() - ohdb.run_backup() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of the ippops homedirs.') - - parser.add_argument('--config_file', - nargs='?', - help='specify the location of a config that matches the ConfigSchema', - default=EXPECTED_CONFIG_FILE) - - return parser.parse_args() +def main(config=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + ohdb = IPPOpsHomedirBackup(config, schema) + ohdb.run_backup_processes() Index: /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py (revision 40962) @@ -3,7 +3,8 @@ from pathlib import Path -import backups.backup as bckup -from backups.ippops_homedir_backup import IPPOpsHomedirBackup -from backups.utils.config_parse_helper import ConfigSchema +import backup as bckup +import ippops_homedir_backup as ihbck_mod +from ippops_homedir_backup import IPPOpsHomedirBackup +from utils.config_parse_helper import ConfigSchema IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__), @@ -31,4 +32,34 @@ ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG) self.assert_defaults(ihb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + ihbck_mod.main(config, schema) + assert True is True Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup.py (revision 40962) @@ -1,12 +1,11 @@ -import argparse import datetime import glob import os.path as path -import backups.utils.config_parse_helper as cfg_help -import backups.utils.errors as errs -import backups.utils.subprocess_utils as sub_utils -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config') @@ -92,20 +91,7 @@ -def main(): - jb = JiraBackup() - jb.run_backup() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of jira.') - - parser.add_argument('--config_file', - nargs='?', - help='specify the location of a config that matches the ConfigSchema', - default=EXPECTED_CONFIG_FILE) - - return parser.parse_args() +def main(config=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + jb = JiraBackup(config, schema) + jb.run_backup_processes() Index: /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py (revision 40962) @@ -7,9 +7,8 @@ import backups.backup as bckup -import backups.jira_backup as jbck_mod -import backups.utils.errors as errs -from backups.jira_backup import JiraBackup -from backups.utils.errors import ValidationError -from backups.utils.config_parse_helper import ConfigSchema +import jira_backup as jbck_mod +from jira_backup import JiraBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') @@ -121,5 +120,5 @@ assert not os.path.exists(default_config) - with pytest.raises(errs.ValidationError) as e: + with pytest.raises(ValidationError) as e: JiraBackup() assert "ValidationError: Config filepath does not exist" in str(e) @@ -174,12 +173,51 @@ def test_nonexistant_config_file_raises_error(self): - with pytest.raises(errs.ValidationError) as e: + with pytest.raises(ValidationError) as e: JiraBackup("not a real file path") assert "ValidationError: Config filepath does not exist" in str(e) def test_main_loads_default_and_fails_due_to_missing_paths(self): - with pytest.raises(errs.ValidationError) as e: + with pytest.raises(ValidationError) as e: jbck_mod.main() assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, jira_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jbck_mod.main(config, schema) + assert True is True Index: /branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py (revision 40962) @@ -1,8 +1,7 @@ -import argparse import os.path as path -import backups.utils.config_parse_helper as cfg_help -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config') @@ -32,20 +31,7 @@ -def main(): - mb = MailmanBackup() - mb.run_backup() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of mailman.') - - parser.add_argument('--config_file', - nargs='?', - help='specify the location of a config that matches the ConfigSchema', - default=EXPECTED_CONFIG_FILE) - - return parser.parse_args() +def main(config=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + mb = MailmanBackup(config, schema) + mb.run_backup_processes() Index: /branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py (revision 40962) @@ -3,7 +3,10 @@ from pathlib import Path -import backups.backup as bckup -from backups.mailman_backup import MailmanBackup -from backups.utils.config_parse_helper import ConfigSchema +import mailman_backup as mailbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from mailman_backup import MailmanBackup +from utils.config_parse_helper import ConfigSchema + MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), @@ -31,4 +34,34 @@ mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG) self.assert_defaults(mb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + mailbck_mod.main(config, schema) + assert True is True @@ -66,6 +99,6 @@ # Setup Schema schema = ConfigSchema(None) - schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) - schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) # check that the rsync is successful (with options) Index: /branches/ipp-259_genericise_backups/tools/backups/svn_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/svn_backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/svn_backup.py (revision 40962) @@ -1,8 +1,7 @@ -import argparse import os.path as path -import backups.utils.config_parse_helper as cfg_help -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +from utils.config_parse_helper import SpecialSchemaProcessing +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config') @@ -11,5 +10,5 @@ [ ConfigSchemaSection('DEFAULT', [ ConfigSchemaLine('backup_paths', True, list, - cfg_help.SpecialSchemaProcessing.commma_separated) + SpecialSchemaProcessing.commma_separated) , ConfigSchemaLine('source_machine', True, str) , ConfigSchemaLine('verbose', False, bool) @@ -18,13 +17,13 @@ , ConfigSchemaSection('COPY', [ ConfigSchemaLine('sources', True, list, - cfg_help.SpecialSchemaProcessing.commma_separated) + SpecialSchemaProcessing.commma_separated) , ConfigSchemaLine('additional_args', False, list, - cfg_help.SpecialSchemaProcessing.space_separated) + SpecialSchemaProcessing.space_separated) ]) , ConfigSchemaSection('RSYNC', [ ConfigSchemaLine('sources', True, list, - cfg_help.SpecialSchemaProcessing.commma_separated) + SpecialSchemaProcessing.commma_separated) , ConfigSchemaLine('additional_args', False, list, - cfg_help.SpecialSchemaProcessing.space_separated) + SpecialSchemaProcessing.space_separated) ]) ] @@ -38,20 +37,7 @@ -def main(): - sb = SvnBackup() - sb.run_backup() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of svn.') - - parser.add_argument('--config_file', - nargs='?', - help='specify the location of a config that matches the ConfigSchema', - default=EXPECTED_CONFIG_FILE) - - return parser.parse_args() +def main(config=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + sb = SvnBackup(config, schema) + sb.run_backup_processes() Index: /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py (revision 40962) @@ -3,7 +3,9 @@ from pathlib import Path -import backups.backup as bckup -from backups.svn_backup import SvnBackup -from backups.utils.config_parse_helper import ConfigSchema +import svn_backup as svnbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, COPY_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from svn_backup import SvnBackup +from utils.config_parse_helper import ConfigSchema SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config') @@ -27,5 +29,5 @@ } assert sb.rsync_dict == \ - { 'sources' : ['/svnroot'] + { 'sources' : ['/export/ippops4.0/svnroot'] , 'additional_args' : ['-a', '--delete-after'] } @@ -68,6 +70,6 @@ # Setup Schema schema = ConfigSchema(None) - schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) - schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(COPY_SCHEMA_SECTION) # check that the rsync is successful (with options) @@ -81,4 +83,42 @@ assert os.path.exists(expected_file_bck1) assert os.path.exists(expected_file_bck2) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + svn_conf_file = "dav_svn.conf" + config['COPY'] = \ + { 'sources' : f'{copy_source}/{svn_conf_file}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, svn_conf_file)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + svnbck_mod.main(config, schema) + assert True is True @@ -124,6 +164,6 @@ # Setup Schema schema = ConfigSchema(None) - schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) - schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) # check that the rsync is successful (with options) Index: /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config (revision 40962) @@ -7,9 +7,10 @@ [COPY] -sources = /var/atlassian/application-data/confluence/export +sources = /var/atlassian/application-data/confluence/backups additional_args = -v +conf_prefix = confluence-xml-backup- [TAR] -sources = /var/atlassian/application-data/confluence/data +sources = /var/atlassian/application-data/confluence/attachments additional_args = --verbose -z -p -cf target_filename = confluence.tar.gz Index: /branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config (revision 40962) @@ -11,4 +11,4 @@ [RSYNC] -sources = /svnroot +sources = /export/ippops4.0/svnroot additional_args = -a --delete-after Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py (revision 40962) @@ -1,5 +1,6 @@ from enum import Enum from configparser import ConfigParser -from backups.utils.errors import ConfigError, ValidationError + +from utils.errors import ConfigError, ValidationError @@ -139,4 +140,5 @@ parsed_items[csl.key] = splitup else: + raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}") else: @@ -150,17 +152,9 @@ -# def parse_config(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: -# return _parse_config_section(config, schema) - def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: parsed_items = {} - print(config) - for c in config: - print(c) for css_name in schema.sections: - print(css_name) css = schema.get_schema_section(css_name) - print(css) # if config.has_section(css_name): Index: /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py (revision 40962) @@ -1,6 +1,6 @@ import pytest -import backups.utils.config_parse_helper as helper -from backups.utils.errors import ConfigError, ValidationError +import utils.config_parse_helper as helper +from utils.errors import ConfigError, ValidationError Index: /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py (revision 40962) @@ -2,9 +2,8 @@ import os.path as path import shutil - from distutils.dir_util import copy_tree -import backups.utils.subprocess_utils as sub_utils -from backups.utils.errors import ValidationError +from .subprocess_utils import simple_unix_wrapper +from .errors import ValidationError @@ -118,5 +117,5 @@ command_and_args = ["mv", fp, destination_dirs[i]] # shutil.move(fp, destination_dirs[i]) - sub_utils.simple_unix_wrapper(command_and_args) + simple_unix_wrapper(command_and_args) except IOError as e: print("IOError copying: ", e.filename, e.filename2) Index: /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils_test.py (revision 40962) @@ -1,7 +1,7 @@ -import backups.utils.generic_utils as utils import os import pytest -from backups.utils.errors import ValidationError +from .generic_utils import * +from .errors import ValidationError from pathlib import Path @@ -19,18 +19,18 @@ def test_copy_fails_with_none_args(self, tmpdir): with pytest.raises(ValidationError): - utils.copy_to_dir(None, tmpdir) + copy_to_dir(None, tmpdir) with pytest.raises(ValidationError): real_file = make_a_real_file(tmpdir, "file.txt") - utils.copy_to_dir(real_file, None) + copy_to_dir(real_file, None) def test_copy_fails_if_source_does_not_exist(self, tmpdir): with pytest.raises(ValidationError): - utils.copy_to_dir("not a real file", tmpdir) + copy_to_dir("not a real file", tmpdir) def test_copy_fails_if_destination_is_not_real(self, tmpdir): with pytest.raises(ValidationError): real_file = make_a_real_file(tmpdir, "real_file.txt") - utils.copy_to_dir(real_file, "not a real directory") + copy_to_dir(real_file, "not a real directory") def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir): @@ -42,5 +42,5 @@ with pytest.raises(ValidationError): - utils.copy_to_dir(real_file, not_a_directory) + copy_to_dir(real_file, not_a_directory) def test_copy_to_dir(self, tmpdir): @@ -60,5 +60,5 @@ # run copy function and confirm copy exists (and original) - result = utils.copy_to_dir(file_to_copy, destination_dir) + result = copy_to_dir(file_to_copy, destination_dir) assert result == 0 assert os.path.exists(file_to_copy) @@ -98,5 +98,5 @@ # Provide only the original tmpdir to the copying function - utils.copy_to_dir(subdir, destination_dir, True) + copy_to_dir(subdir, destination_dir, True) # assert that the subdirs and files all exist @@ -110,11 +110,11 @@ with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(None, None) - - with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(tmpdir, None) - - with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(None, tmpdir) + copy_multiple_items_to_multiple_dirs(None, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpdir, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, tmpdir) def test_copy_requires_args_to_be_in_list_format(self, tmpdir): @@ -122,9 +122,9 @@ not_a_list = "not a list" with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) + copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) tmpfile = make_a_real_file(tmpdir, "a_file.txt") with pytest.raises(ValidationError): - utils.copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) + copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) def test_copy_is_fine_with_single_items(self, tmpdir): @@ -136,5 +136,5 @@ assert not os.path.exists(expected_file) - result = utils.copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) + result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) assert result == 0 assert os.path.exists(expected_file) @@ -168,5 +168,5 @@ assert len(expected_files) == 9 - result = utils.copy_multiple_items_to_multiple_dirs(source_files, dest_paths) + result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths) assert result == 0 for ef in expected_files: @@ -212,5 +212,5 @@ # Provide only the original tmpdir to the copying function - utils.copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) + copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) # assert that the subdirs and files all exist Index: /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py (revision 40962) @@ -2,5 +2,5 @@ # Plans are to flesh this out a bit more to make some smarter copying etc -from backups.utils.errors import ValidationError, SubprocessError +from .errors import ValidationError, SubprocessError Index: /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py (revision 40962) @@ -5,5 +5,5 @@ from pathlib import Path -from backups.utils.errors import SubprocessError, ValidationError +from .errors import SubprocessError, ValidationError Index: /branches/ipp-259_genericise_backups/tools/backups/website_backup.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/website_backup.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/website_backup.py (revision 40962) @@ -1,8 +1,7 @@ -import argparse import os.path as path -import backups.utils.config_parse_helper as cfg_help -from backups.backup import Backup -from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config') @@ -32,20 +31,7 @@ -def main(): - wb = WebsiteBackup() - wb.run_backup() - - -def parse_args(args): - - parser = argparse.ArgumentParser( - description='Performs backup of the panstarrs website.') - - parser.add_argument('--config_file', - nargs='?', - help='specify the location of a config that matches the ConfigSchema', - default=EXPECTED_CONFIG_FILE) - - return parser.parse_args() +def main(config=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + wb = WebsiteBackup(config, schema) + wb.run_backup_processes() Index: /branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py =================================================================== --- /branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py (revision 40961) +++ /branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py (revision 40962) @@ -3,7 +3,8 @@ from pathlib import Path -import backups.backup as bckup -from backups.website_backup import WebsiteBackup -from backups.utils.config_parse_helper import ConfigSchema +import backup as bckup +import website_backup as webbck_mod +from website_backup import WebsiteBackup +from utils.config_parse_helper import ConfigSchema WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__), @@ -31,4 +32,34 @@ wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG) self.assert_defaults(wb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + webbck_mod.main(config, schema) + assert True is True