IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40962


Ignore:
Timestamp:
Oct 22, 2019, 2:30:49 PM (7 years ago)
Author:
fairlamb
Message:

fixed calling as main + tests, removed argsparse from most

Location:
branches/ipp-259_genericise_backups/tools/backups
Files:
22 edited

Legend:

Unmodified
Added
Removed

  • branches/ipp-259_genericise_backups/tools/backups/backup.py

    r40952 r40962  
    77from configparser import ConfigParser
    88
    9 import backups.utils.config_parse_helper as cfg_help
    10 import backups.utils.errors as errs
    11 import backups.utils.subprocess_utils as sub_utils
    12 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig
     9import utils.errors as errs
     10import utils.subprocess_utils as sub_utils
     11from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig, SpecialSchemaProcessing
    1312
    1413DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT',
    15     [ ConfigSchemaLine('backup_paths', True, list, cfg_help.SpecialSchemaProcessing.commma_separated)
     14    [ ConfigSchemaLine('backup_paths', True, list, SpecialSchemaProcessing.commma_separated)
    1615    , ConfigSchemaLine('source_machine', True, str)
    1716    , ConfigSchemaLine('verbose', False, bool)
    … …  
    2019)
    2120COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY',
    22     [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated)
    23     , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated)
     21    [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated)
     22    , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated)
    2423    ]
    2524)
    2625TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR',
    27     [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated)
    28     , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated)
     26    [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated)
     27    , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated)
    2928    , ConfigSchemaLine('target_filename', True, str)
    3029    , ConfigSchemaLine('prefix_date', True, bool)
    … …  
    3231)
    3332RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC',
    34     [ ConfigSchemaLine('sources', True, list, cfg_help.SpecialSchemaProcessing.commma_separated)
    35     , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated)
     33    [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated)
     34    , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated)
    3635    ]
    3736)
    … …  
    4140    , ConfigSchemaLine('password', True, str)
    4241    , ConfigSchemaLine('db_name', True, str)
    43     , ConfigSchemaLine('additional_args', False, list, cfg_help.SpecialSchemaProcessing.space_separated)
     42    , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated)
    4443    , ConfigSchemaLine('target_filename', True, str)
    4544    , ConfigSchemaLine('prefix_date', True, bool)
    … …  
    148147                if not os.path.exists(backup_path):
    149148                    os.makedirs(backup_path)
     149                    sub_utils.chmod_wrapper(['777', backup_path])
    150150
    151151    def _run_copy(self):

  • branches/ipp-259_genericise_backups/tools/backups/backup_test.py

    r40951 r40962  
    11import os
    22import pytest
     3import stat
    34import socket
    45from configparser import ConfigParser
    56from pathlib import Path
    67
    7 import backups.backup as bckup
    8 import backups.utils.errors as errs
    9 from backups.backup import Backup
    10 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     8import backup as bckup
     9import utils.errors as errs
     10from backup import Backup
     11from utils.config_parse_helper import ConfigSchema
    1112
    1213TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config')
    … …  
    207208        assert os.path.exists(bck2_path)
    208209
     210        # Check permissions are 777 too
     211        new_stats = os.stat(bck2_path)
     212        assert bool(new_stats.st_mode & stat.S_IRUSR) is True
     213        assert bool(new_stats.st_mode & stat.S_IWUSR) is True
     214        assert bool(new_stats.st_mode & stat.S_IXUSR) is True
     215        assert bool(new_stats.st_mode & stat.S_IRGRP) is True
     216        assert bool(new_stats.st_mode & stat.S_IWGRP) is True
     217        assert bool(new_stats.st_mode & stat.S_IXGRP) is True
     218        assert bool(new_stats.st_mode & stat.S_IROTH) is True
     219        assert bool(new_stats.st_mode & stat.S_IWOTH) is True
     220        assert bool(new_stats.st_mode & stat.S_IXOTH) is True
     221
    209222
    210223class TestRunningCopy():

  • branches/ipp-259_genericise_backups/tools/backups/confluence_backup.py

    r40951 r40962  
    1 import argparse
    21import datetime
    32import glob
    43import os.path as path
    54
    6 import backups.utils.config_parse_helper as cfg_help
    7 import backups.utils.errors as errs
    8 import backups.utils.subprocess_utils as sub_utils
    9 from backups.backup import Backup
    10 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     5import utils.config_parse_helper as cfg_help
     6import utils.errors as errs
     7import utils.subprocess_utils as sub_utils
     8from backup import Backup
     9from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
    1110
    1211EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config')
    … …  
    7170            source = self.copy_dict['sources'][0]
    7271
    73             confluence_wildcard_search = f"{source}/{ConfluenceBackup.get_confluence_backup_date_format()}*"
     72            confluence_wildcard_search = f"{source}/*{ConfluenceBackup.get_confluence_backup_date_format()}*"
    7473            confluence_xml_filepaths = glob.glob(confluence_wildcard_search)
    7574            if len(confluence_xml_filepaths) == 0:
    … …  
    9291
    9392
    94 def main():
    95     cb = ConfluenceBackup()
    96     cb.run_backup()
    97 
    98 
    99 def parse_args(args):
    100 
    101     parser = argparse.ArgumentParser(
    102         description='Performs backup of confluence.')
    103 
    104     parser.add_argument('--config_file',
    105         nargs='?',
    106         help='specify the location of a config that matches the ConfigSchema',
    107         default=EXPECTED_CONFIG_FILE)
    108 
    109     return parser.parse_args()
     93def main(config=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA):
     94    cb = ConfluenceBackup(config, schema)
     95    cb.run_backup_processes()
    11096
    11197

  • branches/ipp-259_genericise_backups/tools/backups/confluence_backup_test.py

    r40952 r40962  
    66from pathlib import Path
    77
    8 import backups.backup as bckup
    9 import backups.confluence_backup as jbck_mod
    10 import backups.utils.errors as errs
    11 from backups.confluence_backup import ConfluenceBackup
    12 from backups.utils.errors import ValidationError
    13 from backups.utils.config_parse_helper import ConfigSchema
     8import backup as bckup
     9import confluence_backup as cbck_mod
     10
     11from confluence_backup import ConfluenceBackup
     12from utils.errors import ValidationError
     13from utils.config_parse_helper import ConfigSchema
    1414
    1515CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config')
    … …  
    6767
    6868    # Create backup files that will be copied
    69     confluence_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_confluence_test_file.zip"
     69    confluence_backup_name = f"confluence-xml-backup-{ConfluenceBackup.get_confluence_backup_date_format()}.zip"
    7070    confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name)
    7171    Path(confluence_backup_filepath).touch()
    … …  
    9696            }
    9797        assert jb.copy_dict == \
    98             { 'sources' : ['/var/atlassian/application-data/confluence/export']
     98            { 'sources' : ['/var/atlassian/application-data/confluence/backups']
    9999            , 'additional_args' : ['-v']
    100100            }
    101101        assert jb.tar_dict == \
    102             { 'sources' : ['/var/atlassian/application-data/confluence/data']
     102            { 'sources' : ['/var/atlassian/application-data/confluence/attachments']
    103103            , 'additional_args' : ['--verbose', '-z', '-p', '-cf']
    104104            , 'target_filename' : 'confluence.tar.gz'
    … …  
    118118
    119119    def test_default_config_fails_to_load_if_the_file_does_not_exist(self):
    120         default_config = jbck_mod.EXPECTED_CONFIG_FILE
     120        default_config = cbck_mod.EXPECTED_CONFIG_FILE
    121121        assert not os.path.exists(default_config)
    122122
    123         with pytest.raises(errs.ValidationError) as e:
     123        with pytest.raises(ValidationError) as e:
    124124            ConfluenceBackup()
    125125        assert "ValidationError: Config filepath does not exist" in str(e)
    … …  
    174174
    175175    def test_nonexistant_config_file_raises_error(self):
    176         with pytest.raises(errs.ValidationError) as e:
     176        with pytest.raises(ValidationError) as e:
    177177            ConfluenceBackup("not a real file path")
    178178        assert "ValidationError: Config filepath does not exist" in str(e)
    179179
    180180    def test_main_loads_default_and_fails_due_to_missing_paths(self):
    181         with pytest.raises(errs.ValidationError) as e:
    182             jbck_mod.main()
     181        with pytest.raises(ValidationError) as e:
     182            cbck_mod.main()
    183183        assert "ValidationError: Config filepath does not exist" in str(e)
    184184
    185185
     186class TestFullRunFromMain(object):
     187
     188    def test_main_loads_and_runs_fine_when_given_config(self, tmpdir):
     189        config = configparser.ConfigParser()
     190        bpath1 = os.path.join(tmpdir, "path/1")
     191        bpath2 = os.path.join(tmpdir, "path/2")
     192        config['DEFAULT'] = \
     193            { 'backup_paths' :  f"{bpath1}, {bpath2}"
     194            , 'source_machine' : 'rocket'
     195            , 'verbose' : 'True'
     196            }
     197        copy_source = os.path.join(tmpdir, "copy/source")
     198        conf_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_conf_test_file.zip"
     199        config['COPY'] = \
     200            { 'sources' : f'{copy_source}'
     201            }
     202        tar_source =  os.path.join(tmpdir, "tar/source")
     203        config['TAR'] = \
     204            { 'sources' : f"{tar_source}"
     205            , 'additional_args' : "--verbose -z -p -cf"
     206            , 'target_filename' : 'conf.tar.gz'
     207            , 'prefix_date' : 'False'
     208            }
     209        os.makedirs(bpath1)
     210        os.makedirs(bpath2)
     211        os.makedirs(copy_source)
     212        os.makedirs(tar_source)
     213        Path(os.path.join(copy_source, conf_backup_name)).touch()
     214
     215        schema = ConfigSchema(None)
     216        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
     217        schema.add_section(bckup.COPY_SCHEMA_SECTION)
     218        schema.add_section(bckup.TAR_SCHEMA_SECTION)
     219
     220        cbck_mod.main(config, schema)
     221        assert True is True
     222
     223
    186224class TestCopyingBackups(object):
    187225
    … …  
    189227
    190228    def test_get_confluence_backup_date_format(self):
    191         jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG)
     229        cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG)
    192230        today = datetime.date.today()
    193231        expected_format = today.strftime("%Y_%m_%d")
    194         assert jb.get_confluence_backup_date_format() == expected_format
     232        assert cb.get_confluence_backup_date_format() == expected_format
    195233
    196234        another_date = datetime.date(1988, 11, 23)
    197235        expected_format = another_date.strftime("%Y_%m_%d")
    198         assert jb.get_confluence_backup_date_format(another_date) == expected_format
     236        assert cb.get_confluence_backup_date_format(another_date) == expected_format
    199237
    200238    def test_copying_fails_if_paths_not_present(self):
    201         jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG)
    202         # return_value = jb._copy_confluence_backup()
    203         with pytest.raises(ValidationError) as e:
    204             jb._run_copy()
     239        cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG)
     240        # return_value = cb._copy_confluence_backup()
     241        with pytest.raises(ValidationError) as e:
     242            cb._run_copy()
    205243        assert "ValidationError: target path does not exist:" in str(e)
    206244
    … …  
    208246        create_items_for_full_confluence_backup_test(tmpdir)
    209247        config = make_config(tmpdir)
    210         jb = ConfluenceBackup(config_file=config)
     248        cb = ConfluenceBackup(config_file=config)
    211249
    212250        today = ConfluenceBackup.get_confluence_backup_date_format()
    213         expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_confluence_test_file.zip" )
    214         expected_backup_1_copy    = os.path.join(tmpdir, "backup_1_path",    f"{today}_confluence_test_file.zip" )
    215         expected_backup_2_copy    = os.path.join(tmpdir, "backup_2_path",    f"{today}_confluence_test_file.zip" )
     251        expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"confluence-xml-backup-{today}.zip" )
     252        expected_backup_1_copy    = os.path.join(tmpdir, "backup_1_path",    f"confluence-xml-backup-{today}.zip" )
     253        expected_backup_2_copy    = os.path.join(tmpdir, "backup_2_path",    f"confluence-xml-backup-{today}.zip" )
    216254        assert not os.path.exists(expected_host_backup_copy)
    217255        assert not os.path.exists(expected_backup_1_copy)
    218256        assert not os.path.exists(expected_backup_2_copy)
    219257
    220         return_value = jb._run_copy()
     258        return_value = cb._run_copy()
    221259        assert return_value == 0
    222260

  • branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup.py

    r40958 r40962  
    1 import argparse
    21import os.path as path
    32
    4 import backups.utils.config_parse_helper as cfg_help
    5 from backups.backup import Backup
    6 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     3import utils.config_parse_helper as cfg_help
     4from backup import Backup
     5from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
    76
    87EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config')
    … …  
    3231
    3332
    34 def main():
    35     ohdb = IPPOpsHomedirBackup()
    36     ohdb.run_backup()
    37 
    38 
    39 def parse_args(args):
    40 
    41     parser = argparse.ArgumentParser(
    42         description='Performs backup of the ippops homedirs.')
    43 
    44     parser.add_argument('--config_file',
    45         nargs='?',
    46         help='specify the location of a config that matches the ConfigSchema',
    47         default=EXPECTED_CONFIG_FILE)
    48 
    49     return parser.parse_args()
     33def main(config=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA):
     34    ohdb = IPPOpsHomedirBackup(config, schema)
     35    ohdb.run_backup_processes()
    5036
    5137

  • branches/ipp-259_genericise_backups/tools/backups/ippops_homedir_backup_test.py

    r40958 r40962  
    33from pathlib import Path
    44
    5 import backups.backup as bckup
    6 from backups.ippops_homedir_backup import IPPOpsHomedirBackup
    7 from backups.utils.config_parse_helper import ConfigSchema
     5import backup as bckup
     6import ippops_homedir_backup as ihbck_mod
     7from ippops_homedir_backup import IPPOpsHomedirBackup
     8from utils.config_parse_helper import ConfigSchema
    89
    910IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__),
    … …  
    3132        ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG)
    3233        self.assert_defaults(ihb)
     34
     35
     36class TestFullRunFromMain(object):
     37
     38    def test_main_loads_and_runs_fine_when_given_config(self, tmpdir):
     39        config = configparser.ConfigParser()
     40        bpath1 = os.path.join(tmpdir, "path/1")
     41        bpath2 = os.path.join(tmpdir, "path/2")
     42        config['DEFAULT'] = \
     43            { 'backup_paths' :  f"{bpath1}, {bpath2}"
     44            , 'source_machine' : 'rocket'
     45            , 'verbose' : 'True'
     46            }
     47        tar_source =  os.path.join(tmpdir, "tar/source")
     48        config['TAR'] = \
     49            { 'sources' : f"{tar_source}"
     50            , 'additional_args' : "--verbose -z -p -cf"
     51            , 'target_filename' : 'svnroot.tar.gz'
     52            , 'prefix_date' : 'False'
     53            }
     54        os.makedirs(bpath1)
     55        os.makedirs(bpath2)
     56        os.makedirs(tar_source)
     57
     58        schema = ConfigSchema(None)
     59        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
     60        schema.add_section(bckup.TAR_SCHEMA_SECTION)
     61
     62        ihbck_mod.main(config, schema)
     63        assert True is True
    3364
    3465

  • branches/ipp-259_genericise_backups/tools/backups/jira_backup.py

    r40950 r40962  
    1 import argparse
    21import datetime
    32import glob
    43import os.path as path
    54
    6 import backups.utils.config_parse_helper as cfg_help
    7 import backups.utils.errors as errs
    8 import backups.utils.subprocess_utils as sub_utils
    9 from backups.backup import Backup
    10 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     5import utils.config_parse_helper as cfg_help
     6import utils.errors as errs
     7import utils.subprocess_utils as sub_utils
     8from backup import Backup
     9from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
    1110
    1211EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config')
    … …  
    9291
    9392
    94 def main():
    95     jb = JiraBackup()
    96     jb.run_backup()
    97 
    98 
    99 def parse_args(args):
    100 
    101     parser = argparse.ArgumentParser(
    102         description='Performs backup of jira.')
    103 
    104     parser.add_argument('--config_file',
    105         nargs='?',
    106         help='specify the location of a config that matches the ConfigSchema',
    107         default=EXPECTED_CONFIG_FILE)
    108 
    109     return parser.parse_args()
     93def main(config=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA):
     94    jb = JiraBackup(config, schema)
     95    jb.run_backup_processes()
    11096
    11197

  • branches/ipp-259_genericise_backups/tools/backups/jira_backup_test.py

    r40952 r40962  
    77
    88import backups.backup as bckup
    9 import backups.jira_backup as jbck_mod
    10 import backups.utils.errors as errs
    11 from backups.jira_backup import JiraBackup
    12 from backups.utils.errors import ValidationError
    13 from backups.utils.config_parse_helper import ConfigSchema
     9import jira_backup as jbck_mod
     10from jira_backup import JiraBackup
     11from utils.errors import ValidationError
     12from utils.config_parse_helper import ConfigSchema
    1413
    1514JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config')
    … …  
    121120        assert not os.path.exists(default_config)
    122121
    123         with pytest.raises(errs.ValidationError) as e:
     122        with pytest.raises(ValidationError) as e:
    124123            JiraBackup()
    125124        assert "ValidationError: Config filepath does not exist" in str(e)
    … …  
    174173
    175174    def test_nonexistant_config_file_raises_error(self):
    176         with pytest.raises(errs.ValidationError) as e:
     175        with pytest.raises(ValidationError) as e:
    177176            JiraBackup("not a real file path")
    178177        assert "ValidationError: Config filepath does not exist" in str(e)
    179178
    180179    def test_main_loads_default_and_fails_due_to_missing_paths(self):
    181         with pytest.raises(errs.ValidationError) as e:
     180        with pytest.raises(ValidationError) as e:
    182181            jbck_mod.main()
    183182        assert "ValidationError: Config filepath does not exist" in str(e)
     183
     184
     185class TestFullRunFromMain(object):
     186
     187    def test_main_loads_and_runs_fine_when_given_config(self, tmpdir):
     188        config = configparser.ConfigParser()
     189        config.add_section("atlassian_backups")
     190        bpath1 = os.path.join(tmpdir, "path/1")
     191        bpath2 = os.path.join(tmpdir, "path/2")
     192        config['DEFAULT'] = \
     193            { 'backup_paths' :  f"{bpath1}, {bpath2}"
     194            , 'source_machine' : 'rocket'
     195            , 'verbose' : 'True'
     196            }
     197        copy_source = os.path.join(tmpdir, "copy/source")
     198        jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip"
     199        config['COPY'] = \
     200            { 'sources' : f'{copy_source}'
     201            }
     202        tar_source =  os.path.join(tmpdir, "tar/source")
     203        config['TAR'] = \
     204            { 'sources' : f"{tar_source}"
     205            , 'additional_args' : "--verbose -z -p -cf"
     206            , 'target_filename' : 'jira.tar.gz'
     207            , 'prefix_date' : 'False'
     208            }
     209        os.makedirs(bpath1)
     210        os.makedirs(bpath2)
     211        os.makedirs(copy_source)
     212        os.makedirs(tar_source)
     213        Path(os.path.join(copy_source, jira_backup_name)).touch()
     214
     215        schema = ConfigSchema(None)
     216        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
     217        schema.add_section(bckup.COPY_SCHEMA_SECTION)
     218        schema.add_section(bckup.TAR_SCHEMA_SECTION)
     219
     220        jbck_mod.main(config, schema)
     221        assert True is True
    184222
    185223

  • branches/ipp-259_genericise_backups/tools/backups/mailman_backup.py

    r40958 r40962  
    1 import argparse
    21import os.path as path
    32
    4 import backups.utils.config_parse_helper as cfg_help
    5 from backups.backup import Backup
    6 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     3import utils.config_parse_helper as cfg_help
     4from backup import Backup
     5from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
    76
    87EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config')
    … …  
    3231
    3332
    34 def main():
    35     mb = MailmanBackup()
    36     mb.run_backup()
    37 
    38 
    39 def parse_args(args):
    40 
    41     parser = argparse.ArgumentParser(
    42         description='Performs backup of mailman.')
    43 
    44     parser.add_argument('--config_file',
    45         nargs='?',
    46         help='specify the location of a config that matches the ConfigSchema',
    47         default=EXPECTED_CONFIG_FILE)
    48 
    49     return parser.parse_args()
     33def main(config=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA):
     34    mb = MailmanBackup(config, schema)
     35    mb.run_backup_processes()
    5036
    5137

  • branches/ipp-259_genericise_backups/tools/backups/mailman_backup_test.py

    r40958 r40962  
    33from pathlib import Path
    44
    5 import backups.backup as bckup
    6 from backups.mailman_backup import MailmanBackup
    7 from backups.utils.config_parse_helper import ConfigSchema
     5import mailman_backup as mailbck_mod
     6import backup as bckup
     7from backup import DEFAULT_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION
     8from mailman_backup import MailmanBackup
     9from utils.config_parse_helper import ConfigSchema
     10
    811
    912MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__),
    … …  
    3134        mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG)
    3235        self.assert_defaults(mb)
     36
     37
     38class TestFullRunFromMain(object):
     39
     40    def test_main_loads_and_runs_fine_when_given_config(self, tmpdir):
     41        config = configparser.ConfigParser()
     42        bpath1 = os.path.join(tmpdir, "path/1")
     43        bpath2 = os.path.join(tmpdir, "path/2")
     44        config['DEFAULT'] = \
     45            { 'backup_paths' :  f"{bpath1}, {bpath2}"
     46            , 'source_machine' : 'rocket'
     47            , 'verbose' : 'True'
     48            }
     49        tar_source =  os.path.join(tmpdir, "tar/source")
     50        config['TAR'] = \
     51            { 'sources' : f"{tar_source}"
     52            , 'additional_args' : "--verbose -z -p -cf"
     53            , 'target_filename' : 'svnroot.tar.gz'
     54            , 'prefix_date' : 'False'
     55            }
     56        os.makedirs(bpath1)
     57        os.makedirs(bpath2)
     58        os.makedirs(tar_source)
     59
     60        schema = ConfigSchema(None)
     61        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
     62        schema.add_section(bckup.TAR_SCHEMA_SECTION)
     63
     64        mailbck_mod.main(config, schema)
     65        assert True is True
    3366
    3467
    … …  
    6699        # Setup Schema
    67100        schema = ConfigSchema(None)
    68         schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
    69         schema.add_section(bckup.RSYNC_SCHEMA_SECTION)
     101        schema.add_section(DEFAULT_SCHEMA_SECTION)
     102        schema.add_section(RSYNC_SCHEMA_SECTION)
    70103
    71104        # check that the rsync is successful (with options)

  • branches/ipp-259_genericise_backups/tools/backups/svn_backup.py

    r40952 r40962  
    1 import argparse
    21import os.path as path
    32
    4 import backups.utils.config_parse_helper as cfg_help
    5 from backups.backup import Backup
    6 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     3from utils.config_parse_helper import SpecialSchemaProcessing
     4from backup import Backup
     5from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
    76
    87EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config')
    … …  
    1110    [ ConfigSchemaSection('DEFAULT',
    1211        [ ConfigSchemaLine('backup_paths',   True,  list,
    13             cfg_help.SpecialSchemaProcessing.commma_separated)
     12            SpecialSchemaProcessing.commma_separated)
    1413        , ConfigSchemaLine('source_machine', True,  str)
    1514        , ConfigSchemaLine('verbose',        False, bool)
    … …  
    1817    , ConfigSchemaSection('COPY',
    1918        [ ConfigSchemaLine('sources',         True,  list,
    20             cfg_help.SpecialSchemaProcessing.commma_separated)
     19            SpecialSchemaProcessing.commma_separated)
    2120        , ConfigSchemaLine('additional_args', False, list,
    22             cfg_help.SpecialSchemaProcessing.space_separated)
     21            SpecialSchemaProcessing.space_separated)
    2322        ])
    2423    , ConfigSchemaSection('RSYNC',
    2524        [ ConfigSchemaLine('sources', True, list,
    26             cfg_help.SpecialSchemaProcessing.commma_separated)
     25            SpecialSchemaProcessing.commma_separated)
    2726        , ConfigSchemaLine('additional_args', False, list,
    28             cfg_help.SpecialSchemaProcessing.space_separated)
     27            SpecialSchemaProcessing.space_separated)
    2928        ])
    3029    ]
    … …  
    3837
    3938
    40 def main():
    41     sb = SvnBackup()
    42     sb.run_backup()
    43 
    44 
    45 def parse_args(args):
    46 
    47     parser = argparse.ArgumentParser(
    48         description='Performs backup of svn.')
    49 
    50     parser.add_argument('--config_file',
    51         nargs='?',
    52         help='specify the location of a config that matches the ConfigSchema',
    53         default=EXPECTED_CONFIG_FILE)
    54 
    55     return parser.parse_args()
     39def main(config=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA):
     40    sb = SvnBackup(config, schema)
     41    sb.run_backup_processes()
    5642
    5743

  • branches/ipp-259_genericise_backups/tools/backups/svn_backup_test.py

    r40958 r40962  
    33from pathlib import Path
    44
    5 import backups.backup as bckup
    6 from backups.svn_backup import SvnBackup
    7 from backups.utils.config_parse_helper import ConfigSchema
     5import svn_backup as svnbck_mod
     6import backup as bckup
     7from backup import DEFAULT_SCHEMA_SECTION, COPY_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION
     8from svn_backup import SvnBackup
     9from utils.config_parse_helper import ConfigSchema
    810
    911SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config')
    … …  
    2729            }
    2830        assert sb.rsync_dict == \
    29             { 'sources' : ['/svnroot']
     31            { 'sources' : ['/export/ippops4.0/svnroot']
    3032            , 'additional_args' : ['-a', '--delete-after']
    3133            }
    … …  
    6870        # Setup Schema
    6971        schema = ConfigSchema(None)
    70         schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
    71         schema.add_section(bckup.COPY_SCHEMA_SECTION)
     72        schema.add_section(DEFAULT_SCHEMA_SECTION)
     73        schema.add_section(COPY_SCHEMA_SECTION)
    7274
    7375        # check that the rsync is successful (with options)
    … …  
    8183        assert os.path.exists(expected_file_bck1)
    8284        assert os.path.exists(expected_file_bck2)
     85
     86
     87class TestFullRunFromMain(object):
     88
     89    def test_main_loads_and_runs_fine_when_given_config(self, tmpdir):
     90        config = configparser.ConfigParser()
     91        bpath1 = os.path.join(tmpdir, "path/1")
     92        bpath2 = os.path.join(tmpdir, "path/2")
     93        config['DEFAULT'] = \
     94            { 'backup_paths' :  f"{bpath1}, {bpath2}"
     95            , 'source_machine' : 'rocket'
     96            , 'verbose' : 'True'
     97            }
     98        copy_source = os.path.join(tmpdir, "copy/source")
     99        svn_conf_file = "dav_svn.conf"
     100        config['COPY'] = \
     101            { 'sources' : f'{copy_source}/{svn_conf_file}'
     102            }
     103        tar_source =  os.path.join(tmpdir, "tar/source")
     104        config['TAR'] = \
     105            { 'sources' : f"{tar_source}"
     106            , 'additional_args' : "--verbose -z -p -cf"
     107            , 'target_filename' : 'svnroot.tar.gz'
     108            , 'prefix_date' : 'False'
     109            }
     110        os.makedirs(bpath1)
     111        os.makedirs(bpath2)
     112        os.makedirs(copy_source)
     113        os.makedirs(tar_source)
     114        Path(os.path.join(copy_source, svn_conf_file)).touch()
     115
     116        schema = ConfigSchema(None)
     117        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
     118        schema.add_section(bckup.COPY_SCHEMA_SECTION)
     119        schema.add_section(bckup.TAR_SCHEMA_SECTION)
     120
     121        svnbck_mod.main(config, schema)
     122        assert True is True
    83123
    84124
    … …  
    124164        # Setup Schema
    125165        schema = ConfigSchema(None)
    126         schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
    127         schema.add_section(bckup.RSYNC_SCHEMA_SECTION)
     166        schema.add_section(DEFAULT_SCHEMA_SECTION)
     167        schema.add_section(RSYNC_SCHEMA_SECTION)
    128168
    129169        # check that the rsync is successful (with options)

  • branches/ipp-259_genericise_backups/tools/backups/testing/confluence_test.config

    r40952 r40962  
    77
    88[COPY]
    9 sources = /var/atlassian/application-data/confluence/export
     9sources = /var/atlassian/application-data/confluence/backups
    1010additional_args = -v
     11conf_prefix = confluence-xml-backup-
    1112
    1213[TAR]
    13 sources = /var/atlassian/application-data/confluence/data
     14sources = /var/atlassian/application-data/confluence/attachments
    1415additional_args = --verbose -z -p -cf
    1516target_filename = confluence.tar.gz

  • branches/ipp-259_genericise_backups/tools/backups/testing/svn_test.config

    r40952 r40962  
    1111
    1212[RSYNC]
    13 sources = /svnroot
     13sources = /export/ippops4.0/svnroot
    1414additional_args = -a --delete-after

  • branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper.py

    r40949 r40962  
    11from enum import Enum
    22from configparser import ConfigParser
    3 from backups.utils.errors import ConfigError, ValidationError
     3
     4from utils.errors import ConfigError, ValidationError
    45
    56
    … …  
    139140                    parsed_items[csl.key] = splitup
    140141                else:
     142
    141143                    raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}")
    142144            else:
    … …  
    150152
    151153
    152 # def parse_config(config: ConfigParser(), schema: ConfigSchemaSection) -> dict:
    153 #     return _parse_config_section(config, schema)
    154 
    155154def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict:
    156155    parsed_items = {}
    157     print(config)
    158     for c in config:
    159         print(c)
    160156
    161157    for css_name in schema.sections:
    162         print(css_name)
    163158        css = schema.get_schema_section(css_name)
    164         print(css)
    165159
    166160        # if config.has_section(css_name):

  • branches/ipp-259_genericise_backups/tools/backups/utils/config_parse_helper_test.py

    r40949 r40962  
    11import pytest
    22
    3 import backups.utils.config_parse_helper as helper
    4 from backups.utils.errors import ConfigError, ValidationError
     3import utils.config_parse_helper as helper
     4from utils.errors import ConfigError, ValidationError
    55
    66

  • branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils.py

    r40912 r40962  
    22import os.path as path
    33import shutil
    4 
    54from distutils.dir_util import copy_tree
    65
    7 import backups.utils.subprocess_utils as sub_utils
    8 from backups.utils.errors import ValidationError
     6from .subprocess_utils import simple_unix_wrapper
     7from .errors import ValidationError
    98
    109
    … …  
    118117                command_and_args = ["mv", fp, destination_dirs[i]]
    119118                # shutil.move(fp, destination_dirs[i])
    120                 sub_utils.simple_unix_wrapper(command_and_args)
     119                simple_unix_wrapper(command_and_args)
    121120        except IOError as e:
    122121            print("IOError copying: ", e.filename, e.filename2)

  • branches/ipp-259_genericise_backups/tools/backups/utils/generic_utils_test.py

    r40898 r40962  
    1 import backups.utils.generic_utils as utils
    21import os
    32import pytest
    43
    5 from backups.utils.errors import ValidationError
     4from .generic_utils import *
     5from .errors import ValidationError
    66from pathlib import Path
    77
    … …  
    1919    def test_copy_fails_with_none_args(self, tmpdir):
    2020        with pytest.raises(ValidationError):
    21             utils.copy_to_dir(None, tmpdir)
     21            copy_to_dir(None, tmpdir)
    2222
    2323        with pytest.raises(ValidationError):
    2424            real_file = make_a_real_file(tmpdir, "file.txt")
    25             utils.copy_to_dir(real_file, None)
     25            copy_to_dir(real_file, None)
    2626
    2727    def test_copy_fails_if_source_does_not_exist(self, tmpdir):
    2828        with pytest.raises(ValidationError):
    29             utils.copy_to_dir("not a real file", tmpdir)
     29            copy_to_dir("not a real file", tmpdir)
    3030
    3131    def test_copy_fails_if_destination_is_not_real(self, tmpdir):
    3232        with pytest.raises(ValidationError):
    3333            real_file = make_a_real_file(tmpdir, "real_file.txt")
    34             utils.copy_to_dir(real_file, "not a real directory")
     34            copy_to_dir(real_file, "not a real directory")
    3535
    3636    def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir):
    … …  
    4242
    4343        with pytest.raises(ValidationError):
    44             utils.copy_to_dir(real_file, not_a_directory)
     44            copy_to_dir(real_file, not_a_directory)
    4545
    4646    def test_copy_to_dir(self, tmpdir):
    … …  
    6060
    6161        # run copy function and confirm copy exists (and original)
    62         result = utils.copy_to_dir(file_to_copy, destination_dir)
     62        result = copy_to_dir(file_to_copy, destination_dir)
    6363        assert result == 0
    6464        assert os.path.exists(file_to_copy)
    … …  
    9898
    9999        # Provide only the original tmpdir to the copying function
    100         utils.copy_to_dir(subdir, destination_dir, True)
     100        copy_to_dir(subdir, destination_dir, True)
    101101
    102102        # assert that the subdirs and files all exist
    … …  
    110110
    111111        with pytest.raises(ValidationError):
    112             utils.copy_multiple_items_to_multiple_dirs(None, None)
    113 
    114         with pytest.raises(ValidationError):
    115             utils.copy_multiple_items_to_multiple_dirs(tmpdir, None)
    116 
    117         with pytest.raises(ValidationError):
    118             utils.copy_multiple_items_to_multiple_dirs(None, tmpdir)
     112            copy_multiple_items_to_multiple_dirs(None, None)
     113
     114        with pytest.raises(ValidationError):
     115            copy_multiple_items_to_multiple_dirs(tmpdir, None)
     116
     117        with pytest.raises(ValidationError):
     118            copy_multiple_items_to_multiple_dirs(None, tmpdir)
    119119
    120120    def test_copy_requires_args_to_be_in_list_format(self, tmpdir):
    … …  
    122122        not_a_list = "not a list"
    123123        with pytest.raises(ValidationError):
    124             utils.copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir)
     124            copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir)
    125125
    126126        tmpfile = make_a_real_file(tmpdir, "a_file.txt")
    127127        with pytest.raises(ValidationError):
    128             utils.copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list)
     128            copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list)
    129129
    130130    def test_copy_is_fine_with_single_items(self, tmpdir):
    … …  
    136136        assert not os.path.exists(expected_file)
    137137
    138         result = utils.copy_multiple_items_to_multiple_dirs([tmpfile], [destination])
     138        result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination])
    139139        assert result == 0
    140140        assert os.path.exists(expected_file)
    … …  
    168168        assert len(expected_files) == 9
    169169
    170         result = utils.copy_multiple_items_to_multiple_dirs(source_files, dest_paths)
     170        result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths)
    171171        assert result == 0
    172172        for ef in expected_files:
    … …  
    212212
    213213        # Provide only the original tmpdir to the copying function
    214         utils.copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True)
     214        copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True)
    215215
    216216        # assert that the subdirs and files all exist

  • branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils.py

    r40932 r40962  
    22
    33# Plans are to flesh this out a bit more to make some smarter copying etc
    4 from backups.utils.errors import ValidationError, SubprocessError
     4from .errors import ValidationError, SubprocessError
    55
    66

  • branches/ipp-259_genericise_backups/tools/backups/utils/subprocess_utils_test.py

    r40932 r40962  
    55
    66from pathlib import Path
    7 from backups.utils.errors import SubprocessError, ValidationError
     7from .errors import SubprocessError, ValidationError
    88
    99

  • branches/ipp-259_genericise_backups/tools/backups/website_backup.py

    r40958 r40962  
    1 import argparse
    21import os.path as path
    32
    4 import backups.utils.config_parse_helper as cfg_help
    5 from backups.backup import Backup
    6 from backups.utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
     3import utils.config_parse_helper as cfg_help
     4from backup import Backup
     5from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine
    76
    87EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config')
    … …  
    3231
    3332
    34 def main():
    35     wb = WebsiteBackup()
    36     wb.run_backup()
    37 
    38 
    39 def parse_args(args):
    40 
    41     parser = argparse.ArgumentParser(
    42         description='Performs backup of the panstarrs website.')
    43 
    44     parser.add_argument('--config_file',
    45         nargs='?',
    46         help='specify the location of a config that matches the ConfigSchema',
    47         default=EXPECTED_CONFIG_FILE)
    48 
    49     return parser.parse_args()
     33def main(config=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA):
     34    wb = WebsiteBackup(config, schema)
     35    wb.run_backup_processes()
    5036
    5137

  • branches/ipp-259_genericise_backups/tools/backups/website_backup_test.py

    r40958 r40962  
    33from pathlib import Path
    44
    5 import backups.backup as bckup
    6 from backups.website_backup import WebsiteBackup
    7 from backups.utils.config_parse_helper import ConfigSchema
     5import backup as bckup
     6import website_backup as webbck_mod
     7from website_backup import WebsiteBackup
     8from utils.config_parse_helper import ConfigSchema
    89
    910WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__),
    … …  
    3132        wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG)
    3233        self.assert_defaults(wb)
     34
     35
     36class TestFullRunFromMain(object):
     37
     38    def test_main_loads_and_runs_fine_when_given_config(self, tmpdir):
     39        config = configparser.ConfigParser()
     40        bpath1 = os.path.join(tmpdir, "path/1")
     41        bpath2 = os.path.join(tmpdir, "path/2")
     42        config['DEFAULT'] = \
     43            { 'backup_paths' :  f"{bpath1}, {bpath2}"
     44            , 'source_machine' : 'rocket'
     45            , 'verbose' : 'True'
     46            }
     47        tar_source =  os.path.join(tmpdir, "tar/source")
     48        config['TAR'] = \
     49            { 'sources' : f"{tar_source}"
     50            , 'additional_args' : "--verbose -z -p -cf"
     51            , 'target_filename' : 'svnroot.tar.gz'
     52            , 'prefix_date' : 'False'
     53            }
     54        os.makedirs(bpath1)
     55        os.makedirs(bpath2)
     56        os.makedirs(tar_source)
     57
     58        schema = ConfigSchema(None)
     59        schema.add_section(bckup.DEFAULT_SCHEMA_SECTION)
     60        schema.add_section(bckup.TAR_SCHEMA_SECTION)
     61
     62        webbck_mod.main(config, schema)
     63        assert True is True
    3364
    3465
Note: See TracChangeset for help on using the changeset viewer.