IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40897


Ignore:
Timestamp:
Oct 3, 2019, 11:51:21 AM (7 years ago)
Author:
fairlamb
Message:

Better errors, chmod wrapper, fixed config loading and tests

Location:
branches/ipp-132_automate_jira_conf_backups
Files:
1 deleted
6 edited

Legend:

Unmodified
Added
Removed

  • branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py

    r40896 r40897  
    1313# a restructure would be needed
    1414# Ideally, the utils should be added to the path.
    15 import utils.subprocess_utils as sub_utils
    16 
    17 
    18 DEFAULT_CONFIG_FILE = path.abspath('./atlassian_backups.config')
     15import subprocess_utils as sub_utils
     16
     17
     18DEFAULT_CONFIG_FILE = path.join(path.dirname(__file__), 'atlassian_backups.config')
    1919
    2020
    … …  
    236236        jira_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_jira_attachments.tar"
    237237        jira_tar_filepath = path.join(latest_paths[0], jira_tar_name)
    238         jira_tar_args = ["-cf", jira_tar_filepath, self.jira_attachments_path]
     238        jira_tar_args = ["-C", "/", "-cf", jira_tar_filepath, self.jira_attachments_path.strip('/')]
    239239        sub_utils.tar_wrapper(jira_tar_args)
    240240        sub_utils.chmod_wrapper(['774', jira_tar_filepath])
    … …  
    246246        confluence_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_confluence_attachments.tar"
    247247        confluence_tar_filepath = path.join(latest_paths[0], confluence_tar_name)
    248         confluence_tar_args = ["-cf", confluence_tar_filepath, self.conf_attachments_path]
     248        confluence_tar_args = ["-C", "/", "-cf", confluence_tar_filepath, self.conf_attachments_path.strip('/')]
    249249        sub_utils.tar_wrapper(confluence_tar_args)
    250250        sub_utils.chmod_wrapper(['774', confluence_tar_filepath])

  • branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py

    r40894 r40897  
    88
    99import full_atlassian_backup_test as fab_test
    10 import utils.subprocess_utils as sub_utils
     10import subprocess_utils as sub_utils
    1111
    1212from backup_atlassian_applications import AtlassianBackups
    1313
    14 TEST_DEFAULT_CONFIG_FILE = "./testing/test.config"
     14TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), './testing/test.config')
    1515
    1616
    … …  
    4747
    4848    def test_default_config_loads_with_no_args(self):
    49         assert os.path.exists("./atlassian_backups.config")
     49        default_config = baa.DEFAULT_CONFIG_FILE
     50        assert os.path.exists(default_config)
    5051        ab = AtlassianBackups()
    5152        assert ab.host_backup_path      == "/export/ippops4.0/atlassian_backups"

  • branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py

    r40893 r40897  
    66
    77from backup_atlassian_applications import AtlassianBackups
    8 
    9 TEST_DEFAULT_CONFIG_FILE = "./testing/test.config"
    10 
    118
    129def create_all_paths_and_folders(ab_class: AtlassianBackups, tmpdir) -> AtlassianBackups:
    … …  
    140137
    141138    def test_entire_process(self, tmpdir):
    142         ab = AtlassianBackups(config_file=TEST_DEFAULT_CONFIG_FILE)
     139        test_config_file = os.path.join(os.path.dirname(__file__), 'testing', 'test.config')
     140
     141        ab = AtlassianBackups(config_file=test_config_file)
    143142        ab = create_all_paths_and_folders(ab, tmpdir)
    144143

  • branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils.py

    r40895 r40897  
    1616
    1717    if source_file is None or destination_path is None:
    18         print("Error: source for copy cannot be None type")
    19         raise ValidationError
     18        raise ValidationError("Error: source for copy cannot be None type")
    2019    if destination_path is None:
    21         print("Error: copy destination cannot be None type")
    22         raise ValidationError
     20        raise ValidationError("Error: copy destination cannot be None type")
    2321    if not path.exists(source_file):
    24         print("Error: Copy Failure: source does not exist: ", source_file)
    25         raise ValidationError
     22        raise ValidationError(f"Error: Copy Failure: source does not exist: {source_file}")
    2623    if not path.exists(destination_path) or not path.isdir(destination_path):
    2724        print("Error: Copy Failure: destination path does not exist: ", destination_path)
    28         raise ValidationError
     25        raise ValidationError(f"Error: Copy Failure: destination path does not exist: {destination_path}")
    2926
    3027    try:
    3128        if verbose:
    32             print("Copying: {0} to {1}".format(source_file, destination_path))
     29            print(f"Copying: {source_file} to {destination_path}")
    3330        if path.isdir(source_file):
    3431            copy_tree(source_file, destination_path)
    … …  
    3633            shutil.copy2(source_file, destination_path)
    3734    except IOError as e:
    38         print("Error: copy failure.\n  copying: {0}\n  to: {1}".format(e.filename, e.filename2))
    39         print(e)
    40         raise IOError
     35        msg = f"Error: copy failure.\n  copying: {source_file}\n  to: {destination_path}"
     36        raise IOError(msg) from e
    4137    return 0
    4238
    … …  
    4844
    4945    if source_paths is None:
    50         print("Error: source for copy cannot be None type")
    51         raise ValidationError
     46        raise ValidationError("Error: source for copy cannot be None type")
    5247    if destination_paths is None:
    53         print("Error: copy destination cannot be None type")
    54         raise ValidationError
     48        raise ValidationError("Error: copy destination cannot be None type")
    5549
    5650    if type(source_paths) is not list or len(source_paths) == 0:
    57         print("Error: sources must be given as a list with at least one entry")
    58         raise ValidationError
     51        raise ValidationError("Error: sources must be given as a list with at least one entry")
    5952    if type(destination_paths) is not list or len(destination_paths) == 0:
    60         print("Error: destinations must be given as a list with at least one entry")
    61         raise ValidationError
     53        raise ValidationError("Error: destinations must be given as a list with at least one entry")
    6254
    6355    for dpath in destination_paths:
    6456        if not path.exists(dpath):
    65             print("Copy Warning: target path does not exist (skipping): ", dpath)
     57            if verbose:
     58                print("Copy Warning: target path does not exist (skipping): ", dpath)
    6659            continue
    6760        if not path.isdir(dpath):
    68             print("Copy Warning: target path is not a directory (skipping): ", )
     61            if verbose:
     62                print("Copy Warning: target path is not a directory (skipping): ", )
    6963            continue
    7064

  • branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py

    r40895 r40897  
    22
    33# Plans are to flesh this out a bit more to make some smarter copying etc
     4from generic_utils import ValidationError
    45
    56
    … …  
    1011def _args_check(args, function_name):
    1112    if args is None:
    12         print("Error: no arguments provided in ", function_name)
    13         return 2
     13        raise ValidationError(f"Error: no arguments provided in {function_name}")
    1414    if type(args) != list or len(args) == 0:
    15         print("Error: at least 1 arg must be given for ", function_name)
    16         return 2
     15        raise ValidationError(f"Error: no arguments provided in {function_name}")
    1716    return 0
    1817
    … …  
    2019def cp_wrapper(args: [str]):
    2120    if args is None:
    22         return 2
     21        raise ValidationError(f"Error: no arguments provided to {cp_wrapper.__name__}")
    2322    command_and_args = ["cp"] + args
    2423    return simple_unix_wrapper(command_and_args)
    … …  
    2726def tar_wrapper(args: [str]):
    2827    if args is None:
    29         return 2
     28        raise ValidationError(f"Error: no arguments provided to {tar_wrapper.__name__}")
    3029    command_and_args = ["tar"] + args
    3130    return simple_unix_wrapper(command_and_args)
    … …  
    3433def chmod_wrapper(args: [str]):
    3534    if args is None:
    36         return 2
     35        raise ValidationError(f"Error: no arguments provided to {chmod_wrapper.__name__}")
    3736    command_and_args = ["chmod"] + args
    3837    return simple_unix_wrapper(command_and_args)
    … …  
    4443Only a single command should be called.
    4544    """
    46     arg_check = _args_check(command_and_args, simple_unix_wrapper.__name__)
    47     if arg_check != 0:
    48         return arg_check
     45    _args_check(command_and_args, simple_unix_wrapper.__name__)
    4946
    5047    try:
    5148        subprocess.check_call(command_and_args)
    52     except subprocess.CalledProcessError as e:
    53         print("CalledProcessError in python subprocess: ")
    54         print(e)
    55         return 2
     49    except (subprocess.CalledProcessError) as cpe:
     50        raise SubprocessError(f"{type(cpe)} in python simple_unix_wrapper") from cpe
     51    except (ValidationError) as ve:
     52        raise ValidationError(f"{type(ve)} in python simple_unix_wrapper") from ve
    5653    return 0

  • branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils_test.py

    r40885 r40897  
    11import subprocess_utils as utils
    22import os
     3import pytest
     4import stat
    35
    46from pathlib import Path
     7from generic_utils import ValidationError
    58
    69
    … …  
    1518class TestCopyWrapper(object):
    1619
     20    def test_copy_raises_validation_error_with_no_args(self):
     21        with pytest.raises(ValidationError):
     22            utils.cp_wrapper(None)
     23
    1724    def test_copy_fails_with_invalid_args(self, tmpdir):
    18         result = utils.cp_wrapper(["not a real file", "not a valid copy_location"])
    19         assert result == 2
     25        with pytest.raises(utils.SubprocessError):
     26            utils.cp_wrapper(["not a real file", "not a valid copy_location"])
    2027
    2128    def test_copy_fails_if_source_does_not_exist(self, tmpdir):
    22         result = utils.cp_wrapper(["not a real file", tmpdir])
    23         assert result == 2
     29        with pytest.raises(utils.SubprocessError):
     30            utils.cp_wrapper(["not a real file", tmpdir])
    2431
    2532    def test_simple_copy(self, tmpdir):
    … …  
    6875        assert not os.path.exists(expected_filepath_b)
    6976
    70         utils.cp_wrapper([subdir, destination_dir])
     77        # fails without -r arg:
     78        with pytest.raises(utils.SubprocessError):
     79            utils.cp_wrapper([subdir, destination_dir])
    7180        assert not os.path.exists(expected_filepath_a)
    7281        assert not os.path.exists(expected_filepath_b)
     82
     83        utils.cp_wrapper(["-r", subdir, destination_dir])
     84        assert os.path.exists(expected_filepath_a)
     85        assert os.path.exists(expected_filepath_b)
    7386
    7487
    7588class TestTarWrapper(object):
    7689
    77     def test_tar_returns_safely_with_none_args(self):
    78 
    79         result = utils.tar_wrapper(None)
    80         assert result == 2
     90    def test_tar_raise_validation_error_with_none_args(self):
     91        with pytest.raises(ValidationError):
     92            utils.tar_wrapper(None)
    8193
    8294    def test_tar_returns_safely_with_too_few_args(self):
    83 
    84         result = utils.tar_wrapper(["only one arg"])
    85         assert result == 2
     95        with pytest.raises(utils.SubprocessError):
     96            utils.tar_wrapper(["only one arg"])
    8697
    8798    def test_tar_return_error_if_invalid_args_given(self, tmpdir):
    88 
    8999        tar_filename = os.path.join(tmpdir, "tar_file.tar")
    90100        fake_file = os.path.join(tmpdir, "fake_file.txt")
    91101        assert not os.path.exists(fake_file)
    92         result = utils.tar_wrapper(["-cf", tar_filename, fake_file])
    93         assert result == 2
     102        with pytest.raises(utils.SubprocessError):
     103            utils.tar_wrapper(["-cf", tar_filename, fake_file])
    94104
    95105    def test_simple_tar_of_files(self, tmpdir):
    … …  
    102112        args = ["-cf", expected_file, file_a, file_b]
    103113        utils.tar_wrapper(args)
     114        assert os.path.exists(expected_file)
    104115
    105         assert os.path.exists(expected_file)
     116
     117class TestChmodWrapper(object):
     118
     119    def test_chmod_gives_validation_error_when_no_args(self):
     120        with pytest.raises(utils.ValidationError):
     121            utils.chmod_wrapper(None)
     122
     123    def test_chmod_wrapper_raises_error_with_invalid_args(self):
     124        with pytest.raises(utils.SubprocessError):
     125            utils.chmod_wrapper(["only one arg"])
     126
     127    def test_chmod_fails_if_no_permissions_given(self, tmpdir):
     128        real_file = make_a_real_file(tmpdir, 'mr_file')
     129
     130        with pytest.raises(utils.SubprocessError):
     131            utils.chmod_wrapper([real_file])
     132
     133        with pytest.raises(utils.SubprocessError):
     134            utils.chmod_wrapper(["dumb_arg", real_file])
     135
     136    def test_can_chmod_file(self, tmpdir):
     137        real_file = make_a_real_file(tmpdir, 'mr_file')
     138
     139        # check initial permissions
     140        org_stats = os.stat(real_file)
     141        # Default should be: rw-rw-r--
     142        assert bool(org_stats.st_mode & stat.S_IRUSR) is True
     143        assert bool(org_stats.st_mode & stat.S_IWUSR) is True
     144        assert bool(org_stats.st_mode & stat.S_IXUSR) is False
     145        assert bool(org_stats.st_mode & stat.S_IRGRP) is True
     146        assert bool(org_stats.st_mode & stat.S_IWGRP) is True
     147        assert bool(org_stats.st_mode & stat.S_IXGRP) is False
     148        assert bool(org_stats.st_mode & stat.S_IROTH) is True
     149        assert bool(org_stats.st_mode & stat.S_IWOTH) is False
     150        assert bool(org_stats.st_mode & stat.S_IXOTH) is False
     151
     152        # change them
     153        utils.chmod_wrapper(["777", real_file])
     154
     155        # check they changed
     156        new_stats = os.stat(real_file)
     157        assert bool(new_stats.st_mode & stat.S_IRUSR) is True
     158        assert bool(new_stats.st_mode & stat.S_IWUSR) is True
     159        assert bool(new_stats.st_mode & stat.S_IXUSR) is True
     160        assert bool(new_stats.st_mode & stat.S_IRGRP) is True
     161        assert bool(new_stats.st_mode & stat.S_IWGRP) is True
     162        assert bool(new_stats.st_mode & stat.S_IXGRP) is True
     163        assert bool(new_stats.st_mode & stat.S_IROTH) is True
     164        assert bool(new_stats.st_mode & stat.S_IWOTH) is True
     165        assert bool(new_stats.st_mode & stat.S_IXOTH) is True
Note: See TracChangeset for help on using the changeset viewer.