IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40967 for trunk/tools


Ignore:
Timestamp:
Oct 24, 2019, 2:41:04 PM (7 years ago)
Author:
fairlamb
Message:

Merge IPP-259 into trunk: Improved backup programs

Location:
trunk
Files:
8 deleted
6 edited
25 copied

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/tools

  • trunk/tools/backups/utils/generic_utils.py

    r40898 r40967  
     1import glob
    12import os.path as path
    23import shutil
     4from distutils.dir_util import copy_tree
    35
    4 from distutils.dir_util import copy_tree
    5 from backups.utils.errors import ValidationError
     6from .subprocess_utils import simple_unix_wrapper
     7from .errors import ValidationError
    68
    79
    … …  
    3537
    3638
     39def _list_checks(sources: [str], targets: [str]):
     40    if sources is None or targets is None:
     41        raise ValidationError("Error: source for copy cannot be None type")
     42    if targets is None:
     43        raise ValidationError("Error: copy destination cannot be None type")
     44    if type(sources) is not list or len(sources) == 0:
     45        raise ValidationError("Error: sources must be given as a list with at least one entry")
     46    if type(targets) is not list or len(targets) == 0:
     47        raise ValidationError("Error: targets must be given as a list with at least one entry")
     48
     49
     50def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False):
     51    _list_checks()
     52
     53    for i in range(0, len(source_paths)):
     54
     55        if not path.exists(destination_paths[i]):
     56            if verbose:
     57                print("Copy Warning: target path does not exist (skipping): ", destination_paths[i])
     58            continue
     59        if not path.isdir(destination_paths[i]):
     60            if verbose:
     61                print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i])
     62            continue
     63
     64        result = copy_to_dir(source_paths[i], destination_paths[i], verbose)
     65        if result != 0:
     66            return result
     67
     68
    3769def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False):
    3870    """ Copies the sources to the destination paths.
    3971
    4072This is a generic function that just wraps shutil.copy2"""
    41 
    42     if source_paths is None:
    43         raise ValidationError("Error: source for copy cannot be None type")
    44     if destination_paths is None:
    45         raise ValidationError("Error: copy destination cannot be None type")
    46 
    47     if type(source_paths) is not list or len(source_paths) == 0:
    48         raise ValidationError("Error: sources must be given as a list with at least one entry")
    49     if type(destination_paths) is not list or len(destination_paths) == 0:
    50         raise ValidationError("Error: destinations must be given as a list with at least one entry")
     73    _list_checks(source_paths, destination_paths)
    5174
    5275    for dpath in destination_paths:
    … …  
    6689
    6790    return 0
     91
     92
     93def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False):
     94    """ Moves all files from inside source paths to the destination dits.
     95It's important to note that it moves index to index
     96i.e. source[0] -> dest[0]; source[1] -> dest[1]
     97
     98This is actually a pretty generic function, should be moved into a utils class.
     99"""
     100
     101    _list_checks(source_dirs, destination_dirs)
     102
     103    if len(source_dirs) != len(destination_dirs):
     104        raise ValidationError("Error: source and target paths are not equal; aborting file movement")
     105
     106    for i in range(0, len(source_dirs)):
     107        # Identify files in path, move each one:
     108        wildcard_search = "{0}/*".format(source_dirs[i])
     109        filepaths = glob.glob(wildcard_search)
     110        if len(filepaths) == 0:
     111            print("Warning: directory is empty: ", source_dirs[i])
     112            continue
     113        try:
     114            for fp in filepaths:
     115                if verbose:
     116                    print("Moving: {0} -> {1}".format(fp, destination_dirs[i]))
     117                command_and_args = ["mv", fp, destination_dirs[i]]
     118                # shutil.move(fp, destination_dirs[i])
     119                simple_unix_wrapper(command_and_args)
     120        except IOError as e:
     121            print("IOError copying: ", e.filename, e.filename2)
     122            return 2
     123
     124    return 0

  • trunk/tools/backups/utils/generic_utils_test.py

    r40898 r40967  
    1 import backups.utils.generic_utils as utils
    21import os
    32import pytest
    43
    5 from backups.utils.errors import ValidationError
     4from .generic_utils import *
     5from .errors import ValidationError
    66from pathlib import Path
    77
    … …  
    1919    def test_copy_fails_with_none_args(self, tmpdir):
    2020        with pytest.raises(ValidationError):
    21             utils.copy_to_dir(None, tmpdir)
     21            copy_to_dir(None, tmpdir)
    2222
    2323        with pytest.raises(ValidationError):
    2424            real_file = make_a_real_file(tmpdir, "file.txt")
    25             utils.copy_to_dir(real_file, None)
     25            copy_to_dir(real_file, None)
    2626
    2727    def test_copy_fails_if_source_does_not_exist(self, tmpdir):
    2828        with pytest.raises(ValidationError):
    29             utils.copy_to_dir("not a real file", tmpdir)
     29            copy_to_dir("not a real file", tmpdir)
    3030
    3131    def test_copy_fails_if_destination_is_not_real(self, tmpdir):
    3232        with pytest.raises(ValidationError):
    3333            real_file = make_a_real_file(tmpdir, "real_file.txt")
    34             utils.copy_to_dir(real_file, "not a real directory")
     34            copy_to_dir(real_file, "not a real directory")
    3535
    3636    def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir):
    … …  
    4242
    4343        with pytest.raises(ValidationError):
    44             utils.copy_to_dir(real_file, not_a_directory)
     44            copy_to_dir(real_file, not_a_directory)
    4545
    4646    def test_copy_to_dir(self, tmpdir):
    … …  
    6060
    6161        # run copy function and confirm copy exists (and original)
    62         result = utils.copy_to_dir(file_to_copy, destination_dir)
     62        result = copy_to_dir(file_to_copy, destination_dir)
    6363        assert result == 0
    6464        assert os.path.exists(file_to_copy)
    … …  
    9898
    9999        # Provide only the original tmpdir to the copying function
    100         utils.copy_to_dir(subdir, destination_dir, True)
     100        copy_to_dir(subdir, destination_dir, True)
    101101
    102102        # assert that the subdirs and files all exist
    … …  
    110110
    111111        with pytest.raises(ValidationError):
    112             utils.copy_multiple_items_to_multiple_dirs(None, None)
    113 
    114         with pytest.raises(ValidationError):
    115             utils.copy_multiple_items_to_multiple_dirs(tmpdir, None)
    116 
    117         with pytest.raises(ValidationError):
    118             utils.copy_multiple_items_to_multiple_dirs(None, tmpdir)
     112            copy_multiple_items_to_multiple_dirs(None, None)
     113
     114        with pytest.raises(ValidationError):
     115            copy_multiple_items_to_multiple_dirs(tmpdir, None)
     116
     117        with pytest.raises(ValidationError):
     118            copy_multiple_items_to_multiple_dirs(None, tmpdir)
    119119
    120120    def test_copy_requires_args_to_be_in_list_format(self, tmpdir):
    … …  
    122122        not_a_list = "not a list"
    123123        with pytest.raises(ValidationError):
    124             utils.copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir)
     124            copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir)
    125125
    126126        tmpfile = make_a_real_file(tmpdir, "a_file.txt")
    127127        with pytest.raises(ValidationError):
    128             utils.copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list)
     128            copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list)
    129129
    130130    def test_copy_is_fine_with_single_items(self, tmpdir):
    … …  
    136136        assert not os.path.exists(expected_file)
    137137
    138         result = utils.copy_multiple_items_to_multiple_dirs([tmpfile], [destination])
     138        result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination])
    139139        assert result == 0
    140140        assert os.path.exists(expected_file)
    … …  
    168168        assert len(expected_files) == 9
    169169
    170         result = utils.copy_multiple_items_to_multiple_dirs(source_files, dest_paths)
     170        result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths)
    171171        assert result == 0
    172172        for ef in expected_files:
    … …  
    212212
    213213        # Provide only the original tmpdir to the copying function
    214         utils.copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True)
     214        copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True)
    215215
    216216        # assert that the subdirs and files all exist

  • trunk/tools/backups/utils/subprocess_utils.py

    r40898 r40967  
    22
    33# Plans are to flesh this out a bit more to make some smarter copying etc
    4 from backups.utils.errors import ValidationError, SubprocessError
     4from .errors import ValidationError, SubprocessError
    55
    66
    … …  
    3434
    3535
     36def local_rsync_wrapper(args: [str]):
     37    if args is None:
     38        raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}")
     39    command_and_args = ["rsync"] + args
     40    return simple_unix_wrapper(command_and_args)
     41
     42
    3643def simple_unix_wrapper(command_and_args: [str]):
    3744    """ This should provide the program name and all arguments in a list.
    … …  
    4249
    4350    try:
    44         subprocess.check_call(command_and_args)
     51        subprocess_exitcode = subprocess.check_call(command_and_args)
    4552    except (subprocess.CalledProcessError) as cpe:
    4653        raise SubprocessError(f"{type(cpe)} in python simple_unix_wrapper") from cpe
    4754    except (ValidationError) as ve:
    4855        raise ValidationError(f"{type(ve)} in python simple_unix_wrapper") from ve
    49     return 0
     56    return subprocess_exitcode

  • trunk/tools/backups/utils/subprocess_utils_test.py

    r40898 r40967  
    55
    66from pathlib import Path
    7 from backups.utils.errors import SubprocessError, ValidationError
     7from .errors import SubprocessError, ValidationError
    88
    99
    … …  
    164164        assert bool(new_stats.st_mode & stat.S_IWOTH) is True
    165165        assert bool(new_stats.st_mode & stat.S_IXOTH) is True
     166
     167
     168class TestLocalRsyncWrapper(object):
     169    """This is essentially a copy of the 'copy' tests above.
     170    """
     171
     172    def test_rsync_raises_validation_error_with_no_args(self):
     173        with pytest.raises(ValidationError):
     174            utils.local_rsync_wrapper(None)
     175
     176    def test_rsync_fails_with_invalid_args(self, tmpdir):
     177        with pytest.raises(SubprocessError):
     178            utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"])
     179
     180    def test_rsync_fails_if_source_does_not_exist(self, tmpdir):
     181        with pytest.raises(SubprocessError):
     182            utils.local_rsync_wrapper(["not a real file", tmpdir])
     183
     184    def test_simple_rsync(self, tmpdir):
     185        real_file = make_a_real_file(tmpdir, "real_file.txt")
     186        expected_filepath = os.path.join(tmpdir, "copied_file.txt")
     187        assert not os.path.isfile(expected_filepath)
     188
     189        result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath])
     190        assert os.path.isfile(expected_filepath)
     191        assert result == 0
     192
     193    def test_simple_rsync_to_directory(self, tmpdir):
     194
     195        # Setup source file and target dir
     196        filename = "real_file.txt"
     197        real_file = make_a_real_file(tmpdir, filename)
     198
     199        destination_dir = os.path.join(tmpdir, "destination")
     200        os.makedirs(destination_dir)
     201        assert os.path.exists(destination_dir)
     202
     203        expected_filepath = os.path.join(destination_dir, filename)
     204        assert not os.path.exists(expected_filepath)
     205
     206        # run rsync function and confirm rsync exists (and original)
     207        result = utils.local_rsync_wrapper([real_file, destination_dir])
     208        assert result == 0
     209        assert os.path.isfile(real_file)
     210        assert os.path.exists(expected_filepath)
     211
     212    def test_rsyncing_directories_and_subdirectories(self, tmpdir):
     213
     214        subdir = os.path.join(tmpdir, "subdir")
     215        subsubdir = os.path.join(subdir, "subsubdir")
     216        os.makedirs(subsubdir)
     217        filename_a = "file_in_subdir.txt"
     218        filename_b = "file_in_subsubdir.txt"
     219        make_a_real_file(subdir, filename_a)
     220        make_a_real_file(subsubdir, filename_b)
     221
     222        destination_dir = os.path.join(tmpdir, "destination")
     223        os.makedirs(destination_dir)
     224        expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a)
     225        expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b)
     226        assert not os.path.exists(expected_filepath_a)
     227        assert not os.path.exists(expected_filepath_b)
     228
     229        # silently passes without -r, just skips over directories
     230        assert not os.path.exists(expected_filepath_a)
     231        assert not os.path.exists(expected_filepath_b)
     232
     233        utils.local_rsync_wrapper([subdir, destination_dir])
     234        assert not os.path.exists(expected_filepath_a)
     235        assert not os.path.exists(expected_filepath_b)
     236
     237        # requires -r to copy directories (and contents)
     238        utils.local_rsync_wrapper(["-r", subdir, destination_dir])
     239        assert os.path.exists(expected_filepath_a)
     240        assert os.path.exists(expected_filepath_b)
Note: See TracChangeset for help on using the changeset viewer.