Index: /branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils.py (revision 40895) +++ /branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils.py (revision 40895) @@ -0,0 +1,76 @@ +import os.path as path +import shutil + +from distutils.dir_util import copy_tree + + +class ValidationError(Exception): + """validation has failed in the generic utils module""" + + +def copy_to_dir(source_file: str, destination_path: str, verbose: bool=False): + """ Copies a single file into the destination directory. If a directory is + given as a source then all the contents of the directory will be copied (the + directory itself will not be). +""" + + if source_file is None or destination_path is None: + print("Error: source for copy cannot be None type") + raise ValidationError + if destination_path is None: + print("Error: copy destination cannot be None type") + raise ValidationError + if not path.exists(source_file): + print("Error: Copy Failure: source does not exist: ", source_file) + raise ValidationError + if not path.exists(destination_path) or not path.isdir(destination_path): + print("Error: Copy Failure: destination path does not exist: ", destination_path) + raise ValidationError + + try: + if verbose: + print("Copying: {0} to {1}".format(source_file, destination_path)) + if path.isdir(source_file): + copy_tree(source_file, destination_path) + else: + shutil.copy2(source_file, destination_path) + except IOError as e: + print("Error: copy failure.\n copying: {0}\n to: {1}".format(e.filename, e.filename2)) + print(e) + raise IOError + return 0 + + +def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False): + """ Copies the sources to the destination paths. + +This is a generic function that just wraps shutil.copy2""" + + if source_paths is None: + print("Error: source for copy cannot be None type") + raise ValidationError + if destination_paths is None: + print("Error: copy destination cannot be None type") + raise ValidationError + + if type(source_paths) is not list or len(source_paths) == 0: + print("Error: sources must be given as a list with at least one entry") + raise ValidationError + if type(destination_paths) is not list or len(destination_paths) == 0: + print("Error: destinations must be given as a list with at least one entry") + raise ValidationError + + for dpath in destination_paths: + if not path.exists(dpath): + print("Copy Warning: target path does not exist (skipping): ", dpath) + continue + if not path.isdir(dpath): + print("Copy Warning: target path is not a directory (skipping): ", ) + continue + + for spath in source_paths: + result = copy_to_dir(spath, dpath, verbose) + if result != 0: + return result + + return 0 Index: /branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils_test.py (revision 40895) +++ /branches/ipp-132_automate_jira_conf_backups/utils/python/generic_utils_test.py (revision 40895) @@ -0,0 +1,224 @@ +import generic_utils as utils +import os +import pytest +import subprocess + +from pathlib import Path + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyingToDir(object): + + def test_copy_fails_with_none_args(self, tmpdir): + with pytest.raises(utils.ValidationError): + utils.copy_to_dir(None, tmpdir) + + with pytest.raises(utils.ValidationError): + real_file = make_a_real_file(tmpdir, "file.txt") + utils.copy_to_dir(real_file, None) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(utils.ValidationError): + utils.copy_to_dir("not a real file", tmpdir) + + def test_copy_fails_if_destination_is_not_real(self, tmpdir): + with pytest.raises(utils.ValidationError): + real_file = make_a_real_file(tmpdir, "real_file.txt") + utils.copy_to_dir(real_file, "not a real directory") + + def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir): + + real_file = make_a_real_file(tmpdir, "not_a_directory.txt") + not_a_directory = real_file + assert os.path.exists(not_a_directory) + assert not os.path.isdir(not_a_directory) + + with pytest.raises(utils.ValidationError): + utils.copy_to_dir(real_file, not_a_directory) + + def test_copy_to_dir(self, tmpdir): + + # Setup source file and target dir + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + file_name = "fun_file.txt" + file_to_copy = os.path.join(tmpdir, file_name) + Path(file_to_copy).touch() + assert os.path.exists(file_to_copy) + + # copied file does not exist yet + expected_copy = os.path.join(tmpdir, "destination", file_name) + assert not os.path.exists(expected_copy) + + # run copy function and confirm copy exists (and original) + result = utils.copy_to_dir(file_to_copy, destination_dir) + assert result == 0 + assert os.path.exists(file_to_copy) + assert os.path.exists(expected_copy) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + filepath_a = make_a_real_file(subdir, filename_a) + filepath_b = make_a_real_file(subsubdir, filename_b) + + print(filepath_a) + print(filepath_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + print(expected_filepath_a) + print(expected_filepath_b) + print(os.path.exists(expected_filepath_a)) + if os.path.exists(expected_filepath_a): + print("apple") + else: + print("banana") + print(os.path.exists(expected_filepath_b)) + + os.path.commonpath + + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # Provide only the original tmpdir to the copying function + utils.copy_to_dir(subdir, destination_dir, True) + + # assert that the subdirs and files all exist + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + +class TestMultiCopy(object): + + def test_copy_fails_if_args_are_none(self, tmpdir): + + with pytest.raises(utils.ValidationError): + utils.copy_multiple_items_to_multiple_dirs(None, None) + + with pytest.raises(utils.ValidationError): + utils.copy_multiple_items_to_multiple_dirs(tmpdir, None) + + with pytest.raises(utils.ValidationError): + utils.copy_multiple_items_to_multiple_dirs(None, tmpdir) + + def test_copy_requires_args_to_be_in_list_format(self, tmpdir): + + not_a_list = "not a list" + with pytest.raises(utils.ValidationError): + utils.copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) + + tmpfile = make_a_real_file(tmpdir, "a_file.txt") + with pytest.raises(utils.ValidationError): + utils.copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) + + def test_copy_is_fine_with_single_items(self, tmpdir): + + tmpfile = make_a_real_file(tmpdir, "single_file.txt") + destination = os.path.join(tmpdir, "destination") + os.makedirs(destination) + expected_file = os.path.join(tmpdir, "destination", "single_file.txt") + assert not os.path.exists(expected_file) + + result = utils.copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) + assert result == 0 + assert os.path.exists(expected_file) + + def test_copy_multiple_items_to_multiple_dirs(self, tmpdir): + + file_a = make_a_real_file(tmpdir, "single_file.txt") + file_b = make_a_real_file(tmpdir, "single_file.txt") + file_c = make_a_real_file(tmpdir, "single_file.txt") + source_files = [ file_a + , file_b + , file_c + ] + + destination_1 = os.path.join(tmpdir, "dest_1") + destination_2 = os.path.join(tmpdir, "dest_2") + destination_3 = os.path.join(tmpdir, "dest_3") + dest_paths = [ destination_1 + , destination_2 + , destination_3 + ] + os.makedirs(destination_1) + os.makedirs(destination_2) + os.makedirs(destination_3) + + expected_files = [] + for dp in dest_paths: + for sf in source_files: + expected_files.append( os.path.join(dp, os.path.basename(sf))) + assert not os.path.exists(expected_files[-1]) + assert len(expected_files) == 9 + + result = utils.copy_multiple_items_to_multiple_dirs(source_files, dest_paths) + assert result == 0 + for ef in expected_files: + assert os.path.exists(ef) + + def test_copying_is_recursive(self, tmpdir): + + # make some subdirs, with sub-subdirs, and files within them + subdir_a = os.path.join(tmpdir, "subdir_a") + subsubdir_aa = os.path.join(subdir_a, "subsubdir_aa") + subsubdir_ab = os.path.join(subdir_a, "subsubdir_ab") + subdir_b = os.path.join(tmpdir, "subdir_b") + subsubdir_ba = os.path.join(subdir_b, "subsubdir_ba") + subsubdir_bb = os.path.join(subdir_b, "subsubdir_bb") + os.makedirs(subsubdir_aa) + os.makedirs(subsubdir_ab) + os.makedirs(subsubdir_ba) + os.makedirs(subsubdir_bb) + make_a_real_file(subdir_a, "file_a.txt") + make_a_real_file(subsubdir_aa, "file_aa.txt") + make_a_real_file(subsubdir_ab, "file_ab.txt") + make_a_real_file(subdir_b, "file_b.txt") + make_a_real_file(subsubdir_ba, "file_ba.txt") + make_a_real_file(subsubdir_bb, "file_bb.txt") + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir_a", "file_a.txt") + expected_filepath_aa = os.path.join(destination_dir, "subdir_a", "subsubdir_aa", "file_aa.txt") + expected_filepath_ab = os.path.join(destination_dir, "subdir_a", "subsubdir_ab", "file_ab.txt") + expected_filepath_b = os.path.join(destination_dir, "subdir_b", "file_b.txt") + expected_filepath_ba = os.path.join(destination_dir, "subdir_b", "subsubdir_ba", "file_ba.txt") + expected_filepath_bb = os.path.join(destination_dir, "subdir_b", "subsubdir_bb", "file_bb.txt") + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_aa) + assert not os.path.exists(expected_filepath_ab) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_ba) + assert not os.path.exists(expected_filepath_bb) + + # Provide only the original tmpdir to the copying function + utils.copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) + + # assert that the subdirs and files all exist + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_aa) + assert os.path.exists(expected_filepath_ab) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_ba) + assert os.path.exists(expected_filepath_bb) Index: /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py (revision 40894) +++ /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py (revision 40895) @@ -2,4 +2,18 @@ # Plans are to flesh this out a bit more to make some smarter copying etc + + +class SubprocessError(Exception): + """Errors in Popen stderr subprocess""" + + +def _args_check(args, function_name): + if args is None: + print("Error: no arguments provided in ", function_name) + return 2 + if type(args) != list or len(args) == 0: + print("Error: at least 1 arg must be given for ", function_name) + return 2 + return 0 @@ -18,11 +32,19 @@ +def chmod_wrapper(args: [str]): + if args is None: + return 2 + command_and_args = ["chmod"] + args + return simple_unix_wrapper(command_and_args) + + def simple_unix_wrapper(command_and_args: [str]): - if command_and_args is None: - print("error: unix wrapper needs some arguments") - return 2 - if type(command_and_args) != list or len(command_and_args) == 0: - print("error: unix wrapper requires at least 1 arg") - return 2 + """ This should provide the program name and all arguments in a list. + +Only a single command should be called. + """ + arg_check = _args_check(command_and_args, simple_unix_wrapper.__name__) + if arg_check != 0: + return arg_check try: