Index: /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py (revision 40885) @@ -0,0 +1,358 @@ +import argparse +import datetime +import glob +import operator +import os +import os.path as path +import shutil +import sys + +# note that this replies on sym link to the 'trunk/tools/utils/python' folder existing +# in this directory with the name 'utils'. This is not an elegant solution, but +# a restructure would be needed +# Ideally, the utils should be added to the path. +import utils.subprocess_utils as sub_utils + + +DEFAULT_ATLASSIAN_HOST = "ippops4" +DEFAULT_BACKUP_MACHINE_1 = "ippops3" +DEFAULT_BACKUP_MACHINE_2 = "ippops5" +DEFAULT_BACKUP_HOST = "/export/{0}.0/atlassian_backups".format(DEFAULT_ATLASSIAN_HOST) +DEFAULT_BACKUP_1_DIR = "/data/{0}.0/atlassian_backups".format(DEFAULT_BACKUP_MACHINE_1) +DEFAULT_BACKUP_2_DIR = "/data/{0}.0/atlassian_backups".format(DEFAULT_BACKUP_MACHINE_2) + +DEFAULT_ATLASSIAN_HOME_DIR = "/var/atlassian/application-data/" +DEFAULT_JIRA_BACKUP_DIR = path.join(DEFAULT_ATLASSIAN_HOME_DIR, "jira/export") +DEFAULT_CONF_BACKUP_DIR = path.join(DEFAULT_ATLASSIAN_HOME_DIR, "confluence/backups") +DEFAULT_JIRA_ATTACHMENTS_DIR = path.join(DEFAULT_ATLASSIAN_HOME_DIR, "jira/data") +DEFAULT_CONF_ATTACHMENTS_DIR = path.join(DEFAULT_ATLASSIAN_HOME_DIR, "confluence/attachments") + +DEFAULT_VERBOSITY = False + + +def _move_dir_contents_to_dir(source_dirs: [str], destination_dirs: [str]): + """ Moves all files from inside source paths to the destination dits. +It's important to note that it moves index to index +i.e. source[0] -> dest[0]; source[1] -> dest[1] + +This is actually a pretty generic function, should be moved into a utils class. +""" + + if source_dirs is None or destination_dirs is None: + print("Error: You need to supply a source and destination") + return 2 + if type(source_dirs) != list or type(destination_dirs) != list: + print("Error: source_dirs and destination_dirs must be type: list") + print(" type {0} and {1} given".format(type(source_dirs), type(destination_dirs))) + return 2 + if len(source_dirs) != len(destination_dirs): + print("Error: source and target paths are not equal; aborting file movement") + return 2 + + for i in range(0, len(source_dirs)): + # Identify files in path, move each one: + wildcard_search = "{0}/*".format(source_dirs[i]) + filepaths = glob.glob(wildcard_search) + if len(filepaths) == 0: + print("Warning: directory is empty: ", source_dirs[i]) + continue + try: + for fp in filepaths: + print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) + command_and_args = ["mv", fp, destination_dirs[i]] + # shutil.move(fp, destination_dirs[i]) + sub_utils.simple_unix_wrapper(command_and_args) + except IOError as e: + print("IOError copying: ", e.filename, e.filename2) + return 2 + + return 0 + + +class AtlassianBackups(): + + def __init__(self, + host_backup_path=None, + backup_1_path=None, + backup_2_path=None, + jira_backup_path=None, + conf_backup_path=None, + jira_attachments_path=None, + conf_attachments_path=None, + verbose=None): + + self.host_backup_path = DEFAULT_BACKUP_HOST if host_backup_path is None else host_backup_path + self.backup_1_path = DEFAULT_BACKUP_1_DIR if backup_1_path is None else backup_1_path + self.backup_2_path = DEFAULT_BACKUP_2_DIR if backup_2_path is None else backup_2_path + self.jira_backup_path = DEFAULT_JIRA_BACKUP_DIR if jira_backup_path is None else jira_backup_path + self.conf_backup_path = DEFAULT_CONF_BACKUP_DIR if conf_backup_path is None else conf_backup_path + self.jira_attachments_path = DEFAULT_JIRA_ATTACHMENTS_DIR if jira_attachments_path is None else jira_attachments_path + self.conf_attachments_path = DEFAULT_CONF_ATTACHMENTS_DIR if conf_attachments_path is None else conf_attachments_path + self.verbose = DEFAULT_VERBOSITY if verbose is None else verbose + + host_backup_path = property(operator.attrgetter('_host_backup_path')) + + @host_backup_path.setter + def host_backup_path(self, value): + self._host_backup_path = value + + backup_1_path = property(operator.attrgetter('_backup_1_path')) + + @backup_1_path.setter + def backup_1_path(self, value): + self._backup_1_path = value + + backup_2_path = property(operator.attrgetter('_backup_2_path')) + + @backup_2_path.setter + def backup_2_path(self, value): + self._backup_2_path = value + + jira_backup_path = property(operator.attrgetter('_jira_backup_path')) + + @jira_backup_path.setter + def jira_backup_path(self, value): + self._jira_backup_path = value + + conf_backup_path = property(operator.attrgetter('_conf_backup_path')) + + @conf_backup_path.setter + def conf_backup_path(self, value): + self._conf_backup_path = value + + jira_attachments_path = property(operator.attrgetter('_jira_attachments_path')) + + @jira_attachments_path.setter + def jira_attachments_path(self, value): + self._jira_attachments_path = value + + conf_attachments_path = property(operator.attrgetter('_conf_attachments_path')) + + @conf_attachments_path.setter + def conf_attachments_path(self, value): + self._conf_attachments_path = value + + verbose = property(operator.attrgetter('_verbose')) + + @verbose.setter + def verbose(self, home_dir): + self._verbose = home_dir + + def target_backup_paths_ok(self) -> bool: + copy_paths = [ self.host_backup_path + , self.backup_1_path + , self.backup_2_path + ] + for cp in copy_paths: + if not path.exists(cp): + return False + self._make_destination_subdirs_if_needed(cp) + + print("Paths ok") + return True + + def _make_destination_subdirs_if_needed(self, copy_path): + subdirs = [ path.join(copy_path, "latest") + , path.join(copy_path, "old") + , path.join(copy_path, "tmp") + ] + for sd in subdirs: + if not path.exists(sd): + print("creating dir: ", sd) + os.makedirs(sd) + + def target_paths(self, subdir): + return [ path.join(self.host_backup_path, subdir) + , path.join(self.backup_1_path, subdir) + , path.join(self.backup_2_path, subdir) + ] + + def perform_atlassian_backups(self): + + # Check all paths are ok first: + if not self.target_backup_paths_ok(): + return 1 + + latest_paths = self.target_paths('latest') + tmp_paths = self.target_paths('tmp') + old_paths = self.target_paths('old') + + # Move latest to tmp (and delete if move ok) + _move_dir_contents_to_dir(latest_paths, tmp_paths) + + # copy jira + jira_return = self._copy_jira_backup() + + # copy confluence + conf_return = self._copy_confluence_backup() + + # tarfiles: + tar_return = self._tar_attachments() + + # make mysql dump + + # If successful: move tmp to old + if jira_return == 0 and conf_return == 0 and tar_return == 0: + _move_dir_contents_to_dir(tmp_paths, old_paths) + else: + # If any failed: delete latest, move tmp back + _move_dir_contents_to_dir(tmp_paths, latest_paths) + + # TODO:clean-up old files (to save space) + + return 0 + + def get_jira_backup_date_format(self, custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + jira_format = "%Y-%b-%d" + if custom_date is not None: + return custom_date.strftime(jira_format) + return datetime.date.today().strftime(jira_format) + + def _copy_jira_backup(self): + if not self.target_backup_paths_ok(): + return 2 + + jira_wildcard_search = "{0}/{1}*".format(self.jira_backup_path, self.get_jira_backup_date_format()) + jira_xml_filepaths = glob.glob(jira_wildcard_search) + print(len(jira_xml_filepaths)) + if len(jira_xml_filepaths) == 0: + print("Copy Failure: No jira backups found at: ", jira_wildcard_search) + return 2 + target_paths = self.target_paths('latest') + for jfile in jira_xml_filepaths: + for tpath in target_paths: + if not path.exists(tpath): + print("Copy Failure: target path does not exist:", tpath) + return 2 + try: + print("Copying: {0} to {1}".format(jfile, tpath)) + shutil.copy2(jfile, tpath) + except IOError as e: + print("Error: copying jira backup.\n copying: {0}\n to: {1}".format(e.filename, e.filename2)) + print(e) + return 2 + return 0 + + def get_confluence_backup_date_format(self, custom_date=None): + # Format is: confluence-xml-backup-2019_08_14.zip + conf_format = "%Y_%m_%d" + if custom_date is not None: + return custom_date.strftime(conf_format) + return datetime.date.today().strftime(conf_format) + + def _copy_confluence_backup(self): + if not self.target_backup_paths_ok(): + print("Copy Failure: target path failure") + return 2 + + conf_wildcard_search = "{0}/*{1}*.zip".format(self.conf_backup_path, self.get_confluence_backup_date_format()) + conf_xml_filepaths = glob.glob(conf_wildcard_search) + print(len(conf_xml_filepaths)) + if len(conf_xml_filepaths) == 0: + print("Copy Failure: No confluence backups found at: ", conf_wildcard_search) + return 2 + target_paths = self.target_paths('latest') + for cfile in conf_xml_filepaths: + for tpath in target_paths: + try: + print("using cp wrapper") + sub_utils.cp_wrapper([cfile, tpath]) + except Exception as e: + print(e) + return 2 + return 0 + + def _tar_attachments(self): + # tar jira + latest_paths = self.target_paths("latest") + if not self.target_backup_paths_ok(): + return 1 + + jira_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_jira_attachments.tar" + jira_tar_filepath = path.join(latest_paths[0], jira_tar_name) + jira_tar_args = ["-cf", jira_tar_filepath, self.jira_attachments_path] + print(jira_tar_args) + sub_utils.tar_wrapper(jira_tar_args) + # copy to backups + sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[1]]) + sub_utils.cp_wrapper([ jira_tar_filepath, latest_paths[2]]) + + # tar confluence + confluence_tar_name = datetime.date.today().strftime("%Y_%m_%d") + "_confluence_attachments.tar" + confluence_tar_filepath = path.join(latest_paths[0], confluence_tar_name) + confluence_tar_args = ["-cf", confluence_tar_filepath, self.conf_attachments_path] + print(confluence_tar_args) + sub_utils.tar_wrapper(confluence_tar_args) + # copy to backups + sub_utils.cp_wrapper([ confluence_tar_filepath, latest_paths[1]]) + sub_utils.cp_wrapper([ confluence_tar_filepath, latest_paths[2]]) + return 0 + + def make_mysql_dump(self): + # do this as the dumper user + + # make sure target file does not exist (or it will fail) + + # verify mysql checksum? + + # move the dump file to the correct locations + return 0 + + def tidy_backup_folders(self): + return 0 + + def run(self): + self.perform_atlassian_backups() + return 0 + + +def main(): + args = parse_args(sys.argv[1:]) + args_dict = vars(args) + ab = AtlassianBackups(**args_dict) + ab.run() + + +def parse_args(args): + + parser = argparse.ArgumentParser( + description='Performs backup of atlassian applications\nYou should run this on the machine hosting the atlassian applications') + + parser.add_argument('--host_backup_path', + nargs='?', + help='absolute path of where the backups should be copied to on the host', + default=DEFAULT_BACKUP_HOST) + parser.add_argument('--backup_1_dir', + nargs='?', + help='directory of where backups should be copied to', + default=DEFAULT_BACKUP_1_DIR) + parser.add_argument('--backup_2_dir', + nargs='?', + help='second directory of where backups should be copied to', + default=DEFAULT_BACKUP_1_DIR) + parser.add_argument('--jira_backup_dir', + nargs='?', + help='directory where the zipped xml backups are located', + default=DEFAULT_JIRA_BACKUP_DIR) + parser.add_argument('--conf_backup_dir', + nargs='?', + help='directory where the confluence xml backups are located', + default=DEFAULT_CONF_BACKUP_DIR) + parser.add_argument('--jira_attachments_dir', + nargs='?', + help='direcory of where the jira attachments are located', + default=DEFAULT_JIRA_ATTACHMENTS_DIR) + parser.add_argument('--conf_attachments_dir', + nargs='?', + help='directory of where the confluence attachments are located', + default=DEFAULT_CONF_ATTACHMENTS_DIR) + parser.add_argument('--verbose', '-v', + action='store_true', + help='will print some verbose messages') + + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py (revision 40885) @@ -0,0 +1,504 @@ +import backup_atlassian_applications as baa +import datetime +import os + +from pathlib import Path +from backup_atlassian_applications import AtlassianBackups + + +class TestArguments(object): + + def test_default_arguments(self): + + ab = AtlassianBackups() + assert ab.host_backup_path == "/export/ippops4.0/atlassian_backups" + assert ab.backup_1_path == "/data/ippops3.0/atlassian_backups" + assert ab.backup_2_path == "/data/ippops5.0/atlassian_backups" + assert ab.jira_backup_path == "/var/atlassian/application-data/jira/export" + assert ab.conf_backup_path == "/var/atlassian/application-data/confluence/backups" + assert ab.jira_attachments_path == "/var/atlassian/application-data/jira/data" + assert ab.conf_attachments_path == "/var/atlassian/application-data/confluence/attachments" + + def test_providing_args(self): + ab = AtlassianBackups(host_backup_path="/host/backup/path", + backup_1_path="/backup/1/path", + backup_2_path="/backup/2/path", + jira_backup_path="/jira/backup/path", + conf_backup_path="/conf/backup/path", + jira_attachments_path="/jira/attachments/path", + conf_attachments_path="/conf/attachments/path", + verbose=True) + + assert ab.host_backup_path == "/host/backup/path" + assert ab.backup_1_path == "/backup/1/path" + assert ab.backup_2_path == "/backup/2/path" + assert ab.jira_backup_path == "/jira/backup/path" + assert ab.conf_backup_path == "/conf/backup/path" + assert ab.jira_attachments_path == "/jira/attachments/path" + assert ab.conf_attachments_path == "/conf/attachments/path" + + +class TestFilepathVerifications(object): + + def test_default_target_paths_do_not_exists_in_testing(self): + ab = AtlassianBackups() + assert ab.target_backup_paths_ok() is False + + def test_an_incorrect_path_then_correct_path(self, tmpdir): + ab = AtlassianBackups() + + assert os.path.exists(ab.jira_backup_path) is False + ab.jira_backup_path = tmpdir + assert os.path.exists(ab.jira_backup_path) is True + + def test_each_path_individually(self, tmpdir): + ab = AtlassianBackups() + + assert os.path.exists(ab.host_backup_path) is False + ab.host_backup_path = tmpdir + assert os.path.exists(ab.host_backup_path) is True + assert ab.target_backup_paths_ok() is False + + assert os.path.exists(ab.backup_1_path) is False + ab.backup_1_path = tmpdir + assert os.path.exists(ab.backup_1_path) is True + assert ab.target_backup_paths_ok() is False + + assert os.path.exists(ab.backup_2_path) is False + ab.backup_2_path = tmpdir + assert os.path.exists(ab.backup_2_path) is True + # Once all backup paths are set, they are ok + assert ab.target_backup_paths_ok() is True + + assert os.path.exists(ab.jira_backup_path) is False + ab.jira_backup_path = tmpdir + assert os.path.exists(ab.jira_backup_path) is True + + assert os.path.exists(ab.conf_backup_path) is False + ab.conf_backup_path = tmpdir + assert os.path.exists(ab.conf_backup_path) is True + + assert os.path.exists(ab.jira_attachments_path) is False + ab.jira_attachments_path = tmpdir + assert os.path.exists(ab.jira_attachments_path) is True + + assert os.path.exists(ab.conf_attachments_path) is False + ab.conf_attachments_path = tmpdir + assert os.path.exists(ab.conf_attachments_path) is True + + def test_backup_subdirs_are_created_if_missing(self, tmpdir): + ab = AtlassianBackups() + ab.jira_path = tmpdir + ab.conf_path = tmpdir + + backup_paths = [ os.path.join(tmpdir, 'host') + , os.path.join(tmpdir, 'backup_1') + , os.path.join(tmpdir, 'backup_2') + ] + + for bp in backup_paths: + + # make the parent dir + os.makedirs(bp) + ab.bp = bp + assert os.path.exists(bp) + + # Expected subdirs + expected_latest = os.path.join(bp, 'latest') + expected_tmp = os.path.join(bp, 'tmp') + expected_old = os.path.join(bp, 'old') + assert not os.path.exists(expected_latest) + assert not os.path.exists(expected_tmp) + assert not os.path.exists(expected_old) + + # all_paths_ok will call subdir creation + ab._make_destination_subdirs_if_needed(bp) + assert os.path.exists(expected_latest) + assert os.path.exists(expected_tmp) + assert os.path.exists(expected_old) + + def test_subsequent_backup_subdirs_not_created_if_first_fails(self, tmpdir): + ab = AtlassianBackups() + ab.jira_path = tmpdir + ab.conf_path = tmpdir + + backup_paths = [ os.path.join(tmpdir, 'host') + , os.path.join(tmpdir, 'backup_1') + , os.path.join(tmpdir, 'backup_2') + ] + + for bp in backup_paths: + + # make the parent dir + os.makedirs(bp) + ab.bp = bp + assert os.path.exists(bp) + + # Expected subdirs + expected_latest = os.path.join(bp, 'latest') + expected_tmp = os.path.join(bp, 'tmp') + expected_old = os.path.join(bp, 'old') + assert not os.path.exists(expected_latest) + assert not os.path.exists(expected_tmp) + assert not os.path.exists(expected_old) + + # all_paths_ok will call subdir creation + ab._make_destination_subdirs_if_needed(bp) + assert os.path.exists(expected_latest) + assert os.path.exists(expected_tmp) + assert os.path.exists(expected_old) + + +class TestMovingBackups(object): + + def test_target_paths_function(self): + ab = AtlassianBackups() + + default = ab.target_paths('default_paths') + + assert default[0] == "/export/ippops4.0/atlassian_backups/default_paths" + assert default[1] == "/data/ippops3.0/atlassian_backups/default_paths" + assert default[2] == "/data/ippops5.0/atlassian_backups/default_paths" + + def test_target_paths_function_after_changing_paths(self): + ab = AtlassianBackups() + + ab.host_backup_path = "/different_path/for_host" + ab.backup_1_path = "/bk/one" + ab.backup_2_path = "/bk/two" + + another_subdir = ab.target_paths('a_different_subdir') + + assert another_subdir[0] == "/different_path/for_host/a_different_subdir" + assert another_subdir[1] == "/bk/one/a_different_subdir" + assert another_subdir[2] == "/bk/two/a_different_subdir" + + def test_move_aborts_if_source_and_destination_paths_not_equal(self): + source_paths = [ "/Just/one/path" ] + destination_paths = [ "/Two", "/paths"] + + return_is_non_zero = baa._move_dir_contents_to_dir(source_paths, destination_paths) + assert return_is_non_zero != 0 + + def test_moving_old_backups(self, tmpdir): + # Need to set up X source_dirs, and X target_subdirs + tmpdir_host = os.path.join(tmpdir, "host") + tmpdir_bak1 = os.path.join(tmpdir, "bak1") + tmpdir_bak2 = os.path.join(tmpdir, "bak2") + os.makedirs(tmpdir_host) + os.makedirs(tmpdir_bak1) + os.makedirs(tmpdir_bak2) + + # Setup some subdirectories + source_paths = [ os.path.join(tmpdir_host, 'source') + , os.path.join(tmpdir_bak1, 'source') + , os.path.join(tmpdir_bak2, 'source') + ] + destination_paths = [ os.path.join(tmpdir_host, 'destination') + , os.path.join(tmpdir_bak1, 'destination') + , os.path.join(tmpdir_bak2, 'destination') + ] + for i in range(0, len(source_paths)): + os.makedirs(source_paths[i]) + os.makedirs(destination_paths[i]) + + # Put some files in the source directories + source_host_file1 = os.path.join(source_paths[0], "host_file1.zip") + source_bak1_file1 = os.path.join(source_paths[1], "bak1_file1.txt") + source_bak1_file2 = os.path.join(source_paths[1], "bak1_file2.txt") + source_bak1_file3 = os.path.join(source_paths[1], "bak1_file3.txt") + source_bak2_file1 = os.path.join(source_paths[2], "bak2_file1.lol") + source_bak2_file2 = os.path.join(source_paths[2], "bak2_file2.lol") + Path(source_host_file1).touch() + Path(source_bak1_file1).touch() + Path(source_bak1_file2).touch() + Path(source_bak1_file3).touch() + Path(source_bak2_file1).touch() + Path(source_bak2_file2).touch() + assert os.path.exists(source_host_file1) + assert os.path.exists(source_bak1_file1) + assert os.path.exists(source_bak1_file2) + assert os.path.exists(source_bak1_file3) + assert os.path.exists(source_bak2_file1) + assert os.path.exists(source_bak2_file2) + + # Assert files do NOT exist in target dirs yet + destination_host_file1 = os.path.join(destination_paths[0], "host_file1.zip") + destination_bak1_file1 = os.path.join(destination_paths[1], "bak1_file1.txt") + destination_bak1_file2 = os.path.join(destination_paths[1], "bak1_file2.txt") + destination_bak1_file3 = os.path.join(destination_paths[1], "bak1_file3.txt") + destination_bak2_file1 = os.path.join(destination_paths[2], "bak2_file1.lol") + destination_bak2_file2 = os.path.join(destination_paths[2], "bak2_file2.lol") + assert not os.path.exists(destination_host_file1) + assert not os.path.exists(destination_bak1_file1) + assert not os.path.exists(destination_bak1_file2) + assert not os.path.exists(destination_bak1_file3) + assert not os.path.exists(destination_bak2_file1) + assert not os.path.exists(destination_bak2_file2) + + # run the move + return_result = baa._move_dir_contents_to_dir(source_paths, destination_paths) + assert return_result == 0 + + # check that source files are no longer there, but the target ones do exist + assert not os.path.exists(source_host_file1) + assert not os.path.exists(source_bak1_file1) + assert not os.path.exists(source_bak1_file2) + assert not os.path.exists(source_bak1_file3) + assert not os.path.exists(source_bak2_file1) + assert not os.path.exists(source_bak2_file2) + assert os.path.exists(destination_host_file1) + assert os.path.exists(destination_bak1_file1) + assert os.path.exists(destination_bak1_file2) + assert os.path.exists(destination_bak1_file3) + assert os.path.exists(destination_bak2_file1) + assert os.path.exists(destination_bak2_file2) + + +class TestCopyingBackups(object): + + def create_fake_paths_and_files(self, atlas_class: AtlassianBackups, directory) -> AtlassianBackups: + ab = atlas_class + + tmpdir_jira = os.path.join(directory, "jira") + tmpdir_conf = os.path.join(directory, "conf") + tmpdir_host = os.path.join(directory, "host") + tmpdir_bak1 = os.path.join(directory, "bak1") + tmpdir_bak2 = os.path.join(directory, "bak2") + os.makedirs(tmpdir_jira) + os.makedirs(tmpdir_conf) + os.makedirs(tmpdir_host) + os.makedirs(tmpdir_bak1) + os.makedirs(tmpdir_bak2) + + jira_date = ab.get_jira_backup_date_format() + jira_file = "{0}--1234.zip".format(jira_date) + jira_fullpath = os.path.join(tmpdir_jira, jira_file) + assert not os.path.exists(jira_fullpath) + Path(jira_fullpath).touch() + assert os.path.exists(jira_fullpath) + + conf_date = ab.get_confluence_backup_date_format() + conf_file = "confluence-xml-backup-{0}.zip".format(conf_date) + conf_fullpath = os.path.join(tmpdir_conf, conf_file) + assert not os.path.exists(conf_fullpath) + Path(conf_fullpath).touch() + assert os.path.exists(conf_fullpath) + + ab.jira_backup_path = tmpdir_jira + ab.conf_backup_path = tmpdir_conf + ab.host_backup_path = tmpdir_host + ab.backup_1_path = tmpdir_bak1 + ab.backup_2_path = tmpdir_bak2 + + assert os.path.exists(jira_fullpath) + assert os.path.exists(conf_fullpath) + + return ab + + def test_get_jira_backup_date_format(self): + ab = AtlassianBackups() + today = datetime.date.today() + expected_format = today.strftime("%Y-%b-%d") + assert ab.get_jira_backup_date_format() == expected_format + + def test_copying_fails_if_paths_not_present(self): + ab = AtlassianBackups() + return_value = ab._copy_jira_backup() + assert return_value != 0 + + return_value = ab._copy_confluence_backup() + assert return_value != 0 + + def test_jira_copy(self, tmpdir): + ab = AtlassianBackups() + self.create_fake_paths_and_files(ab, tmpdir) + + return_value = ab._copy_jira_backup() + assert return_value == 0 + + def test_get_confluence_backup_date_format(self): + ab = AtlassianBackups() + today = datetime.date.today() + expected_format = today.strftime("%Y_%m_%d") + assert ab.get_confluence_backup_date_format() == expected_format + + def test_confluence_copy(self, tmpdir): + ab = AtlassianBackups() + self.create_fake_paths_and_files(ab, tmpdir) + + return_value = ab._copy_confluence_backup() + assert return_value == 0 + + def test_full_copy(self, tmpdir): + ab = AtlassianBackups() + self.create_fake_paths_and_files(ab, tmpdir) + + result = ab.perform_atlassian_backups() + + assert result == 0 + + def test_atlsssian_copying_fails_when_paths_do_not_exist(self, tmpdir): + ab = AtlassianBackups() + assert not os.path.exists(ab.jira_backup_path) + assert not os.path.exists(ab.conf_backup_path) + assert not os.path.exists(ab.host_backup_path) + assert not os.path.exists(ab.backup_1_path) + assert not os.path.exists(ab.backup_2_path) + + result = ab.perform_atlassian_backups() + assert result == 1 + + +class TestTaringOfAttachments(object): + + def test_attachments_tar(self, tmpdir): + + ab = AtlassianBackups() + ab = create_all_paths_and_folders(ab, tmpdir) + + today = datetime.date.today().strftime("%Y_%m_%d") + jira_tar_name = today + "_jira_attachments.tar" + conf_tar_name = today + "_confluence_attachments.tar" + expected_jira_host_tar = os.path.join(ab.host_backup_path, "latest", jira_tar_name) + expected_conf_host_tar = os.path.join(ab.host_backup_path, "latest", conf_tar_name) + expected_jira_bak1_tar = os.path.join(ab.backup_1_path, "latest", jira_tar_name) + expected_conf_bak1_tar = os.path.join(ab.backup_1_path, "latest", conf_tar_name) + expected_jira_bak2_tar = os.path.join(ab.backup_2_path, "latest", jira_tar_name) + expected_conf_bak2_tar = os.path.join(ab.backup_2_path, "latest", conf_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_conf_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_conf_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + assert not os.path.exists(expected_conf_bak2_tar) + + tar_result = ab._tar_attachments() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_conf_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_conf_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) + assert os.path.exists(expected_conf_bak2_tar) + +# class TestMysqlBackup(object): + +# def test_mysql_dump_is_ok(self): +# assert False is True + + +# class TestTidyingOfFiles(object): + +# def test_tidying(self): +# assert False is True + +def create_all_paths_and_folders(ab_class: AtlassianBackups, tmpdir) -> AtlassianBackups: + + # Create all folders needed + + host_backup_path = os.path.join(tmpdir, "host_backup") + backup_1_path = os.path.join(tmpdir, "backup_1") + backup_2_path = os.path.join(tmpdir, "backup_2") + jira_backup_path = os.path.join(tmpdir, "jira_home/zip_in_here") + conf_backup_path = os.path.join(tmpdir, "conf_home/zip_in_here") + jira_attachments_path = os.path.join(tmpdir, "jira_home/attachments") + conf_attachments_path = os.path.join(tmpdir, "conf_home/attachments") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(conf_backup_path) + os.makedirs(jira_attachments_path) + os.makedirs(conf_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(conf_backup_path) + assert os.path.exists(jira_attachments_path) + assert os.path.exists(conf_attachments_path) + + # And the some of the subdirs (only one of the backups has all of them) + os.makedirs(os.path.join(host_backup_path, "latest")) + os.makedirs(os.path.join(backup_1_path, "old")) + os.makedirs(os.path.join(backup_2_path, "old")) + os.makedirs(os.path.join(backup_2_path, "tmp")) + os.makedirs(os.path.join(backup_2_path, "latest")) + assert os.path.exists(os.path.join(host_backup_path, "latest")) + assert os.path.exists(os.path.join(backup_1_path, "old")) + assert os.path.exists(os.path.join(backup_2_path, "old")) + assert os.path.exists(os.path.join(backup_2_path, "tmp")) + assert os.path.exists(os.path.join(backup_2_path, "latest")) + + # set them on the class + ab_class.host_backup_path = host_backup_path + ab_class.backup_1_path = backup_1_path + ab_class.backup_2_path = backup_2_path + ab_class.jira_backup_path = jira_backup_path + ab_class.conf_backup_path = conf_backup_path + ab_class.jira_attachments_path = jira_attachments_path + ab_class.conf_attachments_path = conf_attachments_path + assert ab_class.host_backup_path == os.path.join(tmpdir, "host_backup") + assert ab_class.backup_1_path == os.path.join(tmpdir, "backup_1") + assert ab_class.backup_2_path == os.path.join(tmpdir, "backup_2") + assert ab_class.jira_backup_path == os.path.join(tmpdir, "jira_home/zip_in_here") + assert ab_class.conf_backup_path == os.path.join(tmpdir, "conf_home/zip_in_here") + assert ab_class.jira_attachments_path == os.path.join(tmpdir, "jira_home/attachments") + assert ab_class.conf_attachments_path == os.path.join(tmpdir, "conf_home/attachments") + + # Create backup files + jira_backup_name = ab_class.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path + jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + conf_backup_name = ab_class.get_confluence_backup_date_format() + "_conf_test_file.zip" + conf_backup_filepath = os.path.join(conf_backup_path + conf_backup_name) + Path(conf_backup_filepath).touch() + assert os.path.exists(conf_backup_filepath) + + # Create some files that are attchments + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path + jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path + jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + conf_attachment_1_name = "attachment_1.zip" + conf_attachment_2_name = "attachment_2.txt" + conf_attachment_1_filepath = os.path.join(conf_attachments_path + conf_attachment_1_name) + conf_attachment_2_filepath = os.path.join(conf_attachments_path + conf_attachment_2_name) + Path(conf_attachment_1_filepath).touch() + Path(conf_attachment_2_filepath).touch() + assert os.path.exists(conf_attachment_1_filepath) + assert os.path.exists(conf_attachment_2_filepath) + + # Have some old and latest versions in the backup dirs: + old_date = datetime.date(2019, 8, 13) + latest_date = datetime.date(2019, 9, 1) + jira_old_backup_name = ab_class.get_jira_backup_date_format(old_date) + "_jira_old_file.zip" + jira_latest_backup_name = ab_class.get_jira_backup_date_format(latest_date) + "_jira_prev_latest_file.zip" + conf_old_backup_name = ab_class.get_confluence_backup_date_format(old_date) + "_conf_old_file.zip" + conf_latest_backup_name = ab_class.get_confluence_backup_date_format(latest_date) + "_conf_prev_latest_file.zip" + + def assert_not_create_assert_exists(root_path, subdir, filename): + filepath = os.path.join(root_path, subdir, filename) + assert not os.path.exists(filepath) + Path(filepath).touch() + assert os.path.exists(filepath) + + assert_not_create_assert_exists(host_backup_path, "latest", jira_latest_backup_name) + assert_not_create_assert_exists(host_backup_path, "latest", conf_latest_backup_name) + assert_not_create_assert_exists(backup_1_path, "old", jira_old_backup_name) + assert_not_create_assert_exists(backup_1_path, "old", conf_old_backup_name) + assert_not_create_assert_exists(backup_2_path, "old", jira_old_backup_name) + assert_not_create_assert_exists(backup_2_path, "old", conf_old_backup_name) + assert_not_create_assert_exists(backup_2_path, "latest", jira_latest_backup_name) + assert_not_create_assert_exists(backup_2_path, "latest", conf_latest_backup_name) + + return ab_class Index: /branches/ipp-132_automate_jira_conf_backups/backups/fixture_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/fixture_test.py (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/backups/fixture_test.py (revision 40885) @@ -0,0 +1,37 @@ +import pytest + + +@pytest.fixture +def mr_fixture(): + + ab = {} + + def _add_to_dict(): + return ab.update({"hiya": 12}) + + def _add_more_to_dict(): + return ab.update({"bonjour": 13}) + + def _full_creation(): + _add_to_dict() + _add_more_to_dict() + print(ab) + return ab + + yield _full_creation() + + def _full_removal(): + ab = {} + print(ab) + + _full_removal() + + +class TestFixtures(object): + + def test_loading_fixture(self, mr_fixture): + ab = mr_fixture + print(mr_fixture) + type(mr_fixture) + assert ab["hiya"] == 12 + assert ab["bonjour"] == 13 Index: /branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py (revision 40885) @@ -0,0 +1,27 @@ + +import backup_atlassian_applications_test as baat + +from backup_atlassian_applications import AtlassianBackups + + +class TestEntireProcess(object): + + def test_entire_process(self, tmpdir): + ab = AtlassianBackups() + ab = baat.create_all_paths_and_folders(ab, tmpdir) + + ab.perform_atlassian_backups() + + # Check backups in correct place + + # old backups have been moved + + # Check tar is ok (inc contents?) + + # old attachments have been moved + + # Check mysql dump + + # check old dump moved + + assert False is True Index: /branches/ipp-132_automate_jira_conf_backups/backups/utils =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/utils (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/backups/utils (revision 40885) @@ -0,0 +1,1 @@ +link ../utils/python Index: /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils.py (revision 40885) @@ -0,0 +1,34 @@ +import subprocess + +# Plans are to flesh this out a bit more to make some smarter copying etc + + +def cp_wrapper(args: [str]): + if args is None: + return 2 + command_and_args = ["cp"] + args + return simple_unix_wrapper(command_and_args) + + +def tar_wrapper(args: [str]): + if args is None: + return 2 + command_and_args = ["tar"] + args + return simple_unix_wrapper(command_and_args) + + +def simple_unix_wrapper(command_and_args: [str]): + if command_and_args is None: + print("error: unix wrapper needs some arguments") + return 2 + if type(command_and_args) != list or len(command_and_args) == 0: + print("error: unix wrapper requires at least 1 arg") + return 2 + + try: + subprocess.check_call(command_and_args) + except subprocess.CalledProcessError as e: + print("CalledProcessError in python subprocess: ") + print(e) + return 2 + return 0 Index: /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils_test.py (revision 40885) +++ /branches/ipp-132_automate_jira_conf_backups/utils/python/subprocess_utils_test.py (revision 40885) @@ -0,0 +1,105 @@ +import subprocess_utils as utils +import os + +from pathlib import Path + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyWrapper(object): + + def test_copy_fails_with_invalid_args(self, tmpdir): + result = utils.cp_wrapper(["not a real file", "not a valid copy_location"]) + assert result == 2 + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + result = utils.cp_wrapper(["not a real file", tmpdir]) + assert result == 2 + + def test_simple_copy(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.cp_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_copy_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run copy function and confirm copy exists (and original) + result = utils.cp_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.cp_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + +class TestTarWrapper(object): + + def test_tar_returns_safely_with_none_args(self): + + result = utils.tar_wrapper(None) + assert result == 2 + + def test_tar_returns_safely_with_too_few_args(self): + + result = utils.tar_wrapper(["only one arg"]) + assert result == 2 + + def test_tar_return_error_if_invalid_args_given(self, tmpdir): + + tar_filename = os.path.join(tmpdir, "tar_file.tar") + fake_file = os.path.join(tmpdir, "fake_file.txt") + assert not os.path.exists(fake_file) + result = utils.tar_wrapper(["-cf", tar_filename, fake_file]) + assert result == 2 + + def test_simple_tar_of_files(self, tmpdir): + file_a = make_a_real_file(tmpdir, "file_a.txt") + file_b = make_a_real_file(tmpdir, "file_b.txt") + + expected_file = os.path.join(tmpdir, "output.tar") + assert not os.path.exists(expected_file) + + args = ["-cf", expected_file, file_a, file_b] + utils.tar_wrapper(args) + + assert os.path.exists(expected_file)