Index: /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py (revision 40891) +++ /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications.py (revision 40892) @@ -6,4 +6,5 @@ import os.path as path import shutil +import subprocess import sys @@ -29,4 +30,8 @@ DEFAULT_VERBOSITY = False + + +class SubprocessError(Exception): + """Errors in Popen stderr subprocess""" @@ -198,4 +203,5 @@ # If any failed: delete latest, move tmp back _move_dir_contents_to_dir(tmp_paths, latest_paths) + return 5 # TODO:clean-up old files (to save space) @@ -290,12 +296,45 @@ return 0 - def make_mysql_dump(self): + def make_mysql_dump(self, dump_password, dump_name=None): # do this as the dumper user + latest_paths = self.target_paths("latest") + if not self.target_backup_paths_ok(): + return 1 + # make sure target file does not exist (or it will fail) - - # verify mysql checksum? + if dump_name is None: + date = datetime.date.today().strftime("%Y_%m_%d") + dump_filepath = path.join(latest_paths[0], '{0}_mysql_dump.sql'.format(date)) + else: + dump_filepath = path.join(latest_paths[0], dump_name) + if path.exists(dump_filepath): + print("Warning: SQL dump already exists: ", dump_filepath) + return 2 + + # Setup the commands: + mysqldump_args = [ 'mysqldump', '-u', 'dumper', '-p{0}'.format(dump_password) + , '--databases', 'jiradb', 'confluencedb' + ] + with open(dump_filepath, "w+") as df: + mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) + dump_result = mysqldump_process.communicate() + + # Get the piped std err. See it contains the word error + stdo, stde = dump_result + if stde is not None: + decoded_stderr = stde.decode() + if decoded_stderr.casefold().find('error'.casefold()) > -1: + raise SubprocessError() # move the dump file to the correct locations + if not path.exists(dump_filepath): + return 2 + cp_1 = sub_utils.cp_wrapper([dump_filepath, latest_paths[1]]) + cp_2 = sub_utils.cp_wrapper([dump_filepath, latest_paths[2]]) + if cp_1 != 0: + return cp_1 + if cp_2 != 0: + return cp_2 return 0 Index: /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py (revision 40891) +++ /branches/ipp-132_automate_jira_conf_backups/backups/backup_atlassian_applications_test.py (revision 40892) @@ -2,6 +2,10 @@ import datetime import os +import pytest from pathlib import Path + +import full_atlassian_backup_test as fab_test + from backup_atlassian_applications import AtlassianBackups @@ -118,39 +122,8 @@ assert os.path.exists(expected_old) - def test_subsequent_backup_subdirs_not_created_if_first_fails(self, tmpdir): - ab = AtlassianBackups() - ab.jira_path = tmpdir - ab.conf_path = tmpdir - - backup_paths = [ os.path.join(tmpdir, 'host') - , os.path.join(tmpdir, 'backup_1') - , os.path.join(tmpdir, 'backup_2') - ] - - for bp in backup_paths: - - # make the parent dir - os.makedirs(bp) - ab.bp = bp - assert os.path.exists(bp) - - # Expected subdirs - expected_latest = os.path.join(bp, 'latest') - expected_tmp = os.path.join(bp, 'tmp') - expected_old = os.path.join(bp, 'old') - assert not os.path.exists(expected_latest) - assert not os.path.exists(expected_tmp) - assert not os.path.exists(expected_old) - - # all_paths_ok will call subdir creation - ab._make_destination_subdirs_if_needed(bp) - assert os.path.exists(expected_latest) - assert os.path.exists(expected_tmp) - assert os.path.exists(expected_old) - class TestMovingBackups(object): - def test_target_paths_function(self): + def test_target_paths_with_defaults_function(self): ab = AtlassianBackups() @@ -174,5 +147,5 @@ assert another_subdir[2] == "/bk/two/a_different_subdir" - def test_move_aborts_if_source_and_destination_paths_not_equal(self): + def test_move_aborts_if_source_and_destination_path_lists_are_different_lengths(self): source_paths = [ "/Just/one/path" ] destination_paths = [ "/Two", "/paths"] @@ -258,43 +231,4 @@ class TestCopyingBackups(object): - def create_fake_paths_and_files(self, atlas_class: AtlassianBackups, directory) -> AtlassianBackups: - ab = atlas_class - - tmpdir_jira = os.path.join(directory, "jira") - tmpdir_conf = os.path.join(directory, "conf") - tmpdir_host = os.path.join(directory, "host") - tmpdir_bak1 = os.path.join(directory, "bak1") - tmpdir_bak2 = os.path.join(directory, "bak2") - os.makedirs(tmpdir_jira) - os.makedirs(tmpdir_conf) - os.makedirs(tmpdir_host) - os.makedirs(tmpdir_bak1) - os.makedirs(tmpdir_bak2) - - jira_date = ab.get_jira_backup_date_format() - jira_file = "{0}--1234.zip".format(jira_date) - jira_fullpath = os.path.join(tmpdir_jira, jira_file) - assert not os.path.exists(jira_fullpath) - Path(jira_fullpath).touch() - assert os.path.exists(jira_fullpath) - - conf_date = ab.get_confluence_backup_date_format() - conf_file = "confluence-xml-backup-{0}.zip".format(conf_date) - conf_fullpath = os.path.join(tmpdir_conf, conf_file) - assert not os.path.exists(conf_fullpath) - Path(conf_fullpath).touch() - assert os.path.exists(conf_fullpath) - - ab.jira_backup_path = tmpdir_jira - ab.conf_backup_path = tmpdir_conf - ab.host_backup_path = tmpdir_host - ab.backup_1_path = tmpdir_bak1 - ab.backup_2_path = tmpdir_bak2 - - assert os.path.exists(jira_fullpath) - assert os.path.exists(conf_fullpath) - - return ab - def test_get_jira_backup_date_format(self): ab = AtlassianBackups() @@ -313,5 +247,5 @@ def test_jira_copy(self, tmpdir): ab = AtlassianBackups() - self.create_fake_paths_and_files(ab, tmpdir) + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) return_value = ab._copy_jira_backup() @@ -326,5 +260,5 @@ def test_confluence_copy(self, tmpdir): ab = AtlassianBackups() - self.create_fake_paths_and_files(ab, tmpdir) + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) return_value = ab._copy_confluence_backup() @@ -333,5 +267,5 @@ def test_full_copy(self, tmpdir): ab = AtlassianBackups() - self.create_fake_paths_and_files(ab, tmpdir) + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) result = ab.perform_atlassian_backups() @@ -341,4 +275,10 @@ def test_atlsssian_copying_fails_when_paths_do_not_exist(self, tmpdir): ab = AtlassianBackups() + fake_path = "/not/a/real/path" + ab.jira_backup_path = fake_path + ab.conf_backup_path = fake_path + ab.host_backup_path = fake_path + ab.backup_1_path = fake_path + ab.backup_2_path = fake_path assert not os.path.exists(ab.jira_backup_path) assert not os.path.exists(ab.conf_backup_path) @@ -356,5 +296,5 @@ ab = AtlassianBackups() - ab = create_all_paths_and_folders(ab, tmpdir) + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) today = datetime.date.today().strftime("%Y_%m_%d") @@ -384,9 +324,52 @@ assert os.path.exists(expected_conf_bak2_tar) -# class TestMysqlBackup(object): - -# def test_mysql_dump_is_ok(self): -# assert False is True - + +@pytest.mark.skip(reason="Requires a fake database to be setup") +class TestMysqlBackup(object): + """ The mySQL dump tests require the machine on which they are +running to have a mysql database running with a user called +'dumper' and the password listed below. + +UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; + +Note that the actual password used for the backups is not included here, +or in the main backup code for security. +""" + + dumper_password = "not_a_real_password" + + def test_mysql_dump_is_ok(self, tmpdir): + ab = AtlassianBackups() + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) + + date = datetime.date.today().strftime("%Y_%m_%d") + expected_dump = os.path.join(ab.host_backup_path, "latest", '{0}_mysql_dump.sql'.format(date)) + print(expected_dump) + assert not os.path.exists(expected_dump) + + mysqldump_result = ab.make_mysql_dump(self.dumper_password) + assert mysqldump_result == 0 + + assert os.path.exists(expected_dump) + + def test_mysql_dump_is_ok_to_custom_file(self, tmpdir): + ab = AtlassianBackups() + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) + + expected_dump = os.path.join(ab.host_backup_path, "latest", "humptydumpy.sql") + print(expected_dump) + assert not os.path.exists(expected_dump) + + mysqldump_result = ab.make_mysql_dump(self.dumper_password, "humptydumpy.sql") + assert mysqldump_result == 0 + + assert os.path.exists(expected_dump) + + def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): + ab = AtlassianBackups() + ab = fab_test.create_all_paths_and_folders(ab, tmpdir) + + with pytest.raises(baa.SubprocessError): + ab.make_mysql_dump("Not the correct password") # class TestTidyingOfFiles(object): @@ -394,111 +377,2 @@ # def test_tidying(self): # assert False is True - -def create_all_paths_and_folders(ab_class: AtlassianBackups, tmpdir) -> AtlassianBackups: - - # Create all folders needed - - host_backup_path = os.path.join(tmpdir, "host_backup") - backup_1_path = os.path.join(tmpdir, "backup_1") - backup_2_path = os.path.join(tmpdir, "backup_2") - jira_backup_path = os.path.join(tmpdir, "jira_home/zip_in_here") - conf_backup_path = os.path.join(tmpdir, "conf_home/zip_in_here") - jira_attachments_path = os.path.join(tmpdir, "jira_home/attachments") - conf_attachments_path = os.path.join(tmpdir, "conf_home/attachments") - os.makedirs(host_backup_path) - os.makedirs(backup_1_path) - os.makedirs(backup_2_path) - os.makedirs(jira_backup_path) - os.makedirs(conf_backup_path) - os.makedirs(jira_attachments_path) - os.makedirs(conf_attachments_path) - assert os.path.exists(host_backup_path) - assert os.path.exists(backup_1_path) - assert os.path.exists(backup_2_path) - assert os.path.exists(jira_backup_path) - assert os.path.exists(conf_backup_path) - assert os.path.exists(jira_attachments_path) - assert os.path.exists(conf_attachments_path) - - # And the some of the subdirs (only one of the backups has all of them) - os.makedirs(os.path.join(host_backup_path, "latest")) - os.makedirs(os.path.join(backup_1_path, "old")) - os.makedirs(os.path.join(backup_2_path, "old")) - os.makedirs(os.path.join(backup_2_path, "tmp")) - os.makedirs(os.path.join(backup_2_path, "latest")) - assert os.path.exists(os.path.join(host_backup_path, "latest")) - assert os.path.exists(os.path.join(backup_1_path, "old")) - assert os.path.exists(os.path.join(backup_2_path, "old")) - assert os.path.exists(os.path.join(backup_2_path, "tmp")) - assert os.path.exists(os.path.join(backup_2_path, "latest")) - - # set them on the class - ab_class.host_backup_path = host_backup_path - ab_class.backup_1_path = backup_1_path - ab_class.backup_2_path = backup_2_path - ab_class.jira_backup_path = jira_backup_path - ab_class.conf_backup_path = conf_backup_path - ab_class.jira_attachments_path = jira_attachments_path - ab_class.conf_attachments_path = conf_attachments_path - assert ab_class.host_backup_path == os.path.join(tmpdir, "host_backup") - assert ab_class.backup_1_path == os.path.join(tmpdir, "backup_1") - assert ab_class.backup_2_path == os.path.join(tmpdir, "backup_2") - assert ab_class.jira_backup_path == os.path.join(tmpdir, "jira_home/zip_in_here") - assert ab_class.conf_backup_path == os.path.join(tmpdir, "conf_home/zip_in_here") - assert ab_class.jira_attachments_path == os.path.join(tmpdir, "jira_home/attachments") - assert ab_class.conf_attachments_path == os.path.join(tmpdir, "conf_home/attachments") - - # Create backup files - jira_backup_name = ab_class.get_jira_backup_date_format() + "_jira_test_file.zip" - jira_backup_filepath = os.path.join(jira_backup_path + jira_backup_name) - Path(jira_backup_filepath).touch() - assert os.path.exists(jira_backup_filepath) - - conf_backup_name = ab_class.get_confluence_backup_date_format() + "_conf_test_file.zip" - conf_backup_filepath = os.path.join(conf_backup_path + conf_backup_name) - Path(conf_backup_filepath).touch() - assert os.path.exists(conf_backup_filepath) - - # Create some files that are attchments - jira_attachment_1_name = "attachment_1.zip" - jira_attachment_2_name = "attachment_2.txt" - jira_attachment_1_filepath = os.path.join(jira_attachments_path + jira_attachment_1_name) - jira_attachment_2_filepath = os.path.join(jira_attachments_path + jira_attachment_2_name) - Path(jira_attachment_1_filepath).touch() - Path(jira_attachment_2_filepath).touch() - assert os.path.exists(jira_attachment_1_filepath) - assert os.path.exists(jira_attachment_2_filepath) - - conf_attachment_1_name = "attachment_1.zip" - conf_attachment_2_name = "attachment_2.txt" - conf_attachment_1_filepath = os.path.join(conf_attachments_path + conf_attachment_1_name) - conf_attachment_2_filepath = os.path.join(conf_attachments_path + conf_attachment_2_name) - Path(conf_attachment_1_filepath).touch() - Path(conf_attachment_2_filepath).touch() - assert os.path.exists(conf_attachment_1_filepath) - assert os.path.exists(conf_attachment_2_filepath) - - # Have some old and latest versions in the backup dirs: - old_date = datetime.date(2019, 8, 13) - latest_date = datetime.date(2019, 9, 1) - jira_old_backup_name = ab_class.get_jira_backup_date_format(old_date) + "_jira_old_file.zip" - jira_latest_backup_name = ab_class.get_jira_backup_date_format(latest_date) + "_jira_prev_latest_file.zip" - conf_old_backup_name = ab_class.get_confluence_backup_date_format(old_date) + "_conf_old_file.zip" - conf_latest_backup_name = ab_class.get_confluence_backup_date_format(latest_date) + "_conf_prev_latest_file.zip" - - def assert_not_create_assert_exists(root_path, subdir, filename): - filepath = os.path.join(root_path, subdir, filename) - assert not os.path.exists(filepath) - Path(filepath).touch() - assert os.path.exists(filepath) - - assert_not_create_assert_exists(host_backup_path, "latest", jira_latest_backup_name) - assert_not_create_assert_exists(host_backup_path, "latest", conf_latest_backup_name) - assert_not_create_assert_exists(backup_1_path, "old", jira_old_backup_name) - assert_not_create_assert_exists(backup_1_path, "old", conf_old_backup_name) - assert_not_create_assert_exists(backup_2_path, "old", jira_old_backup_name) - assert_not_create_assert_exists(backup_2_path, "old", conf_old_backup_name) - assert_not_create_assert_exists(backup_2_path, "latest", jira_latest_backup_name) - assert_not_create_assert_exists(backup_2_path, "latest", conf_latest_backup_name) - - return ab_class Index: /branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py =================================================================== --- /branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py (revision 40891) +++ /branches/ipp-132_automate_jira_conf_backups/backups/full_atlassian_backup_test.py (revision 40892) @@ -1,6 +1,135 @@ +import datetime +import os -import backup_atlassian_applications_test as baat +from pathlib import Path from backup_atlassian_applications import AtlassianBackups + + +def create_all_paths_and_folders(ab_class: AtlassianBackups, tmpdir) -> AtlassianBackups: + """This creates all paths and files needed for thorough testing of the +atlassian backup class. The class returned will have all the paths set too. +Assertions are made that the files exist and they are set on the class object. + + The directory strucutre is as follows: + + tmpdir/host_backup/latest/2019-09-01_jira_prev_latest_file.zip + /2019_09_01_conf_prev_latest_file.zip + /backup_1/old/2019-08-13_jira_old_file.zip + /2019_08_13_conf_old_file.zip + /backup_2/latest/2019-09-01_jira_prev_latest_file.zip + /2019_09_01_conf_prev_latest_file.zip + /old/2019-08-13_jira_old_file.zip + /2019_08_13_conf_old_file.zip + /tmp + + # Directories where the original files come from (date will be todays) + tmpdir/jira_home/zip_in_here/2019-11-23_jira_test_file.zip + /attachments/attachment_1.zip + /attachments/attachment_2.txt + tmpdir/conf_home/zip_in_here/2019_11_23_conf_test_file.zip + /attachments/attachment_1.zip + /attachments/attachment_2.txt + +""" + + # Create all main paths required + host_backup_path = os.path.join(tmpdir, "host_backup") + backup_1_path = os.path.join(tmpdir, "backup_1") + backup_2_path = os.path.join(tmpdir, "backup_2") + jira_backup_path = os.path.join(tmpdir, "jira_home", "zip_in_here") + conf_backup_path = os.path.join(tmpdir, "conf_home", "zip_in_here") + jira_attachments_path = os.path.join(tmpdir, "jira_home", "attachments") + conf_attachments_path = os.path.join(tmpdir, "conf_home", "attachments") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(conf_backup_path) + os.makedirs(jira_attachments_path) + os.makedirs(conf_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(conf_backup_path) + assert os.path.exists(jira_attachments_path) + assert os.path.exists(conf_attachments_path) + + # And the some of the subdirs (only one of the backups has all of them) + os.makedirs(os.path.join(host_backup_path, "latest")) + os.makedirs(os.path.join(backup_1_path, "old")) + os.makedirs(os.path.join(backup_2_path, "old")) + os.makedirs(os.path.join(backup_2_path, "tmp")) + os.makedirs(os.path.join(backup_2_path, "latest")) + assert os.path.exists(os.path.join(host_backup_path, "latest")) + assert os.path.exists(os.path.join(backup_1_path, "old")) + assert os.path.exists(os.path.join(backup_2_path, "old")) + assert os.path.exists(os.path.join(backup_2_path, "tmp")) + assert os.path.exists(os.path.join(backup_2_path, "latest")) + + # set them on the class + ab_class.host_backup_path = host_backup_path + ab_class.backup_1_path = backup_1_path + ab_class.backup_2_path = backup_2_path + ab_class.jira_backup_path = jira_backup_path + ab_class.conf_backup_path = conf_backup_path + ab_class.jira_attachments_path = jira_attachments_path + ab_class.conf_attachments_path = conf_attachments_path + + # Create backup files + jira_backup_name = ab_class.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + conf_backup_name = ab_class.get_confluence_backup_date_format() + "_conf_test_file.zip" + conf_backup_filepath = os.path.join(conf_backup_path, conf_backup_name) + Path(conf_backup_filepath).touch() + assert os.path.exists(conf_backup_filepath) + + # Create some files that are attchments + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + conf_attachment_1_name = "attachment_1.zip" + conf_attachment_2_name = "attachment_2.txt" + conf_attachment_1_filepath = os.path.join(conf_attachments_path, conf_attachment_1_name) + conf_attachment_2_filepath = os.path.join(conf_attachments_path, conf_attachment_2_name) + Path(conf_attachment_1_filepath).touch() + Path(conf_attachment_2_filepath).touch() + assert os.path.exists(conf_attachment_1_filepath) + assert os.path.exists(conf_attachment_2_filepath) + + # Have some old and latest versions in the backup dirs: + old_date = datetime.date(2019, 8, 13) + latest_date = datetime.date(2019, 9, 1) + jira_old_backup_name = ab_class.get_jira_backup_date_format(old_date) + "_jira_old_file.zip" + jira_latest_backup_name = ab_class.get_jira_backup_date_format(latest_date) + "_jira_prev_latest_file.zip" + conf_old_backup_name = ab_class.get_confluence_backup_date_format(old_date) + "_conf_old_file.zip" + conf_latest_backup_name = ab_class.get_confluence_backup_date_format(latest_date) + "_conf_prev_latest_file.zip" + + def assert_not_create_assert_exists(root_path, subdir, filename): + filepath = os.path.join(root_path, subdir, filename) + assert not os.path.exists(filepath) + Path(filepath).touch() + assert os.path.exists(filepath) + + assert_not_create_assert_exists(host_backup_path, "latest", jira_latest_backup_name) + assert_not_create_assert_exists(host_backup_path, "latest", conf_latest_backup_name) + assert_not_create_assert_exists(backup_1_path, "old", jira_old_backup_name) + assert_not_create_assert_exists(backup_1_path, "old", conf_old_backup_name) + assert_not_create_assert_exists(backup_2_path, "old", jira_old_backup_name) + assert_not_create_assert_exists(backup_2_path, "old", conf_old_backup_name) + assert_not_create_assert_exists(backup_2_path, "latest", jira_latest_backup_name) + assert_not_create_assert_exists(backup_2_path, "latest", conf_latest_backup_name) + + return ab_class @@ -9,19 +138,7 @@ def test_entire_process(self, tmpdir): ab = AtlassianBackups() - ab = baat.create_all_paths_and_folders(ab, tmpdir) + ab = create_all_paths_and_folders(ab, tmpdir) - ab.perform_atlassian_backups() + result = ab.perform_atlassian_backups() - # Check backups in correct place - - # old backups have been moved - - # Check tar is ok (inc contents?) - - # old attachments have been moved - - # Check mysql dump - - # check old dump moved - - assert False is True + assert result == 0