Index: /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/include/photdbc.h =================================================================== --- /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/include/photdbc.h (revision 41069) +++ /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/include/photdbc.h (revision 41070) @@ -44,4 +44,12 @@ int SKIP_IMAGES; int ONLY_IMAGES; + +int SKIP_MEASURE; +int SKIP_MISSING; +int SKIP_LENSING; +int SKIP_LENSOBJ; +int SKIP_STARPAR; +int SKIP_GALPHOT; + int SHOW_PARAMS; int VERBOSE; Index: /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/args.c =================================================================== --- /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/args.c (revision 41069) +++ /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/args.c (revision 41070) @@ -29,4 +29,36 @@ } ONLY_IMAGES = TRUE; + remove_argument (N, &argc, argv); + } + + /** allow only certain tables to be copied **/ + SKIP_MEASURE = FALSE; + if ((N = get_argument (argc, argv, "-skip-measure"))) { + SKIP_MEASURE = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_MISSING = FALSE; + if ((N = get_argument (argc, argv, "-skip-missing"))) { + SKIP_MISSING = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_LENSING = FALSE; + if ((N = get_argument (argc, argv, "-skip-lensing"))) { + SKIP_LENSING = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_LENSOBJ = FALSE; + if ((N = get_argument (argc, argv, "-skip-lensobj"))) { + SKIP_LENSOBJ = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_STARPAR = FALSE; + if ((N = get_argument (argc, argv, "-skip-starpar"))) { + SKIP_STARPAR = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_GALPHOT = FALSE; + if ((N = get_argument (argc, argv, "-skip-galphot"))) { + SKIP_GALPHOT = TRUE; remove_argument (N, &argc, argv); } @@ -246,4 +278,36 @@ remove_argument (N, &argc, argv); MAX_MIN_MAG = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + /** allow only certain tables to be copied **/ + SKIP_MEASURE = FALSE; + if ((N = get_argument (argc, argv, "-skip-measure"))) { + SKIP_MEASURE = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_MISSING = FALSE; + if ((N = get_argument (argc, argv, "-skip-missing"))) { + SKIP_MISSING = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_LENSING = FALSE; + if ((N = get_argument (argc, argv, "-skip-lensing"))) { + SKIP_LENSING = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_LENSOBJ = FALSE; + if ((N = get_argument (argc, argv, "-skip-lensobj"))) { + SKIP_LENSOBJ = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_STARPAR = FALSE; + if ((N = get_argument (argc, argv, "-skip-starpar"))) { + SKIP_STARPAR = TRUE; + remove_argument (N, &argc, argv); + } + SKIP_GALPHOT = FALSE; + if ((N = get_argument (argc, argv, "-skip-galphot"))) { + SKIP_GALPHOT = TRUE; remove_argument (N, &argc, argv); } Index: /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/make_subcatalog.c =================================================================== --- /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/make_subcatalog.c (revision 41069) +++ /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/make_subcatalog.c (revision 41070) @@ -1,9 +1,3 @@ # include "photdbc.h" - -# define SKIP_MEASURE FALSE -# define SKIP_LENSING FALSE -# define SKIP_LENSOBJ FALSE -# define SKIP_STARPAR FALSE -# define SKIP_GALPHOT FALSE // copy a catalog to a new subcatalog, applying some filters Index: /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/photdbc_catalogs.c =================================================================== --- /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/photdbc_catalogs.c (revision 41069) +++ /branches/ccl_branches/ipp-20190806/Ohana/src/photdbc/src/photdbc_catalogs.c (revision 41070) @@ -152,4 +152,11 @@ if (CATMODE) { strextend (&command, "-set-mode %s", CATMODE); } + if (SKIP_MEASURE) { strextend (&command, "-skip-measure"); } + if (SKIP_MISSING) { strextend (&command, "-skip-missing"); } + if (SKIP_LENSING) { strextend (&command, "-skip-lensing"); } + if (SKIP_LENSOBJ) { strextend (&command, "-skip-lensobj"); } + if (SKIP_GALPHOT) { strextend (&command, "-skip-galphot"); } + if (SKIP_STARPAR) { strextend (&command, "-skip-starpar"); } + if (PARALLEL_OUTHOSTS) { tmppath = abspath (table_output->hosts[i].pathname, DVO_MAX_PATH); Index: /branches/ccl_branches/ipp-20190806/backups/README =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/README (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/README (revision 41070) @@ -0,0 +1,13 @@ +README + +This module contains files that perform backups of various services used by +the IPP. Each file has it's own backup class, which is inherited from the base +class. To perform a backup there needs to a config that matches the expected +schema. + +Run 'deploy_backups.py' to deploy the relevant scripts and utils to the machines +that the services live on. + +Check a config is in place for each backup. + +If automated backup is desired run the script periodically in a crontab. Index: /branches/ccl_branches/ipp-20190806/backups/backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/backup.py (revision 41070) @@ -0,0 +1,301 @@ +import configparser +import datetime +import os +import os.path as path +import socket +import subprocess +from configparser import ConfigParser + +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine, ParsedConfig, SpecialSchemaProcessing + +DEFAULT_SCHEMA_SECTION = ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ] +) +COPY_SCHEMA_SECTION = ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + ] +) +TAR_SCHEMA_SECTION = ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) +RSYNC_SCHEMA_SECTION = ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, SpecialSchemaProcessing.commma_separated) + ] +) + +MYSQLDUMP_SCHEMA_SECTION = ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ] +) + + +def get_backup_schema() -> ConfigSchema: + backup_schema = ConfigSchema(None) + backup_schema.add_section(DEFAULT_SCHEMA_SECTION) + backup_schema.add_section(COPY_SCHEMA_SECTION) + backup_schema.add_section(TAR_SCHEMA_SECTION) + backup_schema.add_section(RSYNC_SCHEMA_SECTION) + backup_schema.add_section(MYSQLDUMP_SCHEMA_SECTION) + return backup_schema + + +class Backup(): + """Base class for creating backups, contains all the generic versions of +functions for copying, taring etc. + """ + + def __init__(self, config_file=None, schema: ConfigSchema=None): + if config_file is None: + raise errs.ValidationError("ValidationError: Backup class must be given a config") + if schema is None: + schema = get_backup_schema() + self.load_config(config_file, schema) + + def load_config(self, config_class_or_filepath, schema): + """Accepts a filepath to a config file, or """ + + config = ConfigParser() + expected_class = configparser.ConfigParser + if config_class_or_filepath is None: + raise errs.ValidationError("ValidationError: config settings or filepath to config must be given") + elif isinstance(config_class_or_filepath, expected_class): + config = config_class_or_filepath + elif not path.exists(config_class_or_filepath): + raise errs.ValidationError(f"ValidationError: Config filepath does not exist (or incorrect class given, expected: {expected_class})") + config.read(config_class_or_filepath) + + self.read_config_and_schema(config, schema) + + def read_config_and_schema(self, config: ConfigParser, schema: ConfigSchema): + parsed_config = ParsedConfig(config, schema) + self.default_dict = parsed_config.get_section('DEFAULT') + self.hostname_is_valid() + self.maybe_make_backup_dirs() + + section = 'COPY' + self.should_run_copy = parsed_config.has_section(section) + if self.should_run_copy: + self.copy_dict = parsed_config.get_section(section) + + section = 'TAR' + self.should_run_tar = parsed_config.has_section(section) + if self.should_run_tar: + self.tar_dict = parsed_config.get_section(section) + + section = 'RSYNC' + self.should_run_rsync = parsed_config.has_section(section) + if self.should_run_rsync: + self.rsync_dict = parsed_config.get_section(section) + + section = 'MYSQLDUMP' + self.should_run_mysqldump = parsed_config.has_section(section) + if self.should_run_mysqldump: + self.mysqldump_dict = parsed_config.get_section(section) + + def run_backup_processes(self): + self._write_begin_log() + if self.should_run_copy: + self._run_copy() + if self.should_run_tar: + self._run_tar() + if self.should_run_rsync: + self._run_rsync() + if self.should_run_mysqldump: + self._run_mysqldump() + self._write_end_log() + + def hostname_is_valid(self) -> bool: + if self.default_dict is None or 'source_machine' not in self.default_dict.keys(): + raise ValueError('No source machine specified') + + actual_hostname = socket.gethostname() + return actual_hostname == self.default_dict['source_machine'] + + def target_backup_paths_ok(self, raise_error=False): + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + if raise_error: + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + return False + return True + + def maybe_make_backup_dirs(self): + if self.default_dict is None or 'make_dirs' not in self.default_dict.keys(): + return + if self.default_dict['make_dirs']: + for backup_path in self.default_dict['backup_paths']: + if not os.path.exists(backup_path): + os.makedirs(backup_path) + sub_utils.chmod_wrapper(['777', backup_path]) + + def _run_copy(self): + if self.should_run_copy is None or False: + return 0 + self.target_backup_paths_ok(raise_error=True) + + for bp in self.default_dict['backup_paths']: + # if not path.exists(bp): + # raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + for s in self.copy_dict['sources']: + if not path.exists(s): + raise errs.ValidationError(f"ValidationError: target path does not exist: {s}") + try: + args = [s, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {s}\n" + f" to: {bp}\n" + ) from e + return 0 + + def _run_tar(self): + if self.should_run_tar is None or False: + return 0 + + backup_paths = self.default_dict['backup_paths'] + + filename = self.tar_dict['target_filename'] + if 'prefix_date' in self.tar_dict.keys() and self.tar_dict['prefix_date']: + filename = f"{get_date()}_{filename}" + filename = path.join(backup_paths[0], filename) # may need some tar suffix magic + + add_args = [] if 'additional_args' not in self.tar_dict.keys() else self.tar_dict['additional_args'] + all_args = add_args + [filename] + self.tar_dict['sources'] + + try: + sub_utils.tar_wrapper(all_args) + sub_utils.chmod_wrapper(['774', filename]) + # copy to backups + if len(backup_paths) == 1: + return 0 + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([ filename, backup_paths[d]]) + except errs.SubprocessError as e: + raise errs.SubprocessError() from e + return 0 + + def _run_rsync(self): + if self.should_run_rsync is None or False: + return 0 + + self.target_backup_paths_ok(raise_error=True) + backup_paths = self.default_dict['backup_paths'] + + sources = self.rsync_dict['sources'] + for i in range(0, len(sources)): + source = sources[i] + if not path.exists(source): + raise errs.ValidationError(f"ValidationError: target path does not exist: {source}") + + for bp in backup_paths: + try: + additional_args = [] + if 'additional_args' in self.rsync_dict.keys(): + additional_args = self.rsync_dict['additional_args'] + + source_and_target_args = [source, bp] + + # sub-dirs option allows items to be backed up to subdirs within + # the source for better naming + if 'sub-dirs' in self.rsync_dict: + sub_dirs = self.rsync_dict['sub-dirs'] + if len(sub_dirs) != len(sources): + raise errs.ValidationError(f"ValidationError: if using " + + "subdirs the number of subdirs must equal the number of sources") + bp_with_subdir = os.path.join(bp, sub_dirs[i]) + if not os.path.exists(bp_with_subdir): + os.makedirs(bp_with_subdir) + source_and_target_args = [source, bp_with_subdir] + + args = source_and_target_args + additional_args + sub_utils.local_rsync_wrapper(args) + + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: rsync\n" + f" from: {source_and_target_args[0]}\n" + f" to: {source_and_target_args[1]}\n" + ) from e + return 0 + + def _run_mysqldump(self): + + backup_paths = self.default_dict['backup_paths'] + + mysqldump_args = [ 'mysqldump' + , '-u', self.mysqldump_dict['user'] + , f"-p{self.mysqldump_dict['password']}" + , '--databases', self.mysqldump_dict['db_name'] + ] + if 'additional_args' in self.mysqldump_dict.keys(): + mysqldump_args = mysqldump_args + self.mysqldump_dict['additional_args'] + + dump_filename = self.mysqldump_dict['target_filename'] + if 'prefix_date' in self.mysqldump_dict.keys() and self.mysqldump_dict['prefix_date']: + dump_filename = f"{get_date()}_{dump_filename}" + dump_filename = path.join(backup_paths[0], dump_filename) + + if path.exists(dump_filename): + raise errs.ValidationError(f"ValidationError: mysqldump already exists {dump_filename}") + + with open(dump_filename, "w+") as df: + mysqldump_process = subprocess.Popen(mysqldump_args, stdout=df, stderr=subprocess.PIPE) + dump_result = mysqldump_process.communicate() + + # Get the piped std err. See if it contains the word error + stdo, stde = dump_result + if stde is not None: + decoded_stderr = stde.decode() + if decoded_stderr.casefold().find('error'.casefold()) > -1: + raise errs.SubprocessError(f"SubprocessError: Errors detected in mysqldump:\n{decoded_stderr}") + + # copy the dump to other locations + for d in range(1, len(backup_paths)): + sub_utils.cp_wrapper([dump_filename, backup_paths[d]]) + return 0 + + def _write_begin_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if path.exists(log_file): + os.remove(log_file) + with open(log_file, "w+") as lf: + lf.write(f'Run started at: {datetime.datetime.now().strftime("%Y_%b_%d_%H:%M")}\n') + + def _write_end_log(self): + backup_paths = self.default_dict["backup_paths"] + for bp in backup_paths: + log_file = path.join(bp, "latest_run.log") + if not path.exists(log_file): + return + with open(log_file, "a") as lf: + lf.write(f'Last successful backup completed at : {datetime.datetime.now().strftime("%Y_%b_%d_%H:%M")}\n') + + +def get_date(custom_date=None, custom_format="%Y_%m_%d"): + # Format is currently: 2019-10-08 + if custom_date is not None: + return custom_date.strftime(custom_format) + return datetime.date.today().strftime(custom_format) Index: /branches/ccl_branches/ipp-20190806/backups/backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/backup_test.py (revision 41070) @@ -0,0 +1,777 @@ +import os +import pytest +import stat +import socket +from configparser import ConfigParser +from pathlib import Path + +import backup as bckup +import utils.errors as errs +from backup import Backup +from utils.config_parse_helper import ConfigSchema + +TEST_DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'testing/backup_test.config') + + +def make_default_config( tmpdir: str + , backup_targs: [str] = ['bck1', 'bck2'] + , verbose: bool = False + ) -> ConfigParser: + config = ConfigParser() + + # make the paths: + for b in backup_targs: + os.makedirs(os.path.join(tmpdir, b)) + + # Get the correct formats for the config + backup_paths = ','.join([os.path.join(tmpdir, b) for b in backup_targs]) + source_machine = socket.gethostname() + config['DEFAULT'] = \ + { 'backup_paths' : backup_paths + , 'source_machine' : source_machine + , 'verbose' : verbose + } + return config + + +class TestArguments(object): + + def test_default_config_file_load_default(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # Default + assert backy.default_dict['backup_paths'] == \ + [ "/export/ippops4.0/atlassian_backups" + , "/data/ippops3.0/atlassian_backups" + , "/data/ippops5.0/atlassian_backups" + ] + assert backy.default_dict['source_machine'] == "hostname_of_computer" + assert backy.default_dict['verbose'] is True + + def test_copy_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.copy_dict['sources'] == \ + [ "/some_dir_to_copy_from/" + , "/some_dir/and_specific_file.jpeg" + ] + assert backy.copy_dict['additional_args'] == ['-v'] + + def test_tar_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.tar_dict['sources'] == \ + [ "/this_dir/" + , "/another_dir/" + , "/and_a_couple_files/subdir/file1.txt" + , "/and_a_couple_files/subdir/file2.png" + , "/and_a_couple_files/subdir/file3.pdf" + ] + assert backy.tar_dict['additional_args'] == [ "--verbose" + , "-z" + , "-p" + ] + assert backy.tar_dict['target_filename'] == "cool_tar" + assert backy.tar_dict['prefix_date'] is True + + def test_rsync_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Rsync] + assert backy.rsync_dict['sources'] == [ "/dir/to/sync"] + assert backy.rsync_dict['additional_args'] == \ + [ "--progress" + , "-r" + ] + + def test_mysqldump_config_file_load(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + # [Mysqldump] + assert backy.mysqldump_dict['user'] == "dumper" + assert backy.mysqldump_dict['password'] == "not_a_real_password" + assert backy.mysqldump_dict['db_name'] == "jiradb" + assert backy.mysqldump_dict['additional_args'] == [''] + assert backy.mysqldump_dict['target_filename'] == "itsa_mysqldump" + assert backy.mysqldump_dict['prefix_date'] is True + + def test_loading_only_defaults(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.default_dict['backup_paths'] == \ + [ os.path.join(tmpdir, 'bck1') + , os.path.join(tmpdir, 'bck2') + ] + assert backy.default_dict['source_machine'] == socket.gethostname() + assert backy.default_dict['verbose'] is False + + def test_failure_if_no_defaults_provided(self): + config = ConfigParser() + with pytest.raises(errs.ConfigError) as e_empty: + Backup(config_file=config) + assert "is required but was not found in config" in str(e_empty) + + with pytest.raises(errs.ValidationError) as e_none: + Backup(config_file=None) + assert "ValidationError: Backup class must be given a config" in str(e_none) + + with pytest.raises(errs.ValidationError) as e_nofile: + Backup(config_file="/not/a/real/file.config") + assert "ValidationError: Config filepath does not exist" in str(e_nofile) + + def test_nothing_should_be_set_to_run_if_not_present(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2'], False) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + assert backy.should_run_copy is False + assert backy.should_run_tar is False + assert backy.should_run_rsync is False + assert backy.should_run_mysqldump is False + + def test_hostname_assertion_fails(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + backy.default_dict['source_machine'] = 'not_a_real_hostname' + assert backy.hostname_is_valid() is False + + def test_hostname_assertion_passes(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + backy = Backup(config, schema) + + actual_hostname = socket.gethostname() + backy.default_dict['source_machine'] = actual_hostname + assert backy.hostname_is_valid() is True + + +class TestFilepathVerifications(object): + + def test_default_target_paths_do_not_exist_in_testing(self): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + assert backy.target_backup_paths_ok() is False + + def test_raising_an_error_if_a_backup_path_does_not_exist(self): + with pytest.raises(errs.ValidationError) as e: + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + backy.target_backup_paths_ok(raise_error=True) + assert "ValidationError: target path does not exist" in str(e) + + def test_an_incorrect_path_then_correct_path(self, tmpdir): + backy = Backup(config_file=TEST_DEFAULT_CONFIG_FILE) + + backy.default_dict['backup_paths'] = ["/not/real/path"] + assert backy.target_backup_paths_ok() is False + + backy.default_dict['backup_paths'] = [tmpdir] + assert backy.target_backup_paths_ok() is True + + def test_backup_paths_created_if_make_dirs_given(self, tmpdir): + config = ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + bck1_path = os.path.join(tmpdir, 'bck1') + bck1b_path = os.path.join(tmpdir, 'bck1/bobby') + bck1bb_path = os.path.join(tmpdir, 'bck1/bobby/backup') + bck2_path = os.path.join(tmpdir, 'bck2') + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + # backup dirs not created automatically + Backup(config, schema) + assert not os.path.exists(bck1_path) + assert not os.path.exists(bck1b_path) + assert not os.path.exists(bck1bb_path) + assert not os.path.exists(bck2_path) + + # make_dirs included in config - when loaded dirs created. + config['DEFAULT'] = \ + { 'backup_paths' : f"{tmpdir}/bck1/bobby/backup, {tmpdir}/bck2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + , 'make_dirs' : 'True' + } + backy = Backup(config, schema) + assert backy.default_dict['make_dirs'] is True + assert os.path.exists(bck1_path) + assert os.path.exists(bck1b_path) + assert os.path.exists(bck1bb_path) + assert os.path.exists(bck2_path) + + # Check permissions are 777 too + new_stats = os.stat(bck2_path) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestRunningCopy(): + + def test_simple_copy(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + # make file to be copied + Path(source).touch() + + backy = Backup(config, schema) + assert backy.should_run_copy is True + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_copy_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v --preserve -r" + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + os.makedirs(source1) + Path(source2).touch() + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_copy_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['COPY'] = {'sources' : f"{source}"} + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_copy() + + def test_copy_requires_dash_r_arg_for_copying_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['COPY'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + with pytest.raises(errs.SubprocessError): + backy = Backup(config, schema) + backy._run_copy() + + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_copy() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + +class TestRunningTar(): + + def test_simply_tar(self, tmpdir): + config = make_default_config(tmpdir, ['bck1']) + source = os.path.join(tmpdir, "folder_to_tar") + os.makedirs(source) + # put a file in it + Path(os.path.join(source, "a_file.txt")) + + config['TAR'] = \ + { 'sources' : f"{source}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_tar1 = os.path.join(tmpdir, 'bck1', 'cool_tar.tar') + print(expected_tar1) + assert not os.path.exists(expected_tar1) + backy._run_tar() + assert os.path.exists(expected_tar1) + + def test_tar_with_multiple_sources_and_date(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "folder_to_tar") + source2 = os.path.join(tmpdir, "another_folder_to_tar") + os.makedirs(source1) + os.makedirs(source2) + # put a file in it + Path(os.path.join(source1, "a_file.txt")) + Path(os.path.join(source2, "other.file")) + + config['TAR'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-cf" + , 'target_filename' : 'cool_tar.tar' + , 'prefix_date' : 'True' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_tar is True + + date = bckup.get_date() + + expected_tar1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_tar.tar') + expected_tar2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_tar.tar') + assert not os.path.exists(expected_tar1) + assert not os.path.exists(expected_tar2) + backy._run_tar() + assert os.path.exists(expected_tar1) + assert os.path.exists(expected_tar2) + + +class TestRunningRsync(): + + def test_simple_rsync(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = {'sources' : f"{source}"} + # make file to be copied + Path(source).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1file) + assert os.path.exists(expected2file) + + def test_rsync_with_additional_args(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source1) + Path(source2).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected1file = os.path.join(tmpdir, 'bck1', 'and_specific_file.jpeg') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + expected2file = os.path.join(tmpdir, 'bck2', 'and_specific_file.jpeg') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) + + def test_rsync_fails_if_source_file_does_not_exist(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(source) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_fails_if_destination_does_not_exist(self, tmpdir): + config = ConfigParser() + + backup_does_not_exist = os.path.join(tmpdir, 'does_not_exist') + + config['DEFAULT'] = \ + { 'backup_paths' : backup_does_not_exist + , 'source_machine': socket.gethostname() + } + source = os.path.join(tmpdir, "and_specific_file.jpeg") + Path(source).touch() + config['RSYNC'] = {'sources' : f"{source}"} + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + assert not os.path.exists(backup_does_not_exist) + backy = Backup(config, schema) + with pytest.raises(errs.ValidationError): + backy._run_rsync() + + def test_rsync_requires_dash_r_arg_for_copying_only_dirs(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "some_dir") + config['RSYNC'] = \ + { 'sources' : f"{source}" + } + os.makedirs(source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + + config['RSYNC'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + + backy = Backup(config, schema) + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir') + expected2dir = os.path.join(tmpdir, 'bck2', 'some_dir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + + def test_rsync_with_delete_after_options(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source_dir = os.path.join(tmpdir, "some_dir") + source_file1 = os.path.join(source_dir, "file1") + source_file2 = os.path.join(source_dir, "file2") + source_file3 = os.path.join(source_dir, "file3") + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : "-v -z -r --progress" + } + + os.makedirs(source_dir) + Path(source_file1).touch() + Path(source_file2).touch() + Path(source_file3).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected1dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file1') + expected2dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file2') + expected3dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file3') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + backy._run_rsync() + print(expected1dir) + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + + # Delete two files, create a new one. + os.remove(source_file2) + os.remove(source_file3) + source_file4 = os.path.join(source_dir, "file4") + Path(source_file4).touch() + expected4dir = os.path.join(tmpdir, 'bck1', 'some_dir', 'file4') + + # Rsync will add 4, and 2 and 3 should still exist + backy._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected2dir) + assert os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + # with delete after option removed files are cleaned up + config['RSYNC'] = \ + { 'sources' : f"{source_dir}" + , 'additional_args' : '-v -z -r --delete-after' + } + backy = Backup(config, schema) + backy._run_rsync() + assert os.path.exists(expected1dir) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected3dir) + assert os.path.exists(expected4dir) + + def test_rsync_with_subdirs_option(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source1 = os.path.join(tmpdir, "some_dir") + source2 = os.path.join(tmpdir, "some_other_dir") + file_in_dir = os.path.join(source1, "and_specific_file.jpeg") + file_in_other_dir = os.path.join(source2, "another_file.jpeg") + os.makedirs(source1) + os.makedirs(source2) + Path(file_in_dir).touch() + Path(file_in_other_dir).touch() + + config['RSYNC'] = \ + { 'sources' : f"{source1}, {source2}" + , 'additional_args' : "-a --progress -v" + , 'sub-dirs' : "subdir_with_better_name1, subdir_with_better_name2" + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_bak1_src1_dir = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir') + expected_bak1_src1_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg') + expected_bak1_src2_dir = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir') + expected_bak1_src2_file = os.path.join(tmpdir, 'bck1', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg') + expected_bak2_src1_dir = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir') + expected_bak2_src1_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name1', 'some_dir', 'and_specific_file.jpeg') + expected_bak2_src2_dir = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir') + expected_bak2_src2_file = os.path.join(tmpdir, 'bck2', 'subdir_with_better_name2', 'some_other_dir', 'another_file.jpeg') + assert not os.path.exists(expected_bak1_src1_dir) + assert not os.path.exists(expected_bak1_src1_file) + assert not os.path.exists(expected_bak1_src2_dir) + assert not os.path.exists(expected_bak1_src2_file) + assert not os.path.exists(expected_bak2_src1_dir) + assert not os.path.exists(expected_bak2_src1_file) + assert not os.path.exists(expected_bak2_src2_dir) + assert not os.path.exists(expected_bak2_src2_file) + backy._run_rsync() + assert os.path.exists(expected_bak1_src1_dir) + assert os.path.exists(expected_bak1_src1_file) + assert os.path.exists(expected_bak1_src2_dir) + assert os.path.exists(expected_bak1_src2_file) + assert os.path.exists(expected_bak2_src1_dir) + assert os.path.exists(expected_bak2_src1_file) + assert os.path.exists(expected_bak2_src2_dir) + assert os.path.exists(expected_bak2_src2_file) + +@pytest.mark.skip(reason="Requires a fake database to be setup") +class TestMysqlBackup(object): + """ The mySQL dump tests require the machine on which they are +running to have a mysql database running with a user called +'dumper' and a password set to 'not_a_real_password'. + +In production, a proper password should be set on the server and in the config +file. DO NOT USE THE ABOVE IN PRODUCTION + +To change the password: +UPDATE mysql.user SET authentication_string = PASSWORD('NEW_USER_PASSWORD')WHERE User = 'user-name' AND Host = 'localhost';FLUSH PRIVILEGES; +""" + + def test_mysql_dump_is_ok(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + assert backy.should_run_mysqldump is True + + date = bckup.get_date() + + expected_dump1 = os.path.join(tmpdir, 'bck1', f'{date}_cool_dump.sql') + expected_dump2 = os.path.join(tmpdir, 'bck2', f'{date}_cool_dump.sql') + assert not os.path.exists(expected_dump1) + assert not os.path.exists(expected_dump2) + result = backy._run_mysqldump() + assert os.path.exists(expected_dump1) + assert os.path.exists(expected_dump2) + + assert result == 0 + + def test_mysql_dump_fails_with_incorrect_password(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'incorrect_password' + , 'db_name' : 'jiradb' + , 'target_filename' : "cool_dump.sql" + , 'prefix_date' : True + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.MYSQLDUMP_SCHEMA_SECTION) + + backy = Backup(config, schema) + with pytest.raises(errs.SubprocessError): + backy._run_mysqldump() + + +# class TestRunningAll(object): + +# def test_running_copy_tar_and_rsyc(self): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + + +class TestSimpleLogWriting(object): + + def test_log_writing_success(self, tmpdir): + config = make_default_config(tmpdir, ['bck1', 'bck2']) + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + backy.run_backup_processes() + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + # Read file + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 2 + assert "Run started at: " in lines[0] + assert "Last successful backup completed at :" in lines[1] + + def test_log_writes_only_attempted_if_failure_encountered(self, tmpdir): + + config = make_default_config(tmpdir, ['bck1', 'bck2']) + source = os.path.join(tmpdir, "and_specific_file.jpeg") + config['COPY'] = \ + { 'sources' : f"{source}" + , 'additional_args' : '-r' + } + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + + backy = Backup(config, schema) + + expected_log1 = os.path.join(tmpdir, 'bck1', 'latest_run.log') + expected_log2 = os.path.join(tmpdir, 'bck2', 'latest_run.log') + assert not os.path.exists(expected_log1) + assert not os.path.exists(expected_log2) + # fails due to missing source file in copy + with pytest.raises(errs.ValidationError): + backy.run_backup_processes() + + assert os.path.exists(expected_log1) + assert os.path.exists(expected_log2) + + with open(expected_log1, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] + + with open(expected_log2, "r") as logf: + lines = logf.readlines() + assert len(lines) == 1 + assert "Run started at: " in lines[0] Index: /branches/ccl_branches/ipp-20190806/backups/confluence_backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/confluence_backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/confluence_backup.py (revision 41070) @@ -0,0 +1,107 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './confluence_backup.config') + +CONFLUENCE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class ConfluenceBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_confluence_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + confluence_format = "%Y_%m_%d" + if custom_date is not None: + return custom_date.strftime(confluence_format) + return datetime.date.today().strftime(confluence_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For confluence backup there should only be one source: + source = self.copy_dict['sources'][0] + + confluence_wildcard_search = f"{source}/*{ConfluenceBackup.get_confluence_backup_date_format()}*" + confluence_xml_filepaths = glob.glob(confluence_wildcard_search) + if len(confluence_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No confluence backups found of form: {confluence_wildcard_search}") + + for jxf in confluence_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=CONFLUENCE_SCHEMA): + cb = ConfluenceBackup(config, schema) + cb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/confluence_backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/confluence_backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/confluence_backup_test.py (revision 41070) @@ -0,0 +1,366 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backup as bckup +import confluence_backup as cbck_mod + +from confluence_backup import ConfluenceBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +CONF_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/confluence_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'confluence_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'True' + } + config['RSYNC'] = \ + { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')}," + f"{os.path.join(tmpdir, 'rsync_source_dir2')}") + , 'additional_args' : '-a --delete-after --exclude exclude_me_if_you_like' + , 'sub-dirs' : 'conf_subdir1, conf_subdir2' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_confluence_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +ConfluenceBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + confluence_backup_path = os.path.join(tmpdir, "confluence_backup_path") + confluence_attachments_path = os.path.join(tmpdir, "confluence_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(confluence_backup_path) + os.makedirs(confluence_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(confluence_backup_path) + assert os.path.exists(confluence_attachments_path) + + # Create backup files that will be copied + confluence_backup_name = f"confluence-xml-backup-{ConfluenceBackup.get_confluence_backup_date_format()}.zip" + confluence_backup_filepath = os.path.join(confluence_backup_path, confluence_backup_name) + Path(confluence_backup_filepath).touch() + assert os.path.exists(confluence_backup_filepath) + + # Create some files that will be tar'd + confluence_attachment_1_name = "attachment_1.zip" + confluence_attachment_2_name = "attachment_2.txt" + confluence_attachment_1_filepath = os.path.join(confluence_attachments_path, confluence_attachment_1_name) + confluence_attachment_2_filepath = os.path.join(confluence_attachments_path, confluence_attachment_2_name) + Path(confluence_attachment_1_filepath).touch() + Path(confluence_attachment_2_filepath).touch() + assert os.path.exists(confluence_attachment_1_filepath) + assert os.path.exists(confluence_attachment_2_filepath) + + # Create a few folders to be rsync'd + rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1") + rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2") + file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync") + file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync") + exluded_file = os.path.join(rsync_source_dir1, "exclude_me_if_you_like") + os.makedirs(rsync_source_dir1) + os.makedirs(rsync_source_dir2) + Path(file_in_rsync_dir1).touch() + Path(file_in_rsync_dir2).touch() + Path(exluded_file).touch() + assert os.path.exists(file_in_rsync_dir1) + assert os.path.exists(file_in_rsync_dir2) + assert os.path.exists(exluded_file) + + +class TestArguments(object): + + def assert_defaults(self, jb: ConfluenceBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/confluence_backups' + , '/data/ippops3.0/backups/confluence_backups' + , '/data/ippops5.0/backups/confluence_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/backups'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/confluence/attachments'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'confluencedb' + , 'target_filename' : 'confluence_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = cbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + ConfluenceBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = ConfluenceBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'confluence.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + ConfluenceBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + cbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + conf_backup_name = ConfluenceBackup.get_confluence_backup_date_format() + "_conf_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'conf.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, conf_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + cbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_confluence_backup_date_format(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y_%m_%d") + assert cb.get_confluence_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + cb = ConfluenceBackup(config_file=CONF_TEST_CONFIG) + # return_value = cb._copy_confluence_backup() + with pytest.raises(ValidationError) as e: + cb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_confluence_copy(self, tmpdir): + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + cb = ConfluenceBackup(config_file=config) + + today = ConfluenceBackup.get_confluence_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"confluence-xml-backup-{today}.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"confluence-xml-backup-{today}.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + return_value = cb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_confluence_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + jb = ConfluenceBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + confluence_tar_name = f"{today}_confluence.tar.gz" + expected_confluence_host_tar = os.path.join(tmpdir, "host_backup_path", confluence_tar_name) + expected_confluence_bak1_tar = os.path.join(tmpdir, "backup_1_path", confluence_tar_name) + expected_confluence_bak2_tar = os.path.join(tmpdir, "backup_2_path", confluence_tar_name) + assert not os.path.exists(expected_confluence_host_tar) + assert not os.path.exists(expected_confluence_bak1_tar) + assert not os.path.exists(expected_confluence_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_confluence_host_tar) + assert os.path.exists(expected_confluence_bak1_tar) + assert os.path.exists(expected_confluence_bak2_tar) + + +class TestRsyncOfFiles(object): + + def test_confluence_rsync_with_subdirs(self, tmpdir): + """This should be similar to the regular Backup version""" + create_items_for_full_confluence_backup_test(tmpdir) + config = make_config(tmpdir) + + cb = ConfluenceBackup(config_file=config) + + # Expected resulting rsync directories + expected_confluence_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path", "conf_subdir1", "rsync_source_dir1") + expected_confluence_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "conf_subdir2", "rsync_source_dir2") + expected_confluence_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path", "conf_subdir2", "rsync_source_dir2") + expected_confluence_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path", "conf_subdir2", "rsync_source_dir2") + assert not os.path.exists(expected_confluence_host_rsync1) + assert not os.path.exists(expected_confluence_bak1_rsync1) + assert not os.path.exists(expected_confluence_bak2_rsync1) + assert not os.path.exists(expected_confluence_host_rsync2) + assert not os.path.exists(expected_confluence_bak1_rsync2) + assert not os.path.exists(expected_confluence_bak2_rsync2) + # Expected resulting rsync files + expected_file_host_dir1 = os.path.join( expected_confluence_host_rsync1, "some_file_to_rsync") + expected_file_bak1_dir1 = os.path.join( expected_confluence_bak1_rsync1, "some_file_to_rsync") + expected_file_bak2_dir1 = os.path.join( expected_confluence_bak2_rsync1, "some_file_to_rsync") + expected_file_host_dir2 = os.path.join( expected_confluence_host_rsync2, "some_other_file_to_rsync") + expected_file_bak1_dir2 = os.path.join( expected_confluence_bak1_rsync2, "some_other_file_to_rsync") + expected_file_bak2_dir2 = os.path.join( expected_confluence_bak2_rsync2, "some_other_file_to_rsync") + assert not os.path.exists(expected_file_host_dir1) + assert not os.path.exists(expected_file_bak1_dir1) + assert not os.path.exists(expected_file_bak2_dir1) + assert not os.path.exists(expected_file_host_dir2) + assert not os.path.exists(expected_file_bak1_dir2) + assert not os.path.exists(expected_file_bak2_dir2) + an_excluded_file = os.path.join(expected_confluence_host_rsync1, "exclude_me_if_you_like") + assert not os.path.exists(an_excluded_file) + + rsync_result = cb._run_rsync() + assert rsync_result == 0 + + # All files Rsync'd + assert os.path.exists(expected_confluence_host_rsync1) + assert os.path.exists(expected_confluence_bak1_rsync1) + assert os.path.exists(expected_confluence_bak2_rsync1) + assert os.path.exists(expected_confluence_host_rsync2) + assert os.path.exists(expected_confluence_bak1_rsync2) + assert os.path.exists(expected_confluence_bak2_rsync2) + assert os.path.exists(expected_file_host_dir1) + assert os.path.exists(expected_file_bak1_dir1) + assert os.path.exists(expected_file_bak2_dir1) + assert os.path.exists(expected_file_host_dir2) + assert os.path.exists(expected_file_bak1_dir2) + assert os.path.exists(expected_file_bak2_dir2) + assert not os.path.exists(an_excluded_file) Index: /branches/ccl_branches/ipp-20190806/backups/deploy_backups.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/deploy_backups.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/deploy_backups.py (revision 41070) @@ -0,0 +1,63 @@ +import argparse +import sys + +import utils.subprocess_utils as sub_utils + +DEFAULT_DEPLOYMENT = \ + { "jira_backup.py" : "ippops4" + , "confluence_backup.py" : "ippops4" + , "svn_backup.py" : "ippops4" + , "mailman_backup.py" : "ippops3" + , "ippops_homedir_backup.py" : "ippops3" + , "website_backup.py" : "ippops3" + } + +ADDITIONAL_FILES = \ + [ './utils' + , 'backup.py' + ] + +PREFIX = "/export/" +SUFFIX = ".0/backups/" + + +def deploy(user=None): + + for file in DEFAULT_DEPLOYMENT: + default_machine = DEFAULT_DEPLOYMENT[file] + + chosen_machine = input(f"Enter machine to copy {file} to [default={default_machine}; ENTER to accept]:") + chosen_machine.strip('\n').strip(' ') + if chosen_machine == '': + chosen_machine = default_machine + + copy_file_and_utils(file, chosen_machine, user) + + +def copy_file_and_utils(file, machine, user): + # copy all the utils across too + user_at = '' if user is None else f'{user}@' + target = f"{user_at}{machine}.ifa.hawaii.edu:{PREFIX}{machine}{SUFFIX}" + args = ['rsync', '-a', '--progress', file, ] + ADDITIONAL_FILES + [target] + print(f"Performing: {' '.join(args)}") + sub_utils.simple_unix_wrapper(args) + + +def main(): + args = parse_args(sys.argv[1:]) + args_dict = vars(args) + deploy(**args_dict) + + +def parse_args(args): + parser = argparse.ArgumentParser( + description='Performs deploys backup files to desired locations') + parser.add_argument('--user' + , nargs='?' + , help='specify the user to transfer files as' + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/ippops_homedir_backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/ippops_homedir_backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/ippops_homedir_backup.py (revision 41070) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './ippops_homedir_backup.config') + +IPPOPS_HOMEDIR_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class IPPOpsHomedirBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=IPPOPS_HOMEDIR_SCHEMA): + ohdb = IPPOpsHomedirBackup(config, schema) + ohdb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/ippops_homedir_backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/ippops_homedir_backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/ippops_homedir_backup_test.py (revision 41070) @@ -0,0 +1,133 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import ippops_homedir_backup as ihbck_mod +from ippops_homedir_backup import IPPOpsHomedirBackup +from utils.config_parse_helper import ConfigSchema + +IPPOPS_HOMEDIR_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/ippops_homedir_test.config') + + +class TestArguments(object): + + def assert_defaults(self, ihb: IPPOpsHomedirBackup): + + assert ihb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/homedir_backups' + , '/data/ippops4.0/backups/homedir_backups' + , '/data/ippops5.0/backups/homedir_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert ihb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/home/panstarrs'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + ihb = IPPOpsHomedirBackup(config_file=IPPOPS_HOMEDIR_TEST_CONFIG) + self.assert_defaults(ihb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + ihbck_mod.main(config, schema) + assert True is True + + +class TestIppopsHomedirRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'home')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "home") + subdir1 = os.path.join(homedir, "some_user") + subdir2 = os.path.join(homedir, "some_other_user") + file1 = os.path.join(subdir1, "file_in_subdir1") + file2 = os.path.join(subdir2, "file_in_subdir2") + os.makedirs(homedir) + os.makedirs(subdir1) + os.makedirs(subdir2) + Path(file1).touch() + Path(file2).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + ihb = IPPOpsHomedirBackup(config, schema) + + expected_sub1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user') + expected_sub1_file1_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user') + expected_sub2_file2_backup1 = os.path.join(tmpdir, 'backup_1_path', 'home', 'some_other_user/file_in_subdir2') + expected_sub1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user') + expected_sub1_file1_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_user/file_in_subdir1') + expected_sub2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user') + expected_sub2_file2_backup2 = os.path.join(tmpdir, 'backup_2_path', 'home', 'some_other_user/file_in_subdir2') + assert not os.path.exists(expected_sub1_backup1) + assert not os.path.exists(expected_sub1_file1_backup1) + assert not os.path.exists(expected_sub2_backup1) + assert not os.path.exists(expected_sub2_file2_backup1) + assert not os.path.exists(expected_sub1_backup2) + assert not os.path.exists(expected_sub1_file1_backup2) + assert not os.path.exists(expected_sub2_backup2) + assert not os.path.exists(expected_sub2_file2_backup2) + ihb._run_rsync() + assert os.path.exists(expected_sub1_backup1) + assert os.path.exists(expected_sub1_file1_backup1) + assert os.path.exists(expected_sub2_backup1) + assert os.path.exists(expected_sub2_file2_backup1) + assert os.path.exists(expected_sub1_backup2) + assert os.path.exists(expected_sub1_file1_backup2) + assert os.path.exists(expected_sub2_backup2) + assert os.path.exists(expected_sub2_file2_backup2) Index: /branches/ccl_branches/ipp-20190806/backups/jira_backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/jira_backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/jira_backup.py (revision 41070) @@ -0,0 +1,107 @@ +import datetime +import glob +import os.path as path + +import utils.config_parse_helper as cfg_help +import utils.errors as errs +import utils.subprocess_utils as sub_utils +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './jira_backup.config') + +JIRA_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('TAR', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('sub-dirs', False, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + ]) + , ConfigSchemaSection('MYSQLDUMP', + [ ConfigSchemaLine('user', True, str) + , ConfigSchemaLine('password', True, str) + , ConfigSchemaLine('db_name', True, str) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + , ConfigSchemaLine('target_filename', True, str) + , ConfigSchemaLine('prefix_date', True, bool) + ]) + ] +) + + +class JiraBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + self.load_config(config_file, schema) + + @staticmethod + def get_jira_backup_date_format(custom_date=None): + # Format is currently: 2019-Aug-13--0325.zip + jira_format = "%Y-%b-%d" + if custom_date is not None: + return custom_date.strftime(jira_format) + return datetime.date.today().strftime(jira_format) + + def _run_copy(self, verbose=False): + if self.should_run_copy is None or False: + return 0 + + for bp in self.default_dict['backup_paths']: + if not path.exists(bp): + raise errs.ValidationError(f"ValidationError: target path does not exist: {bp}") + + # For jira backup there should only be one source: + source = self.copy_dict['sources'][0] + + jira_wildcard_search = f"{source}/{JiraBackup.get_jira_backup_date_format()}*" + jira_xml_filepaths = glob.glob(jira_wildcard_search) + if len(jira_xml_filepaths) == 0: + raise errs.ValidationError(f"ValidationError: No jira backups found of form: {jira_wildcard_search}") + + for jxf in jira_xml_filepaths: + if not path.exists(jxf): + raise errs.ValidationError(f"ValidationError: target path does not exist: {jxf}") + try: + args = [jxf, bp] + if 'additional_args' in self.copy_dict.keys(): + args = args + self.copy_dict['additional_args'] + sub_utils.cp_wrapper(args) + except errs.SubprocessError as e: + raise errs.SubprocessError( f"Error: copying\n" + f" from: {jxf}\n" + f" to: {bp}\n" + ) from e + return 0 + + +def main(config=EXPECTED_CONFIG_FILE, schema=JIRA_SCHEMA): + jb = JiraBackup(config, schema) + jb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/jira_backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/jira_backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/jira_backup_test.py (revision 41070) @@ -0,0 +1,361 @@ +import configparser +import datetime +import os +import pytest + +from pathlib import Path + +import backups.backup as bckup +import jira_backup as jbck_mod +from jira_backup import JiraBackup +from utils.errors import ValidationError +from utils.config_parse_helper import ConfigSchema + +JIRA_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/jira_test.config') + + +def make_config(tmpdir) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config['DEFAULT'] = \ + { 'backup_paths' : (f"{os.path.join(tmpdir, 'host_backup_path')}," + f"{os.path.join(tmpdir, 'backup_1_path')}," + f"{os.path.join(tmpdir, 'backup_2_path')}") + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_backup_path')}" + } + config['TAR'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'jira_attachments_path')}" + , 'additional_args' : '-czf' + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'True' + } + config['RSYNC'] = \ + { 'sources' : (f"{os.path.join(tmpdir, 'rsync_source_dir1')}," + f"{os.path.join(tmpdir, 'rsync_source_dir2')}") + , 'additional_args' : '-a --delete-after' + , 'sub-dirs' : 'jira_subdir1, jira_subdir2' + } + config['MYSQLDUMP'] = \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : 'True' + } + return config + + +def create_items_for_full_jira_backup_test(tmpdir): + """This creates all paths and files needed for thorough testing of the +JiraBackup class. +""" + + host_backup_path = os.path.join(tmpdir, "host_backup_path") + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + jira_backup_path = os.path.join(tmpdir, "jira_backup_path") + jira_attachments_path = os.path.join(tmpdir, "jira_attachments_path") + os.makedirs(host_backup_path) + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + os.makedirs(jira_backup_path) + os.makedirs(jira_attachments_path) + assert os.path.exists(host_backup_path) + assert os.path.exists(backup_1_path) + assert os.path.exists(backup_2_path) + assert os.path.exists(jira_backup_path) + assert os.path.exists(jira_attachments_path) + + # Create backup files that will be copied + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + jira_backup_filepath = os.path.join(jira_backup_path, jira_backup_name) + Path(jira_backup_filepath).touch() + assert os.path.exists(jira_backup_filepath) + + # Create some files that will be tar'd + jira_attachment_1_name = "attachment_1.zip" + jira_attachment_2_name = "attachment_2.txt" + jira_attachment_1_filepath = os.path.join(jira_attachments_path, jira_attachment_1_name) + jira_attachment_2_filepath = os.path.join(jira_attachments_path, jira_attachment_2_name) + Path(jira_attachment_1_filepath).touch() + Path(jira_attachment_2_filepath).touch() + assert os.path.exists(jira_attachment_1_filepath) + assert os.path.exists(jira_attachment_2_filepath) + + # Create a few folders to be rsync'd + rsync_source_dir1 = os.path.join(tmpdir, "rsync_source_dir1") + rsync_source_dir2 = os.path.join(tmpdir, "rsync_source_dir2") + file_in_rsync_dir1 = os.path.join(rsync_source_dir1, "some_file_to_rsync") + file_in_rsync_dir2 = os.path.join(rsync_source_dir2, "some_other_file_to_rsync") + os.makedirs(rsync_source_dir1) + os.makedirs(rsync_source_dir2) + Path(file_in_rsync_dir1).touch() + Path(file_in_rsync_dir2).touch() + assert os.path.exists(file_in_rsync_dir1) + assert os.path.exists(file_in_rsync_dir2) + + +class TestArguments(object): + + def assert_defaults(self, jb: JiraBackup): + + assert jb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/jira_backups' + , '/data/ippops3.0/backups/jira_backups' + , '/data/ippops5.0/backups/jira_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert jb.copy_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/export'] + , 'additional_args' : ['-v'] + } + assert jb.tar_dict == \ + { 'sources' : ['/var/atlassian/application-data/jira/data'] + , 'additional_args' : ['--verbose', '-z', '-p', '-cf'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : True + } + assert jb.mysqldump_dict == \ + { 'user' : 'dumper' + , 'password' : 'not_a_real_password' + , 'db_name' : 'jiradb' + , 'target_filename' : 'jira_mysqldump.sql' + , 'prefix_date' : True + } + + def test_load_from_config_file(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_default_config_fails_to_load_if_the_file_does_not_exist(self): + default_config = jbck_mod.EXPECTED_CONFIG_FILE + assert not os.path.exists(default_config) + + with pytest.raises(ValidationError) as e: + JiraBackup() + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_default_arguments(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + self.assert_defaults(jb) + + def test_providing_config_directly(self): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + config['DEFAULT'] = \ + { 'backup_paths' : "/path/1, /path/2" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : "/copy/source" + } + config['TAR'] = \ + { 'sources' : "/tar/source" + , 'additional_args' : "--verbose" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jb = JiraBackup(config, schema) + + assert jb.default_dict == \ + { 'backup_paths' : [ '/path/1' + , '/path/2' + ] + , 'source_machine' : 'rocket' + , 'verbose' : True + } + + assert jb.copy_dict == \ + { 'sources' : ['/copy/source'] + } + + assert jb.tar_dict == \ + { 'sources' : ['/tar/source'] + , 'additional_args' : ['--verbose'] + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : False + } + + def test_nonexistant_config_file_raises_error(self): + with pytest.raises(ValidationError) as e: + JiraBackup("not a real file path") + assert "ValidationError: Config filepath does not exist" in str(e) + + def test_main_loads_default_and_fails_due_to_missing_paths(self): + with pytest.raises(ValidationError) as e: + jbck_mod.main() + assert "ValidationError: Config filepath does not exist" in str(e) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + config.add_section("atlassian_backups") + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + jira_backup_name = JiraBackup.get_jira_backup_date_format() + "_jira_test_file.zip" + config['COPY'] = \ + { 'sources' : f'{copy_source}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'jira.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, jira_backup_name)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + jbck_mod.main(config, schema) + assert True is True + + +class TestCopyingBackups(object): + + # This should behanve differently to the regular backups: + + def test_get_jira_backup_date_format(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + today = datetime.date.today() + expected_format = today.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format() == expected_format + + another_date = datetime.date(1988, 11, 23) + expected_format = another_date.strftime("%Y-%b-%d") + assert jb.get_jira_backup_date_format(another_date) == expected_format + + def test_copying_fails_if_paths_not_present(self): + jb = JiraBackup(config_file=JIRA_TEST_CONFIG) + # return_value = jb._copy_jira_backup() + with pytest.raises(ValidationError) as e: + jb._run_copy() + assert "ValidationError: target path does not exist:" in str(e) + + def test_jira_copy(self, tmpdir): + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = JiraBackup.get_jira_backup_date_format() + expected_host_backup_copy = os.path.join(tmpdir, "host_backup_path", f"{today}_jira_test_file.zip" ) + expected_backup_1_copy = os.path.join(tmpdir, "backup_1_path", f"{today}_jira_test_file.zip" ) + expected_backup_2_copy = os.path.join(tmpdir, "backup_2_path", f"{today}_jira_test_file.zip" ) + assert not os.path.exists(expected_host_backup_copy) + assert not os.path.exists(expected_backup_1_copy) + assert not os.path.exists(expected_backup_2_copy) + + assert jb.should_run_copy is True + return_value = jb._run_copy() + assert return_value == 0 + + assert os.path.exists(expected_host_backup_copy) + assert os.path.exists(expected_backup_1_copy) + assert os.path.exists(expected_backup_2_copy) + + +class TestTaringOfAttachments(object): + + def test_jira_attachments_tar(self, tmpdir): + """This should be identical to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + jb = JiraBackup(config_file=config) + + today = datetime.date.today().strftime("%Y_%m_%d") + jira_tar_name = f"{today}_jira.tar.gz" + expected_jira_host_tar = os.path.join(tmpdir, "host_backup_path", jira_tar_name) + expected_jira_bak1_tar = os.path.join(tmpdir, "backup_1_path", jira_tar_name) + expected_jira_bak2_tar = os.path.join(tmpdir, "backup_2_path", jira_tar_name) + assert not os.path.exists(expected_jira_host_tar) + assert not os.path.exists(expected_jira_bak1_tar) + assert not os.path.exists(expected_jira_bak2_tar) + + tar_result = jb._run_tar() + assert tar_result == 0 + + assert os.path.exists(expected_jira_host_tar) + assert os.path.exists(expected_jira_bak1_tar) + assert os.path.exists(expected_jira_bak2_tar) + + +class TestRsyncOfFiles(object): + + def test_jira_rsync_with_subdirs(self, tmpdir): + """This should be similar to the regular Backup version""" + create_items_for_full_jira_backup_test(tmpdir) + config = make_config(tmpdir) + + jb = JiraBackup(config_file=config) + + # Expected resulting rsync directories + expected_jira_host_rsync1 = os.path.join(tmpdir, "host_backup_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_bak1_rsync1 = os.path.join(tmpdir, "backup_1_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_bak2_rsync1 = os.path.join(tmpdir, "backup_2_path", "jira_subdir1", "rsync_source_dir1") + expected_jira_host_rsync2 = os.path.join(tmpdir, "host_backup_path", "jira_subdir2", "rsync_source_dir2") + expected_jira_bak1_rsync2 = os.path.join(tmpdir, "backup_1_path", "jira_subdir2", "rsync_source_dir2") + expected_jira_bak2_rsync2 = os.path.join(tmpdir, "backup_2_path", "jira_subdir2", "rsync_source_dir2") + assert not os.path.exists(expected_jira_host_rsync1) + assert not os.path.exists(expected_jira_bak1_rsync1) + assert not os.path.exists(expected_jira_bak2_rsync1) + assert not os.path.exists(expected_jira_host_rsync2) + assert not os.path.exists(expected_jira_bak1_rsync2) + assert not os.path.exists(expected_jira_bak2_rsync2) + # Expected resulting rsync files + expected_file_host_dir1 = os.path.join( expected_jira_host_rsync1, "some_file_to_rsync") + expected_file_bak1_dir1 = os.path.join( expected_jira_bak1_rsync1, "some_file_to_rsync") + expected_file_bak2_dir1 = os.path.join( expected_jira_bak2_rsync1, "some_file_to_rsync") + expected_file_host_dir2 = os.path.join( expected_jira_host_rsync2, "some_other_file_to_rsync") + expected_file_bak1_dir2 = os.path.join( expected_jira_bak1_rsync2, "some_other_file_to_rsync") + expected_file_bak2_dir2 = os.path.join( expected_jira_bak2_rsync2, "some_other_file_to_rsync") + assert not os.path.exists(expected_file_host_dir1) + assert not os.path.exists(expected_file_bak1_dir1) + assert not os.path.exists(expected_file_bak2_dir1) + assert not os.path.exists(expected_file_host_dir2) + assert not os.path.exists(expected_file_bak1_dir2) + assert not os.path.exists(expected_file_bak2_dir2) + + rsync_result = jb._run_rsync() + assert rsync_result == 0 + + # All files Rsync'd + assert os.path.exists(expected_jira_host_rsync1) + assert os.path.exists(expected_jira_bak1_rsync1) + assert os.path.exists(expected_jira_bak2_rsync1) + assert os.path.exists(expected_jira_host_rsync2) + assert os.path.exists(expected_jira_bak1_rsync2) + assert os.path.exists(expected_jira_bak2_rsync2) + assert os.path.exists(expected_file_host_dir1) + assert os.path.exists(expected_file_bak1_dir1) + assert os.path.exists(expected_file_bak2_dir1) + assert os.path.exists(expected_file_host_dir2) + assert os.path.exists(expected_file_bak1_dir2) + assert os.path.exists(expected_file_bak2_dir2) Index: /branches/ccl_branches/ipp-20190806/backups/mailman_backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/mailman_backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/mailman_backup.py (revision 41070) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './mailman_backup.config') + +MAILMAN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class MailmanBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=MAILMAN_SCHEMA): + mb = MailmanBackup(config, schema) + mb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/mailman_backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/mailman_backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/mailman_backup_test.py (revision 41070) @@ -0,0 +1,119 @@ +import configparser +import os +from pathlib import Path + +import mailman_backup as mailbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from mailman_backup import MailmanBackup +from utils.config_parse_helper import ConfigSchema + + +MAILMAN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/mailman_test.config') + + +class TestArguments(object): + + def assert_defaults(self, mb: MailmanBackup): + + assert mb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/mailman_backups' + , '/data/ippops4.0/backups/mailman_backups' + , '/data/ippops5.0/backups/mailman_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert mb.rsync_dict == \ + { 'sources' : ['/var/lib/mailman'] + , 'additional_args' : ['-a', '--delete-after', '--copy-links'] + } + + def test_load_default_config_file(self): + mb = MailmanBackup(config_file=MAILMAN_TEST_CONFIG) + self.assert_defaults(mb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + mailbck_mod.main(config, schema) + assert True is True + + +class TestMailmanRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'mailman')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + mailman_root = os.path.join(tmpdir, "mailman") + subdir = os.path.join(mailman_root, "some_dir") + file = os.path.join(subdir, "file_in_subdir") + os.makedirs(mailman_root) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + mb = MailmanBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir') + expected1file = os.path.join(tmpdir, 'backup_1_path', 'mailman', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir') + expected2file = os.path.join(tmpdir, 'backup_2_path', 'mailman', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file) + mb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file) Index: /branches/ccl_branches/ipp-20190806/backups/svn_backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/svn_backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/svn_backup.py (revision 41070) @@ -0,0 +1,45 @@ +import os.path as path + +from utils.config_parse_helper import SpecialSchemaProcessing +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './svn_backup.config') + +SVN_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('COPY', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class SvnBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=SVN_SCHEMA): + sb = SvnBackup(config, schema) + sb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/svn_backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/svn_backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/svn_backup_test.py (revision 41070) @@ -0,0 +1,193 @@ +import configparser +import os +from pathlib import Path + +import svn_backup as svnbck_mod +import backup as bckup +from backup import DEFAULT_SCHEMA_SECTION, COPY_SCHEMA_SECTION, RSYNC_SCHEMA_SECTION +from svn_backup import SvnBackup +from utils.config_parse_helper import ConfigSchema + +SVN_TEST_CONFIG = os.path.join(os.path.dirname(__file__), './testing/svn_test.config') + + +class TestArguments(object): + + def assert_defaults(self, sb: SvnBackup): + + assert sb.default_dict == \ + { 'backup_paths' : [ '/export/ippops4.0/backups/svn_backups' + , '/data/ippops3.0/backups/svn_backups' + , '/data/ippops5.0/backups/svn_backups' + ] + , 'source_machine' : 'ippops4' + , 'verbose' : False + } + assert sb.copy_dict == \ + { 'sources' : ['/etc/apache2/mods-enabled/dav_svn.conf'] + , 'additional_args' : ['-v'] + } + assert sb.rsync_dict == \ + { 'sources' : ['/export/ippops4.0/svnroot'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + sb = SvnBackup(config_file=SVN_TEST_CONFIG) + self.assert_defaults(sb) + + +class TestSvnCopy(object): + """Simple inherit of base class Backup""" + + # Setup config + def test_copy(self, tmpdir): + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['COPY'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'apache2/mods-enabled/dav_svn.conf')}" + , 'additional_args' : "-v" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + apache_folder = os.path.join(tmpdir, "apache2") + subdir = os.path.join(apache_folder, "mods-enabled") + file = os.path.join(subdir, "dav_svn.conf") + os.makedirs(apache_folder) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(COPY_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'dav_svn.conf') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'dav_svn.conf') + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_file_bck2) + sb._run_copy() + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_file_bck2) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + copy_source = os.path.join(tmpdir, "copy/source") + svn_conf_file = "dav_svn.conf" + config['COPY'] = \ + { 'sources' : f'{copy_source}/{svn_conf_file}' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(copy_source) + os.makedirs(tar_source) + Path(os.path.join(copy_source, svn_conf_file)).touch() + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.COPY_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + svnbck_mod.main(config, schema) + assert True is True + + +class TestSvnRsync(object): + """Simple inherit of base class Backup""" + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'svnroot')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + svnroot = os.path.join(tmpdir, "svnroot") + subdir = os.path.join(svnroot, "some_dir") + file1 = os.path.join(svnroot, "file_at_root_level") + file2 = os.path.join(subdir, "file_in_subdir") + os.makedirs(svnroot) + os.makedirs(subdir) + Path(file1).touch() + Path(file2).touch() + + # File that will be removed because of --delete-after + os.makedirs(os.path.join(backup_1_path, "svnroot")) + file_that_already_existed = os.path.join(backup_1_path, "svnroot", "file_to_disappear") + Path(file_that_already_existed).touch() + assert os.path.exists(file_that_already_existed) + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(DEFAULT_SCHEMA_SECTION) + schema.add_section(RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + sb = SvnBackup(config, schema) + + expected1dir = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir') + expected1file1 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'file_at_root_level') + expected1file2 = os.path.join(tmpdir, 'backup_1_path', 'svnroot', 'some_dir/file_in_subdir') + expected2dir = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir') + expected2file1 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'file_at_root_level') + expected2file2 = os.path.join(tmpdir, 'backup_2_path', 'svnroot', 'some_dir/file_in_subdir') + assert not os.path.exists(expected1dir) + assert not os.path.exists(expected1file1) + assert not os.path.exists(expected1file2) + assert not os.path.exists(expected2dir) + assert not os.path.exists(expected2file1) + assert not os.path.exists(expected2file2) + assert os.path.exists(file_that_already_existed) + sb._run_rsync() + assert os.path.exists(expected1dir) + assert os.path.exists(expected1file1) + assert os.path.exists(expected1file2) + assert os.path.exists(expected2dir) + assert os.path.exists(expected2file1) + assert os.path.exists(expected2file2) + # file that does not exist in the source, is removed from the target + assert not os.path.exists(file_that_already_existed) Index: /branches/ccl_branches/ipp-20190806/backups/testing/backup_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/backup_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/backup_test.config (revision 41070) @@ -0,0 +1,34 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/atlassian_backups, + /data/ippops3.0/atlassian_backups, + /data/ippops5.0/atlassian_backups +source_machine = hostname_of_computer +verbose = True + +[COPY] +sources = /some_dir_to_copy_from/, + /some_dir/and_specific_file.jpeg +additional_args = -v + +[TAR] +sources = /this_dir/, + /another_dir/, + /and_a_couple_files/subdir/file1.txt, + /and_a_couple_files/subdir/file2.png, + /and_a_couple_files/subdir/file3.pdf +additional_args = --verbose -z -p +target_filename = cool_tar +prefix_date = True + + +[RSYNC] +sources = /dir/to/sync +additional_args = --progress -r + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +additional_args = +target_filename = itsa_mysqldump +prefix_date = True Index: /branches/ccl_branches/ipp-20190806/backups/testing/confluence_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/confluence_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/confluence_test.config (revision 41070) @@ -0,0 +1,29 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/confluence_backups, + /data/ippops3.0/backups/confluence_backups, + /data/ippops5.0/backups/confluence_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/confluence/backups +additional_args = -v +conf_prefix = confluence-xml-backup- + +[TAR] +sources = /var/atlassian/application-data/confluence/attachments +additional_args = --verbose -z -p -cf +target_filename = confluence.tar.gz +prefix_date = True + +[RSYNC] +sources = /opt/atlassian, /var/atlassian +additional_args = -a --delete-after +sub-dirs = conf-install, conf-home + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = confluencedb +target_filename = confluence_mysqldump.sql +prefix_date = True Index: /branches/ccl_branches/ipp-20190806/backups/testing/ippops_homedir_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/ippops_homedir_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/ippops_homedir_test.config (revision 41070) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/homedir_backups, + /data/ippops4.0/backups/homedir_backups, + /data/ippops5.0/backups/homedir_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/home/panstarrs +additional_args = -a --delete-after Index: /branches/ccl_branches/ipp-20190806/backups/testing/jira_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/jira_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/jira_test.config (revision 41070) @@ -0,0 +1,28 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/jira_backups, + /data/ippops3.0/backups/jira_backups, + /data/ippops5.0/backups/jira_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /var/atlassian/application-data/jira/export +additional_args = -v + +[TAR] +sources = /var/atlassian/application-data/jira/data +additional_args = --verbose -z -p -cf +target_filename = jira.tar.gz +prefix_date = True + +[RSYNC] +sources = /opt/atlassian, /var/atlassian +additional_args = -a --delete-after +sub-dirs = jira-install, jira-home + +[MYSQLDUMP] +user = dumper +password = not_a_real_password +db_name = jiradb +target_filename = jira_mysqldump.sql +prefix_date = True Index: /branches/ccl_branches/ipp-20190806/backups/testing/mailman_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/mailman_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/mailman_test.config (revision 41070) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/mailman_backups, + /data/ippops4.0/backups/mailman_backups, + /data/ippops5.0/backups/mailman_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /var/lib/mailman +additional_args = -a --delete-after --copy-links Index: /branches/ccl_branches/ipp-20190806/backups/testing/svn_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/svn_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/svn_test.config (revision 41070) @@ -0,0 +1,14 @@ +[DEFAULT] +backup_paths = /export/ippops4.0/backups/svn_backups, + /data/ippops3.0/backups/svn_backups, + /data/ippops5.0/backups/svn_backups +source_machine = ippops4 +verbose = False + +[COPY] +sources = /etc/apache2/mods-enabled/dav_svn.conf +additional_args = -v + +[RSYNC] +sources = /export/ippops4.0/svnroot +additional_args = -a --delete-after Index: /branches/ccl_branches/ipp-20190806/backups/testing/website_test.config =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/testing/website_test.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/testing/website_test.config (revision 41070) @@ -0,0 +1,10 @@ +[DEFAULT] +backup_paths = /export/ippops3.0/backups/website_backups, + /data/ippops4.0/backups/website_backups, + /data/ippops5.0/backups/website_backups +source_machine = ippops3 +verbose = False + +[RSYNC] +sources = /export/ippc20.0/www +additional_args = -a --delete-after Index: /branches/ccl_branches/ipp-20190806/backups/utils/config_parse_helper.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/config_parse_helper.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/config_parse_helper.py (revision 41070) @@ -0,0 +1,180 @@ +from enum import Enum +from configparser import ConfigParser + +from utils.errors import ConfigError, ValidationError + + +class SpecialSchemaProcessing(Enum): + """These enums define how processing should be applied to a string to +separate it out into list. + """ + commma_separated = 1 + space_separated = 2 + + +class ConfigSchemaLine(): + """Defines a config line by what the key is, if it is required or optional, +the type that the value should be processed as, and any special processing required +""" + + def __init__(self, key: str, required: bool, expected_type, special_processing: SpecialSchemaProcessing=None): + self.key = key + self.required = required + self.expected_type = expected_type + if expected_type is list: + if type(special_processing) is not SpecialSchemaProcessing: + raise ValidationError("ValidationError: special_processing must be defined if implementing a list in config") + self.special_processing = special_processing + + def __str__(self): + return (f'<{self.__class__.__name__} - ' + f'Key:{self.key}, ' + f'Required:{self.required}, ' + f'Type:{self.expected_type}, ' + f'Processing: {self.special_processing}' + f'>') + + +class ConfigSchemaSection(): + """Defines a schema to be used. Consists of a single section name and +multiple config lines""" + + def __init__(self, section_name, options: [ConfigSchemaLine]): + self.section_name = section_name + self.options = {} + self._process_options(options) + + def _process_options(self, options): + if options is None: + return + if type(options) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} options must be '[ConfigSchemaLine]' or 'None'") + if len(options) == 0: + return + for o in options: + self.add_line(o) + + def add_line(self, line: ConfigSchemaLine): + self.options[line.key] = line + + def key_in_options(self, key: str) -> bool: + return key in self.options.keys() + + def get_schema_line(self, key: str) -> ConfigSchemaLine: + if not self.key_in_options(key): + raise KeyError(f"KeyError: '{key}' is not present in schema section") + return self.options[key] + + def __str__(self): + options_strs = [ o.__str__() for o in self.options] + return (f'{self.__class__.__name__} named "{self.section_name}" with {options_strs}') + + +class ConfigSchema(): + + def __init__(self, sections: [ConfigSchemaSection]): + self.sections = {} + self._process_sections(sections) + + def _process_sections(self, sections): + if sections is None: + return + if type(sections) is not list: + raise ValidationError(f"ValidationError: {self.__class__.__name__} sections must be '[ConfigSchemaSection]' or 'None'") + if len(sections) == 0: + return + for sec in sections: + self.add_section(sec) + + def add_section(self, section: ConfigSchemaSection): + self.sections[section.section_name] = section + + def section_exists(self, section_name: str) -> bool: + return section_name in self.sections.keys() + + def get_schema_section(self, section_name: str) -> ConfigSchemaSection: + if not self.section_exists(section_name): + raise KeyError(f"KeyError: '{section_name}' is not present in schema") + return self.sections[section_name] + + def __str__(self): + section_strs = [ o.__str__() for o in self.sections] + return (f'{self.__class__.__name__} with {section_strs}') + + +def _split_string_to_list(items: str, separator=',') -> [str]: + split_items = items.split(separator) + processed_items = [ i.strip(' ').strip('\n') for i in split_items] + return processed_items + + +def _parse_config_section(config: ConfigParser(), schema: ConfigSchemaSection) -> dict: + """For a given config it will extract all values defined by the schema +and return them as a dictionary of the keys and the assoicated value as the +type defined by the schema +""" + parsed_items = {} + + section = schema.section_name + for csl_key in schema.options: + csl = schema.get_schema_line(csl_key) + + # Parses each line + if config.has_option(section, csl_key): + if csl.expected_type is bool: + parsed_items[csl.key] = config.getboolean(section, csl.key) + elif csl.expected_type is int: + parsed_items[csl.key] = config.getint(section, csl.key) + elif csl.expected_type is float: + parsed_items[csl.key] = config.getfloat(section, csl.key) + elif csl.expected_type is str: + parsed_items[csl.key] = config[section][csl.key] + elif csl.expected_type is list: + if csl.special_processing == SpecialSchemaProcessing.commma_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ',') + parsed_items[csl.key] = splitup + elif csl.special_processing == SpecialSchemaProcessing.space_separated: + items = config[section][csl.key] + splitup = _split_string_to_list(items, ' ') + parsed_items[csl.key] = splitup + else: + + raise ConfigError(config, f"ConfigError: Don't know how to process special schema {csl.special_processing}") + else: + raise ConfigError(config, f"ConfigError: Type '{csl.expected_type}' cannot be processed - need to modify {__name__}") + + else: + if csl.required: + raise ConfigError(config, f"ConfigError: Key '{csl.key}' is required but was not found in config") + + return parsed_items + + +def _parse_config(config: ConfigParser(), schema: ConfigSchema) -> dict: + parsed_items = {} + + for css_name in schema.sections: + css = schema.get_schema_section(css_name) + + # if config.has_section(css_name): + parsed_section = _parse_config_section(config, css) + parsed_items[css_name] = parsed_section + # else: + # raise ConfigError(config, f"ConfigError: Expected section '{css_name}' from schema not present in config") + + return parsed_items + + +class ParsedConfig(): + + def __init__(self, config: ConfigParser, schema: ConfigSchema): + self._section_dict = _parse_config(config, schema) + + def has_section(self, section_name): + return section_name in self._section_dict + + def get_section(self, section_name): + if self.has_section(section_name): + return self._section_dict[section_name] + return None Index: /branches/ccl_branches/ipp-20190806/backups/utils/config_parse_helper_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/config_parse_helper_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/config_parse_helper_test.py (revision 41070) @@ -0,0 +1,295 @@ +import pytest + +import utils.config_parse_helper as helper +from utils.errors import ConfigError, ValidationError + + +class TestSpecialSchemaProcessingEnum(): + + def test_all_enums(self): + cs = helper.SpecialSchemaProcessing.commma_separated + assert cs.name == "commma_separated" + assert cs.value == 1 + + ss = helper.SpecialSchemaProcessing.space_separated + assert ss.name == "space_separated" + assert ss.value == 2 + + assert len(list(helper.SpecialSchemaProcessing)) == 2 + + +class TestConfigSchemaLine(): + + def test_initialising_simple_schema_lines(self): + csl = helper.ConfigSchemaLine('key1', True, bool) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is bool + assert csl.special_processing is None + + def test_initialising_list_in_schema(self): + csl = helper.ConfigSchemaLine('key1', True, list, helper.SpecialSchemaProcessing.commma_separated) + assert csl.key == 'key1' + assert csl.required is True + assert csl.expected_type is list + assert csl.special_processing is helper.SpecialSchemaProcessing.commma_separated + + def test_init_list_without_special_processing_fails(self): + not_a_valid_enum = 3 + with pytest.raises(ValidationError): + helper.ConfigSchemaLine('key1', True, list, not_a_valid_enum) + + +class TestConfigSchemaSection(): + + def test_initalising_schema(self): + csl1 = helper.ConfigSchemaLine(key='is_cool', required=False, expected_type=bool) + csl2 = helper.ConfigSchemaLine('fav_word', True, str) + csl3 = helper.ConfigSchemaLine('some_float', False, float) + + schema = helper.ConfigSchemaSection( 'section_name', [csl1, csl2, csl3] ) + assert schema.options == { 'is_cool' : csl1 + , 'fav_word' : csl2 + , 'some_float' : csl3 + } + + def test_empty_schema_section_ok(self): + helper.ConfigSchemaSection("ok_with_none", None) + helper.ConfigSchemaSection("ok_with_empty", []) + + def test_schema_section_fails_if_options_are_not_a_list(self): + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("fails_if_not_a_list", True) + + csl = helper.ConfigSchemaLine('key1', True, bool) + with pytest.raises(ValidationError): + helper.ConfigSchemaSection("even_fails_on_a_single_csl", csl) + + def test_adding_to_schema(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl1 = helper.ConfigSchemaLine('key1', True, bool) + csl2 = helper.ConfigSchemaLine('key2', False, int) + csl3 = helper.ConfigSchemaLine('key3', True, str) + config.add_line(csl1) + assert config.options == { 'key1': csl1 } + + config.add_line(csl2) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + } + + config.add_line(csl3) + assert config.options == { 'key1': csl1 + , 'key2': csl2 + , 'key3': csl3 + } + + def test_adding_a_duplicate_key_to_schema_replaces_previous(self): + config = helper.ConfigSchemaSection("adding", []) + assert config.options == {} + + csl_ori = helper.ConfigSchemaLine('key', True, bool) + csl_alt = helper.ConfigSchemaLine('key', False, int) + + config.add_line(csl_ori) + assert config.options == { 'key': csl_ori } + + config.add_line(csl_alt) + assert config.options == { 'key': csl_alt } + + +class TestStringSplitting(): + + def test_split_comma_separated_list(self): + before_parse = "/hello/sir/, /glorius/filename,/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected + + def test_removal_of_newlines_too(self): + before_parse = "\n/hello/sir/, /glorius/filename\n,\n/some/actual/file.file" + expected = [ "/hello/sir/" + , "/glorius/filename" + , "/some/actual/file.file" + ] + result = helper._split_string_to_list(before_parse, ',') + assert result == expected +# def _split_string_to_list(items: str, separator=',') -> [str]: + + def test_space_separated_split(self): + before_parse = "--verbose --progress -a -f -z" + expected = [ "--verbose" + , "--progress" + , "-a" + , "-f" + , "-z" + ] + result = helper._split_string_to_list(before_parse, ' ') + assert result == expected + + +class TestParsingConfig(): + + def make_simple_config(self, section_name='DEFAULT') -> helper.ConfigParser: + config = helper.ConfigParser() + config[section_name] = \ + { 'inty' : '3' + , 'floaty' : '3.141' + , 'mr_bool' : 'True' + , 'stringer' : "bell" + , 'listo_comma' : "i,am,comma,separated" + , 'listo_space' : "yo it's time for space!" + } + return config + + def test_parsing_fails_with_missing_required_key(self): + config = helper.ConfigParser() + config['DEFAULT'] = { 'not_the_required_key' : False } + + csl1 = helper.ConfigSchemaLine('Required_key', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_schema_section_with_only_optional_values_is_ok(self): + csl1 = helper.ConfigSchemaLine('not_required', False, bool) + csl2 = helper.ConfigSchemaLine('not_required_either', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1, csl2] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == {} + + def test_parsing_schema_section_with_ints(self): + csl1 = helper.ConfigSchemaLine('inty', False, int) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'inty' : 3} + + def test_parsing_schema_section_with_floats(self): + csl1 = helper.ConfigSchemaLine('floaty', False, float) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + + def test_parsing_schema_section_with_bools(self): + csl1 = helper.ConfigSchemaLine('mr_bool', True, bool) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'mr_bool' : True } + + def test_parsing_schema_section_with_strs(self): + csl1 = helper.ConfigSchemaLine('stringer', True, str) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'stringer' : 'bell' } + + def test_parsing_schema_section_with_lists_comma_separated(self): + csl1 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_comma' : [ "i" + , "am" + , "comma" + , "separated" + ] + } + + def test_parsing_schema_section_with_lists_space_separated(self): + csl1 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + result = helper._parse_config_section(config, schema) + assert result == { 'listo_space' : [ "yo" + , "it's" + , "time" + , "for" + , "space!" + ] + } + + def test_parsing_as_different_types_always_ok_for_str_and_list(self): + config = self.make_simple_config() + + csl_float = helper.ConfigSchemaLine('floaty', True, float) + csl_int = helper.ConfigSchemaLine('floaty', True, int) + csl_bool = helper.ConfigSchemaLine('floaty', True, bool) + csl_str = helper.ConfigSchemaLine('floaty', True, str) + csl_list = helper.ConfigSchemaLine('floaty', True, list, + helper.SpecialSchemaProcessing.commma_separated) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_float]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : 3.141 } + assert type(result['floaty']) is float + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_int]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_bool]) + with pytest.raises(ValueError): + helper._parse_config_section(config, schema) + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_str]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : '3.141' } + assert type(result['floaty']) is str + + schema = helper.ConfigSchemaSection('DEFAULT', [csl_list]) + result = helper._parse_config_section(config, schema) + assert result == { 'floaty' : ['3.141'] } + assert type(result['floaty']) is list + + def test_parsing_schema_line_fails_with_unknown_type(self): + csl1 = helper.ConfigSchemaLine('floaty', True, dict) + schema = helper.ConfigSchemaSection('DEFAULT', [csl1] ) + config = self.make_simple_config() + with pytest.raises(ConfigError): + helper._parse_config_section(config, schema) + + def test_all_special_processing_options_are_handled(self): + config = self.make_simple_config() + config['DEFAULT'] = {'enum_checker' : "comma, and spaces"} + + for e in helper.SpecialSchemaProcessing: + csl = helper.ConfigSchemaLine('enum_checker', False, list, e) + schema = helper.ConfigSchemaSection('DEFAULT', [csl]) + + result = helper._parse_config_section(config, schema) + assert type(result['enum_checker']) is list + + def test_parsing_all(self): + config = self.make_simple_config('FULL_TEST') + + csl1 = helper.ConfigSchemaLine('inty', False, int) + csl2 = helper.ConfigSchemaLine('floaty', False, float) + csl3 = helper.ConfigSchemaLine('mr_bool', True, bool) + csl4 = helper.ConfigSchemaLine('stringer', True, str) + csl5 = helper.ConfigSchemaLine('listo_comma', False, list, + helper.SpecialSchemaProcessing.commma_separated) + csl6 = helper.ConfigSchemaLine('listo_space', False, list, + helper.SpecialSchemaProcessing.space_separated) + schema = helper.ConfigSchemaSection('FULL_TEST', [csl1, csl2, csl3, csl4, csl5, csl6]) + + result = helper._parse_config_section(config, schema) + assert result == \ + { 'inty' : 3 + , 'floaty' : 3.141 + , 'mr_bool' : True + , 'stringer' : "bell" + , 'listo_comma' : ["i", "am", "comma", "separated"] + , 'listo_space' : ["yo", "it's", "time", "for", "space!"] + } Index: /branches/ccl_branches/ipp-20190806/backups/utils/errors.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/errors.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/errors.py (revision 41070) @@ -0,0 +1,16 @@ + + +class ConfigError(Exception): + """Error in loading the config file""" + + def __init__(self, config_item, message): + self.config_item = config_item + self.message = message + + +class ValidationError(Exception): + """Validation errors in utils functions. Raised when arguments are not valid""" + + +class SubprocessError(Exception): + """Errors in Popen stderr subprocess in subprocess_utils""" Index: /branches/ccl_branches/ipp-20190806/backups/utils/generic_utils.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/generic_utils.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/generic_utils.py (revision 41070) @@ -0,0 +1,124 @@ +import glob +import os.path as path +import shutil +from distutils.dir_util import copy_tree + +from .subprocess_utils import simple_unix_wrapper +from .errors import ValidationError + + +def copy_to_dir(source_file: str, destination_path: str, verbose: bool=False): + """ Copies a single file into the destination directory. If a directory is + given as a source then all the contents of the directory will be copied (the + directory itself will not be). +""" + + if source_file is None or destination_path is None: + raise ValidationError("Error: source for copy cannot be None type") + if destination_path is None: + raise ValidationError("Error: copy destination cannot be None type") + if not path.exists(source_file): + raise ValidationError(f"Error: Copy Failure: source does not exist: {source_file}") + if not path.exists(destination_path) or not path.isdir(destination_path): + print("Error: Copy Failure: destination path does not exist: ", destination_path) + raise ValidationError(f"Error: Copy Failure: destination path does not exist: {destination_path}") + + try: + if verbose: + print(f"Copying: {source_file} to {destination_path}") + if path.isdir(source_file): + copy_tree(source_file, destination_path) + else: + shutil.copy2(source_file, destination_path) + except IOError as e: + msg = f"Error: copy failure.\n copying: {source_file}\n to: {destination_path}" + raise IOError(msg) from e + return 0 + + +def _list_checks(sources: [str], targets: [str]): + if sources is None or targets is None: + raise ValidationError("Error: source for copy cannot be None type") + if targets is None: + raise ValidationError("Error: copy destination cannot be None type") + if type(sources) is not list or len(sources) == 0: + raise ValidationError("Error: sources must be given as a list with at least one entry") + if type(targets) is not list or len(targets) == 0: + raise ValidationError("Error: targets must be given as a list with at least one entry") + + +def copy_multiple_items_as_one_to_one(source_paths: [str], destination_paths: [str], verbose: bool=False): + _list_checks() + + for i in range(0, len(source_paths)): + + if not path.exists(destination_paths[i]): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", destination_paths[i]) + continue + if not path.isdir(destination_paths[i]): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", destination_paths[i]) + continue + + result = copy_to_dir(source_paths[i], destination_paths[i], verbose) + if result != 0: + return result + + +def copy_multiple_items_to_multiple_dirs(source_paths: [str], destination_paths: [str], verbose: bool=False): + """ Copies the sources to the destination paths. + +This is a generic function that just wraps shutil.copy2""" + _list_checks(source_paths, destination_paths) + + for dpath in destination_paths: + if not path.exists(dpath): + if verbose: + print("Copy Warning: target path does not exist (skipping): ", dpath) + continue + if not path.isdir(dpath): + if verbose: + print("Copy Warning: target path is not a directory (skipping): ", ) + continue + + for spath in source_paths: + result = copy_to_dir(spath, dpath, verbose) + if result != 0: + return result + + return 0 + + +def move_dirs(source_dirs: [str], destination_dirs: [str], verbose: bool=False): + """ Moves all files from inside source paths to the destination dits. +It's important to note that it moves index to index +i.e. source[0] -> dest[0]; source[1] -> dest[1] + +This is actually a pretty generic function, should be moved into a utils class. +""" + + _list_checks(source_dirs, destination_dirs) + + if len(source_dirs) != len(destination_dirs): + raise ValidationError("Error: source and target paths are not equal; aborting file movement") + + for i in range(0, len(source_dirs)): + # Identify files in path, move each one: + wildcard_search = "{0}/*".format(source_dirs[i]) + filepaths = glob.glob(wildcard_search) + if len(filepaths) == 0: + print("Warning: directory is empty: ", source_dirs[i]) + continue + try: + for fp in filepaths: + if verbose: + print("Moving: {0} -> {1}".format(fp, destination_dirs[i])) + command_and_args = ["mv", fp, destination_dirs[i]] + # shutil.move(fp, destination_dirs[i]) + simple_unix_wrapper(command_and_args) + except IOError as e: + print("IOError copying: ", e.filename, e.filename2) + return 2 + + return 0 Index: /branches/ccl_branches/ipp-20190806/backups/utils/generic_utils_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/generic_utils_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/generic_utils_test.py (revision 41070) @@ -0,0 +1,224 @@ +import os +import pytest + +from .generic_utils import * +from .errors import ValidationError +from pathlib import Path + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyingToDir(object): + + def test_copy_fails_with_none_args(self, tmpdir): + with pytest.raises(ValidationError): + copy_to_dir(None, tmpdir) + + with pytest.raises(ValidationError): + real_file = make_a_real_file(tmpdir, "file.txt") + copy_to_dir(real_file, None) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(ValidationError): + copy_to_dir("not a real file", tmpdir) + + def test_copy_fails_if_destination_is_not_real(self, tmpdir): + with pytest.raises(ValidationError): + real_file = make_a_real_file(tmpdir, "real_file.txt") + copy_to_dir(real_file, "not a real directory") + + def test_copy_fails_if_destination_is_not_a_directory(self, tmpdir): + + real_file = make_a_real_file(tmpdir, "not_a_directory.txt") + not_a_directory = real_file + assert os.path.exists(not_a_directory) + assert not os.path.isdir(not_a_directory) + + with pytest.raises(ValidationError): + copy_to_dir(real_file, not_a_directory) + + def test_copy_to_dir(self, tmpdir): + + # Setup source file and target dir + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + file_name = "fun_file.txt" + file_to_copy = os.path.join(tmpdir, file_name) + Path(file_to_copy).touch() + assert os.path.exists(file_to_copy) + + # copied file does not exist yet + expected_copy = os.path.join(tmpdir, "destination", file_name) + assert not os.path.exists(expected_copy) + + # run copy function and confirm copy exists (and original) + result = copy_to_dir(file_to_copy, destination_dir) + assert result == 0 + assert os.path.exists(file_to_copy) + assert os.path.exists(expected_copy) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + filepath_a = make_a_real_file(subdir, filename_a) + filepath_b = make_a_real_file(subsubdir, filename_b) + + print(filepath_a) + print(filepath_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + print(expected_filepath_a) + print(expected_filepath_b) + print(os.path.exists(expected_filepath_a)) + if os.path.exists(expected_filepath_a): + print("apple") + else: + print("banana") + print(os.path.exists(expected_filepath_b)) + + os.path.commonpath + + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # Provide only the original tmpdir to the copying function + copy_to_dir(subdir, destination_dir, True) + + # assert that the subdirs and files all exist + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + +class TestMultiCopy(object): + + def test_copy_fails_if_args_are_none(self, tmpdir): + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpdir, None) + + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(None, tmpdir) + + def test_copy_requires_args_to_be_in_list_format(self, tmpdir): + + not_a_list = "not a list" + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(not_a_list, tmpdir) + + tmpfile = make_a_real_file(tmpdir, "a_file.txt") + with pytest.raises(ValidationError): + copy_multiple_items_to_multiple_dirs(tmpfile, not_a_list) + + def test_copy_is_fine_with_single_items(self, tmpdir): + + tmpfile = make_a_real_file(tmpdir, "single_file.txt") + destination = os.path.join(tmpdir, "destination") + os.makedirs(destination) + expected_file = os.path.join(tmpdir, "destination", "single_file.txt") + assert not os.path.exists(expected_file) + + result = copy_multiple_items_to_multiple_dirs([tmpfile], [destination]) + assert result == 0 + assert os.path.exists(expected_file) + + def test_copy_multiple_items_to_multiple_dirs(self, tmpdir): + + file_a = make_a_real_file(tmpdir, "single_file.txt") + file_b = make_a_real_file(tmpdir, "single_file.txt") + file_c = make_a_real_file(tmpdir, "single_file.txt") + source_files = [ file_a + , file_b + , file_c + ] + + destination_1 = os.path.join(tmpdir, "dest_1") + destination_2 = os.path.join(tmpdir, "dest_2") + destination_3 = os.path.join(tmpdir, "dest_3") + dest_paths = [ destination_1 + , destination_2 + , destination_3 + ] + os.makedirs(destination_1) + os.makedirs(destination_2) + os.makedirs(destination_3) + + expected_files = [] + for dp in dest_paths: + for sf in source_files: + expected_files.append( os.path.join(dp, os.path.basename(sf))) + assert not os.path.exists(expected_files[-1]) + assert len(expected_files) == 9 + + result = copy_multiple_items_to_multiple_dirs(source_files, dest_paths) + assert result == 0 + for ef in expected_files: + assert os.path.exists(ef) + + def test_copying_is_recursive(self, tmpdir): + + # make some subdirs, with sub-subdirs, and files within them + subdir_a = os.path.join(tmpdir, "subdir_a") + subsubdir_aa = os.path.join(subdir_a, "subsubdir_aa") + subsubdir_ab = os.path.join(subdir_a, "subsubdir_ab") + subdir_b = os.path.join(tmpdir, "subdir_b") + subsubdir_ba = os.path.join(subdir_b, "subsubdir_ba") + subsubdir_bb = os.path.join(subdir_b, "subsubdir_bb") + os.makedirs(subsubdir_aa) + os.makedirs(subsubdir_ab) + os.makedirs(subsubdir_ba) + os.makedirs(subsubdir_bb) + make_a_real_file(subdir_a, "file_a.txt") + make_a_real_file(subsubdir_aa, "file_aa.txt") + make_a_real_file(subsubdir_ab, "file_ab.txt") + make_a_real_file(subdir_b, "file_b.txt") + make_a_real_file(subsubdir_ba, "file_ba.txt") + make_a_real_file(subsubdir_bb, "file_bb.txt") + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + + expected_filepath_a = os.path.join(destination_dir, "subdir_a", "file_a.txt") + expected_filepath_aa = os.path.join(destination_dir, "subdir_a", "subsubdir_aa", "file_aa.txt") + expected_filepath_ab = os.path.join(destination_dir, "subdir_a", "subsubdir_ab", "file_ab.txt") + expected_filepath_b = os.path.join(destination_dir, "subdir_b", "file_b.txt") + expected_filepath_ba = os.path.join(destination_dir, "subdir_b", "subsubdir_ba", "file_ba.txt") + expected_filepath_bb = os.path.join(destination_dir, "subdir_b", "subsubdir_bb", "file_bb.txt") + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_aa) + assert not os.path.exists(expected_filepath_ab) + assert not os.path.exists(expected_filepath_b) + assert not os.path.exists(expected_filepath_ba) + assert not os.path.exists(expected_filepath_bb) + + # Provide only the original tmpdir to the copying function + copy_multiple_items_to_multiple_dirs([tmpdir], [destination_dir], True) + + # assert that the subdirs and files all exist + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_aa) + assert os.path.exists(expected_filepath_ab) + assert os.path.exists(expected_filepath_b) + assert os.path.exists(expected_filepath_ba) + assert os.path.exists(expected_filepath_bb) Index: /branches/ccl_branches/ipp-20190806/backups/utils/subprocess_utils.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/subprocess_utils.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/subprocess_utils.py (revision 41070) @@ -0,0 +1,56 @@ +import subprocess + +# Plans are to flesh this out a bit more to make some smarter copying etc +from .errors import ValidationError, SubprocessError + + +def _args_check(args, function_name): + if args is None: + raise ValidationError(f"Error: no arguments provided in {function_name}") + if type(args) != list or len(args) == 0: + raise ValidationError(f"Error: no arguments provided in {function_name}") + return 0 + + +def cp_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {cp_wrapper.__name__}") + command_and_args = ["cp"] + args + return simple_unix_wrapper(command_and_args) + + +def tar_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {tar_wrapper.__name__}") + command_and_args = ["tar"] + args + return simple_unix_wrapper(command_and_args) + + +def chmod_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error: no arguments provided to {chmod_wrapper.__name__}") + command_and_args = ["chmod"] + args + return simple_unix_wrapper(command_and_args) + + +def local_rsync_wrapper(args: [str]): + if args is None: + raise ValidationError(f"Error:no arguments provided to {local_rsync_wrapper.__name__}") + command_and_args = ["rsync"] + args + return simple_unix_wrapper(command_and_args) + + +def simple_unix_wrapper(command_and_args: [str]): + """ This should provide the program name and all arguments in a list. + +Only a single command should be called. + """ + _args_check(command_and_args, simple_unix_wrapper.__name__) + + try: + subprocess_exitcode = subprocess.check_call(command_and_args) + except (subprocess.CalledProcessError) as cpe: + raise SubprocessError(f"{type(cpe)} in python simple_unix_wrapper") from cpe + except (ValidationError) as ve: + raise ValidationError(f"{type(ve)} in python simple_unix_wrapper") from ve + return subprocess_exitcode Index: /branches/ccl_branches/ipp-20190806/backups/utils/subprocess_utils_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/utils/subprocess_utils_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/utils/subprocess_utils_test.py (revision 41070) @@ -0,0 +1,240 @@ +import backups.utils.subprocess_utils as utils +import os +import pytest +import stat + +from pathlib import Path +from .errors import SubprocessError, ValidationError + + +def make_a_real_file(file_path, file_name): + real_file = os.path.join(file_path, file_name) + Path(real_file).touch() + assert os.path.exists(real_file) + assert os.path.isfile(real_file) + return real_file + + +class TestCopyWrapper(object): + + def test_copy_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.cp_wrapper(None) + + def test_copy_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.cp_wrapper(["not a real file", "not a valid copy_location"]) + + def test_copy_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.cp_wrapper(["not a real file", tmpdir]) + + def test_simple_copy(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.cp_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_copy_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run copy function and confirm copy exists (and original) + result = utils.cp_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_copying_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # fails without -r arg: + with pytest.raises(SubprocessError): + utils.cp_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.cp_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) + + +class TestTarWrapper(object): + + def test_tar_raise_validation_error_with_none_args(self): + with pytest.raises(ValidationError): + utils.tar_wrapper(None) + + def test_tar_returns_safely_with_too_few_args(self): + with pytest.raises(SubprocessError): + utils.tar_wrapper(["only one arg"]) + + def test_tar_return_error_if_invalid_args_given(self, tmpdir): + tar_filename = os.path.join(tmpdir, "tar_file.tar") + fake_file = os.path.join(tmpdir, "fake_file.txt") + assert not os.path.exists(fake_file) + with pytest.raises(SubprocessError): + utils.tar_wrapper(["-cf", tar_filename, fake_file]) + + def test_simple_tar_of_files(self, tmpdir): + file_a = make_a_real_file(tmpdir, "file_a.txt") + file_b = make_a_real_file(tmpdir, "file_b.txt") + + expected_file = os.path.join(tmpdir, "output.tar") + assert not os.path.exists(expected_file) + + args = ["-cf", expected_file, file_a, file_b] + utils.tar_wrapper(args) + assert os.path.exists(expected_file) + + +class TestChmodWrapper(object): + + def test_chmod_gives_validation_error_when_no_args(self): + with pytest.raises(utils.ValidationError): + utils.chmod_wrapper(None) + + def test_chmod_wrapper_raises_error_with_invalid_args(self): + with pytest.raises(SubprocessError): + utils.chmod_wrapper(["only one arg"]) + + def test_chmod_fails_if_no_permissions_given(self, tmpdir): + real_file = make_a_real_file(tmpdir, 'mr_file') + + with pytest.raises(SubprocessError): + utils.chmod_wrapper([real_file]) + + with pytest.raises(SubprocessError): + utils.chmod_wrapper(["dumb_arg", real_file]) + + def test_can_chmod_file(self, tmpdir): + real_file = make_a_real_file(tmpdir, 'mr_file') + + # check initial permissions + org_stats = os.stat(real_file) + # Default should be: rw-rw-r-- + assert bool(org_stats.st_mode & stat.S_IRUSR) is True + assert bool(org_stats.st_mode & stat.S_IWUSR) is True + assert bool(org_stats.st_mode & stat.S_IXUSR) is False + assert bool(org_stats.st_mode & stat.S_IRGRP) is True + assert bool(org_stats.st_mode & stat.S_IWGRP) is True + assert bool(org_stats.st_mode & stat.S_IXGRP) is False + assert bool(org_stats.st_mode & stat.S_IROTH) is True + assert bool(org_stats.st_mode & stat.S_IWOTH) is False + assert bool(org_stats.st_mode & stat.S_IXOTH) is False + + # change them + utils.chmod_wrapper(["777", real_file]) + + # check they changed + new_stats = os.stat(real_file) + assert bool(new_stats.st_mode & stat.S_IRUSR) is True + assert bool(new_stats.st_mode & stat.S_IWUSR) is True + assert bool(new_stats.st_mode & stat.S_IXUSR) is True + assert bool(new_stats.st_mode & stat.S_IRGRP) is True + assert bool(new_stats.st_mode & stat.S_IWGRP) is True + assert bool(new_stats.st_mode & stat.S_IXGRP) is True + assert bool(new_stats.st_mode & stat.S_IROTH) is True + assert bool(new_stats.st_mode & stat.S_IWOTH) is True + assert bool(new_stats.st_mode & stat.S_IXOTH) is True + + +class TestLocalRsyncWrapper(object): + """This is essentially a copy of the 'copy' tests above. + """ + + def test_rsync_raises_validation_error_with_no_args(self): + with pytest.raises(ValidationError): + utils.local_rsync_wrapper(None) + + def test_rsync_fails_with_invalid_args(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", "not a valid rsync_location"]) + + def test_rsync_fails_if_source_does_not_exist(self, tmpdir): + with pytest.raises(SubprocessError): + utils.local_rsync_wrapper(["not a real file", tmpdir]) + + def test_simple_rsync(self, tmpdir): + real_file = make_a_real_file(tmpdir, "real_file.txt") + expected_filepath = os.path.join(tmpdir, "copied_file.txt") + assert not os.path.isfile(expected_filepath) + + result = utils.local_rsync_wrapper(["-v", real_file, expected_filepath]) + assert os.path.isfile(expected_filepath) + assert result == 0 + + def test_simple_rsync_to_directory(self, tmpdir): + + # Setup source file and target dir + filename = "real_file.txt" + real_file = make_a_real_file(tmpdir, filename) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + assert os.path.exists(destination_dir) + + expected_filepath = os.path.join(destination_dir, filename) + assert not os.path.exists(expected_filepath) + + # run rsync function and confirm rsync exists (and original) + result = utils.local_rsync_wrapper([real_file, destination_dir]) + assert result == 0 + assert os.path.isfile(real_file) + assert os.path.exists(expected_filepath) + + def test_rsyncing_directories_and_subdirectories(self, tmpdir): + + subdir = os.path.join(tmpdir, "subdir") + subsubdir = os.path.join(subdir, "subsubdir") + os.makedirs(subsubdir) + filename_a = "file_in_subdir.txt" + filename_b = "file_in_subsubdir.txt" + make_a_real_file(subdir, filename_a) + make_a_real_file(subsubdir, filename_b) + + destination_dir = os.path.join(tmpdir, "destination") + os.makedirs(destination_dir) + expected_filepath_a = os.path.join(destination_dir, "subdir", filename_a) + expected_filepath_b = os.path.join(destination_dir, "subdir", "subsubdir", filename_b) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # silently passes without -r, just skips over directories + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + utils.local_rsync_wrapper([subdir, destination_dir]) + assert not os.path.exists(expected_filepath_a) + assert not os.path.exists(expected_filepath_b) + + # requires -r to copy directories (and contents) + utils.local_rsync_wrapper(["-r", subdir, destination_dir]) + assert os.path.exists(expected_filepath_a) + assert os.path.exists(expected_filepath_b) Index: /branches/ccl_branches/ipp-20190806/backups/website_backup.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/website_backup.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/website_backup.py (revision 41070) @@ -0,0 +1,39 @@ +import os.path as path + +import utils.config_parse_helper as cfg_help +from backup import Backup +from utils.config_parse_helper import ConfigSchema, ConfigSchemaSection, ConfigSchemaLine + +EXPECTED_CONFIG_FILE = path.join(path.dirname(__file__), './website_backup.config') + +WEBSITE_SCHEMA = ConfigSchema( + [ ConfigSchemaSection('DEFAULT', + [ ConfigSchemaLine('backup_paths', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('source_machine', True, str) + , ConfigSchemaLine('verbose', False, bool) + , ConfigSchemaLine('make_dirs', False, bool) + ]) + , ConfigSchemaSection('RSYNC', + [ ConfigSchemaLine('sources', True, list, + cfg_help.SpecialSchemaProcessing.commma_separated) + , ConfigSchemaLine('additional_args', False, list, + cfg_help.SpecialSchemaProcessing.space_separated) + ]) + ] +) + + +class WebsiteBackup(Backup): + + def __init__(self, config_file=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + self.load_config(config_file, schema) + + +def main(config=EXPECTED_CONFIG_FILE, schema=WEBSITE_SCHEMA): + wb = WebsiteBackup(config, schema) + wb.run_backup_processes() + + +if __name__ == "__main__": + main() Index: /branches/ccl_branches/ipp-20190806/backups/website_backup_test.py =================================================================== --- /branches/ccl_branches/ipp-20190806/backups/website_backup_test.py (revision 41070) +++ /branches/ccl_branches/ipp-20190806/backups/website_backup_test.py (revision 41070) @@ -0,0 +1,116 @@ +import configparser +import os +from pathlib import Path + +import backup as bckup +import website_backup as webbck_mod +from website_backup import WebsiteBackup +from utils.config_parse_helper import ConfigSchema + +WEBSITE_TEST_CONFIG = os.path.join(os.path.dirname(__file__), + './testing/website_test.config') + + +class TestArguments(object): + + def assert_defaults(self, wb: WebsiteBackup): + + assert wb.default_dict == \ + { 'backup_paths' : [ '/export/ippops3.0/backups/website_backups' + , '/data/ippops4.0/backups/website_backups' + , '/data/ippops5.0/backups/website_backups' + ] + , 'source_machine' : 'ippops3' + , 'verbose' : False + } + assert wb.rsync_dict == \ + { 'sources' : ['/export/ippc20.0/www'] + , 'additional_args' : ['-a', '--delete-after'] + } + + def test_load_default_config_file(self): + wb = WebsiteBackup(config_file=WEBSITE_TEST_CONFIG) + self.assert_defaults(wb) + + +class TestFullRunFromMain(object): + + def test_main_loads_and_runs_fine_when_given_config(self, tmpdir): + config = configparser.ConfigParser() + bpath1 = os.path.join(tmpdir, "path/1") + bpath2 = os.path.join(tmpdir, "path/2") + config['DEFAULT'] = \ + { 'backup_paths' : f"{bpath1}, {bpath2}" + , 'source_machine' : 'rocket' + , 'verbose' : 'True' + } + tar_source = os.path.join(tmpdir, "tar/source") + config['TAR'] = \ + { 'sources' : f"{tar_source}" + , 'additional_args' : "--verbose -z -p -cf" + , 'target_filename' : 'svnroot.tar.gz' + , 'prefix_date' : 'False' + } + os.makedirs(bpath1) + os.makedirs(bpath2) + os.makedirs(tar_source) + + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.TAR_SCHEMA_SECTION) + + webbck_mod.main(config, schema) + assert True is True + + +class TestWebsiteRsync(object): + + def test_rsync_with_additional_args(self, tmpdir): + + # Setup config + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + config = configparser.ConfigParser() + config['DEFAULT'] = \ + { 'backup_paths' : f"{backup_1_path}, {backup_2_path}" + , 'source_machine' : 'ippops3' + , 'verbose' : 'True' + } + config['RSYNC'] = \ + { 'sources' : f"{os.path.join(tmpdir, 'www')}" + , 'additional_args' : "-a --delete-after" + } + backup_1_path = os.path.join(tmpdir, "backup_1_path") + backup_2_path = os.path.join(tmpdir, "backup_2_path") + os.makedirs(backup_1_path) + os.makedirs(backup_2_path) + + # Create files to be Rync'd + homedir = os.path.join(tmpdir, "www") + subdir = os.path.join(homedir, "subdir") + file = os.path.join(subdir, "website_file") + os.makedirs(homedir) + os.makedirs(subdir) + Path(file).touch() + + # Setup Schema + schema = ConfigSchema(None) + schema.add_section(bckup.DEFAULT_SCHEMA_SECTION) + schema.add_section(bckup.RSYNC_SCHEMA_SECTION) + + # check that the rsync is successful (with options) + wb = WebsiteBackup(config, schema) + + expected_subdir_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir') + expected_file_bck1 = os.path.join(tmpdir, 'backup_1_path', 'www', 'subdir/website_file') + expected_subdir_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir') + expected_file_bck2 = os.path.join(tmpdir, 'backup_2_path', 'www', 'subdir/website_file') + assert not os.path.exists(expected_subdir_bck1) + assert not os.path.exists(expected_file_bck1) + assert not os.path.exists(expected_subdir_bck2) + assert not os.path.exists(expected_file_bck2) + wb._run_rsync() + assert os.path.exists(expected_subdir_bck1) + assert os.path.exists(expected_file_bck1) + assert os.path.exists(expected_subdir_bck2) + assert os.path.exists(expected_file_bck2) Index: /branches/ccl_branches/ipp-20190806/dbconfig/add.md =================================================================== --- /branches/ccl_branches/ipp-20190806/dbconfig/add.md (revision 41069) +++ /branches/ccl_branches/ipp-20190806/dbconfig/add.md (revision 41070) @@ -17,5 +17,4 @@ minidvodb_group STR 64 minidvodb_name STR 64 - hostname STR 64 END Index: /branches/ccl_branches/ipp-20190806/dbconfig/minidvodb.md =================================================================== --- /branches/ccl_branches/ipp-20190806/dbconfig/minidvodb.md (revision 41069) +++ /branches/ccl_branches/ipp-20190806/dbconfig/minidvodb.md (revision 41070) @@ -6,5 +6,4 @@ state STR 64 creation_date UTC 0001-01-01T00:00:00Z - hostname STR 64 END Index: /branches/ccl_branches/ipp-20190806/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/ccl_branches/ipp-20190806/ippMonitor/raw/czartool_labels.php (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippMonitor/raw/czartool_labels.php (revision 41070) @@ -81,12 +81,15 @@ $servers=array( ##### should be the same as Pantasks.pm #### + "addstar", + "cleanup", + "stdscience", ##### standard pantasks ###### - "cleanup", - "distribution", - "pstamp", - "registration", - "stack", - "stdscience", - "summitcopy", +# "cleanup", +# "distribution", +# "pstamp", +# "registration", +# "stack", +# "stdscience", +# "summitcopy", ##### additional pantasks ###### # "addstar", @@ -98,5 +101,5 @@ # "replication", # "update", - "temp" +# "temp" ); Index: /branches/ccl_branches/ipp-20190806/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/ccl_branches/ipp-20190806/ippScripts/scripts/camera_exp.pl (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippScripts/scripts/camera_exp.pl (revision 41070) @@ -134,4 +134,5 @@ # Get list of component files +# get FWHM statistics from input chip cmf files using ppStatsFromMetadata my $files; # Array of component files { Index: /branches/ccl_branches/ipp-20190806/ippScripts/scripts/night_report.pl =================================================================== --- /branches/ccl_branches/ipp-20190806/ippScripts/scripts/night_report.pl (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippScripts/scripts/night_report.pl (revision 41070) @@ -17,4 +17,5 @@ 'very-verbose' => \$verbose2, # Print to stdout 'chunks' => \$chunks, # give information on the current status of the chunks + 'mopsreport' => \$mopsreport, # return a report in the new style requested by MOPS ) or pod2usage( 2 ); @@ -40,7 +41,4 @@ $dataGroupDate=~ s/\.//g; -print "\n\n\n"; -print "--- Report for $dbname : $date ---\n\n"; - # set up the database connection: my $dsn = "DBI:mysql:host=$dbhost;database=$dbname"; @@ -69,4 +67,261 @@ ################## +# testing how to get more information parsed by chunk +if ($mopsreport or $report) { + print "\n\n\n"; + print "--- MOPS report for $dbname : $date ---\n\n"; + + my $badcamflag = 0; + my $badwarpflag = 0; + my $baddiffflag = 0; + + #do a query of raw/chip/cam/warp counts for this night, split by chunk. + #We want to stick to only MOPS products, so we do labels %SS%, %EU% and %Bright.% (to avoid BrightTwi and Bright3PI which are not in quads) + #we need to do a subquery to find the total number of skycells in warpSkyfile, as well as the number of those with a bad quality + my $query = "SELECT suba.chunk,suba.Nvis1,suba.Nvis2,suba.Nvis3,suba.Nvis4,suba.Ncamgood,suba.Ncambad,subb.Nwarpgood,subb.Nwarpbad FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(if(rawExp.comment LIKE '%visit 1%',1,NULL)) AS Nvis1,"; + $query .= " count(if(rawExp.comment LIKE '%visit 2%',1,NULL)) AS Nvis2,count(if(rawExp.comment LIKE '%visit 3%',1,NULL)) AS Nvis3,"; + $query .= " count(if(rawExp.comment LIKE '%visit 4%',1,NULL)) AS Nvis4, count(if(camProcessedExp.sigma_ra <= 5 AND camProcessedExp.sigma_dec <= 5 AND camProcessedExp.quality = 0,1,NULL)) AS Ncamgood,"; + $query .= " count(if(camProcessedExp.sigma_ra > 5 OR camProcessedExp.sigma_dec > 5 OR camProcessedExp.quality > 0,1,NULL)) AS Ncambad"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) JOIN camProcessedExp USING (cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY chunk ORDER BY dateobs) as suba "; + $query .= "LEFT JOIN"; + $query .= " (SELECT subc.chunk, count(if(subc.Nwarpskycellbad < subc.Nwarpskycell,1,NULL)) AS Nwarpgood, count(if(subc.Nwarpskycellbad = subc.Nwarpskycell,1,NULL)) AS Nwarpbad"; + $query .= " FROM "; + $query .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(warpSkyfile.quality) AS Nwarpskycell,count(if(warpSkyfile.quality > 0,1,NULL)) AS Nwarpskycellbad"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) JOIN warpSkyfile USING (warp_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY exp_name) as subc"; + $query .= " GROUP BY chunk) AS subb "; + $query .= "ON suba.chunk=subb.chunk"; + + my $result = &mysql_select ($query); + + printf "%18s | %11s | %4s | %4s | %7s | %4s | %7s | %4s | %7s |\n", "chunk", "Nobs", "Ncam","Nwrp","Nwrp_pub", "Nwwd", "Nwwd_pub", "Nwsd", "Nwsd_pub"; + while (@row = $result->fetchrow_array()) { + my $Nvis1 = 0; + my $Nvis2 = 0; + my $Nvis3 = 0; + my $Nvis4 = 0; + my $Ncamgood = 0; + my $Ncambad = 0; + my $Nwarpgood = 0; + my $Nwarpbad = 0; + my $NexpWWdiff = 0; + my $Npubwarps = 0; + my $uniwarps = undef; + + my $chunkname = $row[0]; + if($row[1] gt 0) {$Nvis1 = $row[1]}; + if($row[2] gt 0) {$Nvis2 = $row[2]}; + if($row[3] gt 0) {$Nvis3 = $row[3]}; + if($row[4] gt 0) {$Nvis4 = $row[4]}; + if($row[5] gt 0) {$Ncamgood = $row[5]}; + if($row[6] gt 0) {$Ncambad = $row[6]}; + if($row[7] gt 0) {$Nwarpgood = $row[7]}; + if($row[8] gt 0) {$Nwarpbad = $row[8]}; + + #set flags if present + if($Ncambad >= 1) {$badcamflag = 1}; + if($Nwarpbad >= 1) {$badwarpflag = 1}; + + #Find the total published number of unique warps for this chunk + #my $query0 = " SELECT count(DISTINCT warpRun.warp_id) FROM "; + #$query0 .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) JOIN publishRun ON (diffInputSkyfile.diff_id = stage_id)"; + #$query0 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%')"; + #$query0 .= " AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL' AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' group by warpRun.warp_id"; + #my $result0 = &mysql_select ($query0); + #while (@row = $result0->fetchrow_array()) {$Npubwarps = $row[0];} + + + #Find the expected number of diffs for this chunk + my $query1 = "SELECT suba.chunk, SUM(IF(suba.Nvisits = 2,1,NULL)) as totvis2, SUM(IF(suba.Nvisits = 3 OR suba.Nvisits = 4,2,NULL)) as totvis34 FROM"; + $query1 .= " (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits FROM"; + $query1 .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) WHERE dateobs like '$date%'"; + $query1 .= " AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query1 .= " AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' group by object) as suba group by chunk;"; + my $result1 = &mysql_select ($query1); + while (@row = $result1->fetchrow_array()) {$NexpWWdiff = $row[1] + $row[2];} + + #find the unique diff_ids for each chunk, to be matched to the diffRun catalog + my $query2 = " SELECT diff_id,warp1,stack1,warp2,stack2 FROM "; + $query2 .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query2 .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%')"; + $query2 .= " AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL' AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%' GROUP By diff_id"; + my $result2 = &mysql_select ($query2); + + my $NWWdiffgood = 0; + my $NWWdiffbad = 0; + my $NWWdiffpub = 0; + + my $NWSdiffgood = 0; + my $NWSdiffbad = 0; + my $NWSdiffpub = 0; + my %wrpcnt = (); + + while (@row2 = $result2->fetchrow_array()) { + #distinguish between WW and WS diffs based on stack2 + if ($row2[4] == 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row2[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]<$row3[0]) {$NWWdiffgood ++;} + if ($row3[1]==$row3[0]) {$NWWdiffbad ++;} + if ($row3[2]==$row3[0]) {$NWWdiffpub ++;} + + #if this wwdiff is published, record its corresponding warp_ids to see how many unique ones there are + if ($row3[2]==$row3[0]) {$wrpcnt{$row2[1]}++;} # record the presence of warp1 + if ($row3[2]==$row3[0]) {$wrpcnt{$row2[3]}++;} # record the presence of warp2 + } + } + + if ($row2[4] > 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row2[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]<$row3[0]) {$NWSdiffgood ++;} + if ($row3[1]==$row3[0]) {$NWSdiffbad ++;} + if ($row3[2]==$row3[0]) {$NWSdiffpub ++;} + } + } + } + #pull out the unique warp_ids + $Npubwarps = keys %wrpcnt; + + #check for bad diffs + if(($NWWdiffbad >= 1) or ($NWSdiffbad >= 1)) {$baddiffflag = 1;} + + if ($Nvis1 ne $Nvis4) { + if ($NWWdiffpub ne $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk incomplete, not done";} + if ($NWWdiffpub eq $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk incomplete, done";} + } + if ($Nvis1 eq $Nvis4) { + if ($NWWdiffpub ne $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk complete, not done";} + if ($NWWdiffpub eq $NexpWWdiff) {printf "%18s | %2s%1s%2s%1s%2s%1s%2s | %2s%1s%1s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %2s%1s%1s | %8s | %20s\n", $chunkname, $Nvis1, "/",$Nvis2, "/",$Nvis3, "/",$Nvis4,$Ncamgood, "/",$Ncambad,$Nwarpgood, "/",$Nwarpbad,$Npubwarps,$NWWdiffgood, "/",$NWWdiffbad,$NWWdiffpub,$NWSdiffgood, "/",$NWSdiffbad,$NWSdiffpub, "chunk complete, done";} + } + } + + print "\n"; + + #----------------------------------------- + #list bad quality cam exposures + if ($badcamflag != 0) { + my $query = "SELECT exp_name,cam_id, camProcessedExp.quality, camProcessedExp.sigma_ra, camProcessedExp.sigma_dec, comment"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun USING (exp_id)"; + $query .= " JOIN camRun USING (chip_id)"; + $query .= " JOIN camProcessedExp USING (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " AND (camProcessedExp.quality > 0 OR camProcessedExp.sigma_ra > 5 OR camProcessedExp.sigma_dec > 5)"; + $query .= " ORDER BY comment"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Bad quality camRun exposures:\n"; + printf "%11s | %7s | %7s | %9s | %9s | %15s\n", "exp_name","cam_id", "quality", "sigma_ra", "sigma_dec", "comment"; + $doHeader = 0; + } + printf "%11s | %7s | %7d | %9s | %9s | %15s\n", $row[0], $row[1], $row[2], $row[3], $row[4], $row[5]; + } + if (!$doHeader) { print "\n"; } + } + + #----------------------------------------- + #list bad quality warp exposures + if ($badwarpflag != 0) { + $query = "SELECT exp_name,warp_id,count(warpSkyfile.quality) AS Nwarpskycell,count(if(warpSkyfile.quality > 0,1,NULL)) AS Nwarpskycellbad, comment"; + $query .= " FROM rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) JOIN warpSkyfile USING (warp_id)"; + $query .= " WHERE dateobs like '$date%' AND exp_id > $refExpID"; + $query .= " AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP BY exp_name ORDER BY comment"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + + while (@row = $result->fetchrow_array()) { + if ($row[2] == $row[3]) { + if ($doHeader) { + print "Bad quality warpRun exposures:\n"; + printf "%11s | %7s | %15s\n", "exp_name", "warp_id", "comment"; + $doHeader = 0; + } + printf "%11s | %7d | %15s \n", $row[0], $row[1], $row[4]; + } + } + if (!$doHeader) { print "\n"; } + } + + #----------------------------------------- + #list bad quality diff exposures + if ($baddiffflag != 0) { + my $query = " SELECT diff_id,warp1,stack1,warp2,stack2,exp_name,comment FROM "; + $query .= " diffInputSkyfile JOIN warpRun ON (warp1=warp_id OR warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query .= " WHERE rawExp.dateobs LIKE '$date%' AND exp_id > $refExpID AND (camRun.label LIKE '%SS%' OR camRun.label LIKE '%EU%' OR camRun.label LIKE '%Bright.%') AND obs_mode NOT LIKE 'ENGINEERING' AND obs_mode NOT LIKE 'MANUAL'"; + $query .= " GROUP By diff_id"; + my $result = &mysql_select ($query); + + $doHeader = 1; + + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Bad quality diffRun exposures:\n"; + printf "%11s | %7s | %7s | %7s | %7s | %7s | %15s\n", "exp_name", "diff_id", "warp1", "stack1", "warp2", "stack2", "comment"; + $doHeader = 0; + } + + #distinguish between WW and WS diffs based on stack2 + if ($row[4] == 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]==$row3[0]) {printf "%11s | %7d | %7d | %7d | %7d | %7d | %15s \n", $row[5], $row[0], $row[1], $row[2], $row[3], $row[4], $row[6];} + } + } + + if ($row[4] > 0) { + #find the total number of skycell and the number of skycell with a bad quality flag as well as the full publish state + my $query3 = " SELECT count(diffSkyfile.quality) AS Ndiff,count(if(diffSkyfile.quality > 0,1,NULL)) AS Ndiffbad,count(if(publishRun.state LIKE 'full',1,NULL)) AS Npub FROM "; + $query3 .= " diffRun JOIN diffSkyfile USING (diff_id) LEFT JOIN publishRun ON (diffRun.diff_id = stage_id)"; + $query3 .= " WHERE diff_id='$row[0]' GROUP BY diffRun.diff_id"; + my $result3 = &mysql_select ($query3); + while (@row3 = $result3->fetchrow_array()) { + #if all skycells have bad quality, this is a bad diff. If all the skycells are in a full state in publishRun, it is fully published + if ($row3[1]==$row3[0]) {printf "%11s | %7d | %7d | %7d | %7d | %7d | %15s \n", $row[5], $row[0], $row[1], $row[2], $row[3], $row[4], $row[6];} + } + } + } + if (!$doHeader) { print "\n"; } + } +} + + +print "\n\n\n"; +print "--- IPP Report for $dbname : $date ---\n\n"; + + + ### testing the diff grouping -- this does not work for the Euclid data taken 2018.07.29 because the OBJECT ### field is missing @@ -433,4 +688,7 @@ print " * $diffExpect diffRuns expected *\n"; if ($verbose) { print "\n"; } + + + } @@ -520,4 +778,84 @@ if ($diffDone != $diffExpect) { print "*** WARNING: Expect $diffExpect diffRuns, complete $diffDone diffRuns: inconsistent ***\n"; + + if ($verbose) { + print "Published diffs split by Nvisit :\n"; + printf "%8s %10s\n", "Nvisit", "N_pubdiff"; + + my $query = "SELECT Nvisit, SUM(Ndiffs) FROM "; + $query .= " (SELECT object, filter, chunk, count(*) as Nvisit,count(DISTINCT diff_id) as Ndiffs FROM"; + $query .= " (SELECT expall.object, expall.filter, expall.chunk as chunk, expall.exp_name,exppub.diff_id from"; + $query .= " ("; + $query .= " SELECT object, filter,exp_name, substr(comment, 1, position(' ' in comment)) as chunk, dateobs, warp_id"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN fakeRun using (cam_id)"; + $query .= " JOIN warpRun using (fake_id)"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND exp_id > $refExpID"; + $query .= " AND (camProcessedExp.quality = 0)"; + $query .= " AND (comment NOT LIKE '%twilight%')"; + $query .= " ) AS expall"; + $query .= " JOIN"; + $query .= " (SELECT * FROM ("; + $query .= " (SELECT exp_name, warpRun.warp_id, diffRun.diff_id, publishRun.state as pubstate FROM"; + $query .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id)"; + $query .= " JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id)"; + $query .= " JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query .= " WHERE rawExp.dateobs like '$date%' AND diffRun.label NOT LIKE '%WS.nigh%' AND diffRun.label NOT LIKE 'goto_cleaned'"; + $query .= " AND stack2 IS NULL AND diffRun.data_group LIKE '%.$dataGroupDate')"; + $query .= " UNION"; + $query .= " (SELECT exp_name, warpRun.warp_id, diffRun.diff_id, publishRun.state as pubstate FROM"; + $query .= " diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id)"; + $query .= " JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id)"; + $query .= " JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)"; + $query .= " WHERE rawExp.dateobs like '$date%' AND diffRun.label NOT LIKE '%WS.nigh%' AND diffRun.label NOT LIKE 'goto_cleaned'"; + $query .= " AND stack2 IS NULL AND diffRun.data_group LIKE '%.$dataGroupDate')"; + $query .= " ) AS diffgather"; + $query .= " GROUP BY diffgather.exp_name)"; + $query .= " AS exppub"; + $query .= " ON expall.exp_name = exppub.exp_name"; + $query .= " ) AS pubdiffgather GROUP BY object, filter, chunk"; + $query .= " ) AS tableVisits GROUP BY Nvisit"; + + my $result = &mysql_select ($query); + while (@row = $result->fetchrow_array()) { + printf "%8s %10s\n", $row[0], $row[1]; + } + + #------------------------------------------------------------- + print "\n"; + print "Listing objects with 1< Nvisits < 4:\n"; + printf "| %12s | %8s | %19s | %8s |\n", "object", "filter", "chunk", "Nvisit"; + + my $query = "SELECT * FROM"; + $query .= " ("; + $query .= " SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + $query .= " ) AS tableVisits WHERE Nvisit > 1 AND Nvisit < 4 ORDER BY chunk,Nvisit "; + my $result = &mysql_select ($query); + + while (@row = $result->fetchrow_array()) { + printf "| %12s | %8s | %19s | %8s |\n", $row[0], $row[1], $row[2], $row[3]; + } + print "\n"; + } + } else { if (not $report) { print " * $diffDone of $diffExpect diffRuns completed *\n"; } @@ -781,5 +1119,5 @@ # count the number of chunks, and their status: -if ($verbose or $chunks) { +if ($verbose or $report) { print "Current status of SS and Euclid chunks:\n"; @@ -854,4 +1192,5 @@ + exit 0; Index: /branches/ccl_branches/ipp-20190806/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/ccl_branches/ipp-20190806/ippTools/share/pxadmin_create_tables.sql (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippTools/share/pxadmin_create_tables.sql (revision 41070) @@ -558,5 +558,4 @@ minidvodb_group VARCHAR(64), minidvodb_name VARCHAR(64), - hostname VARCHAR(64), PRIMARY KEY(add_id), KEY(add_id), @@ -2035,5 +2034,4 @@ epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, fault SMALLINT NOT NULL, - hostname varchar(64), UNIQUE KEY (minidvodb_id), KEY(minidvodb_id), Index: /branches/ccl_branches/ipp-20190806/ippTools/src/addtool.c =================================================================== --- /branches/ccl_branches/ipp-20190806/ippTools/src/addtool.c (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippTools/src/addtool.c (revision 41070) @@ -1668,5 +1668,4 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(hostname, config->args,"-hostname", false, false); @@ -1692,6 +1691,5 @@ minidvodbpath, "new", - 0, - hostname + 0 )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/ccl_branches/ipp-20190806/ippconfig/dvo.photcodes =================================================================== --- /branches/ccl_branches/ipp-20190806/ippconfig/dvo.photcodes (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippconfig/dvo.photcodes (revision 41070) @@ -21,4 +21,5 @@ # PS1 TC3: g: 3280-3296, r: 3380-3396, i: 3480-3496, z: 3580-3596, y: 3680-3696, # PS1 GPC1: g: 3201-3264, r: 3301-3364, i: 3401-3464, z: 3501-3564, y: 3601-3664, w: 3201-3464 +# PS2 GPC2: g: 3201-3264, r: 3301-3364, i: 3401-3464, z: 3501-3564, y: 3601-3664, w: 3201-3464 # NOTE 2012.03.30 EAM : I have added 'poor' and 'bad' bitsets for GPC1 data using psphot @@ -191,4 +192,6 @@ 2073 UKIRT.K.03 dep 22.570 0.050 0.000 - - 0.0000 0 8 0.080 0.000 5.000 0.000 0x0000 0x0000 0x0000 0x0000 2074 UKIRT.K.04 dep 22.570 0.050 0.000 - - 0.0000 0 8 0.080 0.000 5.000 0.000 0x0000 0x0000 0x0000 0x0000 + + 2080 UKIRT_UHS_J ref 0.000 0.000 0.000 - - 0.0000 0 6 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 # ICRF QSOs have astrometric error stored in dX,dY as 1/100th mas @@ -953,4 +956,7 @@ 12400 GPC1.y.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 12500 GPC1.w.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 9 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + + + 160 MOSAIC2.B.00 dep 25.107 0.000 0.000 - - 0.0000 0 1 0.000 0.000 1.000 0.000 0x0000 0x3888 0x0000 0x0000 161 MOSAIC2.B.01 dep 25.120 0.000 0.000 - - 0.0000 0 1 0.000 0.000 1.000 0.000 0x0000 0x3888 0x0000 0x0000 @@ -1779,2 +1785,376 @@ 26111 HSC.NB816.111 dep 25.000 0.000 0.000 - - 0.0000 0 - 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + + 30001 GPC2.g.XY01 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30002 GPC2.g.XY02 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30003 GPC2.g.XY03 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30004 GPC2.g.XY04 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30005 GPC2.g.XY05 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30006 GPC2.g.XY06 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30010 GPC2.g.XY10 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30011 GPC2.g.XY11 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30012 GPC2.g.XY12 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30013 GPC2.g.XY13 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30014 GPC2.g.XY14 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30015 GPC2.g.XY15 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30016 GPC2.g.XY16 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30017 GPC2.g.XY17 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30020 GPC2.g.XY20 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30021 GPC2.g.XY21 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30022 GPC2.g.XY22 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30023 GPC2.g.XY23 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30024 GPC2.g.XY24 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30025 GPC2.g.XY25 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30026 GPC2.g.XY26 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30027 GPC2.g.XY27 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30030 GPC2.g.XY30 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30031 GPC2.g.XY31 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30032 GPC2.g.XY32 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30033 GPC2.g.XY33 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30034 GPC2.g.XY34 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30035 GPC2.g.XY35 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30036 GPC2.g.XY36 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30037 GPC2.g.XY37 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30040 GPC2.g.XY40 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30041 GPC2.g.XY41 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30042 GPC2.g.XY42 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30043 GPC2.g.XY43 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30044 GPC2.g.XY44 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30045 GPC2.g.XY45 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30046 GPC2.g.XY46 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30047 GPC2.g.XY47 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30050 GPC2.g.XY50 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30051 GPC2.g.XY51 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30052 GPC2.g.XY52 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30053 GPC2.g.XY53 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30054 GPC2.g.XY54 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30055 GPC2.g.XY55 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30056 GPC2.g.XY56 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30057 GPC2.g.XY57 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30060 GPC2.g.XY60 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30061 GPC2.g.XY61 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30062 GPC2.g.XY62 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30063 GPC2.g.XY63 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30064 GPC2.g.XY64 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30065 GPC2.g.XY65 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30066 GPC2.g.XY66 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30067 GPC2.g.XY67 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30071 GPC2.g.XY71 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30072 GPC2.g.XY72 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30073 GPC2.g.XY73 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30074 GPC2.g.XY74 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30075 GPC2.g.XY75 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30076 GPC2.g.XY76 dep 24.563 -0.147 0.000 - - 0.0000 0 1 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30101 GPC2.r.XY01 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30102 GPC2.r.XY02 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30103 GPC2.r.XY03 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30104 GPC2.r.XY04 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30105 GPC2.r.XY05 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30106 GPC2.r.XY06 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30110 GPC2.r.XY10 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30111 GPC2.r.XY11 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30112 GPC2.r.XY12 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30113 GPC2.r.XY13 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30114 GPC2.r.XY14 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30115 GPC2.r.XY15 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30116 GPC2.r.XY16 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30117 GPC2.r.XY17 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30120 GPC2.r.XY20 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30121 GPC2.r.XY21 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30122 GPC2.r.XY22 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30123 GPC2.r.XY23 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30124 GPC2.r.XY24 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30125 GPC2.r.XY25 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30126 GPC2.r.XY26 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30127 GPC2.r.XY27 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30130 GPC2.r.XY30 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30131 GPC2.r.XY31 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30132 GPC2.r.XY32 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30133 GPC2.r.XY33 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30134 GPC2.r.XY34 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30135 GPC2.r.XY35 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30136 GPC2.r.XY36 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30137 GPC2.r.XY37 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30140 GPC2.r.XY40 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30141 GPC2.r.XY41 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30142 GPC2.r.XY42 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30143 GPC2.r.XY43 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30144 GPC2.r.XY44 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30145 GPC2.r.XY45 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30146 GPC2.r.XY46 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30147 GPC2.r.XY47 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30150 GPC2.r.XY50 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30151 GPC2.r.XY51 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30152 GPC2.r.XY52 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30153 GPC2.r.XY53 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30154 GPC2.r.XY54 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30155 GPC2.r.XY55 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30156 GPC2.r.XY56 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30157 GPC2.r.XY57 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30160 GPC2.r.XY60 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30161 GPC2.r.XY61 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30162 GPC2.r.XY62 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30163 GPC2.r.XY63 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30164 GPC2.r.XY64 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30165 GPC2.r.XY65 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30166 GPC2.r.XY66 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30167 GPC2.r.XY67 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30171 GPC2.r.XY71 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30172 GPC2.r.XY72 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30173 GPC2.r.XY73 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30174 GPC2.r.XY74 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30175 GPC2.r.XY75 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30176 GPC2.r.XY76 dep 24.750 -0.085 0.000 - - 0.0000 0 2 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30201 GPC2.i.XY01 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30202 GPC2.i.XY02 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30203 GPC2.i.XY03 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30204 GPC2.i.XY04 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30205 GPC2.i.XY05 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30206 GPC2.i.XY06 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30210 GPC2.i.XY10 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30211 GPC2.i.XY11 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30212 GPC2.i.XY12 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30213 GPC2.i.XY13 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30214 GPC2.i.XY14 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30215 GPC2.i.XY15 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30216 GPC2.i.XY16 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30217 GPC2.i.XY17 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30220 GPC2.i.XY20 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30221 GPC2.i.XY21 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30222 GPC2.i.XY22 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30223 GPC2.i.XY23 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30224 GPC2.i.XY24 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30225 GPC2.i.XY25 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30226 GPC2.i.XY26 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30227 GPC2.i.XY27 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30230 GPC2.i.XY30 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30231 GPC2.i.XY31 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30232 GPC2.i.XY32 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30233 GPC2.i.XY33 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30234 GPC2.i.XY34 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30235 GPC2.i.XY35 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30236 GPC2.i.XY36 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30237 GPC2.i.XY37 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30240 GPC2.i.XY40 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30241 GPC2.i.XY41 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30242 GPC2.i.XY42 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30243 GPC2.i.XY43 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30244 GPC2.i.XY44 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30245 GPC2.i.XY45 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30246 GPC2.i.XY46 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30247 GPC2.i.XY47 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30250 GPC2.i.XY50 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30251 GPC2.i.XY51 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30252 GPC2.i.XY52 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30253 GPC2.i.XY53 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30254 GPC2.i.XY54 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30255 GPC2.i.XY55 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30256 GPC2.i.XY56 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30257 GPC2.i.XY57 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30260 GPC2.i.XY60 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30261 GPC2.i.XY61 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30262 GPC2.i.XY62 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30263 GPC2.i.XY63 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30264 GPC2.i.XY64 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30265 GPC2.i.XY65 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30266 GPC2.i.XY66 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30267 GPC2.i.XY67 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30271 GPC2.i.XY71 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30272 GPC2.i.XY72 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30273 GPC2.i.XY73 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30274 GPC2.i.XY74 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30275 GPC2.i.XY75 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30276 GPC2.i.XY76 dep 24.611 -0.044 0.000 - - 0.0000 0 3 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30301 GPC2.z.XY01 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30302 GPC2.z.XY02 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30303 GPC2.z.XY03 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30304 GPC2.z.XY04 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30305 GPC2.z.XY05 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30306 GPC2.z.XY06 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30310 GPC2.z.XY10 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30311 GPC2.z.XY11 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30312 GPC2.z.XY12 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30313 GPC2.z.XY13 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30314 GPC2.z.XY14 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30315 GPC2.z.XY15 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30316 GPC2.z.XY16 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30317 GPC2.z.XY17 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30320 GPC2.z.XY20 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30321 GPC2.z.XY21 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30322 GPC2.z.XY22 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30323 GPC2.z.XY23 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30324 GPC2.z.XY24 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30325 GPC2.z.XY25 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30326 GPC2.z.XY26 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30327 GPC2.z.XY27 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30330 GPC2.z.XY30 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30331 GPC2.z.XY31 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30332 GPC2.z.XY32 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30333 GPC2.z.XY33 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30334 GPC2.z.XY34 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30335 GPC2.z.XY35 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30336 GPC2.z.XY36 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30337 GPC2.z.XY37 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30340 GPC2.z.XY40 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30341 GPC2.z.XY41 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30342 GPC2.z.XY42 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30343 GPC2.z.XY43 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30344 GPC2.z.XY44 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30345 GPC2.z.XY45 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30346 GPC2.z.XY46 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30347 GPC2.z.XY47 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30350 GPC2.z.XY50 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30351 GPC2.z.XY51 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30352 GPC2.z.XY52 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30353 GPC2.z.XY53 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30354 GPC2.z.XY54 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30355 GPC2.z.XY55 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30356 GPC2.z.XY56 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30357 GPC2.z.XY57 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30360 GPC2.z.XY60 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30361 GPC2.z.XY61 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30362 GPC2.z.XY62 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30363 GPC2.z.XY63 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30364 GPC2.z.XY64 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30365 GPC2.z.XY65 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30366 GPC2.z.XY66 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30367 GPC2.z.XY67 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30371 GPC2.z.XY71 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30372 GPC2.z.XY72 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30373 GPC2.z.XY73 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30374 GPC2.z.XY74 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30375 GPC2.z.XY75 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30376 GPC2.z.XY76 dep 24.250 -0.033 0.000 - - 0.0000 0 4 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30401 GPC2.y.XY01 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30402 GPC2.y.XY02 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30403 GPC2.y.XY03 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30404 GPC2.y.XY04 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30405 GPC2.y.XY05 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30406 GPC2.y.XY06 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30410 GPC2.y.XY10 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30411 GPC2.y.XY11 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30412 GPC2.y.XY12 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30413 GPC2.y.XY13 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30414 GPC2.y.XY14 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30415 GPC2.y.XY15 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30416 GPC2.y.XY16 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30417 GPC2.y.XY17 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30420 GPC2.y.XY20 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30421 GPC2.y.XY21 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30422 GPC2.y.XY22 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30423 GPC2.y.XY23 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30424 GPC2.y.XY24 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30425 GPC2.y.XY25 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30426 GPC2.y.XY26 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30427 GPC2.y.XY27 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30430 GPC2.y.XY30 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30431 GPC2.y.XY31 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30432 GPC2.y.XY32 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30433 GPC2.y.XY33 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30434 GPC2.y.XY34 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30435 GPC2.y.XY35 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30436 GPC2.y.XY36 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30437 GPC2.y.XY37 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30440 GPC2.y.XY40 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30441 GPC2.y.XY41 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30442 GPC2.y.XY42 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30443 GPC2.y.XY43 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30444 GPC2.y.XY44 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30445 GPC2.y.XY45 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30446 GPC2.y.XY46 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30447 GPC2.y.XY47 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30450 GPC2.y.XY50 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30451 GPC2.y.XY51 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30452 GPC2.y.XY52 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30453 GPC2.y.XY53 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30454 GPC2.y.XY54 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30455 GPC2.y.XY55 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30456 GPC2.y.XY56 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30457 GPC2.y.XY57 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30460 GPC2.y.XY60 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30461 GPC2.y.XY61 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30462 GPC2.y.XY62 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30463 GPC2.y.XY63 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30464 GPC2.y.XY64 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30465 GPC2.y.XY65 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30466 GPC2.y.XY66 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30467 GPC2.y.XY67 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30471 GPC2.y.XY71 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30472 GPC2.y.XY72 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30473 GPC2.y.XY73 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30474 GPC2.y.XY74 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30475 GPC2.y.XY75 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30476 GPC2.y.XY76 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30501 GPC2.w.XY01 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30502 GPC2.w.XY02 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30503 GPC2.w.XY03 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30504 GPC2.w.XY04 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30505 GPC2.w.XY05 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30506 GPC2.w.XY06 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30510 GPC2.w.XY10 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30511 GPC2.w.XY11 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30512 GPC2.w.XY12 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30513 GPC2.w.XY13 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30514 GPC2.w.XY14 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30515 GPC2.w.XY15 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30516 GPC2.w.XY16 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30517 GPC2.w.XY17 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30520 GPC2.w.XY20 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30521 GPC2.w.XY21 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30522 GPC2.w.XY22 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30523 GPC2.w.XY23 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30524 GPC2.w.XY24 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30525 GPC2.w.XY25 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30526 GPC2.w.XY26 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30527 GPC2.w.XY27 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30530 GPC2.w.XY30 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30531 GPC2.w.XY31 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30532 GPC2.w.XY32 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30533 GPC2.w.XY33 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30534 GPC2.w.XY34 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30535 GPC2.w.XY35 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30536 GPC2.w.XY36 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30537 GPC2.w.XY37 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30540 GPC2.w.XY40 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30541 GPC2.w.XY41 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30542 GPC2.w.XY42 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30543 GPC2.w.XY43 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30544 GPC2.w.XY44 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30545 GPC2.w.XY45 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30546 GPC2.w.XY46 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30547 GPC2.w.XY47 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30550 GPC2.w.XY50 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30551 GPC2.w.XY51 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30552 GPC2.w.XY52 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30553 GPC2.w.XY53 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30554 GPC2.w.XY54 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30555 GPC2.w.XY55 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30556 GPC2.w.XY56 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30557 GPC2.w.XY57 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30560 GPC2.w.XY60 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30561 GPC2.w.XY61 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30562 GPC2.w.XY62 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30563 GPC2.w.XY63 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30564 GPC2.w.XY64 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30565 GPC2.w.XY65 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30566 GPC2.w.XY66 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30567 GPC2.w.XY67 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30571 GPC2.w.XY71 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30572 GPC2.w.XY72 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30573 GPC2.w.XY73 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30574 GPC2.w.XY74 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30575 GPC2.w.XY75 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 30576 GPC2.w.XY76 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 31000 GPC2.g.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 31100 GPC2.r.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 31200 GPC2.i.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 31300 GPC2.z.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 31400 GPC2.y.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 31500 GPC2.w.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 9 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 32000 GPC2.g.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 32100 GPC2.r.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 32200 GPC2.i.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 32300 GPC2.z.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 32400 GPC2.y.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 32500 GPC2.w.ForcedWarp dep 25.000 0.000 0.000 - - 0.0000 0 9 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + Index: /branches/ccl_branches/ipp-20190806/ippconfig/gpc1/psastro.config =================================================================== --- /branches/ccl_branches/ipp-20190806/ippconfig/gpc1/psastro.config (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippconfig/gpc1/psastro.config (revision 41070) @@ -620,4 +620,31 @@ END +STATICSKY_POLE METADATA + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + DVO.GETSTAR.FIXED.ZEROPT F32 25.0 + DVO.GETSTAR.MIN.MAG.INST F32 -15.0 +# PSASTRO.CATDIR STR PS1.REF.20120524 +# PSASTRO.CATDIR STR PS1.REF.20140605 +# PSASTRO.CATDIR STR PS1.REF.20140713 +# PSASTRO.CATDIR STR PS1.REF.20170919 + PSASTRO.CATDIR STR PS1.GAIA.POLEREF.20190619 + ZERO.POINT.USE.MEAN BOOL TRUE + + ## for skycell, radius shouldn't need to be as large as 90 pixel default. deeper stacks, crowded fields can be problems + PSASTRO.MATCH.RADIUS.N0 F32 15 + PSASTRO.MATCH.RADIUS.N1 F32 10 + PSASTRO.MATCH.RADIUS.N2 F32 5 + PSASTRO.MATCH.RADIUS.N3 F32 1 + PSASTRO.MATCH.RADIUS.N4 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 5 + + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, if larger than 0.2" on skycell then a problem.. + + TRANSFORM.GALAXY.SHAPE.ERRORS BOOL T +END # revised setup for MD nightly stack inputs, should work for case of N~2-30 Index: /branches/ccl_branches/ipp-20190806/ippconfig/gpc2/camera.config =================================================================== --- /branches/ccl_branches/ipp-20190806/ippconfig/gpc2/camera.config (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippconfig/gpc2/camera.config (revision 41070) @@ -155,5 +155,5 @@ # PSVIDEOPHOT STR gpc2/psvideophot.config # REJECTIONS STR gpc2/rejections.config # rejection recipe -# PPSTACK STR gpc2/ppStack.config + PPSTACK STR gpc2/ppStack.config END Index: /branches/ccl_branches/ipp-20190806/ippconfig/gpc2/ppStack.config =================================================================== --- /branches/ccl_branches/ipp-20190806/ippconfig/gpc2/ppStack.config (revision 41070) +++ /branches/ccl_branches/ipp-20190806/ippconfig/gpc2/ppStack.config (revision 41070) @@ -0,0 +1,468 @@ +PSF.MODEL STR PS_MODEL_PS1_V1 # Model for PSF generation +BACKGROUND.MODEL BOOL F + +ZP.AIRMASS METADATA # Airmass terms by filter + g.00002 F32 0.0 + r.00002 F32 0.0 + i.00002 F32 0.0 + z.00002 F32 0.0 + y.00002 F32 0.0 + w.00002 F32 0.0 +END +ZP.TARGET METADATA # Target zero point by filter + g.00002 F32 25.0 + r.00002 F32 25.0 + i.00002 F32 25.0 + z.00002 F32 25.0 + y.00002 F32 25.0 + w.00002 F32 25.0 +END + +PS1_REFERENCE METADATA # Refstack for diffims, typically best of best IQ/images and N>20 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + PSF.INPUT.MAX F32 6.0 # never really want >6 pixels in refstack + PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set even smaller?) + PSF.INPUT.THRESH F32 0.0 + PSF.INPUT.ASYMMETRY F32 0.0 +END + +STACK_DEEP METADATA # N>>20 inputs, typically MD fields + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + PSF.INPUT.MAX F32 10.0 + PSF.INPUT.CLIP.NSIGMA F32 1.5 + PSF.INPUT.THRESH F32 5.0 + PSF.INPUT.ASYMMETRY F32 0.2 +END + +QUICKSTACK_DEEP METADATA # Unconvolved and deep stacks only + CONVOLVE BOOL FALSE + COMBINE.REJ F32 3.5 # Rejection threshold in combination (sigma). tune? + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean. tune? + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + PSF.INPUT.MAX F32 10.0 # since not convolved limit, relax/tune? + PSF.INPUT.CLIP.NSIGMA F32 1.5 + PSF.INPUT.THRESH F32 5.0 + PSF.INPUT.ASYMMETRY F32 0.2 +END + +STACK_NIGHTLY METADATA # Typically MD 8 input nightly stacks + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL FALSE +END + +STACK_THREEPI METADATA + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL TRUE + PSF.INPUT.MAX F32 10.0 # input cut more restrictive now 12->10 pix, limit target PSF in convolved stacks + PSF.INPUT.CLIP.NSIGMA F32 1.5 # additional cut if majority of inputs much smaller than INPUT.MAX + PSF.INPUT.THRESH F32 10.0 + PSF.INPUT.ASYMMETRY F32 0.2 + PSF.TARGET.AS.MAX BOOL TRUE + MATCH.REJ F32 4.0 + SAFE BOOL F + +END + +STACK_THREEPI_ALT METADATA + OUTPUT.NOCOMP BOOL TRUE ## MD test mod + OUTPUT.LOGFLUX BOOL FALSE ## MD test mod + STACK.TYPE STR DEEP_STACK ## MD test mod + PSF.INPUT.MAX F32 10.0 # input cut more restrictive now 12->10 pix, limit target PSF in convolved stacks + PSF.INPUT.CLIP.NSIGMA F32 1.5 # additional cut if majority of inputs much smaller than INPUT.MAX + PSF.INPUT.THRESH F32 10.0 + PSF.INPUT.ASYMMETRY F32 0.2 + MATCH.REJ F32 4.0 + SAFE BOOL F + +END + +STACK_CNP METADATA ## Based on STACK_THREEPI and STACK_MD_DEEP_1DG + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + + MATCH.REJ F32 4.0 + SAFE BOOL F + + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + +END + + +STACK_MD_REF METADATA ## revised config for MD refstacks vs PS1_REFERENCE above after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 6.0 # never really want >6 pixels in refstack + PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set even smaller?) + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE +END + +STACK_MD_DEEP METADATA ## revised config for MD deepstack above after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE +END + +STACK_ALLDEEP METADATA # no cuts, everything in changes after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 100.0 + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE +END + +STACK_NIGHTLY_1DG METADATA # Revised MD 8 input nightly stacks and using 1DG convolution + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL FALSE + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 12.0 # what FWHM does diffim become mostly useless, if any? + PSF.INPUT.CLIP.NSIGMA F32 1000.0 # want to use up to MAX, but sig could possibly be small? + MATCH.REJ F32 4.0 # need to tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 +END + +STACK_MD_IQ_1DG METADATA ## revised config for MD refstacks vs PS1_REFERENCE above after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # center/edge can vary 1-2 pix, prefer <6 pix but needs to be <10 pix for edges until code improved.. + PSF.INPUT.CLIP.NSIGMA F32 100.0 # any change to inputs can greatly impact edge filling factor.. + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL TRUE # HACK for removing bscale offset from input warps because of compression (0.5*BSCALE) +END + +STACK_MD_REF_1DG METADATA ## revised config for MD refstacks vs PS1_REFERENCE above after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 7.0 # center/edge can vary 1-2 pix, prefer <6 pix but needs to be <7 pix for edges + PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set even smaller?) + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL TRUE # HACK for removing bscale offset from input warps because of compression (0.5*BSCALE) +END + +STACK_MD_DEEP_1DG METADATA ## revised config for MD deepstack above after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 30 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL TRUE # HACK for removing bscale offset from input warps because of compression (0.5*BSCALE) +END + +STACK_ALLDEEP_1DG METADATA # no real cuts, for changes after ppStack mods 5/03/13 + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 14.0 # greater than 14 is more often junk + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.5 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.5 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image +END + +STACK_MDTHREEPI_DEEP_1DG METADATA ## revised config for MD+3PI + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.01 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 2.5 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image +END + + + + +STACK_CNP_PV3_IQ_1DG METADATA ## revised -- based on STACK_THREEPI/STACK_MD_DEEP_1DG + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 + SAFE BOOL F + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 3 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_CNP_PV3_REF_1DG METADATA ## revised CNP PV3 REF stack -- + OUTPUT.NOCOMP BOOL TRUE # pointed field -- no compression, no logflux + OUTPUT.LOGFLUX BOOL FALSE # + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # CNP large FWHM, center/edge vary 1-2 pix? + PSF.INPUT.CLIP.NSIGMA F32 0.0 # sample typically on rising side of distribution + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 3 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_CNP_PV3_DEEP_1DG METADATA ## -- + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 16.0 # large FWHM images up to 20 -- + PSF.INPUT.CLIP.NSIGMA F32 100.0 # do not want to clip really + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 3 # Minimum input per pixel -- more and get crazy structures.. + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels -- many, dont trust SUSPECT + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 (PV2 bias hack fix) +END + + +STACK_MD_PV3_NIGHT_1DG METADATA ## Revised MD 8 input NIGHT stacks for PV3 -- no time to test, so similar to original but 1DG + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL FALSE + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 12.0 # what FWHM does diffim become mostly useless, if any? + PSF.INPUT.CLIP.NSIGMA F32 1000.0 # want to use up to MAX, but sig could possibly be small? + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL TRUE # Play safe when combining small number of values? + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + ## -- extra masking not tested and likely not want for small Nin stack + #THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + #COMBINE.ITER F32 0.5 # Number of rejection iterations per input + #COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + #COMBINE.SYS F32 0.1 # Relative systematic error in combination + #COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + #IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + #MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + ## -- + NMINPIX S32 1 # Minimum input per pixel + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_MD_PV3_IQ_1DG METADATA ## revised config for MD PV3 IQ stacks -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # center/edge can vary 1-2 pix, prefer <6 pix but needs to be <10 pix for edges until code improved.. + PSF.INPUT.CLIP.NSIGMA F32 100.0 # any change to inputs can greatly impact edge filling factor.. + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_MD_PV3_REF_1DG METADATA ## revised config for MD PV3 refstacks -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 7.0 # center/edge can vary 1-2 pix, prefer <6 pix but needs to be <7 pix for edges + PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set even smaller?) + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_MD_PV3_DEEP_1DG METADATA ## revised config for MD PV3 deepstack -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 30 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + + + +# Recipe overrides for STACK +STACKPHOT METADATA + PSF.MODEL STR PS_MODEL_QGAUSS # Model for PSF match +END Index: /branches/ccl_branches/ipp-20190806/ippconfig/recipes/psastro.config =================================================================== --- /branches/ccl_branches/ipp-20190806/ippconfig/recipes/psastro.config (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippconfig/recipes/psastro.config (revision 41070) @@ -328,4 +328,8 @@ END +STATICSKY_POLE METADATA + PSASTRO.SAVE.CFF BOOL TRUE # save a cff file (input for psphotFullForce) +END + STATICSKY_TESTCAL METADATA END Index: /branches/ccl_branches/ipp-20190806/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/ccl_branches/ipp-20190806/ippconfig/recipes/reductionClasses.mdc (revision 41069) +++ /branches/ccl_branches/ipp-20190806/ippconfig/recipes/reductionClasses.mdc (revision 41070) @@ -411,5 +411,6 @@ STACKPHOT_PPSUB STR STACKPHOT STACKPHOT_PPSTACK STR STACKPHOT - STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_POLE BACKGROUND_PPBACKGROUND STR BACKGROUND BACKGROUND_PSWARP STR BACKGROUND @@ -1879,4 +1880,6 @@ JPEG_BIN2 STR PPIMAGE_J2 PSASTRO STR DEFAULT_RECIPE -END - + # this is for running addstar + ADDSTAR STR ADDSTAR +END + Index: /branches/ccl_branches/ipp-20190806/psModules/src/concepts/pmConceptsStandard.c =================================================================== --- /branches/ccl_branches/ipp-20190806/psModules/src/concepts/pmConceptsStandard.c (revision 41069) +++ /branches/ccl_branches/ipp-20190806/psModules/src/concepts/pmConceptsStandard.c (revision 41070) @@ -931,5 +931,5 @@ if (!item || item->type != PS_DATA_S32) { - psWarning("Unable to find %s --- assuming UTC", timesysName); + psWarning("Unable to find %s in format file --- assuming UTC", timesysName); return PS_TIME_UTC; } @@ -982,5 +982,5 @@ if (concept->type != PS_DATA_STRING || strlen(sys) <= 0) { // XXX is this too low verbosity? - psWarning("Can't interpret %s --- assuming UTC.", pattern->name); + psWarning("Can't interpret %s --- assuming UTC (other options: TAI, UT1, TT).", pattern->name); } else if (strcasecmp(sys, "TAI") == 0) { timeSys = PS_TIME_TAI; @@ -993,5 +993,5 @@ } else { // XXX is this too low verbosity? - psWarning("Can't interpret %s --- assuming UTC.", pattern->name); + psWarning("Can't interpret %s --- assuming UTC (other options: TAI, UT1, TT).", pattern->name); } @@ -1464,4 +1464,5 @@ const pmCell *cell) { + psAssert (concept->type == PS_DATA_TIME, "programming error: concept not supplied with psTime pointer\n"); psTime *time = concept->data.V; // The time