Index: branches/pap/ippTools/src/Makefile.am =================================================================== --- branches/pap/ippTools/src/Makefile.am (revision 23948) +++ branches/pap/ippTools/src/Makefile.am (revision 25027) @@ -19,7 +19,7 @@ regtool \ stacktool \ - warptool - -# receivetool + warptool \ + receivetool \ + pubtool @@ -60,5 +60,6 @@ regtool.h \ stacktool.h \ - warptool.h + warptool.h \ + pubtool.h lib_LTLIBRARIES = libpxtools.la @@ -221,4 +222,10 @@ receivetoolConfig.c +pubtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pubtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pubtool_SOURCES = \ + pubtool.c \ + pubtoolConfig.c + clean-local: -rm -f TAGS Index: branches/pap/ippTools/src/camtool.c =================================================================== --- branches/pap/ippTools/src/camtool.c (revision 23948) +++ branches/pap/ippTools/src/camtool.c (revision 25027) @@ -491,4 +491,6 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // generate restrictions @@ -614,5 +616,5 @@ // since there is only one exp per 'new' set camRun.state = 'full' - if (!pxcamRunSetState(config, row->cam_id, "full")) { + if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) { psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); psFree(row); @@ -766,5 +768,5 @@ pxcamGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "camProcessedExp.fault", "=="); @@ -953,5 +955,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); psString query = pxDataGet("camtool_pendingcleanuprun.sql"); Index: branches/pap/ippTools/src/camtoolConfig.c =================================================================== --- branches/pap/ippTools/src/camtoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/camtoolConfig.c (revision 25027) @@ -167,4 +167,5 @@ psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); // -processedexp @@ -187,5 +188,5 @@ pxcamSetSearchArgs(revertprocessedexpArgs); psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by camRun reduction class", NULL); psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); Index: branches/pap/ippTools/src/chiptool.c =================================================================== --- branches/pap/ippTools/src/chiptool.c (revision 23948) +++ branches/pap/ippTools/src/chiptool.c (revision 25027) @@ -473,5 +473,5 @@ PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // default values @@ -601,5 +601,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); - PXOPT_COPY_S32(config->args, where, "-magicked", "chipRun.magicked", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); if (!psListLength(where->list) && @@ -676,6 +676,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); - // require a single label - PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "chipProcessedImfile.fault", "=="); Index: branches/pap/ippTools/src/chiptoolConfig.c =================================================================== --- branches/pap/ippTools/src/chiptoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/chiptoolConfig.c (revision 25027) @@ -169,5 +169,5 @@ psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); - psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); pxchipSetSearchArgs(revertprocessedimfileArgs); psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); Index: branches/pap/ippTools/src/detselect.c =================================================================== --- branches/pap/ippTools/src/detselect.c (revision 23948) +++ branches/pap/ippTools/src/detselect.c (revision 25027) @@ -71,4 +71,26 @@ } +# define PXOPT_COPY_NULLTEST_F32(from, to, oldname, newname, comment) \ +{ \ + bool status = false; \ + psF32 var = psMetadataLookupF32(&status, from, oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \ + return false; \ + } \ + if (!isnan(var)) { \ + if (!psMetadataAddF32(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + if (!psMetadataAddTime(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, "==", NULL)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add NULL test " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + static bool searchMode(pxConfig *config) { @@ -89,14 +111,14 @@ // airmass_min < airmass < airmass_max - PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<="); - PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">="); + PXOPT_COPY_NULLTEST_F32(config->args, where, "-airmass", "airmass_min", "<="); + PXOPT_COPY_NULLTEST_F32(config->args, where, "-airmass", "airmass_max", ">="); // exp_time_min < exp_time < exp_time_max - PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<="); - PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">="); + PXOPT_COPY_NULLTEST_F32(config->args, where, "-exp_time", "exp_time_min", "<="); + PXOPT_COPY_NULLTEST_F32(config->args, where, "-exp_time", "exp_time_max", ">="); // ccd_temp_min < ccd_temp < ccd_temp_max - PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); - PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); + PXOPT_COPY_NULLTEST_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); + PXOPT_COPY_NULLTEST_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<="); Index: branches/pap/ippTools/src/dettool.c =================================================================== --- branches/pap/ippTools/src/dettool.c (revision 23948) +++ branches/pap/ippTools/src/dettool.c (revision 25027) @@ -276,6 +276,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-sun_angle_max", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); @@ -521,6 +521,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-sun_angle_max", false, false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -557,6 +557,16 @@ PXOPT_COPY_F64(config->args, where, "-select_posang_min", "posang", ">="); PXOPT_COPY_F64(config->args, where, "-select_posang_max", "posang", "<="); - PXOPT_COPY_F64(config->args, where, "-select_solang_min", "solang", ">="); - PXOPT_COPY_F64(config->args, where, "-select_solang_max", "solang", "<="); + PXOPT_COPY_F64(config->args, where, "-select_sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F64(config->args, where, "-select_sun_angle_max", "sun_angle", "<="); + PXOPT_COPY_F64(config->args, where, "-select_sun_alt_min", "sun_alt", ">="); + PXOPT_COPY_F64(config->args, where, "-select_sun_alt_max", "sun_alt", "<="); + + PXOPT_COPY_F64(config->args, where, "-select_moon_angle_min", "moon_angle", ">="); + PXOPT_COPY_F64(config->args, where, "-select_moon_angle_max", "moon_angle", "<="); + PXOPT_COPY_F64(config->args, where, "-select_moon_alt_min", "moon_alt", ">="); + PXOPT_COPY_F64(config->args, where, "-select_moon_alt_max", "moon_alt", "<="); + PXOPT_COPY_F64(config->args, where, "-select_moon_phase_min", "moon_phase", ">="); + PXOPT_COPY_F64(config->args, where, "-select_moon_phase_max", "moon_phase", "<="); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); if (!psListLength(where->list)) { @@ -767,6 +777,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-set_posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-set_posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-set_solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-set_solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-set_sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-set_sun_angle_max", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-set_time_begin", false, false); @@ -1605,6 +1615,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-sun_angle_max", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); Index: branches/pap/ippTools/src/dettoolConfig.c =================================================================== --- branches/pap/ippTools/src/dettoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/dettoolConfig.c (revision 25027) @@ -68,6 +68,7 @@ psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); @@ -99,6 +100,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); @@ -124,6 +125,17 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_alt_min", 0, "define min solar altitude", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_alt_max", 0, "define max solar altitude", NAN); + + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_min", 0, "define min moon angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_max", 0, "define max moon angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_min", 0, "define min moon alt", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_max", 0, "define max moon alt", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_min", 0, "define min moon phase", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class for processing", NULL); @@ -154,6 +166,6 @@ psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_min", 0, "define rotator position angle", NAN); psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_max", 0, "define rotator position angle", NAN); - psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_min", 0, "define solar angle", NAN); - psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_max", 0, "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_sun_angle_min", 0, "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_sun_angle_max", 0, "define solar angle", NAN); psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, "start of period to apply detrend too", NULL); @@ -830,6 +842,6 @@ psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); Index: branches/pap/ippTools/src/difftool.c =================================================================== --- branches/pap/ippTools/src/difftool.c (revision 23948) +++ branches/pap/ippTools/src/difftool.c (revision 25027) @@ -40,5 +40,5 @@ static bool revertdiffskyfileMode(pxConfig *config); static bool definepoprunMode(pxConfig *config); -static bool definebyqueryMode(pxConfig *config); +static bool definewarpstackMode(pxConfig *config); static bool definewarpwarpMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); @@ -49,5 +49,5 @@ static bool importrunMode(pxConfig *config); -static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked); +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); static bool diffRunComplete(pxConfig *config); @@ -79,5 +79,5 @@ MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -116,7 +116,8 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); + PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); // default @@ -133,15 +134,16 @@ registered, tess_id, - exp_id, + bothways, + exposure, false ); if (!run) { psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object"); - return true; + return false; } if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(run); - return true; + return false; } @@ -150,7 +152,7 @@ if (!diffRunPrintObject(stdout, run, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(run); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; } @@ -487,5 +489,4 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false); @@ -511,5 +512,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); if (!psDBTransaction(config->dbh)) { @@ -521,5 +522,4 @@ diff_id, skycell_id, - uri, path_base, "full", @@ -582,5 +582,5 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); @@ -662,4 +662,10 @@ PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -695,5 +701,5 @@ psFree(query); - psLogMsg("diftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); } @@ -739,5 +745,5 @@ -static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked) +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -749,5 +755,5 @@ } - char *query = "UPDATE diffRun SET state = '%s', magicked = %d WHERE diff_id = %"PRId64; + char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { @@ -771,5 +777,4 @@ psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none - psS64 exp_id, // exposure id for input_warp_id (if defined) pxConfig *config // Configuration ) @@ -802,6 +807,7 @@ registered, tess_id, - exp_id, - false // magicked + false, + false, + 0 // magicked ); @@ -882,5 +888,4 @@ PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false); PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -902,9 +907,4 @@ psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No input has been defined (-input_stack_id or -input_warp_id)"); - return false; - } - if (input_warp_id && !exp_id) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, - "-exp_id is required with -input_warp_id."); return false; } @@ -917,5 +917,4 @@ template_warp_id ? template_warp_id : PS_MAX_S64, template_stack_id ? template_stack_id : PS_MAX_S64, - exp_id ? exp_id : PS_MAX_S64, config)) { psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun"); @@ -935,5 +934,5 @@ -static bool definebyqueryMode(pxConfig *config) +static bool definewarpstackMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -944,8 +943,8 @@ PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "=="); PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "=="); PXOPT_COPY_F32(config->args, warpWhere, "-good_frac", "warpSkyfile.good_frac", ">="); @@ -960,7 +959,8 @@ PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); // find all things to queue - psString query = pxDataGet("difftool_definebyquery_part1.sql"); + psString query = pxDataGet("difftool_definewarpstack_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -1057,6 +1057,17 @@ } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + // create temporary table - query = pxDataGet("difftool_definebyquery_temp_create.sql"); + query = pxDataGet("difftool_definewarpstack_temp_create.sql"); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1070,5 +1081,5 @@ query = NULL; - psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql"); + psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql"); psArray *list = psArrayAllocEmpty(16); // List of runs, to print @@ -1183,6 +1194,7 @@ registered, tess_id, - exp_id, - false // magicked + false, // bothways + true, // exposure + 0 // magicked ); @@ -1282,7 +1294,40 @@ "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<="); + // other where restrictions: + PXOPT_COPY_TIME(config->args, selectWhere, "-dateobs_begin", "inputRawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, selectWhere, "-dateobs_end", "inputRawExp.dateobs", "<="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_min", "inputRawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_max", "inputRawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_min", "inputRawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_max", "inputRawExp.decl", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-sat_pixel_frac_min", "inputRawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-sat_pixel_frac_max", "inputRawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-airmass_min", "inputRawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-airmass_max", "inputRawExp.airmass", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-exp_time_min", "inputRawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-exp_time_max", "inputRawExp.exp_time", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_min", "inputRawExp.bg", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_max", "inputRawExp.bg", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_stdev_min", "inputRawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_stdev_max", "inputRawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_mean_stdev_min", "inputRawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_mean_stdev_max", "inputRawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-alt_min", "inputRawExp.alt", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-alt_max", "inputRawExp.alt", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-az_min", "inputRawExp.az", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-az_max", "inputRawExp.az", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-ccd_temp_min", "inputRawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-ccd_temp_max", "inputRawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-posang_min", "inputRawExp.posang", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-posang_max", "inputRawExp.posang", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-sun_angle_min", "inputRawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-sun_angle_max", "inputRawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, selectWhere, "-comment", "inputRawExp.comment", "LIKE"); + // Haversine formula for great circle distance PXOPT_COPY_F32(config->args, selectWhere, "-distance", - "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<="); + "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + " + "COS(inputRawExp.decl)*COS(templateRawExp.decl)*" + "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<="); PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false); @@ -1308,7 +1353,76 @@ PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(selectWhere); + psFree(insertWhere); + return false; + } + + if (!rerun) { + // Need to build table of exposures with diffs + psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL + if (!tempCreate) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (!p_psDBRunQuery(config->dbh, tempCreate)) { + psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate); + psFree(tempCreate); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(tempCreate); + + psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table + if (!tempInsert) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString where = psStringCopy(""); // WHERE for insertion + if (label) { + psStringAppend(&where, "\nAND diffRun.label = '%s'", label); + } + + if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) { + psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert); + psFree(tempInsert); + psFree(where); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(where); + psFree(tempInsert); + } + + // Get list of warps to diff psString select = pxDataGet("difftool_definewarpwarp_select.sql"); if (!select) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1323,26 +1437,17 @@ psFree(selectWhere); + if (!available) { + psStringAppend(&whereClause, + "\n%s inputWarpRun.state = 'full'" + "AND templateWarpRun.state = 'full'", + whereClause ? "AND" : "WHERE"); + } + if (!rerun) { - psStringAppend(&whereClause, "\n%s (diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE"); - if (label) { - psStringAppend(&whereClause, " OR diffRun.label != %s", label); - } - psStringAppend(&whereClause, ")"); - } - - if (!available) { - psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE"); - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(select); - psFree(whereClause); - psFree(insertWhere); - return false; + psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE"); } if (!p_psDBRunQueryF(config->dbh, select, - !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun? + !rerun ? "\n" : "", // Activate LEFT JOIN against diffs? whereClause)) { psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause); @@ -1414,9 +1519,8 @@ psMetadata *row = results->data[i]; // Result row from query - psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id"); psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id"); const char *template = psMetadataLookupStr(NULL, row, "template_warp_id"); const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); - if (!exp_id || !input_id || !template || !tess_id) { + if (!input_id || !template || !tess_id) { psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); psFree(list); @@ -1430,5 +1534,5 @@ diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, - tess_id, exp_id, false); // Run to insert + tess_id, true, true, false); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1511,5 +1615,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); psString query = pxDataGet("difftool_pendingcleanuprun.sql"); @@ -1576,5 +1680,5 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); } - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); @@ -1738,5 +1842,5 @@ psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - bool magicked = psMetadataLookupBool(NULL, row, "magicked"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); // set diffRun.state to 'stop' Index: branches/pap/ippTools/src/difftool.h =================================================================== --- branches/pap/ippTools/src/difftool.h (revision 23948) +++ branches/pap/ippTools/src/difftool.h (revision 25027) @@ -34,5 +34,5 @@ DIFFTOOL_MODE_REVERTDIFFSKYFILE, DIFFTOOL_MODE_DEFINEPOPRUN, - DIFFTOOL_MODE_DEFINEBYQUERY, + DIFFTOOL_MODE_DEFINEWARPSTACK, DIFFTOOL_MODE_DEFINEWARPWARP, DIFFTOOL_MODE_PENDINGCLEANUPRUN, Index: branches/pap/ippTools/src/difftoolConfig.c =================================================================== --- branches/pap/ippTools/src/difftoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/difftoolConfig.c (revision 25027) @@ -49,4 +49,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); @@ -90,5 +92,4 @@ psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); - psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); @@ -132,4 +133,5 @@ psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdiffskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); // -definepoprun @@ -142,5 +144,4 @@ psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID for template", 0); psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0, "define warp ID for template", 0); psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0, "define stack ID for template", 0); @@ -148,22 +149,23 @@ psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0, "define stack ID for input", 0); - // -definebyquery - psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -definewarpstack + psMetadata *definewarpstackArgs = psMetadataAlloc(); + psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); // -definewarpwarp @@ -188,4 +190,33 @@ psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); // -pendingcleanuprun @@ -239,5 +270,5 @@ PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); - PXOPT_ADD_MODE("-definebyquery", "", DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definewarpstack", "", DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackArgs); PXOPT_ADD_MODE("-definewarpwarp", "", DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); Index: branches/pap/ippTools/src/disttool.c =================================================================== --- branches/pap/ippTools/src/disttool.c (revision 23948) +++ branches/pap/ippTools/src/disttool.c (revision 25027) @@ -47,4 +47,17 @@ static bool pendingdestMode(pxConfig *config); +static bool definetargetMode(pxConfig *config); +static bool updatetargetMode(pxConfig *config); +static bool listtargetMode(pxConfig *config); + +static bool definedsproductMode(pxConfig *config); +static bool updatedsproductMode(pxConfig *config); + +static bool definedestinationMode(pxConfig *config); +static bool updatedestinationMode(pxConfig *config); + +static bool defineinterestMode(pxConfig *config); +static bool updateinterestMode(pxConfig *config); + # define MODECASE(caseName, func) \ case caseName: \ @@ -81,4 +94,13 @@ MODECASE(DISTTOOL_MODE_REVERTRCRUN, revertrcrunMode); MODECASE(DISTTOOL_MODE_PENDINGDEST, pendingdestMode); + MODECASE(DISTTOOL_MODE_DEFINETARGET, definetargetMode); + MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode); + MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode); + MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode); + MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode); + MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode); + MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode); + MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode); + MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode); default: psAbort("invalid option (this should not happen)"); @@ -151,10 +173,11 @@ PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // optional PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); - - // optional PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); @@ -166,5 +189,4 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); @@ -181,4 +203,9 @@ psFree(where); return false; + } + + // for raw stage we select by chipRun.label because raw labels typically aren't set + if (label) { + psStringAppend(&query, " AND (chipRun.label = '%s')", label); } } else if (!strcmp(stage, "chip")) { @@ -323,20 +350,26 @@ } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("disttool", PS_LOG_INFO, "no rows found"); psFree(output); return true; } - // XXX Remove this debugging output at some point - if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } + if (dry_run) { + if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } psFree(output); return true; } + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *list = psArrayAllocEmpty(limit); for (long i=0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; @@ -345,4 +378,5 @@ psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); + psString target_label = psMetadataLookupStr(NULL, md, "label"); bool clean = psMetadataLookupBool(NULL, md, "clean"); @@ -350,5 +384,8 @@ psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); - if (!distRunInsert(config->dbh, + if (set_label == NULL) { + set_label = target_label; + } + distRunRow *row = distRunRowAlloc( 0, // dist_id target_id, @@ -362,5 +399,13 @@ NULL, // time_stamp 0 // fault - )) { + ); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate distRunRow"); + psFree(outroot); + psFree(output); + return false; + } + if (!distRunInsertObject(config->dbh, row)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(outroot); @@ -369,6 +414,21 @@ } psFree(outroot); - } - + psS64 dist_id = psDBLastInsertID(config->dbh); + row->dist_id = dist_id; + psArrayAdd(list, list->n, row); + psFree(row); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!distRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); psFree(output); @@ -436,5 +496,9 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "distComponent.fault", "=="); + + // we need to disambiguate fault so make a copy of the where list before adding fault + psMetadata *whereComponent = psMetadataCopy(NULL, where); + PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); + PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "=="); // It might be useful to be able to query by the parameters of the underlying runs @@ -481,15 +545,4 @@ numUpdated = psDBAffectedRows(config->dbh); - -#ifdef notdef - // don't need this. distRun.state may still be in 'new' state - if (numUpdated < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } -#endif } @@ -508,6 +561,6 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (psListLength(whereComponent->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -530,4 +583,5 @@ psFree(where); + psFree(whereComponent); if (!psDBCommit(config->dbh)) { @@ -546,5 +600,5 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -667,5 +721,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -811,5 +865,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1109,5 +1163,5 @@ - PXOPT_LOOKUP_STR(last_fileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(last_fileset, config->args, "-set_last_fileset", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -1213,2 +1267,443 @@ return true; } + +static bool definetargetMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + // optional + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + + distTargetRow *row = distTargetRowAlloc( + 0, // target_id + label, + filter, + stage, + clean, + state ? state : "enabled", + comment + ); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate distTarget object"); + return false; + } + if (!distTargetInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + // get the assigned target_id + row->target_id = psDBLastInsertID(config->dbh); + + if (!distTargetPrintObject(stdout, row, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(row); + return false; + } + + psFree(row); + + return true; +} +static bool updatetargetMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + psString query = psStringCopy("UPDATE distTarget SET state = '%s'"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool listtargetMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (clean && full) { + psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); + return false; + } + + psString query = psStringCopy("SELECT * FROM distTarget"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + if (clean) { + psStringAppend(&query, " AND (clean)"); + } else if (full) { + psStringAppend(&query, " AND (!clean)"); + } + } else if (clean) { + psStringAppend(&query, " WHERE clean"); + } else if (full) { + psStringAppend(&query, " WHERE !clean"); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "distTarget", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definedsproductMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); + PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); + + // XXX: should we insure that these names do not contatin any whitespace? + + rcDSProductRow *row = rcDSProductRowAlloc( + 0, // prod_id + name, + dbname, + dbhost + ); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object"); + return false; + } + if (!rcDSProductInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + // get the assigned target_id + row->prod_id = psDBLastInsertID(config->dbh); + + if (!rcDSProductPrintObject(stdout, row, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(row); + return false; + } + + psFree(row); + + return true; +} +static bool updatedsproductMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + + PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false); + PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false); + + if (!(dbname || dbhost)) { + psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required"); + psFree(where); + return false; + } + psString query = psStringCopy("UPDATE rcDSProduct SET"); + psString sep = ""; + if (dbname) { + psStringAppend(&query, " dbname = '%s'", dbname); + sep = ","; + } + if (dbhost) { + psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool definedestinationMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + + // optional + PXOPT_LOOKUP_STR(status_uri, config->args, "-status_uri", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(last_fileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + // XXX: should we insure that these names do not contatin any whitespace? + + rcDestinationRow *row = rcDestinationRowAlloc( + 0, // dest_id + prod_id, + name, + status_uri, + comment, + last_fileset, + state ? state : "enabled" + ); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDestination object"); + return false; + } + if (!rcDestinationInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + // get the assigned target_id + row->dest_id = psDBLastInsertID(config->dbh); + + if (!rcDestinationPrintObject(stdout, row, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(row); + return false; + } + + psFree(row); + + return true; +} + +static bool updatedestinationMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); +#ifdef ALLOW_UPDATE_LAST_FILESET + PXOPT_LOOKUP_STR(last_fileset, config->args, "-set_last_fileset", false, false); + if (!(state || last_fileset)) { + psError(PS_ERR_UNKNOWN, true, "one or more of -set_state or -set_last_fileset is required"); +# else + if (!state) { +#endif + psFree(where); + return false; + } + psString query = psStringCopy("UPDATE rcDestination SET"); + psString sep = ""; + if (state) { + psStringAppend(&query, " state = '%s'", state); + sep = ","; + } +#ifdef ALLOW_UPDATE_LAST_FILESET + // last_fileset normally gets set by updatercrunMode + // Allowing it to be set here might cause problems + // especially since we are allowing selection by prod_id + if (last_fileset) { + psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset); + } +#endif + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool defineinterestMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); + PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); + + // optional + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + // XXX: should we insure that these names do not contatin any whitespace? + + rcInterestRow *row = rcInterestRowAlloc( + 0, // int_id + dest_id, + target_id, + state ? state : "enabled" + ); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object"); + return false; + } + if (!rcInterestInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + // get the assigned target_id + row->int_id = psDBLastInsertID(config->dbh); + + if (!rcInterestPrintObject(stdout, row, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(row); + return false; + } + + psFree(row); + + return true; +} + +static bool updateinterestMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + if (!state) { + psError(PS_ERR_UNKNOWN, true, "-set_state is required"); + psFree(where); + return false; + } + psString query = NULL; + psStringAppend(&query, "UPDATE rcInterest SET state = '%s'", state); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + Index: branches/pap/ippTools/src/disttool.h =================================================================== --- branches/pap/ippTools/src/disttool.h (revision 23948) +++ branches/pap/ippTools/src/disttool.h (revision 25027) @@ -40,4 +40,13 @@ DISTTOOL_MODE_REVERTRCRUN, DISTTOOL_MODE_PENDINGDEST, + DISTTOOL_MODE_DEFINEDESTINATION, + DISTTOOL_MODE_UPDATEDESTINATION, + DISTTOOL_MODE_DEFINEDSPRODUCT, + DISTTOOL_MODE_UPDATEDSPRODUCT, + DISTTOOL_MODE_DEFINETARGET, + DISTTOOL_MODE_UPDATETARGET, + DISTTOOL_MODE_LISTTARGET, + DISTTOOL_MODE_DEFINEINTEREST, + DISTTOOL_MODE_UPDATEINTEREST, } disttoolMode; Index: branches/pap/ippTools/src/disttoolConfig.c =================================================================== --- branches/pap/ippTools/src/disttoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/disttoolConfig.c (revision 25027) @@ -59,7 +59,7 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define chip_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "select by obs_mode", NULL); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definerun @@ -97,5 +97,5 @@ psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); - psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", 0, "limit results to label", NULL); + psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -120,5 +120,5 @@ psMetadata *toadvanceArgs = psMetadataAlloc(); psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", 0, "limit updates to label", NULL); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit updates to label", NULL); psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -127,5 +127,5 @@ psMetadata *pendingfilesetArgs = psMetadataAlloc(); psMetadataAddS64(pendingfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", 0, "limit results to label", NULL); + psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -144,5 +144,5 @@ psMetadataAddU64(pendingdestArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingdestArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddStr(pendingdestArgs, PS_LIST_TAIL, "-label", 0, "limit results to label", NULL); + psMetadataAddStr(pendingdestArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); // -revertfileset @@ -178,5 +178,5 @@ psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-status_fs_name", 0, "define status fileset name", NULL); psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL); - psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-last_fileset",0, "new value for last_fileset", NULL); + psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-set_last_fileset",0, "new value for last_fileset", NULL); psMetadataAddS16(updatercrunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", -1); @@ -188,4 +188,80 @@ psMetadataAddS16(revertrcrunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); + + // -definedsproduct + psMetadata *definedsproductArgs = psMetadataAlloc(); + psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name", 0, "define product name", NULL); + psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); + psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); + + // -updatedsproduct + // does this mode make sense? + psMetadata *updatedsproductArgs = psMetadataAlloc(); + psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id", 0, "select by prod_id", 0); + // can't select by name because it isn't necssarily unique + psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); + psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); + + // -definedestination + psMetadata *definedestinationArgs = psMetadataAlloc(); + psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id (required)", 0); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-last_fileset",0, "define last_fileset", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -updatedestination + psMetadata *updatedestinationArgs = psMetadataAlloc(); + psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); + psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); + psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name", NULL); + psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); + psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); +// last_fileset gets updated by -updatercrun +// psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL); + psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -definetarget + psMetadata *definetargetArgs = psMetadataAlloc(); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-label", 0, "define label (required)", NULL); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter (required)", NULL); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddBool(definetargetArgs, PS_LIST_TAIL,"-clean", 0, "define clean", false); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + + // -updatetarget + psMetadata *updatetargetArgs = psMetadataAlloc(); + psMetadataAddS64(updatetargetArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); + + // -listtarget + psMetadata *listtargetArgs = psMetadataAlloc(); + psMetadataAddS64(listtargetArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); + psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-label", 0, "list targets for label", NULL); + psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); + psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); + psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); + psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-full", 0, "list full targets", false); + psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-state", 0, "list targets in state", NULL); + psMetadataAddU64(listtargetArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); + psMetadataAddBool(listtargetArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -defineinterest + psMetadata *defineinterestArgs = psMetadataAlloc(); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id (required)", 0); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -updateinterest + psMetadata *updateinterestArgs = psMetadataAlloc(); + psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-int_id", 0, "define int_id", 0); + psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); + psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); psMetadata *argSets = psMetadataAlloc(); @@ -208,4 +284,20 @@ PXOPT_ADD_MODE("-pendingdest", "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs); + PXOPT_ADD_MODE("-definedsproduct", "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs); + PXOPT_ADD_MODE("-updatedsproduct", "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs); +// PXOPT_ADD_MODE("-listdsproduct", "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs); + + PXOPT_ADD_MODE("-definedestination", "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs); + PXOPT_ADD_MODE("-updatedestination", "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs); +// PXOPT_ADD_MODE("-listdestination", "", DISTTOOL_MODE_LISTDESTINATION, listdestinationArgs); + + PXOPT_ADD_MODE("-definetarget", "", DISTTOOL_MODE_DEFINETARGET, definetargetArgs); + PXOPT_ADD_MODE("-updatetarget", "", DISTTOOL_MODE_UPDATETARGET, updatetargetArgs); + PXOPT_ADD_MODE("-listtarget", "", DISTTOOL_MODE_LISTTARGET, listtargetArgs); + + PXOPT_ADD_MODE("-defineinterest", "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs); + PXOPT_ADD_MODE("-updateinterest", "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs); +// PXOPT_ADD_MODE("-listinterest", "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { psError(PS_ERR_UNKNOWN, true, "option parsing failed"); Index: branches/pap/ippTools/src/faketool.c =================================================================== --- branches/pap/ippTools/src/faketool.c (revision 23948) +++ branches/pap/ippTools/src/faketool.c (revision 25027) @@ -134,8 +134,8 @@ PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "decl", "<"); PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); @@ -157,6 +157,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); @@ -235,8 +235,8 @@ bool status; char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); - if (end_stage && strcasecmp(end_stage, "warp")) continue; + if (end_stage && strcasecmp(end_stage, "warp")) continue; char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); - if (raw_tess_id || tess_id) continue; + if (raw_tess_id || tess_id) continue; char *label = psMetadataLookupStr(&status, md, "label"); @@ -244,5 +244,5 @@ if (!status) { - psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); + psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); psFree(output); return false; @@ -302,8 +302,8 @@ PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "decl", "<"); PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); @@ -325,6 +325,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); @@ -455,6 +455,6 @@ if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "fakeRun"); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -563,5 +563,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); @@ -632,6 +632,6 @@ psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); @@ -646,8 +646,8 @@ PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "decl", "<"); PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); @@ -669,6 +669,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); if (!psListLength(where->list) @@ -895,5 +895,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); psString query = pxDataGet("faketool_pendingcleanuprun.sql"); @@ -1157,10 +1157,11 @@ // pxwarpQueueByFakeID() can only be run after fakeRun.state has been set to stop if (!pxwarpQueueByFakeID(config, - fakeRun->fake_id, - fakeRun->workdir, - fakeRun->label, - fakeRun->dvodb, - fakeRun->tess_id, - fakeRun->end_stage + fakeRun->fake_id, + fakeRun->workdir, + fakeRun->label, + fakeRun->dvodb, + fakeRun->tess_id, + fakeRun->reduction, + fakeRun->end_stage )) { psError(PS_ERR_UNKNOWN, false, "failed to queue warpRun"); Index: branches/pap/ippTools/src/faketoolConfig.c =================================================================== --- branches/pap/ippTools/src/faketoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/faketoolConfig.c (revision 25027) @@ -61,8 +61,8 @@ psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); - psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees)", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees)", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); @@ -84,6 +84,6 @@ psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -114,8 +114,8 @@ psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); - psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); - psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); - psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); - psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees)", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees)", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); @@ -137,6 +137,6 @@ psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); @@ -198,5 +198,5 @@ psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); - psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); @@ -212,8 +212,8 @@ psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); - psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); - psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); - psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); - psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees)", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees)", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); @@ -235,6 +235,6 @@ psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); Index: branches/pap/ippTools/src/flatcorr.c =================================================================== --- branches/pap/ippTools/src/flatcorr.c (revision 23948) +++ branches/pap/ippTools/src/flatcorr.c (revision 25027) @@ -535,4 +535,7 @@ PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + psString query = pxDataGet("flatcorr_pendingprocess.sql"); if (!query) { @@ -540,4 +543,11 @@ return false; } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" Index: branches/pap/ippTools/src/flatcorrConfig.c =================================================================== --- branches/pap/ippTools/src/flatcorrConfig.c (revision 23948) +++ branches/pap/ippTools/src/flatcorrConfig.c (revision 25027) @@ -101,4 +101,5 @@ psMetadataAddU64 (pendingprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(pendingprocessArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -addprocess (XXX need to add fault and stats) @@ -106,5 +107,5 @@ psMetadataAddS64 (addprocessArgs, PS_LIST_TAIL, "-corr_id", 0, "add complete run for specified corr_id (required)", 0); psMetadataAddStr (addprocessArgs, PS_LIST_TAIL, "-hostname", 0, "set hostname", NULL); - psMetadataAddS16 (addprocessArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16 (addprocessArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); // -updaterun Index: branches/pap/ippTools/src/magicdstool.c =================================================================== --- branches/pap/ippTools/src/magicdstool.c (revision 23948) +++ branches/pap/ippTools/src/magicdstool.c (revision 25027) @@ -2,5 +2,5 @@ * magicdstool.c * - * Copyright (C) 2006-2007 IfA + * Copyright (C) 2006-2009 IfA * * This program is free software; you can redistribute it and/or modify it @@ -27,4 +27,5 @@ #include #include +#include #include "pxtools.h" @@ -38,4 +39,7 @@ static bool revertdestreakedfileMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); +static bool toremoveMode(pxConfig *config); +static bool torestoreMode(pxConfig *config); +static bool torevertMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -68,4 +72,7 @@ MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); + MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); + MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); + MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); default: psAbort("invalid option (this should not happen)"); @@ -91,119 +98,114 @@ static bool definebyqueryMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_UNKNOWN, true, "definebyquery not implelmented yet"); - - return false; -#ifdef notyet // Required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + // Optional PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); - - // Optional - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); + PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - // Create temporary table of the best diffs - { - psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // search args + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); + + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + + ippStage stageNum = ippStringToStage(stage); + + psString query = NULL; + switch (stageNum) { + case IPP_STAGE_RAW: + query = pxDataGet("magicdstool_definebyquery_raw.sql"); + break; + case IPP_STAGE_CHIP: + query = pxDataGet("magicdstool_definebyquery_chip.sql"); + break; + case IPP_STAGE_CAMERA: + query = pxDataGet("magicdstool_definebyquery_camera.sql"); + break; + case IPP_STAGE_WARP: + query = pxDataGet("magicdstool_definebyquery_warp.sql"); + break; + case IPP_STAGE_DIFF: + query = pxDataGet("magicdstool_definebyquery_diff.sql"); + break; + case IPP_STAGE_FAKE: + case IPP_STAGE_STACK: + psError(PXTOOLS_ERR_DATA, true, "%sRuns do not need to be destreaked", stage); + return false; + case IPP_STAGE_NONE: + psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage); + return false; + default: + psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s", + stageNum, stage); + return false; + } + + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + char *rerun_flag = rerun ? "\n1 " : "\n0" ; + + if (stageNum != IPP_STAGE_DIFF) { + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag)) { psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); - } - - // Insert list of best diffs into temporary table - { - psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - psString whereClause = NULL; // WHERE conditions - if (psListLength(where->list)) { - whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); - } - psFree(where); - - if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereClause); psFree(query); return false; } - psFree(whereClause); - psFree(query); - } - - // Get list of exposures ready to magic - { - psString query = pxDataGet("magictool_definebyquery_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString magicSkyCellNumsWhere = NULL; // WHERE conditions for magicSkyCellNums - { - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - if (psListLength(where->list)) { - magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&magicSkyCellNumsWhere, "\n AND "); - } - psFree(where); - } - - // "available" means only concern ourselves with exposures that have all diffs completed, unless we're - // told to only take what's available. - // "new" means we want a new run even if there's already a magic run defined - PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); - PXOPT_LOOKUP_BOOL(new, config->args, "-new", false); - - psString queryWhere = NULL; // WHERE conditions for entire query - if (available) { - psStringAppend(&queryWhere, " WHERE num_done = num_todo"); - } - if (new) { - const char *newWhere = " magic_id IS NULL"; // String to add - if (queryWhere) { - psStringAppend(&queryWhere, " AND %s", newWhere); - } else { - psStringAppend(&queryWhere, " WHERE %s", newWhere); - } - } - if (queryWhere) { - psStringAppend(&query, " %s", queryWhere); - psFree(queryWhere); - } - - - if (!p_psDBRunQueryF(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) { + } else { + // diff stage query has two types bothways and !bothways + // so we need to send the rerun flag and the where data twice + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag, whereString, rerun_flag, whereString)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(magicSkyCellNumsWhere); + psFree(whereString); psFree(query); return false; } - psFree(magicSkyCellNumsWhere); - psFree(query); - } + psFree(whereString); + } + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); @@ -227,72 +229,71 @@ } - // Parse the list of exposures ready to magic - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + // Parse the list of runs ready to be destreaked + + if (!dry_run && !psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } psArray *list = psArrayAllocEmpty(16); // List of runs, to print for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; // Row of interest - psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier + psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psS64 inv_magic_id = psMetadataLookupS64(NULL, row, "inv_magic_id"); + psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); + psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); + + // if workdir is not supplied use the magicRun's + if (!workdir) { + workdir = magicRunWorkdir; + } + + psString outroot = NULL; + // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip + psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage); // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, "new", workdir, "dirty", label, dvodb, registered, 0); + magicDSRunRow *run = magicDSRunRowAlloc( + 0, // magic_ds_id + magic_id, + inv_magic_id, + "new", + stage, + stage_id, + cam_id, + set_label ? set_label : magicRunLabel, + outroot, + recoveryroot, + re_place, + 0); // remove + + psFree(outroot); if (!run) { - psAbort("failed to alloc magicRun object"); - } - - if (!magicRunInsertObject(config->dbh, run)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(run); - psFree(insert); - psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { + psAbort("failed to alloc magicDSRun object"); + } + + if (!dry_run) { + if (!magicDSRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; } - return false; - } - - psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier - run->magic_id = magic_id; + psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_ds_id = magic_ds_id; + } psArrayAdd(list, list->n, run); psFree(run); - - // Create a suitable insertion query for this run - psString thisInsert = psStringCopy(insert); - { - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, magic_id); - psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); - psFree(idString); - } - { - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, exp_id); - psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); - psFree(idString); - } - - if (!p_psDBRunQueryF(config->dbh, thisInsert, magic_id, exp_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(thisInsert); - psFree(insert); - psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(thisInsert); - } - - if (!psDBCommit(config->dbh)) { + } + + if (!dry_run && !psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -300,5 +301,5 @@ psFree(output); - if (!magicRunPrintObjects(stdout, list, !simple)) { + if (!magicDSRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -309,5 +310,4 @@ return true; -#endif // notyet } @@ -338,4 +338,5 @@ 0, // ID magic_id, + 0, // inv_magic_id "new", // state stage, @@ -398,5 +399,6 @@ PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -495,23 +497,34 @@ psMetadata *row = output->data[0]; - psString stage = psMetadataLookupStr(NULL, row, "stage"); + psString stage= psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + ippStage stageNum = ippStringToStage(stage); // chose the appropriate query based on the stage - if (!strcmp(stage, "raw")) { - query = "UPDATE rawImfile SET magicked = 1 where exp_id = %" PRId64 " AND class_id = '%s'"; - } else if (!strcmp(stage, "chip")) { - query = "UPDATE chipProcessedImfile SET magicked = 1 where chip_id = %" PRId64 " AND class_id = '%s'"; - } else if (!strcmp(stage, "warp")) { - query = "UPDATE warpSkyfile SET magicked = 1 where warp_id = %" PRId64 " AND skycell_id = '%s'"; - } else if (!strcmp(stage, "diff")) { - query = "UPDATE diffSkyfile SET magicked = 1 where diff_id = %" PRId64 " AND skycell_id = '%s'"; - } else { + switch (stageNum) { + case IPP_STAGE_RAW: + query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; + break; + case IPP_STAGE_CHIP: + query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; + break; + case IPP_STAGE_CAMERA: + // no there is no magicked column in camProcessedExp so we have nothing to do + psFree(output); + return true; + case IPP_STAGE_WARP: + query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'"; + break; + case IPP_STAGE_DIFF: + query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'"; + break; + default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); psFree(output); return false; } - if (!p_psDBRunQueryF(config->dbh, query, stage_id, component)) { + + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -560,20 +573,29 @@ psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + ippStage stageNum = ippStringToStage(stage); // chose the appropriate query based on the stage - if (!strcmp(stage, "raw")) { - query = "UPDATE rawExp SET magicked = 1 where exp_id = %" PRId64; - } else if (!strcmp(stage, "chip")) { - query = "UPDATE chipRun SET magicked = 1 where chip_id = %" PRId64; - } else if (!strcmp(stage, "warp")) { - query = "UPDATE warpRun SET magicked = 1 where warp_id = %" PRId64; - } else if (!strcmp(stage, "diff")) { - query = "UPDATE diffRun SET magicked = 1 where diff_id = %" PRId64; - } else { + switch (stageNum) { + case IPP_STAGE_RAW: + query = "UPDATE rawExp SET magicked = %" PRId64 " where exp_id = %" PRId64; + break; + case IPP_STAGE_CHIP: + query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64; + break; + case IPP_STAGE_CAMERA: + query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64; + break; + case IPP_STAGE_WARP: + query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64; + break; + case IPP_STAGE_DIFF: + query = "UPDATE diffRun SET magicked = %" PRId64 " where diff_id = %" PRId64; + break; + default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); psFree(output); return false; } - if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -625,5 +647,5 @@ } - if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault)) { + if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) { // rollback if (!psDBRollback(config->dbh)) { @@ -658,23 +680,9 @@ PS_ASSERT_PTR_NON_NULL(cam_id, false); - if (!strcmp(stage, "diff")) { - // don't need these ids for diff stage because diff_id is in the magicRun - *stage_id = 0; - *cam_id = 0; - return true; - } - - int stageNum; - if (!strcmp(stage, "raw")) { - stageNum = 0; - } else if (!strcmp(stage, "chip")) { - stageNum = 1; - } else if (!strcmp(stage, "warp")) { - stageNum = 2; - } else { - psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage); - return false; - } - + int stageNum = ippStringToStage(stage);; + if (stageNum == IPP_STAGE_NONE) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage); + return false; + } psString query = pxDataGet("magicdstool_getrunids.sql"); @@ -709,10 +717,20 @@ *cam_id = psMetadataLookupS64(NULL, row, "cam_id"); - if (stageNum == 0) { + switch (stageNum) { + case IPP_STAGE_RAW: *stage_id = psMetadataLookupS64(NULL, row, "exp_id"); - } else if (stageNum == 1) { + break; + case IPP_STAGE_CHIP: *stage_id = psMetadataLookupS64(NULL, row, "chip_id"); - } else if (stageNum == 2) { + break; + case IPP_STAGE_CAMERA: + *stage_id = *cam_id; + return true; + case IPP_STAGE_WARP: *stage_id = psMetadataLookupS64(NULL, row, "warp_id"); + break; + case IPP_STAGE_DIFF: + *stage_id = psMetadataLookupS64(NULL, row, "diff_id"); + break; } @@ -779,9 +797,10 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - - psString query = psStringCopy("DELETE FROM magicDSFile WHERE fault != 0"); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = psStringCopy("DELETE FROM magicDSFile USING magicDSFile, magicDSRun WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) AND magicDSFile.fault != 0"); if (psListLength(where->list)) { @@ -889,2 +908,231 @@ } +static bool toremoveMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magicdstool_toremove.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool torestoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magicdstool_torestore.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool torevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_torevert_%s.sql", stage); + + psString query = pxDataGet(sql_file); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "torevert", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + Index: branches/pap/ippTools/src/magicdstool.h =================================================================== --- branches/pap/ippTools/src/magicdstool.h (revision 23948) +++ branches/pap/ippTools/src/magicdstool.h (revision 25027) @@ -32,4 +32,7 @@ MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, MAGICDSTOOL_MODE_GETSKYCELLS, + MAGICDSTOOL_MODE_TOREMOVE, + MAGICDSTOOL_MODE_TORESTORE, + MAGICDSTOOL_MODE_TOREVERT, } MAGICDStoolMode; Index: branches/pap/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/pap/ippTools/src/magicdstoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/magicdstoolConfig.c (revision 25027) @@ -49,21 +49,28 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); -#ifdef notyet - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -#endif + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one arleady exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define output directory (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for run (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); @@ -88,5 +95,6 @@ psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); - psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -106,4 +114,5 @@ psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); // -getskycells @@ -113,4 +122,29 @@ psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); psMetadataAddBool(getskycellsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -toremove + psMetadata *toremoveArgs = psMetadataAlloc(); + psMetadataAddS64(toremoveArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(toremoveArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(toremoveArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -torestore + psMetadata *torestoreArgs = psMetadataAlloc(); + psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -torevert + psMetadata *torevertArgs = psMetadataAlloc(); + psMetadataAddS64(torevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(torevertArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(torevertArgs, PS_LIST_TAIL, "-stage", 0, "define output directory (required)", NULL); + psMetadataAddStr(torevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -133,4 +167,10 @@ PXOPT_ADD_MODE("-getskycells", "get skycell files ", MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsArgs); + PXOPT_ADD_MODE("-toremove", "backup images pending removal", + MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs); + PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels", + MAGICDSTOOL_MODE_TORESTORE, torestoreArgs); + PXOPT_ADD_MODE("-torevert", "faulted images to revert", + MAGICDSTOOL_MODE_TOREVERT, torevertArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/pap/ippTools/src/magictool.c =================================================================== --- branches/pap/ippTools/src/magictool.c (revision 23948) +++ branches/pap/ippTools/src/magictool.c (revision 25027) @@ -47,9 +47,5 @@ static bool revertmaskMode(pxConfig *config); static bool maskMode(pxConfig *config); -static bool diffskyfileMode(pxConfig *config); -static bool warpskyfileMode(pxConfig *config); -static bool chipprocessedimfileMode(pxConfig *config); -static bool rawimfileMode(pxConfig *config); - +static bool censorrunMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -90,8 +86,5 @@ MODECASE(MAGICTOOL_MODE_REVERTMASK, revertmaskMode); MODECASE(MAGICTOOL_MODE_MASK, maskMode); - MODECASE(MAGICTOOL_MODE_DIFFSKYFILE, diffskyfileMode); - MODECASE(MAGICTOOL_MODE_WARPSKYFILE, warpskyfileMode); - MODECASE(MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileMode); - MODECASE(MAGICTOOL_MODE_RAWIMFILE, rawimfileMode); + MODECASE(MAGICTOOL_MODE_CENSORRUN, censorrunMode); default: psAbort("invalid option (this should not happen)"); @@ -127,9 +120,12 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns + PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "=="); + PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "=="); + + psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else + PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "=="); // Get list of exposures ready to magic @@ -146,36 +142,46 @@ PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - psString queryWhere = NULL; // WHERE conditions for entire query + psString diffWhereStr = NULL; // WHERE conditions for diffRuns + psString queryWhereStr = NULL; // WHERE conditions for entire query if (!available) { - psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'"); - } else { - // what if no skycells for the diff run completed? - } + psStringAppend(&diffWhereStr, "\nAND diffRun.state = 'full'"); + } + // what if no skycells for the diff run completed? + if (!rerun) { - const char *newWhere = " magic_id IS NULL"; // String to add - if (queryWhere) { - psStringAppend(&queryWhere, " AND %s", newWhere); - } else { - psStringAppend(&queryWhere, "\nWHERE %s", newWhere); - } - } + psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE"); + } + // now add the user specified qualifiers - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause); + if (psListLength(diffWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(diffWhere, NULL); + psStringAppend(&diffWhereStr, "\nAND %s", whereClause); psFree(whereClause); } - psFree(where); - if (!queryWhere) { - psStringAppend(&queryWhere, " "); - } - - if (!p_psDBRunQueryF(config->dbh, query, queryWhere)) { + if (psListLength(queryWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(queryWhere, NULL); + psStringAppend(&queryWhereStr, "\n%s %s", queryWhereStr ? "AND" : "WHERE", whereClause); + psFree(whereClause); + } + psFree(diffWhere); + psFree(queryWhere); + + // Ensure the WHERE strings have something + if (!diffWhereStr) { + diffWhereStr = psStringCopy(""); + } + if (!queryWhereStr) { + queryWhereStr = psStringCopy(""); + } + + if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(queryWhere); + psFree(diffWhereStr); + psFree(queryWhereStr); psFree(query); return false; } - psFree(queryWhere); + psFree(diffWhereStr); + psFree(queryWhereStr); psFree(query); } @@ -201,6 +207,18 @@ } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // Parse the list of exposures ready to magic - if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -215,7 +233,9 @@ psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier + bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types! // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0); + magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label, + dvodb, registered, 0); if (!run) { psAbort("failed to alloc magicRun object"); @@ -296,4 +316,5 @@ // optional + PXOPT_LOOKUP_BOOL(inverse, config->args, "-inverse", false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); @@ -305,4 +326,5 @@ exp_id, diff_id ? diff_id : PS_MAX_S64, + inverse, "reg", // state workdir, @@ -363,5 +385,4 @@ // required PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); @@ -369,5 +390,4 @@ config->dbh, magic_id, - diff_id, node ); @@ -457,5 +477,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -562,6 +582,7 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - - psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0"); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0"); if (psListLength(where->list)) { @@ -584,7 +605,13 @@ PS_ASSERT_PTR_NON_NULL(config, false); - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + // Regrettably, there are multiple WHERE hooks which call the same things different names + psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template + psMetadata *magicWhere = psMetadataAlloc(); // WHERE for selecting magic runs + + PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "=="); + + PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -597,10 +624,20 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template + psString magicWhereStr = psStringCopy(""); // WHERE for selecting magic runs + + if (psListLength(templatesWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL); + psStringAppend(&templatesWhereStr, "\nAND %s", whereClause); psFree(whereClause); } - psFree(where); + psFree(templatesWhere); + + if (psListLength(magicWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL); + psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause); + psFree(whereClause); + } + psFree(magicWhere); // treat limit == 0 as "no limit" @@ -611,9 +648,15 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, + templatesWhereStr, templatesWhereStr, + magicWhereStr, magicWhereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(templatesWhereStr); + psFree(magicWhereStr); psFree(query); return false; } + psFree(templatesWhereStr); + psFree(magicWhereStr); psFree(query); @@ -704,9 +747,9 @@ bool status = false; - psS32 done = psMetadataLookupS32(&status, data, "done"); + psS64 done = psMetadataLookupS64(&status, data, "done"); if (!status) { psAbort("failed to lookup value for done column"); } - psS16 bad = psMetadataLookupS16(&status, data, "bad"); + psS64 bad = psMetadataLookupS64(&status, data, "bad"); if (!status) { psAbort("failed to lookup value for bad column"); @@ -738,8 +781,9 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); // look for "inputs" that need to processed @@ -750,13 +794,24 @@ } - psString whereClause = NULL; + psString whereString = NULL; if (psListLength(where->list)) { - whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); } psFree(where); - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + // treat limit == 0 as "no limit" + if (limit) { + // cut limit in half + // hack to prevent pending leaf nodes from blocking branch nodes + psString limitString = psDBGenerateLimitSQL((limit + 1) / 2); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); psFree(query); return false; @@ -782,6 +837,20 @@ } + if (limit) { + if (psArrayLength(output) >= limit) { + // we've found enough (note that limit was applied to the query so '> limit' won't happen) + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + } + // look for tree nodes that need to be processed - query = pxDataGet("magictool_toprocess_tree.sql"); + query = pxDataGet("magictool_toprocess_runs.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -789,21 +858,21 @@ } - if (whereClause) { - psString new = NULL; - psStringAppend(&new, "\n AND %s", whereClause); - psFree(whereClause); - whereClause = new; - } - if (!p_psDBRunQueryF(config->dbh, query, whereClause ? whereClause : "")) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereClause); + { + psString limitString = psDBGenerateLimitSQL( 100 ); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); psFree(query); return false; } - psFree(whereClause); + psFree(whereString); psFree(query); - psArray *magicTree = p_psDBFetchResult(config->dbh); - if (!magicTree) { + psArray *magicRuns = p_psDBFetchResult(config->dbh); + if (!magicRuns) { psErrorCode err = psErrorCodeLast(); switch (err) { @@ -818,39 +887,60 @@ return false; } - if (!psArrayLength(magicTree)) { + if (!psArrayLength(magicRuns)) { psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(magicTree); + psFree(magicRuns); return true; } - // entries are ordered by magic_id - long index = 0; - while (index < psArrayLength(magicTree)) { + query = pxDataGet("magictool_toprocess_tree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + for (psS64 index = 0; index < psArrayLength(magicRuns); index++) { + if (limit && (psArrayLength(output) >= limit)) { + break; + } bool status; - psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id"); + psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id"); if (!status) { psAbort("failed to lookup value for magic_id column"); } - - // find the end of this block - long first = index; - long last = index; - for (long i = index + 1; i < psArrayLength(magicTree); i++) { - psS64 magic_id = psMetadataLookupS64(&status, magicTree->data[i], "magic_id"); - if (!status) { - psAbort("failed to lookup value for magic_id column"); + + whereString = NULL; + psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id); + if (!p_psDBRunQueryF(config->dbh, query, whereString )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + psFree(query); + return false; + } + psFree(whereString); + psArray *magicTree = p_psDBFetchResult(config->dbh); + if (!magicTree) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - if (magic_id != current_magic_id) { - break; - } - last = i; - } - - index = last + 1; - - psHash *forest = psHashAlloc(last - first + 1); + + return false; + } + psS64 length = psArrayLength(magicTree); + if (!length) { + psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id); + psFree(magicTree); + continue; + } + + psHash *forest = psHashAlloc(length); // convert the array of metadata into a pxTree structure - for (long i = first; i <= last; i++) { + for (long i = 0; i < length; i++) { bool status; psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); @@ -877,8 +967,15 @@ pxTreeCrawl(root, findReadyNodes, output); psFree(root); - + psFree(magicTree); } if (psArrayLength(output)) { + if (limit && (limit < psArrayLength(output))) { + // truncate the array + long arrayLength = psArrayLength(output); + for (long i = arrayLength - 1; i >= limit; i--) { + psArrayRemoveIndex(output, i); + } + } // negative simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { @@ -889,5 +986,4 @@ } - psFree(magicTree); psFree(output); @@ -905,5 +1001,5 @@ // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); // default values @@ -913,5 +1009,5 @@ magic_id, node, - uri, + path_base, fault )) { @@ -931,5 +1027,5 @@ PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "magicNodeResult.fault", "=="); psString query = pxDataGet("magictool_revertnode.sql"); @@ -1226,266 +1322,4 @@ } - -static bool diffskyfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_diffskyfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool warpskyfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_warpskyfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicWarpSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool chipprocessedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_chipprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicChipProcessedImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool rawimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_rawimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicRawImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state) { @@ -1558,2 +1392,111 @@ return true; } + +static bool censorStage(pxConfig *config, psString stage, psString whereClause) +{ + psString queryFile = NULL; + psStringAppend(&queryFile, "magictool_censor_%s.sql", stage); + psString query = pxDataGet(queryFile); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psFree(queryFile); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(queryFile); + + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + return true; +} + +static bool censorrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet"); + return false; + + psMetadata *where = psMetadataAlloc(); + + // at least one of these required + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "either -exp_id or -magic_id is required"); + psFree(where); + return false; + } + + psString query = psStringCopy("UPDATE magicRun SET state = 'censored'"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicRun"); + psFree(where); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(query); + + psS32 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated == 0) { + psError(PS_ERR_UNKNOWN, false, "failed to censor magicRun"); + psFree(whereClause); + return false; + } + + // Now queue any destreaked files to be re-verted + + // note: on failure censorStage issues the rollback + if (!censorStage(config, "raw", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "chip", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "camera", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "warp", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "diff", whereClause)) { + psFree(whereClause); + return false; + } + + psFree(whereClause); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: branches/pap/ippTools/src/magictool.h =================================================================== --- branches/pap/ippTools/src/magictool.h (revision 23948) +++ branches/pap/ippTools/src/magictool.h (revision 25027) @@ -41,8 +41,5 @@ MAGICTOOL_MODE_REVERTMASK, MAGICTOOL_MODE_MASK, - MAGICTOOL_MODE_DIFFSKYFILE, - MAGICTOOL_MODE_WARPSKYFILE, - MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, - MAGICTOOL_MODE_RAWIMFILE, + MAGICTOOL_MODE_CENSORRUN, } MAGICtoolMode; Index: branches/pap/ippTools/src/magictoolConfig.c =================================================================== --- branches/pap/ippTools/src/magictoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/magictoolConfig.c (revision 25027) @@ -53,4 +53,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL); @@ -58,4 +59,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false); // -definerun @@ -65,4 +67,5 @@ // following argument is for compatability psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff_id", 0); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-inverse", 0, "use the inverse subtraction?", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); @@ -78,5 +81,4 @@ psMetadata *addinputskyfileArgs = psMetadataAlloc(); psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0); psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); @@ -93,5 +95,5 @@ psMetadataAddS64(totreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddU64(totreeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddStr(totreeArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(totreeArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddBool(totreeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -106,4 +108,5 @@ psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); // -inputs @@ -118,5 +121,5 @@ psMetadataAddS64(toprocessArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddU64(toprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddStr(toprocessArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(toprocessArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddBool(toprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -125,5 +128,5 @@ psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); - psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base", NULL); psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -158,28 +161,8 @@ psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - // -diffskyfile - psMetadata *diffskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -warpskyfile - psMetadata *warpskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(warpskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddBool(warpskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddStr(warpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); - - // -chipprocessedimfile - psMetadata *chipprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(chipprocessedimfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(chipprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); - psMetadataAddBool(chipprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -rawimfile - psMetadata *rawimfileArgs = psMetadataAlloc(); - psMetadataAddS64(rawimfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(rawimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); - psMetadataAddBool(rawimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -censorun + psMetadata *censorrunArgs = psMetadataAlloc(); + psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); + psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); psFree(now); @@ -204,8 +187,5 @@ PXOPT_ADD_MODE("-revertmask", "", MAGICTOOL_MODE_REVERTMASK, revertmaskArgs); PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); - PXOPT_ADD_MODE("-diffskyfile", "", MAGICTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); - PXOPT_ADD_MODE("-warpskyfile", "", MAGICTOOL_MODE_WARPSKYFILE, warpskyfileArgs); - PXOPT_ADD_MODE("-chipprocessedimfile", "", MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileArgs); - PXOPT_ADD_MODE("-rawimfile", "", MAGICTOOL_MODE_RAWIMFILE, rawimfileArgs); + PXOPT_ADD_MODE("-censorrun", "", MAGICTOOL_MODE_CENSORRUN, censorrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/pap/ippTools/src/pstamptool.c =================================================================== --- branches/pap/ippTools/src/pstamptool.c (revision 23948) +++ branches/pap/ippTools/src/pstamptool.c (revision 25027) @@ -34,17 +34,17 @@ static bool datastoreMode(pxConfig *config); static bool moddatastoreMode(pxConfig *config); -static bool addReqMode(pxConfig *config); -static bool completedReqMode(pxConfig *config); -static bool listReqMode(pxConfig *config); -static bool pendingReqMode(pxConfig *config); -static bool updateReqMode(pxConfig *config); -static bool revertReqMode(pxConfig *config); -static bool addJobMode(pxConfig *config); -static bool listJobMode(pxConfig *config); -static bool pendingJobMode(pxConfig *config); -static bool updateJobMode(pxConfig *config); -static bool addProjectMode(pxConfig *config); +static bool addreqMode(pxConfig *config); +static bool completedreqMode(pxConfig *config); +static bool listreqMode(pxConfig *config); +static bool pendingreqMode(pxConfig *config); +static bool updatereqMode(pxConfig *config); +static bool revertreqMode(pxConfig *config); +static bool addjobMode(pxConfig *config); +static bool listjobMode(pxConfig *config); +static bool pendingjobMode(pxConfig *config); +static bool updatejobMode(pxConfig *config); +static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); -static bool modProjectMode(pxConfig *config); +static bool modprojectMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -70,16 +70,16 @@ MODECASE(PSTAMPTOOL_MODE_DATASTORE, datastoreMode); MODECASE(PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreMode); - MODECASE(PSTAMPTOOL_MODE_ADDREQ, addReqMode); - MODECASE(PSTAMPTOOL_MODE_COMPLETEDREQ, completedReqMode); - MODECASE(PSTAMPTOOL_MODE_LISTREQ, listReqMode); - MODECASE(PSTAMPTOOL_MODE_PENDINGREQ, pendingReqMode); - MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updateReqMode); - MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertReqMode); - MODECASE(PSTAMPTOOL_MODE_ADDJOB, addJobMode); - MODECASE(PSTAMPTOOL_MODE_LISTJOB, listJobMode); - MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingJobMode); - MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updateJobMode); - MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addProjectMode); - MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modProjectMode); + MODECASE(PSTAMPTOOL_MODE_ADDREQ, addreqMode); + MODECASE(PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqMode); + MODECASE(PSTAMPTOOL_MODE_LISTREQ, listreqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGREQ, pendingreqMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updatereqMode); + MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertreqMode); + MODECASE(PSTAMPTOOL_MODE_ADDJOB, addjobMode); + MODECASE(PSTAMPTOOL_MODE_LISTJOB, listjobMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode); + MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); + MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); default: @@ -220,5 +220,5 @@ } -static bool addReqMode(pxConfig *config) +static bool addreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -251,5 +251,5 @@ } -static bool pendingReqMode(pxConfig *config) +static bool pendingreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -311,5 +311,5 @@ } -static bool listReqMode(pxConfig *config) +static bool listreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -359,5 +359,5 @@ } -static bool completedReqMode(pxConfig *config) +static bool completedreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -408,5 +408,5 @@ } -static bool updateReqMode(pxConfig *config) +static bool updatereqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -472,5 +472,5 @@ } -static bool revertReqMode(pxConfig *config) +static bool revertreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -493,5 +493,5 @@ } -static bool addJobMode(pxConfig *config) +static bool addjobMode(pxConfig *config) { bool stampJob = false; @@ -500,10 +500,8 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(uri, config->args, "-uri", true, false); PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); PXOPT_LOOKUP_STR(rownum, config->args, "-rownum", true, false); PXOPT_LOOKUP_STR(job_type, config->args, "-job_type", false, false); PXOPT_LOOKUP_STR(outputBase, config->args, "-outputBase", true, false); - PXOPT_LOOKUP_STR(argString, config->args, "-args", false, false); PXOPT_LOOKUP_STR(stateString, config->args, "-state", false, false); PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); @@ -519,10 +517,4 @@ return false; } - - if (stampJob && !argString) { - psError(PS_ERR_UNKNOWN, true, "-args is required for stamp job"); - return false; - } - if (stampJob) { query = pxDataGet("pstamptool_addjob_stampjob.sql"); @@ -535,5 +527,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, uri, exp_id, outputBase, fault, argString)) { + if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, exp_id, outputBase, fault)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -554,5 +546,5 @@ } -static bool listJobMode(pxConfig *config) +static bool listjobMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -626,5 +618,5 @@ } -static bool pendingJobMode(pxConfig *config) +static bool pendingjobMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -688,5 +680,5 @@ } -static bool updateJobMode(pxConfig *config) +static bool updatejobMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -723,5 +715,5 @@ return true; } -static bool addProjectMode(pxConfig *config) +static bool addprojectMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -801,5 +793,5 @@ return true; } -static bool modProjectMode(pxConfig *config) +static bool modprojectMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); Index: branches/pap/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/pap/ippTools/src/pstamptoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/pstamptoolConfig.c (revision 25027) @@ -100,10 +100,8 @@ // -addjob psMetadata *addjobArgs = psMetadataAlloc(); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-uri", 0, "define job file uri", NULL); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-args", 0, "define job args", NULL); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); @@ -128,5 +126,4 @@ psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - // psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-result", 0, "new result", NULL); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); Index: branches/pap/ippTools/src/pubtool.c =================================================================== --- branches/pap/ippTools/src/pubtool.c (revision 25027) +++ branches/pap/ippTools/src/pubtool.c (revision 25027) @@ -0,0 +1,336 @@ +/* + * pubtool.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pubtool.h" + +static bool defineclientMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool pendingMode(pxConfig *config); +static bool addMode(pxConfig *config); +static bool revertMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pubtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PUBTOOL_MODE_DEFINECLIENT, defineclientMode); + MODECASE(PUBTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(PUBTOOL_MODE_PENDING, pendingMode); + MODECASE(PUBTOOL_MODE_ADD, addMode); + MODECASE(PUBTOOL_MODE_REVERT, revertMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool defineclientMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(product, config->args, "-product", true, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + + if (!publishClientInsert(config->dbh, 0, product, stage, workdir, comment)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + + // required + + // optional + PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + psString query = pxDataGet("pubtool_definerun.sql"); // Query to run + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(where); + return false; + } + + psString whereClause = psStringCopy(""); // Additional constraints to add to query + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\n AND %s", clause); + psFree(clause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + psFree(whereClause); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + psFree(query); + psFree(whereClause); + + psArray *output = p_psDBFetchResult(config->dbh); // Output of SELECT statement + if (!output) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("pubtool", PS_LOG_INFO, "No rows found"); + psFree(output); + return true; + } + + for (int i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 client = psMetadataLookupS64(NULL, row, "client_id"); // Client identifier + psS64 stage = psMetadataLookupS64(NULL, row, "stage_id"); // Stage identifier + + if (!publishRunInsert(config->dbh, 0, client, stage, label, "new")) { + psError(PS_ERR_UNKNOWN, false, "Unable to add fileset"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + + // required + PXOPT_COPY_STR(config->args, where, "-client_id", "publishClient.client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "=="); + + // optional + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pubtool_pending.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psStringCopy(""); // WHERE conditions to add + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\nAND %s", clause); + psFree(clause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pubtool", PS_LOG_INFO, "No rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "publishRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "Failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool addMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add file"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + + if (fault == 0) { + if (!p_psDBRunQueryF(config->dbh, + "UPDATE publishRun SET state = 'full' WHERE pub_id = %" PRId64, + pub_id)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + + + + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + PXOPT_COPY_S64(config->args, where, "-pub_id", "publishRun.pub_id", "=="); + PXOPT_COPY_S32(config->args, where, "-fault", "publishDone.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-client_id", "publishClient.client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "=="); + + psString query = pxDataGet("pubtool_revert.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + Index: branches/pap/ippTools/src/pubtool.h =================================================================== --- branches/pap/ippTools/src/pubtool.h (revision 25027) +++ branches/pap/ippTools/src/pubtool.h (revision 25027) @@ -0,0 +1,36 @@ +/* + * pubtool.h + * + * Copyright (C) 2008 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PUBTOOL_H +#define PUBTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PUBTOOL_MODE_NONE = 0x0, + PUBTOOL_MODE_DEFINECLIENT, + PUBTOOL_MODE_DEFINERUN, + PUBTOOL_MODE_PENDING, + PUBTOOL_MODE_ADD, + PUBTOOL_MODE_REVERT, +} pubtoolMode; + +pxConfig *pubtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PUBTOOL_H Index: branches/pap/ippTools/src/pubtoolConfig.c =================================================================== --- branches/pap/ippTools/src/pubtoolConfig.c (revision 25027) +++ branches/pap/ippTools/src/pubtoolConfig.c (revision 25027) @@ -0,0 +1,111 @@ +/* + * pubtoolConfig.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pubtool.h" + +pxConfig *pubtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!"); + psFree(config); + return NULL; + } + + // -defineclient + psMetadata *defineclientArgs = psMetadataAlloc(); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "set and search by label", NULL); + + // -pending + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-stage", 0, "search on source", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); + psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); + + // -add + psMetadata *addArgs = psMetadataAlloc(); + psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0); + psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); + psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + + // -revert + psMetadata *revertArgs = psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-pub_id", 0, "search on pub_id", 0); + psMetadataAddS32(revertArgs, PS_LIST_TAIL, "-fault", 0, "search on fault code", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-defineclient", "", PUBTOOL_MODE_DEFINECLIENT, defineclientArgs); + PXOPT_ADD_MODE("-definerun", "", PUBTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-pending", "", PUBTOOL_MODE_PENDING, pendingArgs); + PXOPT_ADD_MODE("-add", "", PUBTOOL_MODE_ADD, addArgs); + PXOPT_ADD_MODE("-revert", "", PUBTOOL_MODE_REVERT, revertArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define database handle, if used + // do this last so we don't setup a connection before CLI options are validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/pap/ippTools/src/pxcam.c =================================================================== --- branches/pap/ippTools/src/pxcam.c (revision 23948) +++ branches/pap/ippTools/src/pxcam.c (revision 25027) @@ -45,22 +45,22 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); @@ -68,6 +68,6 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); return true; @@ -76,48 +76,48 @@ bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where) { - PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); - PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); - PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); - PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); - PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); - PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); - PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); - PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); - PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); - PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); - PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); - - return true; -} - -bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state) + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + + return true; +} + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state, psS64 magicked) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -131,6 +131,6 @@ } - char *query = "UPDATE camRun SET state = '%s' WHERE cam_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, cam_id)) { + char *query = "UPDATE camRun SET state = '%s', magicked = %" PRId64 " WHERE cam_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, cam_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for cam_id %" PRId64, cam_id); Index: branches/pap/ippTools/src/pxcam.h =================================================================== --- branches/pap/ippTools/src/pxcam.h (revision 23948) +++ branches/pap/ippTools/src/pxcam.h (revision 25027) @@ -25,5 +25,5 @@ #include "pxtools.h" -bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state); +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state, psS64 magicked); bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label); Index: branches/pap/ippTools/src/pxchip.c =================================================================== --- branches/pap/ippTools/src/pxchip.c (revision 23948) +++ branches/pap/ippTools/src/pxchip.c (revision 25027) @@ -47,29 +47,29 @@ psMetadataAddF32(md, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); - psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "search by max solar angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); @@ -99,8 +99,8 @@ PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); @@ -122,6 +122,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); @@ -175,5 +175,5 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); - if (where) { + if (where && psListLength(where->list) > 0) { psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(&query, " %s", whereClause); @@ -216,5 +216,5 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); - if (where) { + if (where && psListLength(where->list) > 0) { psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(&query, " %s", whereClause); @@ -264,5 +264,5 @@ dvodb, tess_id, - end_stage, + end_stage, 0 // magicked ) @@ -277,5 +277,5 @@ psS64 chip_id = psDBLastInsertID(config->dbh); - // Create rows in chipImfile table for each input exposure. + // Create rows in chipImfile table for each input exposure. // This creates the chip_image_id values psString query = "INSERT INTO chipImfile " @@ -295,5 +295,5 @@ return 0; } - + return chip_id; } Index: branches/pap/ippTools/src/pxtools.h =================================================================== --- branches/pap/ippTools/src/pxtools.h (revision 23948) +++ branches/pap/ippTools/src/pxtools.h (revision 25027) @@ -231,4 +231,22 @@ } +// convert the supplied value from degrees (external) to radians (internal) +#define PXOPT_COPY_RADEC(from, to, oldname, newname, comment) \ +{ \ + bool status = false; \ + psF64 var = psMetadataLookupF64(&status, from, oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \ + return false; \ + } \ + if (!isnan(var)) { \ + if (!psMetadataAddF64(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, PS_RAD_DEG*var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + #define PXOPT_COPY_F32(from, to, oldname, newname, comment) \ PXOPT_COPY_F(from, to, F32, oldname, newname, comment) Index: branches/pap/ippTools/src/pxwarp.c =================================================================== --- branches/pap/ippTools/src/pxwarp.c (revision 23948) +++ branches/pap/ippTools/src/pxwarp.c (revision 25027) @@ -126,10 +126,11 @@ bool pxwarpQueueByFakeID(pxConfig *config, - psS64 fake_id, - char *workdir, - char *label, - char *dvodb, - char *tess_id, - char *end_stage) + psS64 fake_id, + const char *workdir, + const char *label, + const char *dvodb, + const char *tess_id, + const char *reduction, + const char *end_stage) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -146,4 +147,5 @@ dvodb, tess_id, + reduction, end_stage, NULL, // registered Index: branches/pap/ippTools/src/pxwarp.h =================================================================== --- branches/pap/ippTools/src/pxwarp.h (revision 23948) +++ branches/pap/ippTools/src/pxwarp.h (revision 25027) @@ -31,10 +31,11 @@ bool pxwarpQueueByFakeID(pxConfig *config, - psS64 fake_id, - char *workdir, - char *label, - char *dvodb, - char *tess_id, - char *end_stage); + psS64 fake_id, + const char *workdir, + const char *label, + const char *dvodb, + const char *tess_id, + const char *reduction, + const char *end_stage); #endif // PXWARP_H Index: branches/pap/ippTools/src/receivetool.c =================================================================== --- branches/pap/ippTools/src/receivetool.c (revision 23948) +++ branches/pap/ippTools/src/receivetool.c (revision 25027) @@ -1,4 +1,4 @@ /* - * disttool.c + * receivetool.c * * Copyright (C) 2008 @@ -36,7 +36,9 @@ static bool updatelastMode(pxConfig *config); static bool pendingfilesetMode(pxConfig *config); +static bool updatefilesetMode(pxConfig *config); static bool addfileMode(pxConfig *config); static bool pendingfileMode(pxConfig *config); static bool addresultMode(pxConfig *config); +static bool toadvanceMode(pxConfig *config); static bool revertMode(pxConfig *config); @@ -65,4 +67,6 @@ MODECASE(RECEIVETOOL_MODE_UPDATELAST, updatelastMode); MODECASE(RECEIVETOOL_MODE_PENDINGFILESET, pendingfilesetMode); + MODECASE(RECEIVETOOL_MODE_UPDATEFILESET, updatefilesetMode); + MODECASE(RECEIVETOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(RECEIVETOOL_MODE_ADDFILE, addfileMode); MODECASE(RECEIVETOOL_MODE_PENDINGFILE, pendingfileMode); @@ -102,6 +106,9 @@ PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); PXOPT_LOOKUP_STR(last, config->args, "-last", false, false); - - if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last)) { + PXOPT_LOOKUP_STR(status_product, config->args, "-status_product", false, false); + PXOPT_LOOKUP_STR(ds_dbname, config->args, "-ds_dbname", false, false); + PXOPT_LOOKUP_STR(ds_dbhost, config->args, "-ds_dbhost", false, false); + + if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last, status_product, ds_dbname, ds_dbhost)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -146,5 +153,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "Database error"); psFree(query); @@ -231,5 +238,5 @@ psFree(output); - if (!receiveFilesetInsert(config->dbh, 0, source_id, fileset)) { + if (!receiveFilesetInsert(config->dbh, 0, source_id, fileset, "reg", NULL, NULL, 0)) { psError(PS_ERR_UNKNOWN, false, "Unable to add fileset"); psFree(source_id_str); @@ -266,5 +273,5 @@ fileset, source_id); - if (!p_psDBRunQueryF(config->dbh, query)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "Database error"); psFree(query); @@ -306,4 +313,8 @@ psFree(where); + // make sure that the data is processed in the order that the filesets + // were posted + psStringAppend(&query, "\nORDER BY fileset_id"); + if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -312,5 +323,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "Database error"); psFree(query); @@ -345,9 +356,13 @@ // required PXOPT_LOOKUP_S64(fileset_id, config->args, "-fileset_id", true, false); - psMetadataItem *files = psMetadataLookup(config->args, "-file"); + PXOPT_LOOKUP_STR(file_list, config->args, "-file_list", true, false); + + unsigned int numBad; + psMetadata *files = psMetadataConfigRead(NULL, &numBad, file_list, false); if (!files) { - psError(PS_ERR_UNKNOWN, true, "-file is required"); - return false; - } + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to cleanly read MDC file with file list."); + return false; + } + if (!psDBTransaction(config->dbh)) { @@ -356,12 +371,26 @@ } - psListIterator *iter = psListIteratorAlloc(files->data.list, PS_LIST_HEAD, false); // Iterator - psMetadataItem *item = NULL; // Item from iteration - while ((item = psListGetAndIncrement(iter))) { - psAssert(item && item->data.V && item->type == PS_DATA_STRING, "Argument is bad"); - - if (!receiveFileInsert(config->dbh, 0, fileset_id, item->data.str)) { + psMetadataIterator *iter = psMetadataIteratorAlloc(files, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; + while ((item = psMetadataGetAndIncrement(iter))) { + psMetadata *md = item->data.md; + psString file = psMetadataLookupStr(NULL, md, "file"); + psS64 bytes = psMetadataLookupS64(NULL, md, "bytes"); + psString md5sum = psMetadataLookupStr(NULL, md, "md5sum"); + psString file_type = psMetadataLookupStr(NULL, md, "file_type"); + psString component = psMetadataLookupStr(NULL, md, "component"); + + if (!file) { + psError(PS_ERR_UNKNOWN, false, "failed to find value for file"); + return false; + } + if (!component) { + psError(PS_ERR_UNKNOWN, false, "failed to find value for component"); + return false; + } + + if (!receiveFileInsert(config->dbh, 0, fileset_id, file, bytes, md5sum, file_type, component)) { psError(PS_ERR_UNKNOWN, false, "Unable to add file"); - if (!psDBTransaction(config->dbh)) { + if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -371,4 +400,5 @@ } } + psFree(iter); @@ -391,5 +421,6 @@ PXOPT_COPY_STR(config->args, where, "-product", "receiveSource.product", "=="); PXOPT_COPY_STR(config->args, where, "-comment", "receiveSource.comment", "LIKE"); - PXOPT_COPY_S64(config->args, where, "-fileset_id", "receiveFile.file_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fileset_id", "receiveFile.fileset_id", "=="); + PXOPT_COPY_S64(config->args, where, "-file_id", "receiveFile.file_id", "=="); // optional @@ -411,4 +442,8 @@ psFree(where); + // make sure that the data is processed in the order that the filesets + // were posted + psStringAppend(&query, "\nORDER BY fileset_id"); + if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -417,5 +452,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "Database error"); psFree(query); @@ -485,10 +520,10 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "Database error"); psFree(query); @@ -499,2 +534,110 @@ return true; } +static bool toadvanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + + // optional + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("receivetool_toadvance.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("receivetool", PS_LOG_INFO, "No rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "toadvanceFilesets", !simple)) { + psError(PS_ERR_UNKNOWN, false, "Failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool updatefilesetMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(fileset_id, config->args, "-fileset_id", true, false); + + // to chanage + PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(destdir, config->args, "-destdir", false, false); + PXOPT_LOOKUP_STR(dirinfo, config->args, "-dirinfo", false, false); + PXOPT_LOOKUP_STR(dbinfo, config->args, "-dbinfo", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!fault && !dirinfo &&!dbinfo && !state) { + psError(PS_ERR_UNKNOWN, true, "at least one of -fault, -dirinfo, -dbinfo, -set_state are required"); + return false; + } + + psString query = NULL; // Query to execute + psStringAppend(&query, "UPDATE receiveFileset SET "); + + psString sep = ""; + if (fault) { + psStringAppend(&query, "%s fault = %d", sep, fault); + sep = ","; + } + if (dirinfo) { + psStringAppend(&query, "%s dirinfo = '%s'", sep, dirinfo); + sep = ","; + } + if (dbinfo) { + psStringAppend(&query, "%s dbinfo = '%s'", sep, dbinfo); + sep = ","; + } + if (state) { + psStringAppend(&query, "%s state = '%s'", sep, state); + sep = ","; + } + + psStringAppend(&query, " WHERE fileset_id = %" PRId64, fileset_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} Index: branches/pap/ippTools/src/receivetool.h =================================================================== --- branches/pap/ippTools/src/receivetool.h (revision 23948) +++ branches/pap/ippTools/src/receivetool.h (revision 25027) @@ -30,4 +30,6 @@ RECEIVETOOL_MODE_UPDATELAST, RECEIVETOOL_MODE_PENDINGFILESET, + RECEIVETOOL_MODE_UPDATEFILESET, + RECEIVETOOL_MODE_TOADVANCE, RECEIVETOOL_MODE_ADDFILE, RECEIVETOOL_MODE_PENDINGFILE, Index: branches/pap/ippTools/src/receivetoolConfig.c =================================================================== --- branches/pap/ippTools/src/receivetoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/receivetoolConfig.c (revision 25027) @@ -50,4 +50,7 @@ psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL); + psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-status_product", 0, "define status_product", NULL); + psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-ds_dbname", 0, "define ds_dbname", NULL); + psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-ds_dbhost", 0, "define ds_dbhost", NULL); // -list @@ -74,11 +77,21 @@ psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-product", 0, "search on product", NULL); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); + psMetadataAddBool(pendingfilesetArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); - psMetadataAddBool(pendingfilesetArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); + psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + + // -updatefileset + psMetadata *updatefilesetArgs = psMetadataAlloc(); + psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fileset_id", 0, "define fileset_id (required)", 0); + psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-destdir", 0, "define destdir", NULL); + psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-dirinfo", 0, "define dirinfo", NULL); + psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-dbinfo", 0, "define dbinfo", NULL); + psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); // -addfile psMetadata *addfileArgs = psMetadataAlloc(); psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-fileset_id", 0, "define fileset_id (required)", 0); - psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-file", PS_META_DUPLICATE_OK, "define file (multiple OK, required)", NULL); + psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-file_list", 0, "define file list (required)", NULL); // -pendingfile @@ -88,10 +101,13 @@ psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); psMetadataAddS64(pendingfileArgs, PS_LIST_TAIL, "-fileset_id", 0, "search on fileset_id", 0); + psMetadataAddS64(pendingfileArgs, PS_LIST_TAIL, "-file_id", 0, "search on file_id", 0); psMetadataAddU64(pendingfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); psMetadataAddBool(pendingfileArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); + psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); // -addresult psMetadata *addresultArgs = psMetadataAlloc(); psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-file_id", 0, "define receive_id (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "path_base for component", NULL); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_copy", 0, "define time to copy", NAN); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_extract", 0, "define time to extract", NAN); @@ -107,4 +123,13 @@ psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); + // -toadvance + psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-fileset_id", 0, "search on fileset_id", 0); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-source_id", 0, "search on source_id", 0); + psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); + psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); + // label is not used but pantasks requires it + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -114,5 +139,7 @@ PXOPT_ADD_MODE("-addfileset", "", RECEIVETOOL_MODE_ADDFILESET, addfilesetArgs); PXOPT_ADD_MODE("-updatelast", "", RECEIVETOOL_MODE_UPDATELAST, updatelastArgs); + PXOPT_ADD_MODE("-updatefileset", "", RECEIVETOOL_MODE_UPDATEFILESET, updatefilesetArgs); PXOPT_ADD_MODE("-pendingfileset", "", RECEIVETOOL_MODE_PENDINGFILESET, pendingfilesetArgs); + PXOPT_ADD_MODE("-toadvance", "", RECEIVETOOL_MODE_TOADVANCE, toadvanceArgs); PXOPT_ADD_MODE("-addfile", "", RECEIVETOOL_MODE_ADDFILE, addfileArgs); PXOPT_ADD_MODE("-pendingfile", "", RECEIVETOOL_MODE_PENDINGFILE, pendingfileArgs); Index: branches/pap/ippTools/src/regtool.c =================================================================== --- branches/pap/ippTools/src/regtool.c (revision 23948) +++ branches/pap/ippTools/src/regtool.c (revision 25027) @@ -455,5 +455,5 @@ PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); @@ -887,42 +887,42 @@ // XX test this out; need to make this consistent with the list in regtoolConfig.c psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); - PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); - PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); - PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); - PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_min", "bg", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_max", "bg", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); - PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); - PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); - PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); - PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); - PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); - PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); - PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); Index: branches/pap/ippTools/src/regtoolConfig.c =================================================================== --- branches/pap/ippTools/src/regtoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/regtoolConfig.c (revision 25027) @@ -74,7 +74,10 @@ ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + // Note: RA & DEC are supplied here in radians, but the query options use degrees. This + // can be justified by the fact that this option (addprocessedimfile) is a software + // interface, but the query mechanisms are human user interfaces. + ADD_OPT(F64, addprocessedimfileArgs, "-ra", "define RA (NOTE: radians)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-decl", "define DEC (NOTE: radians)", NAN); ADD_OPT(F32, addprocessedimfileArgs, "-airmass", "define airmass", NAN); - ADD_OPT(F64, addprocessedimfileArgs, "-ra", "define RA", NAN); - ADD_OPT(F64, addprocessedimfileArgs, "-decl", "define DEC", NAN); ADD_OPT(F32, addprocessedimfileArgs, "-exp_time", "define exposure time", NAN); ADD_OPT(F32, addprocessedimfileArgs, "-sat_pixel_frac", "define fraction of saturated pixels", NAN); @@ -135,4 +138,5 @@ ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); + ADD_OPT(Bool, processedimfileArgs, "-ordered_by_date", "force output to be in DATE order", false); // -revertprocessedimfile @@ -150,5 +154,5 @@ ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); ADD_OPT(F64, updateprocessedimfileArgs, "-user_1", "set user stat (1)", NAN); - ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code (required)", INT16_MAX); + ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); // -pendingexp @@ -181,7 +185,10 @@ ADD_OPT(Str, addprocessedexpArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + // Note: RA & DEC are supplied here in radians, but the query options use degrees. This + // can be justified by the fact that this option (addprocessedexp) is a software + // interface, but the query mechanisms are human user interfaces. + ADD_OPT(F64, addprocessedexpArgs, "-ra", "define RA (NOTE: radians)", NAN); + ADD_OPT(F64, addprocessedexpArgs, "-decl", "define DEC (NOTE: radians)", NAN); ADD_OPT(F32, addprocessedexpArgs, "-airmass", "define airmass", NAN); - ADD_OPT(F64, addprocessedexpArgs, "-ra", "define RA", NAN); - ADD_OPT(F64, addprocessedexpArgs, "-decl", "define DEC", NAN); ADD_OPT(F32, addprocessedexpArgs, "-exp_time", "define exposure time", NAN); ADD_OPT(F32, addprocessedexpArgs, "-sat_pixel_frac", "define fraction of saturated pixels", NAN); @@ -244,22 +251,22 @@ psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); - psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); @@ -267,6 +274,6 @@ psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/pap/ippTools/src/stacktool.c =================================================================== --- branches/pap/ippTools/src/stacktool.c (revision 23948) +++ branches/pap/ippTools/src/stacktool.c (revision 25027) @@ -133,10 +133,14 @@ PXOPT_COPY_F32(config->args, where, "-select_posang_min", "rawExp.posang", ">="); PXOPT_COPY_F32(config->args, where, "-select_posang_max", "rawExp.posang", "<="); - PXOPT_COPY_F32(config->args, where, "-select_solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-select_solang_max", "rawExp.solang", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-select_sun_angle_max", "rawExp.sun_angle", "<="); + PXOPT_COPY_F32(config->args, where, "-select_fwhm_major_min", "camProcessedExp.fwhm_major", ">="); + PXOPT_COPY_F32(config->args, where, "-select_fwhm_major_max", "camProcessedExp.fwhm_major", "<="); + PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); + PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-select_label", "warpRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-select_label", "warpRun.label", "LIKE"); // these are used to build the HAVING restriction @@ -149,4 +153,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -232,5 +237,17 @@ } if (!psArrayLength(output)) { - psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psWarning("stacktool: no rows found"); + psFree(output); + psFree(where); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } psFree(output); psFree(where); @@ -358,10 +375,20 @@ psFree(idString); + // replace @FILTER@, @SKYCELL_ID@, @RANDOM_LIMIT@ + psStringSubstitute(&thisInsert, filter, "@FILTER@"); + psStringSubstitute(&thisInsert, skycell_id, "@SKYCELL_ID@"); + + if (randomLimit > 0) { + psString limString = NULL; + psStringAppend(&limString, "%d", randomLimit); + psStringSubstitute(&thisInsert, limString, "@RANDOM_LIMIT@"); + psFree(limString); + } + // XXX this insert uses a select to generate the list of warp_ids for the stack, // we have applied a set of criteria above (WHERE) to select the relevant warps // this insert needs to use exactly the same restrictions (race condition is probably not critical) // the insert below seems to only restrict matches to the skycell, tess, and filter - if ((randomLimit > 0 && !p_psDBRunQueryF(config->dbh, thisInsert, skycell_id, filter, randomLimit)) || - (randomLimit <= 0 && !p_psDBRunQueryF(config->dbh, thisInsert, skycell_id, filter))) { + if (!p_psDBRunQuery(config->dbh, thisInsert)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(thisInsert); @@ -375,4 +402,40 @@ } psFree(thisInsert); + +# if (0) + { + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("stacktool (definebyquery, insert): no rows found"); + psFree(output); + psFree(where); + return true; + } + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } +# endif + } psFree(output); @@ -765,10 +828,12 @@ } - if (!setstackRunState(config, stack_id, "full")) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); - return false; + if (fault == 0) { + if (!setstackRunState(config, stack_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); + return false; + } } @@ -792,4 +857,6 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -866,94 +933,33 @@ PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - return false; - } - - int numStacks; // Number of stacks affected - - // Update state to 'new' - { - psString update = pxDataGet("stacktool_revertsumskyfile_update.sql"); - if (!update) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&update, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, update)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(update); - psFree(where); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(update); - - numStacks = psDBAffectedRows(config->dbh); - - if (numStacks < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; } // Delete product - { - psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); - if (!delete) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&delete, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, delete)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(delete); - psFree(where); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(delete); - - if (psDBAffectedRows(config->dbh) != numStacks) { - psError(PS_ERR_UNKNOWN, true, "Updated and deleted different number of entries!"); - psFree(where); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - } - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); psFree(where); @@ -991,5 +997,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); @@ -1055,5 +1061,5 @@ PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); } - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); Index: branches/pap/ippTools/src/stacktoolConfig.c =================================================================== --- branches/pap/ippTools/src/stacktoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/stacktoolConfig.c (revision 25027) @@ -55,5 +55,5 @@ psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", 0, "search for label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by warpRun label (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); @@ -71,6 +71,10 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_major_min", 0, "define min fwhm (major axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_major_max", 0, "define max fwhm (major axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -78,4 +82,5 @@ psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-min_frac", 0, "minumum fraction of new inputs", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -154,4 +159,6 @@ psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); @@ -164,4 +171,5 @@ psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); // -pendingcleanuprun Index: branches/pap/ippTools/src/warptool.c =================================================================== --- branches/pap/ippTools/src/warptool.c (revision 23948) +++ branches/pap/ippTools/src/warptool.c (revision 25027) @@ -38,7 +38,9 @@ static bool tooverlapMode(pxConfig *config); static bool addoverlapMode(pxConfig *config); +static bool revertoverlapMode(pxConfig *config); static bool scmapMode(pxConfig *config); static bool towarpedMode(pxConfig *config); static bool addwarpedMode(pxConfig *config); +static bool advancerunMode(pxConfig *config); static bool warpedMode(pxConfig *config); static bool revertwarpedMode(pxConfig *config); @@ -85,7 +87,9 @@ MODECASE(WARPTOOL_MODE_TOOVERLAP, tooverlapMode); MODECASE(WARPTOOL_MODE_ADDOVERLAP, addoverlapMode); + MODECASE(WARPTOOL_MODE_REVERTOVERLAP, revertoverlapMode); MODECASE(WARPTOOL_MODE_SCMAP, scmapMode); MODECASE(WARPTOOL_MODE_TOWARPED, towarpedMode); MODECASE(WARPTOOL_MODE_ADDWARPED, addwarpedMode); + MODECASE(WARPTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(WARPTOOL_MODE_WARPED, warpedMode); MODECASE(WARPTOOL_MODE_REVERTWARPED, revertwarpedMode); @@ -135,4 +139,5 @@ PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS) + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // required (no default TESS) PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -155,7 +160,8 @@ dvodb, tess_id, + reduction, end_stage, registered, - false // magicked + 0 // magicked ); if (!warpRun) { @@ -190,46 +196,46 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); - PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); - PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); - PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); - PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); - PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); - PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); - PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); - PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); - PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); - PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); - PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); - PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); - PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); if (!psListLength(where->list) && @@ -245,4 +251,5 @@ PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); @@ -306,13 +313,12 @@ } - if (!tess_id && !row->tess_id) { - psError(PS_ERR_UNKNOWN, false, "cannot queue warp run without a defined tess id: label: %s, fake_id %" PRId64, row->label, row->fake_id); + if (!tess_id && !row->tess_id) { + psError(PS_ERR_UNKNOWN, false, "cannot queue warp run without a defined tess id: label: %s, fake_id %" PRId64, row->label, row->fake_id); psFree(output); return false; - } + } psFree(row); } - psFree(output); // loop over our list of fakeRun rows @@ -329,10 +335,11 @@ // queue the exp if (!pxwarpQueueByFakeID(config, - row->fake_id, - workdir ? workdir : row->workdir, - label ? label : row->label, - dvodb ? dvodb : row->dvodb, - tess_id ? tess_id : row->tess_id, - end_stage ? end_stage : row->end_stage)) + row->fake_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + reduction ? reduction : row->reduction, + end_stage ? end_stage : row->end_stage)) { psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); @@ -649,7 +656,7 @@ warpSkyCellMapInsert(config->dbh, warp_id, - NULL, // skycell_id - NULL, // tess_id - NULL, // class_id + "faulted", // skycell_id + "faulted", // tess_id + "faulted", // class_id fault // fault ); @@ -664,4 +671,61 @@ return true; } + +static bool revertoverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyCellMap.tess_id", "=="); + // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyCellMap.fault", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + int numDeleted; // Number deleted + { + psString query = pxDataGet("warptool_revertoverlap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + numDeleted = psDBAffectedRows(config->dbh); + } + + psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkycellMap", numDeleted); + + psFree(where); + + return true; +} + @@ -917,5 +981,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // default values @@ -959,12 +1023,4 @@ } - if (!warpCompletedRuns(config)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -976,7 +1032,13 @@ } -bool warpCompletedRuns(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +static bool advancerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); psString query = pxDataGet("warptool_finished_run_select.sql"); @@ -985,4 +1047,11 @@ return false; } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -1016,4 +1085,73 @@ return false; } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked"); + psFree(output); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(query); + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + psFree(output); + return false; + } + } + psFree(output); + psFree(query); + + return true; +} + +bool warpCompletedRuns(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("warptool_finished_run_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + query = pxDataGet("warptool_finish_run.sql"); + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + bool status; + psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id"); + psFree(output); + psFree(query); + return false; + } psS32 magicked = psMetadataLookupS64(&status, row, "magicked"); if (!status) { @@ -1051,4 +1189,6 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_skyfile_id", "warpImfile.warp_skyfile_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); @@ -1132,5 +1272,6 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); @@ -1147,53 +1288,4 @@ return false; } - - - // Update state to 'new' - int numUpdated; // Number updated - { - // This query is no longer necessary because we do not set warpRun's to full statte - // if they have faulted skyfiles. - // We do have runs in the DB that follow the old convention so we leave this in for - // now - psString query = pxDataGet("warptool_revertwarped_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numUpdated = psDBAffectedRows(config->dbh); - -#ifdef notdef - // new warpRuns won't get changed (they're already new) so don't require an update - if (numUpdated < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } -#endif - } - - psLogMsg("warptool", PS_LOG_INFO, "Updated %d warp runs", numUpdated); // Delete product @@ -1318,5 +1410,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); psString query = pxDataGet("warptool_pendingcleanuprun.sql"); @@ -1381,5 +1473,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); @@ -1668,5 +1760,5 @@ if (clean) { - bool success = true; + bool success = true; if (!strcmp(tables[i].tableName, "warpRun")) { success = pxSetStateCleaned("warpRun", "state", output); Index: branches/pap/ippTools/src/warptool.h =================================================================== --- branches/pap/ippTools/src/warptool.h (revision 23948) +++ branches/pap/ippTools/src/warptool.h (revision 25027) @@ -34,7 +34,9 @@ WARPTOOL_MODE_TOOVERLAP, WARPTOOL_MODE_ADDOVERLAP, + WARPTOOL_MODE_REVERTOVERLAP, WARPTOOL_MODE_SCMAP, WARPTOOL_MODE_TOWARPED, WARPTOOL_MODE_ADDWARPED, + WARPTOOL_MODE_ADVANCERUN, WARPTOOL_MODE_WARPED, WARPTOOL_MODE_REVERTWARPED, Index: branches/pap/ippTools/src/warptoolConfig.c =================================================================== --- branches/pap/ippTools/src/warptoolConfig.c (revision 23948) +++ branches/pap/ippTools/src/warptoolConfig.c (revision 25027) @@ -66,22 +66,22 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); @@ -89,6 +89,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "search by max solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by fakeRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on fakeRun label", NULL); @@ -98,4 +98,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -111,5 +112,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tess_id", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tess_id (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, "define end stage", NULL); psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -156,4 +158,14 @@ psMetadataAddS64(addoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "set warp ID", 0); psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -revertoverlap + psMetadata *revertoverlapArgs = psMetadataAlloc(); + psMetadataAddS64(revertoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(revertoverlapArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(revertoverlapArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(revertoverlapArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddS16(revertoverlapArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertoverlapArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + // -scmap @@ -196,12 +208,13 @@ // -warped psMetadata *warpedArgs = psMetadataAlloc(); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", 0); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure tag", 0); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_skyfile_id", 0, "search by skyfile ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertwarped @@ -215,7 +228,13 @@ psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction", 0, "search by warpRun reduction class", NULL); - psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label", 0, "search by warpRun label", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertwarpedArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -advancerun + psMetadata *advancerunArgs = psMetadataAlloc(); + psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL); + psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit exposures to advance to N items", 0); // -block @@ -294,7 +313,9 @@ PXOPT_ADD_MODE("-tooverlap", "", WARPTOOL_MODE_TOOVERLAP, tooverlapArgs); PXOPT_ADD_MODE("-addoverlap", "", WARPTOOL_MODE_ADDOVERLAP, addoverlapArgs); + PXOPT_ADD_MODE("-revertoverlap", "", WARPTOOL_MODE_REVERTOVERLAP, revertoverlapArgs); PXOPT_ADD_MODE("-scmap", "", WARPTOOL_MODE_SCMAP, scmapArgs); PXOPT_ADD_MODE("-towarped", "", WARPTOOL_MODE_TOWARPED, towarpedArgs); PXOPT_ADD_MODE("-addwarped", "", WARPTOOL_MODE_ADDWARPED, addwarpedArgs); + PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs);