Index: branches/eam_branches/ipp-20110213/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/addtool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/addtool.c (revision 31083) @@ -684,5 +684,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); @@ -1179,4 +1179,7 @@ PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -1230,5 +1233,7 @@ dtime_resort, dtime_relphot, + dtime_verify, dtime_merge, + dtime_script, epoch, mergedvodb_path, @@ -1293,5 +1298,5 @@ psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); - printf("%s", query4); + //printf("%s", query4); if (!p_psDBRunQuery(config->dbh, query4)) { // rollback @@ -1511,5 +1516,6 @@ PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); - +PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); +PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); @@ -1562,4 +1568,18 @@ cnt++; } +if (dtime_verify) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_verify = %f", dtime_verify); + cnt++; + } +if (dtime_script) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_script = %f", dtime_script); + cnt++; + } psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -1567,5 +1587,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error \n%s\n",query ); psFree(query); return false; Index: branches/eam_branches/ipp-20110213/ippTools/src/addtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/addtoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/addtoolConfig.c (revision 31083) @@ -195,5 +195,7 @@ psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); @@ -220,6 +222,7 @@ psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); - + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); Index: branches/eam_branches/ipp-20110213/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/camtool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/camtool.c (revision 31083) @@ -983,4 +983,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); psMetadata *where = psMetadataAlloc(); @@ -990,5 +991,5 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault)) { + if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault, quality)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); Index: branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c (revision 31083) @@ -244,4 +244,5 @@ psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", INT16_MAX); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); // -block Index: branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c (revision 31083) @@ -963,6 +963,7 @@ if (!state) { PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault, quality)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); return false; Index: branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c (revision 31083) @@ -235,5 +235,5 @@ psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); - + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); // -processedExp psMetadata *listrunArgs = psMetadataAlloc(); Index: branches/eam_branches/ipp-20110213/ippTools/src/detselect.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/detselect.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/detselect.c (revision 31083) @@ -152,5 +152,5 @@ // unless explicitly specified by the user, list all possible matches if (!unlimit) { - psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); + psStringAppend(&query, " ORDER BY registered DESC, iteration DESC LIMIT 1"); } Index: branches/eam_branches/ipp-20110213/ippTools/src/dettool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/dettool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/dettool.c (revision 31083) @@ -1314,21 +1314,60 @@ PS_ASSERT_PTR_NON_NULL(config, false); + bool updating = false; + long rows = 0; + psMetadata *where = psMetadataAlloc(); PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + + psMetadata *values = psMetadataAlloc(); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-set_time_begin", false, false); + if (time_begin) { + updating = true; + PXOPT_COPY_TIME(config->args, values, "-set_time_begin", "time_begin", "=="); + } + PXOPT_LOOKUP_TIME(time_end, config->args, "-set_time_end", false, false); + if (time_end) { + updating = true; + PXOPT_COPY_TIME(config->args, values, "-set_time_end", "time_end", "=="); + } + PXOPT_LOOKUP_TIME(use_begin, config->args, "-set_use_begin", false, false); + if (use_begin) { + updating = true; + PXOPT_COPY_TIME(config->args, values, "-set_use_begin", "use_begin", "=="); + } + PXOPT_LOOKUP_TIME(use_end, config->args, "-set_use_end", false, false); + if (use_end) { + updating = true; + PXOPT_COPY_TIME(config->args, values, "-set_use_end", "use_end", "=="); + } + if (state) { + updating = true; + if (!isValidDetRunState (state)) return false; + PXOPT_COPY_STR(config->args, values, "-state", "state", "=="); + } + + // either -rerun or -state must be specified - if (!(again || state)) { - psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); - return false; - } - if (again && state) { - psError(PS_ERR_UNKNOWN, true, "-accept and -reject are exclusive"); - return false; - } - - if (state) { - // set detRun.state to state - return setDetRunState(config, det_id, state); + if (!(again || updating)) { + psError(PS_ERR_UNKNOWN, true, "either -again or update parameters must be specified"); + return false; + } + if (again && updating) { + psError(PS_ERR_UNKNOWN, true, "-again and update parameters are exclusive"); + return false; + } + + if (updating) { + rows = psDBUpdateRows(config->dbh,"detRun",where,values); + if (rows) { + return(true); + } + else { + return(false); + } } Index: branches/eam_branches/ipp-20110213/ippTools/src/dettoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/dettoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/dettoolConfig.c (revision 31083) @@ -848,4 +848,8 @@ psMetadataAddBool(updatedetrunArgs, PS_LIST_TAIL, "-again", 0, "start a new iteration of this detrend run", false); psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-state", 0, "set the state of this detrend run", false); + psMetadataAddTime(updatedetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, "start of period to apply detrend too", NULL); + psMetadataAddTime(updatedetrunArgs, PS_LIST_TAIL, "-set_time_end", 0, "end of period to apply detrend too", NULL); + psMetadataAddTime(updatedetrunArgs, PS_LIST_TAIL, "-set_use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(updatedetrunArgs, PS_LIST_TAIL, "-set_use_end", 0, "end of detrend run applicable period", NULL); // -rerun Index: branches/eam_branches/ipp-20110213/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/difftool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/difftool.c (revision 31083) @@ -2681,4 +2681,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); psMetadata *where = psMetadataAlloc(); @@ -2686,5 +2687,5 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) { + if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); Index: branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c (revision 31083) @@ -411,6 +411,7 @@ psMetadata *updatediffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); - psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); // -tocleanedskyfile Index: branches/eam_branches/ipp-20110213/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/disttool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/disttool.c (revision 31083) @@ -141,4 +141,5 @@ PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false); PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); @@ -150,4 +151,24 @@ PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + // select arguments + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-target_id", "distTarget.target_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "==");; + PXOPT_COPY_STR(config->args, where, "-dist_group", "distTarget.dist_group", "==");; + + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + + // queuing no_magic runs can be quite dangerous. Require confirmation + if (no_magic && !pretend && !i_am_sure) { + psError(PXTOOLS_ERR_SYS, true, "-i_am_sure required to queue no_magic distribution runs"); + return false; + } + if (use_alternate) { if (strcmp(stage, "raw")) { @@ -161,13 +182,15 @@ } - // select arguments - psMetadata *where = psMetadataAlloc(); - - PXOPT_COPY_S64(config->args, where, "-target_id", "distTarget.target_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + // We want to possible to distribute unmagicked raw exposures for the purpose + // of distributing detrend inputs + // To make it less likely that unmagicked raw stage exposures from being distributed + // accidentally we require that dist_group (to select target and destination with + // an interest) + // and exp_id to be supplied. We could add -exp_type and dateobs cuts to make it easier + if (!strcmp(stage, "raw") && no_magic && !(exp_id && dist_group && exp_type)) { + psError(PXTOOLS_ERR_SYS, true, "exp_id, exp_type, and dist_group are required for raw stage if no_magic"); + return false; + } + psString query = NULL; @@ -182,5 +205,9 @@ runJoinStr = "camRun.exp_id"; } - query = pxDataGet("disttool_definebyquery_raw.sql"); + if (!no_magic) { + query = pxDataGet("disttool_definebyquery_raw.sql"); + } else { + query = pxDataGet("disttool_definebyquery_raw_no_magic.sql"); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -189,10 +216,13 @@ } - // for raw stage we select by camRun.label and dist_group because rawExp doesn't have those columns - if (label) { - psStringAppend(&query, " AND (magicDSRun.label = '%s')", label); - } - if (dist_group) { - psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); + if (!no_magic) { + if (label) { + psStringAppend(&query, " AND (magicDSRun.label = '%s')", label); + } + // for raw stage we select by camRun.label and dist_group because rawExp + // doesn't have those columns + if (dist_group) { + psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); + } } } else if (!strcmp(stage, "chip")) { Index: branches/eam_branches/ipp-20110213/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/disttoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/disttoolConfig.c (revision 31083) @@ -49,4 +49,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirmation needed when -no_magic is supplied", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); @@ -57,7 +58,8 @@ // select args - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "select by target_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exp_id", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "select by exp_type", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "select by chip_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by run label", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group", 0, "select by dist_group", NULL); Index: branches/eam_branches/ipp-20110213/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/faketool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/faketool.c (revision 31083) @@ -706,5 +706,5 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, fault)) { + if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree(where); Index: branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c (revision 31083) @@ -602,5 +602,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); Index: branches/eam_branches/ipp-20110213/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/magicdstoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/magicdstoolConfig.c (revision 31083) @@ -106,5 +106,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define label (LIKE) comparison", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "data_group (LIKE) comparision", NULL); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-noreplace", 0, "only update runs with replace not set", false); Index: branches/eam_branches/ipp-20110213/ippTools/src/pxfault.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pxfault.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pxfault.c (revision 31083) @@ -26,5 +26,5 @@ #include "pxtools.h" -bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code) +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code, psS16 quality) { PS_ASSERT_PTR_NON_NULL(dbh, false); @@ -48,4 +48,11 @@ return false; } + if (quality) { + if (!psMetadataAddS16(values, PS_LIST_HEAD, "quality", 0, NULL, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to add metadata item quality"); + psFree(values); + return false; + } + } long rowsAffected = psDBUpdateRows(dbh, tableName, where, values); Index: branches/eam_branches/ipp-20110213/ippTools/src/pxinject.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pxinject.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pxinject.c (revision 31083) @@ -92,4 +92,5 @@ if (!newExpInsert(config->dbh, 0, // exp_id + 0, // summit_id tmp_exp_name, tmp_camera, Index: branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h (revision 31083) @@ -75,5 +75,5 @@ bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op); -bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code, psS16 quality); bool pxExportVersion(pxConfig *config, FILE *f); bool pxCheckImportVersion(pxConfig *config, psMetadata *md); Index: branches/eam_branches/ipp-20110213/ippTools/src/pzgetexp.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pzgetexp.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pzgetexp.c (revision 31083) @@ -215,5 +215,6 @@ char *query = "INSERT INTO summitExp" - " SElECT" + " SELECT" + " NULL," // summit_id " incoming.*," " NULL," // imfiles Index: branches/eam_branches/ipp-20110213/ippTools/src/pzgetimfiles.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pzgetimfiles.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pzgetimfiles.c (revision 31083) @@ -99,7 +99,31 @@ int status = pclose(output); + // We have an id value now, let's get it and use it + char *id_query = "SELECT summit_id FROM summitExp WHERE exp_name = '%s' AND camera = '%s' AND telescope = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, id_query, filesetid, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *id_output = p_psDBFetchResult(config->dbh); + if (!id_output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (psArrayLength(id_output) != 1) { + psError(PS_ERR_UNKNOWN, false, "database error: incorrect number of results"); + return false; + } + + psMetadata *id_result = id_output->data[0]; + + psS64 summit_id = psMetadataLookupS64(NULL, id_result, "summit_id"); + psFree(id_output); + // End of summit_id block. + if (status != 0) { // mark the summitExp row as faulted - if (!p_psDBRunQueryF(config->dbh, "UPDATE summitExp SET fault = %d WHERE exp_name = '%s' AND camera = '%s' AND telescope = '%s'", WEXITSTATUS(status), filesetid, camera, telescope)) { + if (!p_psDBRunQueryF(config->dbh, "UPDATE summitExp SET fault = %d WHERE summit_id = %ld", WEXITSTATUS(status), summit_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); } @@ -117,5 +141,5 @@ // download an "empty" fileset. // mark the summitExp row as faulted - if (!p_psDBRunQueryF(config->dbh, "UPDATE summitExp SET fault = %d WHERE exp_name = '%s' AND camera = '%s' AND telescope = '%s'", 250, filesetid, camera, telescope)) { + if (!p_psDBRunQueryF(config->dbh, "UPDATE summitExp SET fault = %d WHERE summit_id = %ld", 250, summit_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); } @@ -142,8 +166,6 @@ "UPDATE summitExp" " SET imfiles = %d" - " WHERE exp_name = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, imfiles, filesetid, camera, telescope)) { + " WHERE summit_id = %ld"; + if (!p_psDBRunQueryF(config->dbh, query, imfiles, summit_id)) { // rollback if (!psDBRollback(config->dbh)) { @@ -159,8 +181,6 @@ "DELETE FROM pzDownloadExp" " WHERE" - " exp_name = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, filesetid, camera, telescope)) { + " summit_id = %ld"; + if (!p_psDBRunQueryF(config->dbh, query, summit_id)) { // rollback if (!psDBRollback(config->dbh)) { @@ -245,4 +265,5 @@ "INSERT IGNORE INTO summitImfile" " SELECT" + " %ld," // summit_id " incoming.exp_name," " incoming.camera," @@ -256,5 +277,5 @@ " NULL" // epoch " FROM incoming"; - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query,summit_id)) { // rollback if (!psDBRollback(config->dbh)) { @@ -272,14 +293,10 @@ " SET imfiles = (SELECT COUNT(*) FROM summitImfile" " WHERE" - " exp_name = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'" + " summit_id = %ld" ")" " WHERE" - " exp_name = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'" + " summit_id = %ld" " AND imfiles IS NULL"; - if (!p_psDBRunQueryF(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { + if (!p_psDBRunQueryF(config->dbh, query, summit_id, summit_id)) { // rollback if (!psDBRollback(config->dbh)) { @@ -308,4 +325,5 @@ "INSERT IGNORE INTO pzDownloadExp" " SELECT" + " %ld," // summit_id " incoming.exp_name," " incoming.camera," @@ -319,5 +337,5 @@ " incoming.telescope"; - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query,summit_id)) { // rollback if (!psDBRollback(config->dbh)) { Index: branches/eam_branches/ipp-20110213/ippTools/src/pztool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pztool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pztool.c (revision 31083) @@ -53,5 +53,5 @@ static psArray *pzGetPendingCameras(pxConfig *config); static psArray *pzArrayZip(psArray *arraySet, psS64 limit); -static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state); +static bool pzDownloadExpSetState(pxConfig *config, const psS64 summit_id, const char *state); # define MODECASE(caseName, func) \ @@ -222,4 +222,5 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-summit_id", "summit_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); @@ -237,9 +238,7 @@ " FROM summitExp" " LEFT JOIN pzDownloadExp" - " USING(exp_name, camera, telescope)" + " USING(summit_id)" " WHERE" - " pzDownloadExp.exp_name IS NULL" - " AND pzDownloadExp.camera IS NULL" - " AND pzDownloadExp.telescope IS NULL" + " pzDownloadExp.summit_id IS NULL" " AND summitExp.fault = 0" ); @@ -260,4 +259,5 @@ if (limit) { psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); psFree(limitString); @@ -299,4 +299,5 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-summit_id", "summit_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); @@ -401,4 +402,5 @@ // required + PXOPT_LOOKUP_S64(summit_id, config->args, "-summit_id", true, false); PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); @@ -428,4 +430,5 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-summit_id", "summit_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); @@ -459,4 +462,5 @@ if (!pzDownloadImfileInsert(config->dbh, + summit_id, exp_name, camera, @@ -505,4 +509,5 @@ psMetadata *where = psMetadataAlloc(); +/* PXOPT_COPY_STR(config->args, where, "-summit_id", "summit_id", "=="); */ PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); @@ -560,4 +565,5 @@ // required + PXOPT_LOOKUP_S64(summit_id, config->args, "-summit_id", true, false); PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); PXOPT_LOOKUP_STR(inst, config->args, "-inst", true, false); @@ -579,4 +585,5 @@ if (!newExpInsert(config->dbh, 0x0, // exp_id + summit_id, // summit_id exp_name, // tmp_exp_name inst, // tmp_camera @@ -597,11 +604,11 @@ } - psS64 exp_id = psDBLastInsertID(config->dbh); - - // insert newImfiles - { - char *query = + psS64 exp_id = psDBLastInsertID(config->dbh); + + // insert newImfiles + { + char *query = "INSERT INTO newImfile" - " SElECT" + " SELECT" " %" PRId64 "," // exp_id " pzDownloadImfile.class_id," // tmp_class_id @@ -616,20 +623,20 @@ " AND pzDownloadImfile.telescope = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, exp_id, exp_name, inst, telescope)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // sanity check: we should have inserted at least one row - psU64 affected = psDBAffectedRows(config->dbh); - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, exp_id, exp_name, inst, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have inserted at least one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); + return false; + } } // set pzDownloadExp.state to 'stop' - if (!pzDownloadExpSetState(config, exp_name, inst, telescope, "stop")) { - psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", exp_name, inst, telescope); + if (!pzDownloadExpSetState(config, summit_id, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %ld", summit_id); return false; } @@ -798,5 +805,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - if (!pxSetFaultCode(config->dbh, "pzDownloadImfile", where, fault)) { + if (!pxSetFaultCode(config->dbh, "pzDownloadImfile", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); @@ -887,5 +894,5 @@ -static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state) +static bool pzDownloadExpSetState(pxConfig *config, const psS64 summit_id, const char *state) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -904,7 +911,7 @@ } - char *query = "UPDATE pzDownloadExp SET state = '%s' WHERE exp_name = '%s' and camera = '%s' and telescope = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, state, exp_name, camera, telescope)) { - psError(PS_ERR_UNKNOWN, false, "failed to change state for %s:%s:%s", exp_name, camera, telescope); + char *query = "UPDATE pzDownloadExp SET state = '%s' WHERE summit_id = %ld"; + if (!p_psDBRunQueryF(config->dbh, query, state, summit_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for %ld", summit_id); return false; } @@ -916,11 +923,9 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); - PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); - PXOPT_LOOKUP_STR(telescope,config->args, "-telescope",true, false); + PXOPT_LOOKUP_S64(summit_id, config->args, "-summit_id", true, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state",true, false); - if (!pzDownloadExpSetState(config,exp_name,camera,telescope,state)) { - psError(PS_ERR_UNKNOWN, false, "failed to change state for %s:%s:%s", exp_name, camera, telescope); + if (!pzDownloadExpSetState(config,summit_id, state)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for %ld", summit_id); return false; } Index: branches/eam_branches/ipp-20110213/ippTools/src/pztoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/pztoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/pztoolConfig.c (revision 31083) @@ -63,4 +63,5 @@ // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-summit_id", 0, "define summit_id", 0); psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); @@ -73,4 +74,5 @@ // -pendingimfile psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-summit_id", 0, "define summit_id", 0); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); @@ -85,4 +87,5 @@ // -copydone psMetadata *copydoneArgs = psMetadataAlloc(); + psMetadataAddS64(copydoneArgs, PS_LIST_TAIL, "-summit_id", 0, "define summit_id (required)", 0); psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); @@ -138,4 +141,5 @@ // -toadvance psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-summit_id", 0, "define summit_id", 0); psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); @@ -147,4 +151,5 @@ // -advance psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddS64(advanceArgs, PS_LIST_TAIL, "-summit_id", 0, "define summit_id", 0); psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); @@ -158,4 +163,5 @@ // -updatepzexp psMetadata *updatepzexpArgs = psMetadataAlloc(); + psMetadataAddS64(updatepzexpArgs, PS_LIST_TAIL, "-summit_id", 0, "define summit_id", 0); psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name (required)", NULL); psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-inst", 0, "search by camera (required)", NULL); Index: branches/eam_branches/ipp-20110213/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/regtool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/regtool.c (revision 31083) @@ -311,7 +311,7 @@ // required - PXOPT_LOOKUP_STR(dateobs_begin, config->args, "-dateobs_begin", true, false); - PXOPT_LOOKUP_STR(dateobs_end, config->args, "-dateobs_end", true, false); - PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + PXOPT_LOOKUP_STR(dateobs_begin, config->args, "-dateobs_begin", true, false); + PXOPT_LOOKUP_STR(dateobs_end, config->args, "-dateobs_end", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); // optional PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -894,5 +894,5 @@ if (fault != INT16_MAX) { // this is fairly dangerous : can set all if the where is not set... - if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault)) { + if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); @@ -1361,12 +1361,15 @@ // set the state for the newExp to stop - if (!pxnewExpSetState(config, exp_id, "stop")) { + // but only if we didn't encounter a fault + if (fault == 0) { + if (!pxnewExpSetState(config, exp_id, "stop")) { // rollback if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error"); } psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %"PRId64, exp_id); - psFree(newExp); - return false; + psFree(newExp); + return false; + } } @@ -1608,5 +1611,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { + if (!pxSetFaultCode(config->dbh, "rawExp", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree(where); Index: branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c (revision 31083) @@ -968,4 +968,5 @@ PXOPT_LOOKUP_F32(dtime_reject, config->args, "-dtime_reject", false, false); PXOPT_LOOKUP_F32(dtime_final, config->args, "-dtime_final", false, false); + PXOPT_LOOKUP_F32(dtime_convolve, config->args, "-dtime_convolve", false, false); PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); @@ -1031,4 +1032,5 @@ dtime_match_mean, dtime_match_stdev, + dtime_convolve, dtime_initial, dtime_reject, @@ -1660,9 +1662,10 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); - if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault)) { + if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault, quality)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); Index: branches/eam_branches/ipp-20110213/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/stacktoolConfig.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/stacktoolConfig.c (revision 31083) @@ -157,4 +157,5 @@ psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_reject", 0, "define rejection time", NAN); psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_final", 0, "define final stack time", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_convolve", 0, "define image convolution time", NAN); psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); @@ -266,4 +267,5 @@ psMetadataAddS64(updatesumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); // -exportrun Index: branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c =================================================================== --- branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c (revision 30812) +++ branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c (revision 31083) @@ -716,5 +716,5 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); - if (!pxSetFaultCode(config->dbh, "staticskyResult", where, fault)) { + if (!pxSetFaultCode(config->dbh, "staticskyResult", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where);