Index: branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c (revision 30579) @@ -838,4 +838,6 @@ PXOPT_COPY_S16(config->args, where, "-fault", "chipProcessedImfile.fault", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -874,4 +876,11 @@ } psFree(query); + + if (!fault) { + // If fault has not been supplied, don't revert update faults with the magic value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query_update, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query_update)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1919,4 +1928,7 @@ } + // we do not update components with the magic fault value. They are non-updateable + psStringAppend(&query, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/eam_branches/ipp-20101205/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/difftool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/difftool.c (revision 30579) @@ -917,4 +917,6 @@ PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); @@ -950,4 +952,10 @@ psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + if (!fault) { + // If fault has not been supplied, don't revert update faults with the magic value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query_updated, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query_updated)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -3201,4 +3209,7 @@ psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); } + // we do not update components with the magic fault value. They are non-updateable + // (But can be recovered with "difftool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT) + psStringAppend(&query, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { Index: branches/eam_branches/ipp-20101205/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/disttool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/disttool.c (revision 30579) @@ -1185,5 +1185,5 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "rcDSFileset.fault", "=="); Index: branches/eam_branches/ipp-20101205/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/disttoolConfig.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/disttoolConfig.c (revision 30579) @@ -197,5 +197,5 @@ psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0); psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); - psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddS16(revertfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); Index: branches/eam_branches/ipp-20101205/ippTools/src/dqstatstool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/dqstatstool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/dqstatstool.c (revision 30579) @@ -92,5 +92,5 @@ } else { - psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); + psStringAppend(&query, " AND (warp_id IS NOT NULL AND invalid IS NULL) "); } @@ -402,7 +402,17 @@ if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& - isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& - isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { + (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"STRICT") == 0)&& + (isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM")))) { psArrayAdd(decisionRules,decisionRules->n,enginedef); + } + else if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& + (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"CDF") == 0)&& + (isfinite(psMetadataLookupF32(&status,enginedef,"CDF00"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"CDF50"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"CDF100")))) { + psArrayAdd(decisionRules,decisionRules->n, enginedef); + } + else if (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"UNUSED") == 0) { } else { @@ -637,4 +647,5 @@ // Calculate if this exposure was "good" or not. bool accept = true; + psF32 quality = 1.0; for (long j = 0; j < decisionRules->n; j++) { psMetadata *rule = decisionRules->data[j]; @@ -644,29 +655,10 @@ psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); return(false); - } - psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); - if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); - return(false); - } - psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); - if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); - return(false); - } - psString filter = psMetadataLookupStr(&status,rule,"FILTER"); - if (!status) { - status = true; - } - if (filter) { - psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); - if (strcmp(filter,imfilter) != 0) { - continue; - } } // Not happy with this being set to a F32. Can this ever be something else? psF32 value = psMetadataLookupF32(&status,tableRow,colname); if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f) in compare with rule.", + colname,value); psFree(colNames); psFree(colTypes); @@ -677,7 +669,92 @@ return(false); } - if ((value > max_value)||(value < min_value)) { - accept = false; - } + psString filter = psMetadataLookupStr(&status,rule,"FILTER"); + if (!status) { + status = true; + } + if (filter) { + psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); + if (strcmp(filter,imfilter) != 0) { + continue; + } + } + + if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"STRICT") == 0) { + psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); + return(false); + } + psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); + return(false); + } + + if ((value > max_value)||(value < min_value)) { + accept = false; + } + } // End STRICT min/max rule parsing + else if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"CDF") == 0) { + // I'm just going to implement a quick linear interpolation over the points that exist. + psF32 min_value,max_value = 99; + psF32 min_cdf = 0.0,max_cdf = 1.0; + psF32 cdf; + psF32 try[22] = {0.00,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.45,0.50, + 0.55,0.60,0.65,0.70,0.75,0.80,0.85,0.90,0.95,0.99,1.00}; + int i; + int done = 0; + status = true; + min_value = psMetadataLookupF32(&status,rule,"CDF00"); + min_cdf = 0.0; + if (!isfinite(value)) { + quality = 0.0; + continue; + } + if (!status) { + status = true; + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find CDF00 value for %s",colname); + } + if (value < min_value) { + if (quality > min_cdf) { + quality = 0.0; + continue; + } + } + + for (i = 1; i < 22; i++) { + psString label = psStringCopy(""); + psStringAppend(&label,"CDF%02d",(int) (100 * try[i])); + + max_value = psMetadataLookupF32(&status,rule,label); + max_cdf = try[i]; + if (!status) { + status = true; + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find %s value for %s",label,colname); + } + if ((value < max_value)&&(value > min_value)) { + cdf = ((value - min_value) * max_cdf + (max_value - value) * min_cdf) / (max_value - min_value); + // fprintf(stderr,"value: %f %f %f %f %f %f %d %d %f\n",value,quality,min_cdf,min_value,max_cdf,max_value, value < max_value, value > min_value,cdf); + if (quality > cdf) { + quality = cdf; + done = 1; + i = 25; + } + } + min_value = max_value; + min_cdf = max_cdf; + } + if (done) { + continue; + } + // Reached end of array without finding the value; + if (value > min_value) { + if (quality > min_cdf) { + quality = 1.0; + } + } + + + } // End CDF rule parsing } bool success; @@ -695,4 +772,17 @@ return(false); } + success = psMetadataAddF32(tableRow,PS_LIST_TAIL,"QUALITY",0,"",quality); + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + return(false); + } // Add the row to the table array. Index: branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c (revision 30579) @@ -1112,6 +1112,8 @@ PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); psString queryFile = NULL; + bool stateIsUpdate = false; if (state) { if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) { @@ -1119,4 +1121,5 @@ } else if (!strcmp(state, "update")) { queryFile = "magicdstool_revertupdated.sql"; + stateIsUpdate = true; } else { psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state); @@ -1141,4 +1144,12 @@ } psFree(where); + + if (stateIsUpdate && !fault) { + // If fault has not been supplied, don't revert update faults with + // the magic "do not update" value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query, " AND magicDSFile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -1744,4 +1755,8 @@ } + // we do not update components with the magic fault value. They are non-updateable + // (But can be recovered with "magicdstool -revertdestreakedfile -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT) + psStringAppend(&query, " AND (magicDSFile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/eam_branches/ipp-20101205/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/magictool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/magictool.c (revision 30579) @@ -627,5 +627,5 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0"); Index: branches/eam_branches/ipp-20101205/ippTools/src/magictoolConfig.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/magictoolConfig.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/magictoolConfig.c (revision 30579) @@ -114,5 +114,5 @@ psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -inputs Index: branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c (revision 30579) @@ -571,4 +571,6 @@ PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-reqType", "reqType", "=="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">="); PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<="); @@ -586,5 +588,5 @@ psStringAppend(&query, ", state = '%s'", state); if (!strcmp(state, "goto_cleaned")) { - psStringAppend(&stateCheck, " AND state != 'cleaned'"); + psStringAppend(&stateCheck, " AND (state != 'cleaned' AND state != 'goto_cleaned')"); } } Index: branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c (revision 30579) @@ -104,4 +104,6 @@ psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "search by reqType", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "search by reqType (LIKE comparsion)", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL); Index: branches/eam_branches/ipp-20101205/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/pxtools.h (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/pxtools.h (revision 30579) @@ -51,4 +51,7 @@ # define MAX_ROWS 10e9 # define PXTOOL_MODE_NONE 0x0 +// we do not revert or update components with this fault value +// This is the same value as PSTAMP_GONE +# define PXTOOL_DO_NOT_REVERT_FAULT 26 bool pxIsValidState(const char *state); Index: branches/eam_branches/ipp-20101205/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/ipp-20101205/ippTools/src/warptool.c (revision 30530) +++ branches/eam_branches/ipp-20101205/ippTools/src/warptool.c (revision 30579) @@ -1485,4 +1485,6 @@ PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -1510,4 +1512,11 @@ } psFree(where); + + if (!fault) { + // If fault has not been supplied, don't revert update faults with the magic value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query_updated, " AND warpSkyfile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -2003,5 +2012,5 @@ // shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile -static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) +static bool change_skyfile_data_state(pxConfig *config, psString data_state) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -2065,17 +2074,17 @@ static bool tocleanedskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "cleaned", "goto_cleaned"); + return change_skyfile_data_state(config, "cleaned"); } static bool tofullskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "full", "update"); + return change_skyfile_data_state(config, "full"); } static bool topurgedskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "purged", "goto_purged"); + return change_skyfile_data_state(config, "purged"); } static bool toscrubbedskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); + return change_skyfile_data_state(config, "scrubbed"); } @@ -2101,11 +2110,11 @@ else { if (strcmp(state,"error_cleaned") == 0) { - change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + change_skyfile_data_state(config,"error_cleaned"); } else if (strcmp(state, "error_scrubbed") == 0) { - change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + change_skyfile_data_state(config,"error_scrubbed"); } else if (strcmp(state, "error_purged") == 0) { - change_skyfile_data_state(config,"error_purged","goto_purged"); + change_skyfile_data_state(config,"error_purged"); } else { @@ -2526,4 +2535,7 @@ psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); } + // we do not update components with the magic fault value. They are non-updateable + // (But can be recovered with "warptool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT) + psStringAppend(&query, " AND (warpSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) {