Index: trunk/ippTools/share/dqstatstool_definebyquery.sql =================================================================== --- trunk/ippTools/share/dqstatstool_definebyquery.sql (revision 27301) +++ trunk/ippTools/share/dqstatstool_definebyquery.sql (revision 27304) @@ -1,5 +1,5 @@ -- query to identify unique data to bundle -SELECT exp_id,chip_id,cam_id,warp_id,invalid from rawExp +SELECT DISTINCT exp_id,chip_id,cam_id,warp_id,invalid from rawExp JOIN chipRun USING(exp_id) JOIN camRun USING(chip_id) @@ -7,3 +7,3 @@ LEFT OUTER JOIN warpRun USING(fake_id) LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id,warp_id) -WHERE (invalid IS NULL OR invalid = 1) +WHERE camRun.state = 'full' Index: trunk/ippTools/share/dqstatstool_get_run.sql =================================================================== --- trunk/ippTools/share/dqstatstool_get_run.sql (revision 27301) +++ trunk/ippTools/share/dqstatstool_get_run.sql (revision 27304) @@ -1,2 +1,4 @@ -SELECT * from dqstatsRun - WHERE state = 'new' AND (fault = 0) +SELECT DISTINCT dqstatsRun.*,rawExp.camera from dqstatsRun + JOIN dqstatsContent USING(dqstats_id) + JOIN rawExp USING(exp_id) + WHERE dqstatsRun.state = 'new' AND (dqstatsRun.fault = 0) Index: trunk/ippTools/src/dqstatstool.c =================================================================== --- trunk/ippTools/src/dqstatstool.c (revision 27301) +++ trunk/ippTools/src/dqstatstool.c (revision 27304) @@ -45,5 +45,5 @@ } - psFree(config); + //psFree(config); pmConfigDone(); psLibFinalize(); @@ -54,5 +54,5 @@ int exit_status = pxerrorGetExitStatus(); - psFree(config); + // psFree(config); pmConfigDone(); psLibFinalize(); @@ -88,11 +88,15 @@ } // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (force) { + psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1 OR invalid = 0) "); + } + else { + psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); + } + if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); - } - if (force) { - psStringAppend(&query, " OR invalid = 0"); } @@ -103,5 +107,5 @@ psFree(limitString); } - + psTrace("debug",3,">> %s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -325,5 +329,5 @@ psFree(output); // This held the information about the Run. - // Read column list from config file. + // Read column list from config file psMetadata *recipe = psMetadataLookupPtr(&status, config->modules->recipes, "DQSTATS"); psMetadata *columns = psMetadataLookupMetadata(&status,recipe,"STATS.COLUMNS"); @@ -342,4 +346,10 @@ psMetadata *colTypes = psMetadataAlloc(); while ((colItem = psMetadataGetAndIncrement(iter))) { + if (strcmp(colItem->name, "COLUMN")) continue; + if (colItem->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool COLUMN not of type METADATA\n"); + return(NULL); + } + bool status; psMetadata *coldef = colItem->data.md; @@ -369,4 +379,44 @@ return(false); } + + // Read decision engine settings from config file. + psMetadata *engine = psMetadataLookupMetadata(&status,recipe,"DECISION.ENGINE"); + if (!engine) { + psError(PS_ERR_UNKNOWN, false, "Could not read DECISION.ENGINE metadata"); + psFree(whereClause); + return(false); + } + psArray *decisionRules = psArrayAllocEmpty(4); // list of screened and verified rules. + psMetadataIterator *iterEngine = psMetadataIteratorAlloc(engine, PS_LIST_HEAD,NULL); + psMetadataItem *engineItem = NULL; + + while ((engineItem = psMetadataGetAndIncrement(iterEngine))) { + if (strcmp(engineItem->name, "RULE")) continue; + if (engineItem->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool RULE not of type METADATA\n"); + return(NULL); + } + + bool status; + psMetadata *enginedef = engineItem->data.md; + + if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& + isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { + psArrayAdd(decisionRules,decisionRules->n,enginedef); + } + else { + psError(PS_ERR_UNKNOWN, false, "Rule invalid."); + } + } + psFree(iterEngine); + + if (decisionRules->n < 1) { + psError(PS_ERR_UNKNOWN, false, "Failed to generate a decision engine"); + psFree(decisionRules); + psFree(engine); + return(false); + } + psFree(engine); // Find the contents that comprise this run @@ -439,4 +489,9 @@ bool status; psString MYquery = psStringCopy(query); + if (!MYquery) { + psTrace("debug",4,"Failed to copy, trying again."); + MYquery = psStringCopy(query); + } + psMetadata *where = psMetadataAlloc(); @@ -445,12 +500,23 @@ psMetadataAddS64(where,PS_LIST_TAIL, "camRun.cam_id", 0, "==", psMetadataLookupS64(&status,row,"cam_id")); // This may not be real, so we need to check first. Is there a better way to check for a null here? - if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != 9223372036854775807)) { + if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != PS_MAX_S64)) { psMetadataAddS64(where,PS_LIST_TAIL, "warpRun.warp_id", 0, "==", psMetadataLookupS64(&status,row,"warp_id")); } psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (!whereClause) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate Where Condition"); + return(false); + } psStringAppend(&MYquery, " WHERE %s", whereClause); - psFree(whereClause); - psFree(where); + +/* psTrace("debug",2,"%ld %ld %ld %ld %s\n",psMetadataLookupS64(&status,row,"exp_id"), */ +/* psMetadataLookupS64(&status,row,"chip_id"), */ +/* psMetadataLookupS64(&status,row,"cam_id"), */ +/* psMetadataLookupS64(&status,row,"warp_id"), */ +/* whereClause); */ + +/* psFree(whereClause); */ +/* psFree(where); */ // printf("%s\n",MYquery); @@ -465,5 +531,4 @@ return(false); } - psFree(MYquery); psArray *statsoutput = p_psDBFetchResult(config->dbh); @@ -478,5 +543,5 @@ } if (!psArrayLength(statsoutput)) { - psWarning("dqstatstool No stats rows found"); + psWarning("dqstatstool No stats rows found: %ld %s %s",i,MYquery,query); psFree(colNames); psFree(colTypes); @@ -487,4 +552,7 @@ return(false); } + psFree(MYquery); + psFree(whereClause); + psFree(where); // Magic happens and the fits table is filled. @@ -546,7 +614,12 @@ success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.str); } - + else if (!strcmp(colType,"PS_DATA_TIME")) { + psString timeString = psTimeToISO(rowItem->data.V); + success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",timeString); + psFree(timeString); + } + if (!success) { - psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row."); + psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row. %s %s",colType,colName); psFree(colNames); psFree(colTypes); @@ -562,5 +635,56 @@ } } - + // Calculate if this exposure was "good" or not. + bool accept = true; + for (long j = 0; j < decisionRules->n; j++) { + psMetadata *rule = decisionRules->data[j]; + bool status = true; + psString colname = psMetadataLookupStr(&status,rule,"COLNAME"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); + return(false); + } + psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); + return(false); + } + psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); + return(false); + } + // Not happy with this being set to a F32. Can this ever be something else? + psF32 value = psMetadataLookupF32(&status,tableRow,colname); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(tableRow); + return(false); + } + if ((value > max_value)||(value < min_value)) { + accept = false; + } + } + bool success; + success = psMetadataAddBool(tableRow,PS_LIST_TAIL,"ACCEPT",0,"",accept); + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + return(false); + } + + // Add the row to the table array. psArrayAdd(outTable,0,tableRow); psFree(tableRow); @@ -600,7 +724,7 @@ static bool updaterunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config,false); - PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", "dqstatsRun.dqstats_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", "state", "=="); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", "fault", "=="); + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); if (state && ! pxIsValidState(state)) { Index: trunk/ippTools/src/dqstatstoolConfig.c =================================================================== --- trunk/ippTools/src/dqstatstoolConfig.c (revision 27301) +++ trunk/ippTools/src/dqstatstoolConfig.c (revision 27304) @@ -49,4 +49,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set dqstatsRun stats", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); // -pendingbundle