Index: /tags/ipp-20101215/ippTools/src/dqstatstool.c =================================================================== --- /tags/ipp-20101215/ippTools/src/dqstatstool.c (revision 30466) +++ /tags/ipp-20101215/ippTools/src/dqstatstool.c (revision 30467) @@ -92,5 +92,5 @@ } else { - psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); + psStringAppend(&query, " AND (warp_id IS NOT NULL AND invalid IS NULL) "); } @@ -402,7 +402,17 @@ if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& - isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& - isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { + (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"STRICT") == 0)&& + (isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM")))) { psArrayAdd(decisionRules,decisionRules->n,enginedef); + } + else if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& + (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"CDF") == 0)&& + (isfinite(psMetadataLookupF32(&status,enginedef,"CDF00"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"CDF50"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"CDF100")))) { + psArrayAdd(decisionRules,decisionRules->n, enginedef); + } + else if (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"UNUSED") == 0) { } else { @@ -637,4 +647,5 @@ // Calculate if this exposure was "good" or not. bool accept = true; + psF32 quality = 1.0; for (long j = 0; j < decisionRules->n; j++) { psMetadata *rule = decisionRules->data[j]; @@ -644,29 +655,10 @@ psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); return(false); - } - psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); - if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); - return(false); - } - psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); - if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); - return(false); - } - psString filter = psMetadataLookupStr(&status,rule,"FILTER"); - if (!status) { - status = true; - } - if (filter) { - psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); - if (strcmp(filter,imfilter) != 0) { - continue; - } } // Not happy with this being set to a F32. Can this ever be something else? psF32 value = psMetadataLookupF32(&status,tableRow,colname); if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f) in compare with rule.", + colname,value); psFree(colNames); psFree(colTypes); @@ -677,7 +669,92 @@ return(false); } - if ((value > max_value)||(value < min_value)) { - accept = false; - } + psString filter = psMetadataLookupStr(&status,rule,"FILTER"); + if (!status) { + status = true; + } + if (filter) { + psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); + if (strcmp(filter,imfilter) != 0) { + continue; + } + } + + if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"STRICT") == 0) { + psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); + return(false); + } + psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); + return(false); + } + + if ((value > max_value)||(value < min_value)) { + accept = false; + } + } // End STRICT min/max rule parsing + else if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"CDF") == 0) { + // I'm just going to implement a quick linear interpolation over the points that exist. + psF32 min_value,max_value = 99; + psF32 min_cdf = 0.0,max_cdf = 1.0; + psF32 cdf; + psF32 try[22] = {0.00,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.45,0.50, + 0.55,0.60,0.65,0.70,0.75,0.80,0.85,0.90,0.95,0.99,1.00}; + int i; + int done = 0; + status = true; + min_value = psMetadataLookupF32(&status,rule,"CDF00"); + min_cdf = 0.0; + if (!isfinite(value)) { + quality = 0.0; + continue; + } + if (!status) { + status = true; + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find CDF00 value for %s",colname); + } + if (value < min_value) { + if (quality > min_cdf) { + quality = 0.0; + continue; + } + } + + for (i = 1; i < 22; i++) { + psString label = psStringCopy(""); + psStringAppend(&label,"CDF%02d",(int) (100 * try[i])); + + max_value = psMetadataLookupF32(&status,rule,label); + max_cdf = try[i]; + if (!status) { + status = true; + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find %s value for %s",label,colname); + } + if ((value < max_value)&&(value > min_value)) { + cdf = ((value - min_value) * max_cdf + (max_value - value) * min_cdf) / (max_value - min_value); + // fprintf(stderr,"value: %f %f %f %f %f %f %d %d %f\n",value,quality,min_cdf,min_value,max_cdf,max_value, value < max_value, value > min_value,cdf); + if (quality > cdf) { + quality = cdf; + done = 1; + i = 25; + } + } + min_value = max_value; + min_cdf = max_cdf; + } + if (done) { + continue; + } + // Reached end of array without finding the value; + if (value > min_value) { + if (quality > min_cdf) { + quality = 1.0; + } + } + + + } // End CDF rule parsing } bool success; @@ -695,4 +772,17 @@ return(false); } + success = psMetadataAddF32(tableRow,PS_LIST_TAIL,"QUALITY",0,"",quality); + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + return(false); + } // Add the row to the table array. Index: /tags/ipp-20101215/ippconfig/recipes/dqStatsTool.config =================================================================== --- /tags/ipp-20101215/ippconfig/recipes/dqStatsTool.config (revision 30466) +++ /tags/ipp-20101215/ippconfig/recipes/dqStatsTool.config (revision 30467) @@ -5,5 +5,177 @@ RULE MULTI RULE METADATA + COLNAME STR CAMRUN_QUALITY + MINIMUM S32 0 + MAXIMUM S32 0 + RULETYPE STR STRICT + END +# CDF data generated from the data in camRun with non-NULL deteff values. +# CZW: How I did this so I can remember next time. +# echo "select zpt_obs,fwhm_major,deteff from camProcessedExp JOIN camRun USING (cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE camRun.label LIKE '%.nightlyscience' AND quality = 0 AND filter = 'FILTER';" | sed 's/FILTER/g.00000/' | mysql -h ippdb01 -u ipp -pipp gpc1 | awk '{print($3)}' | grep -v deteff | grep -v NULL | mk_cdf.pl | awk '{print($2,$1)}' | spline - `echo 0.0 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.4 0.45 0.5 0.55 0.6 0.65 0.7 0.75 0.8 0.85 0.9 0.95 0.99 1.0` + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR g.00000 + CDF00 F32 13.0214 + CDF05 F32 18.4582 + CDF10 F32 18.8631 + CDF15 F32 19.1020 + CDF20 F32 19.3304 + CDF25 F32 19.4613 + CDF30 F32 19.5324 + CDF35 F32 19.6094 + CDF40 F32 19.6878 + CDF45 F32 19.7477 + CDF50 F32 19.8058 + CDF55 F32 19.8737 + CDF60 F32 19.9409 + CDF65 F32 20.0108 + CDF70 F32 20.0860 + CDF75 F32 20.1694 + CDF80 F32 20.2681 + CDF85 F32 20.3974 + CDF90 F32 20.5609 + CDF95 F32 20.7193 + CDF99 F32 20.9785 + CDF100 F32 21.6135 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR r.00000 + CDF00 F32 13.0000 + CDF05 F32 17.9079 + CDF10 F32 18.4030 + CDF15 F32 18.7973 + CDF20 F32 19.0507 + CDF25 F32 19.2644 + CDF30 F32 19.4433 + CDF35 F32 19.5680 + CDF40 F32 19.6728 + CDF45 F32 19.7609 + CDF50 F32 19.8393 + CDF55 F32 19.9125 + CDF60 F32 19.9751 + CDF65 F32 20.0395 + CDF70 F32 20.0926 + CDF75 F32 20.1673 + CDF80 F32 20.2457 + CDF85 F32 20.3496 + CDF90 F32 20.4967 + CDF95 F32 20.7552 + CDF99 F32 21.2945 + CDF100 F32 21.6046 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR i.00000 + CDF00 F32 16.4751 + CDF05 F32 18.3041 + CDF10 F32 18.5018 + CDF15 F32 18.6497 + CDF20 F32 18.7602 + CDF25 F32 18.8521 + CDF30 F32 18.9267 + CDF35 F32 19.0105 + CDF40 F32 19.0948 + CDF45 F32 19.1563 + CDF50 F32 19.2215 + CDF55 F32 19.296 + CDF60 F32 19.3689 + CDF65 F32 19.4438 + CDF70 F32 19.5261 + CDF75 F32 19.6127 + CDF80 F32 19.7071 + CDF85 F32 19.8518 + CDF90 F32 20.0447 + CDF95 F32 20.5386 + CDF99 F32 21.1344 + CDF100 F32 21.5592 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR z.00000 + CDF00 F32 12.1771 + CDF05 F32 17.5262 + CDF10 F32 17.7958 + CDF15 F32 17.9979 + CDF20 F32 18.1753 + CDF25 F32 18.3201 + CDF30 F32 18.4335 + CDF35 F32 18.5478 + CDF40 F32 18.6470 + CDF45 F32 18.7494 + CDF50 F32 18.8370 + CDF55 F32 18.9166 + CDF60 F32 19.0028 + CDF65 F32 19.0912 + CDF70 F32 19.1886 + CDF75 F32 19.2912 + CDF80 F32 19.4137 + CDF85 F32 19.5244 + CDF90 F32 19.6437 + CDF95 F32 19.9135 + CDF99 F32 20.6482 + CDF100 F32 21.0925 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR y.00000 + CDF00 F32 11.6109 + CDF05 F32 16.7278 + CDF10 F32 17.0561 + CDF15 F32 17.2778 + CDF20 F32 17.4367 + CDF25 F32 17.5495 + CDF30 F32 17.6469 + CDF35 F32 17.7392 + CDF40 F32 17.8160 + CDF45 F32 17.8927 + CDF50 F32 17.9779 + CDF55 F32 18.0514 + CDF60 F32 18.1292 + CDF65 F32 18.2153 + CDF70 F32 19.2903 + CDF75 F32 18.3627 + CDF80 F32 18.4325 + CDF85 F32 18.5321 + CDF90 F32 18.6403 + CDF95 F32 18.7523 + CDF99 F32 18.8892 + CDF100 F32 19.064 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR w.00000 + CDF00 F32 17.4082 + CDF05 F32 19.4333 + CDF10 F32 19.9698 + CDF15 F32 20.2711 + CDF20 F32 20.3919 + CDF25 F32 20.5038 + CDF30 F32 20.6192 + CDF35 F32 20.7273 + CDF40 F32 20.8184 + CDF45 F32 20.9186 + CDF50 F32 21.0151 + CDF55 F32 21.1326 + CDF60 F32 21.2442 + CDF65 F32 21.3412 + CDF70 F32 21.4394 + CDF75 F32 21.5105 + CDF80 F32 21.5567 + CDF85 F32 21.6281 + CDF90 F32 21.6908 + CDF95 F32 21.7566 + CDF99 F32 21.8907 + CDF100 F32 23.6312 + END + RULE METADATA COLNAME STR FWHM_MAJOR + RULETYPE STR UNUSED MINIMUM F32 0.0 MAXIMUM F32 9.3 # 90-percentile 2010-02-14 through 2010-06-20 = 2.4" @@ -11,4 +183,5 @@ RULE METADATA COLNAME STR FWHM_MINOR + RULETYPE STR UNUSED MINIMUM F32 0.0 MAXIMUM F32 8.44 # 90-percentile 2010-02-14 through 2010-06-20 = 2.18" @@ -16,4 +189,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR g.00000 MINIMUM F32 24.3051 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -22,4 +196,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR r.00000 MINIMUM F32 24.5294 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -28,4 +203,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR i.00000 MINIMUM F32 24.4641 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -34,4 +210,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR z.00000 MINIMUM F32 24.1128 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -40,4 +217,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR y.00000 MINIMUM F32 23.1503 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -46,11 +224,9 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR w.00000 MINIMUM F32 25.8944 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. END -# TYPE VAL COLNAME MINIMUM MAXIMUM -# RULE VAL FWHM_MAJOR 0.0 5.0 -# RULE VAL FWHM_MINOR 0.0 5.0 END @@ -107,4 +283,6 @@ COLUMN VAL camProcessedExp iq_m4_lq PS_DATA_F32 IQ_M4_LQ COLUMN VAL camProcessedExp iq_m4_uq PS_DATA_F32 IQ_M4_UQ + COLUMN VAL camProcessedExp deteff PS_DATA_F32 DETEFF + COLUMN VAL camProcessedExp quality PS_DATA_S32 CAMRUN_QUALITY END