Index: /branches/eam_branches/ipp-20101205/DataStoreServer/scripts/dsreg =================================================================== --- /branches/eam_branches/ipp-20101205/DataStoreServer/scripts/dsreg (revision 30580) +++ /branches/eam_branches/ipp-20101205/DataStoreServer/scripts/dsreg (revision 30581) @@ -338,8 +338,13 @@ ## then copy or symlink the files into place foreach my $ref (@files) { - my $src = (defined $abspath ? '' : "$datapath/") . "$ref->{file}"; + my $src_path = (defined $abspath ? '' : "$datapath/") . "$ref->{file}"; my $filename = fileparse($ref->{file}, ()); my $dest = "$fileset_dir/$filename"; + my $src = $ipprc->file_resolve($src_path); + if (!$src) { + print STDERR "failed to resolve file $src_path\n"; + exit $PS_EXIT_SYS_ERROR; + } if ( !-e $src or !-r $src) { print STDERR "source file $src does not exist or is not accessible\n"; Index: /branches/eam_branches/ipp-20101205/Ohana/src/getstar/src/MatchCoords.c =================================================================== --- /branches/eam_branches/ipp-20101205/Ohana/src/getstar/src/MatchCoords.c (revision 30580) +++ /branches/eam_branches/ipp-20101205/Ohana/src/getstar/src/MatchCoords.c (revision 30581) @@ -53,7 +53,6 @@ } -#ifdef notdef - printf("%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); -#endif + + if (VERBOSE) fprintf(stderr, "%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) { Index: /branches/eam_branches/ipp-20101205/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/eam_branches/ipp-20101205/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 30580) +++ /branches/eam_branches/ipp-20101205/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 30581) @@ -34,4 +34,5 @@ $PSTAMP_NO_WAIT_FOR_UPDATE $PSTAMP_SUCCESS + $PSTAMP_FIRST_ERROR_CODE $PSTAMP_SYSTEM_ERROR $PSTAMP_NOT_IMPLEMENTED @@ -74,4 +75,5 @@ # NOTE: these must match the values in pstamp/src/pstamp.h our $PSTAMP_SUCCESS = 0; +our $PSTAMP_FIRST_ERROR_CODE = 10; our $PSTAMP_SYSTEM_ERROR = 10; our $PSTAMP_NOT_IMPLEMENTED = 11; Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/dqStatsTool.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/dqStatsTool.config (revision 30580) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/dqStatsTool.config (revision 30581) @@ -5,5 +5,177 @@ RULE MULTI RULE METADATA + COLNAME STR CAMRUN_QUALITY + MINIMUM S32 0 + MAXIMUM S32 0 + RULETYPE STR STRICT + END +# CDF data generated from the data in camRun with non-NULL deteff values. +# CZW: How I did this so I can remember next time. +# echo "select zpt_obs,fwhm_major,deteff from camProcessedExp JOIN camRun USING (cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE camRun.label LIKE '%.nightlyscience' AND quality = 0 AND filter = 'FILTER';" | sed 's/FILTER/g.00000/' | mysql -h ippdb01 -u ipp -pipp gpc1 | awk '{print($3)}' | grep -v deteff | grep -v NULL | mk_cdf.pl | awk '{print($2,$1)}' | spline - `echo 0.0 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.4 0.45 0.5 0.55 0.6 0.65 0.7 0.75 0.8 0.85 0.9 0.95 0.99 1.0` + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR g.00000 + CDF00 F32 13.0214 + CDF05 F32 18.4582 + CDF10 F32 18.8631 + CDF15 F32 19.1020 + CDF20 F32 19.3304 + CDF25 F32 19.4613 + CDF30 F32 19.5324 + CDF35 F32 19.6094 + CDF40 F32 19.6878 + CDF45 F32 19.7477 + CDF50 F32 19.8058 + CDF55 F32 19.8737 + CDF60 F32 19.9409 + CDF65 F32 20.0108 + CDF70 F32 20.0860 + CDF75 F32 20.1694 + CDF80 F32 20.2681 + CDF85 F32 20.3974 + CDF90 F32 20.5609 + CDF95 F32 20.7193 + CDF99 F32 20.9785 + CDF100 F32 21.6135 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR r.00000 + CDF00 F32 13.0000 + CDF05 F32 17.9079 + CDF10 F32 18.4030 + CDF15 F32 18.7973 + CDF20 F32 19.0507 + CDF25 F32 19.2644 + CDF30 F32 19.4433 + CDF35 F32 19.5680 + CDF40 F32 19.6728 + CDF45 F32 19.7609 + CDF50 F32 19.8393 + CDF55 F32 19.9125 + CDF60 F32 19.9751 + CDF65 F32 20.0395 + CDF70 F32 20.0926 + CDF75 F32 20.1673 + CDF80 F32 20.2457 + CDF85 F32 20.3496 + CDF90 F32 20.4967 + CDF95 F32 20.7552 + CDF99 F32 21.2945 + CDF100 F32 21.6046 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR i.00000 + CDF00 F32 16.4751 + CDF05 F32 18.3041 + CDF10 F32 18.5018 + CDF15 F32 18.6497 + CDF20 F32 18.7602 + CDF25 F32 18.8521 + CDF30 F32 18.9267 + CDF35 F32 19.0105 + CDF40 F32 19.0948 + CDF45 F32 19.1563 + CDF50 F32 19.2215 + CDF55 F32 19.296 + CDF60 F32 19.3689 + CDF65 F32 19.4438 + CDF70 F32 19.5261 + CDF75 F32 19.6127 + CDF80 F32 19.7071 + CDF85 F32 19.8518 + CDF90 F32 20.0447 + CDF95 F32 20.5386 + CDF99 F32 21.1344 + CDF100 F32 21.5592 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR z.00000 + CDF00 F32 12.1771 + CDF05 F32 17.5262 + CDF10 F32 17.7958 + CDF15 F32 17.9979 + CDF20 F32 18.1753 + CDF25 F32 18.3201 + CDF30 F32 18.4335 + CDF35 F32 18.5478 + CDF40 F32 18.6470 + CDF45 F32 18.7494 + CDF50 F32 18.8370 + CDF55 F32 18.9166 + CDF60 F32 19.0028 + CDF65 F32 19.0912 + CDF70 F32 19.1886 + CDF75 F32 19.2912 + CDF80 F32 19.4137 + CDF85 F32 19.5244 + CDF90 F32 19.6437 + CDF95 F32 19.9135 + CDF99 F32 20.6482 + CDF100 F32 21.0925 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR y.00000 + CDF00 F32 11.6109 + CDF05 F32 16.7278 + CDF10 F32 17.0561 + CDF15 F32 17.2778 + CDF20 F32 17.4367 + CDF25 F32 17.5495 + CDF30 F32 17.6469 + CDF35 F32 17.7392 + CDF40 F32 17.8160 + CDF45 F32 17.8927 + CDF50 F32 17.9779 + CDF55 F32 18.0514 + CDF60 F32 18.1292 + CDF65 F32 18.2153 + CDF70 F32 19.2903 + CDF75 F32 18.3627 + CDF80 F32 18.4325 + CDF85 F32 18.5321 + CDF90 F32 18.6403 + CDF95 F32 18.7523 + CDF99 F32 18.8892 + CDF100 F32 19.064 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR w.00000 + CDF00 F32 17.4082 + CDF05 F32 19.4333 + CDF10 F32 19.9698 + CDF15 F32 20.2711 + CDF20 F32 20.3919 + CDF25 F32 20.5038 + CDF30 F32 20.6192 + CDF35 F32 20.7273 + CDF40 F32 20.8184 + CDF45 F32 20.9186 + CDF50 F32 21.0151 + CDF55 F32 21.1326 + CDF60 F32 21.2442 + CDF65 F32 21.3412 + CDF70 F32 21.4394 + CDF75 F32 21.5105 + CDF80 F32 21.5567 + CDF85 F32 21.6281 + CDF90 F32 21.6908 + CDF95 F32 21.7566 + CDF99 F32 21.8907 + CDF100 F32 23.6312 + END + RULE METADATA COLNAME STR FWHM_MAJOR + RULETYPE STR UNUSED MINIMUM F32 0.0 MAXIMUM F32 9.3 # 90-percentile 2010-02-14 through 2010-06-20 = 2.4" @@ -11,4 +183,5 @@ RULE METADATA COLNAME STR FWHM_MINOR + RULETYPE STR UNUSED MINIMUM F32 0.0 MAXIMUM F32 8.44 # 90-percentile 2010-02-14 through 2010-06-20 = 2.18" @@ -16,4 +189,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR g.00000 MINIMUM F32 24.3051 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -22,4 +196,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR r.00000 MINIMUM F32 24.5294 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -28,4 +203,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR i.00000 MINIMUM F32 24.4641 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -34,4 +210,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR z.00000 MINIMUM F32 24.1128 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -40,4 +217,5 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR y.00000 MINIMUM F32 23.1503 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes @@ -46,11 +224,9 @@ RULE METADATA COLNAME STR ZEROPOINT + RULETYPE STR UNUSED FILTER STR w.00000 MINIMUM F32 25.8944 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. END -# TYPE VAL COLNAME MINIMUM MAXIMUM -# RULE VAL FWHM_MAJOR 0.0 5.0 -# RULE VAL FWHM_MINOR 0.0 5.0 END @@ -65,4 +241,5 @@ COLUMN VAL rawExp decl PS_DATA_F64 DEC # Declination COLUMN VAL rawExp exp_time PS_DATA_F32 EXPTIME # Exposure time + COLUMN VAL rawExp obs_mode PS_DATA_STRING OBS_MODE COLUMN VAL camProcessedExp zpt_obs PS_DATA_F32 ZEROPOINT @@ -107,4 +284,6 @@ COLUMN VAL camProcessedExp iq_m4_lq PS_DATA_F32 IQ_M4_LQ COLUMN VAL camProcessedExp iq_m4_uq PS_DATA_F32 IQ_M4_UQ + COLUMN VAL camProcessedExp deteff PS_DATA_F32 DETEFF + COLUMN VAL camProcessedExp quality PS_DATA_S32 CAMRUN_QUALITY END Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/filerules-split.mdc (revision 30580) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/filerules-split.mdc (revision 30581) @@ -341,8 +341,14 @@ CENSOR.OUTPUT OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF -MAGIC.ORIGINAL.PNG OUTPUT {OUTPUT}_original.png JPEG NONE FPA TRUE NONE -MAGIC.RESIDUAL.PNG OUTPUT {OUTPUT}_residual.png JPEG NONE FPA TRUE NONE -MAGIC.MASK.PNG OUTPUT {OUTPUT}_mask.png JPEG NONE FPA TRUE NONE -MAGIC.DUPLICATE.PNG OUTPUT {OUTPUT}_duplicate.png JPEG NONE FPA TRUE NONE +MAGIC.ORIGINAL.PNG OUTPUT {OUTPUT}.original.png JPEG NONE FPA TRUE NONE +MAGIC.RESIDUAL.PNG OUTPUT {OUTPUT}.residual.png JPEG NONE FPA TRUE NONE +MAGIC.MASK.PNG OUTPUT {OUTPUT}.mask.png JPEG NONE FPA TRUE NONE +MAGIC.DUPLICATE.PNG OUTPUT {OUTPUT}.duplicate.png JPEG NONE FPA TRUE NONE + +# TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +DIST.OUTPUT.CHIP.BUNDLE OUTPUT {OUTPUT}.{CHIP.NAME}.tgz TEXT NONE CHIP TRUE NONE +DIST.OUTPUT.BUNDLE OUTPUT {OUTPUT}.tgz TEXT NONE FPA TRUE NONE +DIST.OUTPUT.DBINFO OUTPUT {OUTPUT}.mdc TEXT NONE FPA TRUE NONE +DIST.OUTPUT.DIRINFO OUTPUT {OUTPUT}.mdc TEXT NONE FPA TRUE NONE PPSMOOTH.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE MEF Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config (revision 30580) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config (revision 30581) @@ -3,5 +3,5 @@ MODE STR CHIP COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ - RETENTION_TIME U16 30 + RETENTION_TIME U16 21 END CLEAN_MODES METADATA @@ -13,5 +13,5 @@ MODE STR DIFF COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ - RETENTION_TIME S16 30 + RETENTION_TIME S16 21 END CLEAN_MODES METADATA @@ -28,10 +28,10 @@ MODE STR BGCHIP COMMAND STR bgtool -dbname @DBNAME@ -updatechip -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ - RETENTION_TIME S16 14 + RETENTION_TIME S16 7 END CLEAN_MODES METADATA MODE STR BGWARP COMMAND STR bgtool -dbname @DBNAME@ -updatewarp -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ - RETENTION_TIME S16 14 + RETENTION_TIME S16 7 END @@ -216,5 +216,5 @@ STACKABLE BOOL FALSE DIFFABLE BOOL TRUE - WARP S16 60 +# WARP S16 60 END TARGETS METADATA @@ -249,6 +249,6 @@ STACKABLE BOOL FALSE DIFFABLE BOOL TRUE - CHIP S16 14 - DIFF S16 14 + CHIP S16 10 + DIFF S16 10 END # This is a hack, and I freely admit it. Index: /branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c =================================================================== --- /branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c (revision 30580) +++ /branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c (revision 30581) @@ -75,5 +75,5 @@ if (sfiles->inWeight) { double recWeightValue = psImageGet(sfiles->recWeight->image, x, y); - if (recWeightValue != NAN) { + if (!isnan(recWeightValue)) { psImageSet(sfiles->inWeight->image, x, y, recWeightValue); } @@ -86,4 +86,8 @@ } while (streakFilesNextExtension(sfiles)); + + // check the weight and mask files for extra extensions that might be in files + // (covariance matrix for example) + copyExtraExtensions(sfiles); // close all files @@ -235,5 +239,5 @@ usage(); } - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0, + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.MASK", 0, "name of input mask image", argv[argnum]); psArgumentRemove(argnum, &argc, argv); @@ -251,5 +255,5 @@ usage(); } - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0, + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.WEIGHT", 0, "name of input weight image", argv[argnum]); psArgumentRemove(argnum, &argc, argv); @@ -321,5 +325,4 @@ addDestreakKeyword(sf->outMask->header, false); - // XXX: TODO setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false); @@ -333,5 +336,5 @@ sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header); addDestreakKeyword(sf->outWeight->header, false); - setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false); + setupImageRefs(sf->outWeight, NULL, sf->inWeight, sf->extnum, false); copyFitsOptions(sf->outWeight, NULL, sf->inWeight, sf->tiles); Index: /branches/eam_branches/ipp-20101205/ppTranslate/src/ppMopsMerge.c =================================================================== --- /branches/eam_branches/ipp-20101205/ppTranslate/src/ppMopsMerge.c (revision 30580) +++ /branches/eam_branches/ipp-20101205/ppTranslate/src/ppMopsMerge.c (revision 30581) @@ -35,5 +35,5 @@ ppMopsDetections *merged = NULL; // Merged list - int num = 1; // Number of merged files + int num = 0; // Number of merged files for (int i = 0; i < detections->n; i++) { ppMopsDetections *det = detections->data[i]; // Detections of interest @@ -163,4 +163,9 @@ } + if (num == 0) { + //All detections were NULL?! + psTrace("ppMops.merge", 3, "All %d detections were NULL\n", detections->n); + return NULL; + } psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num); Index: /branches/eam_branches/ipp-20101205/psModules/src/detrend/pmFringeStats.c =================================================================== --- /branches/eam_branches/ipp-20101205/psModules/src/detrend/pmFringeStats.c (revision 30580) +++ /branches/eam_branches/ipp-20101205/psModules/src/detrend/pmFringeStats.c (revision 30581) @@ -1008,16 +1008,68 @@ } } + // Allocate array of vectors to hold data. + psArray *bins = psArrayAlloc(4000); + for (int j = 0; j < bins->n; j++) { + bins->data[j] = psVectorAllocEmpty(1,PS_TYPE_F32); + // psVector *v = psVectorAllocEmpty(1,PS_TYPE_F32); + // bins = psArrayAdd(bins,1,v); + } + + // Fill vectors + pmFringeStats *fringe = fringes->data[0]; + for (int j = 0; j < numRegions; j++) { + if (mask->data.PS_TYPE_VECTOR_MASK_DATA[j] == 0) { + int array_bin = (int) ((fringe->f->data.F32[j] - -0.1) / 5e-5); + psVector *bin = bins->data[array_bin]; + psVectorAppend(bin,science->f->data.F32[j]); + } + } + psVector *fringe_positions = psVectorAllocEmpty(4000,PS_TYPE_F32); + psVector *science_values = psVectorAllocEmpty(4000,PS_TYPE_F32); + psVector *science_errors = psVectorAllocEmpty(4000,PS_TYPE_F32); + + psStats *binStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + for (int i = 0; i < 4000; i++) { + psVector *bin = bins->data[i]; + if (bin->n > 2) { + psStatsInit(binStats); + + psVectorStats(binStats,bin,NULL,NULL,1); + + if (isfinite(binStats->robustStdev) && + isfinite(binStats->robustMedian) && + binStats->robustStdev > 0) { + psVectorAppend(fringe_positions,-0.1 + i * 5e-5); + psVectorAppend(science_values, binStats->robustMedian); + psVectorAppend(science_errors, binStats->robustStdev); + } + } + psFree(bins->data[i]); + } + psFree(bins); + psFree(binStats); + + for (int i = 0; i < fringe_positions->n; i++) { + psTrace("psModules.detrend",7,"FITDATA: %f %f %f\n", + fringe_positions->data.F32[i], + science_values->data.F32[i], + science_errors->data.F32[i]); + } /* // Begin switch from old outlier removal and fitting code. */ psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); - pmFringeStats *fringe = fringes->data[0]; - psVector *errors = psVectorAlloc(science->df->n,PS_TYPE_F32); - for (int j = 0; j < errors->n; j++) { - errors->data.F32[j] = 1 / science->df->data.F32[j]; - } - psVectorFitPolynomial1D(poly,mask,0xff,science->f,errors,fringe->f); - + // pmFringeStats *fringe = fringes->data[0]; +/* psVector *errors = psVectorAlloc(science->df->n,PS_TYPE_F32); */ +/* for (int j = 0; j < errors->n; j++) { */ +/* errors->data.F32[j] = 1 / science->df->data.F32[j]; */ +/* } */ +/* psVectorFitPolynomial1D(poly,mask,0xff,science->f,errors,fringe->f); */ + psVectorFitPolynomial1D(poly,NULL,0xff,science_values,science_errors,fringe_positions); + psFree(fringe_positions); + psFree(science_values); + psFree(science_errors); + for (int i = 0; i <= poly->nX; i++) { scale->coeff->data.F32[i] = poly->coeff[i]; @@ -1027,5 +1079,5 @@ psFree(poly); // psFree(fringe); - psFree(errors); + // psFree(errors); psFree(median);