Index: /branches/czw_branch/20101203/Ohana/src/addstar/include/addstar.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/include/addstar.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/include/addstar.h (revision 30631) @@ -265,6 +265,4 @@ int InitStar PROTO((Stars *star)); -short ShortPixels PROTO((float pixels)); -short ShortDegree PROTO((float degrees)); int args_skycells (int argc, char **argv); Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsFITS.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsFITS.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsFITS.c (revision 30631) @@ -98,7 +98,7 @@ stars[i].measure.photFlags = (smpdata[i].dophot << 16); - stars[i].measure.FWx = ShortPixels (smpdata[i].fx); - stars[i].measure.FWy = ShortPixels (smpdata[i].fy); - stars[i].measure.theta = ShortDegree (smpdata[i].df); + stars[i].measure.FWx = ToShortPixels (smpdata[i].fx); + stars[i].measure.FWy = ToShortPixels (smpdata[i].fy); + stars[i].measure.theta = ToShortDegrees (smpdata[i].df); } *nstars = Nstars; @@ -127,6 +127,6 @@ stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { @@ -140,7 +140,7 @@ stars[i].measure.dSky = ps1data[i].dSky; - stars[i].measure.FWx = ShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ShortPixels(ps1data[i].fy); - stars[i].measure.theta = ShortDegree(ps1data[i].df); + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); stars[i].measure.psfChisq = ps1data[i].psfChisq; @@ -176,6 +176,6 @@ stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { @@ -189,7 +189,7 @@ stars[i].measure.dSky = ps1data[i].dSky; - stars[i].measure.FWx = ShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ShortPixels(ps1data[i].fy); - stars[i].measure.theta = ShortDegree(ps1data[i].df); + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); stars[i].measure.psfChisq = ps1data[i].psfChisq; @@ -238,8 +238,8 @@ stars[i].measure.Xccd = ps1data[i].X; stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); - - stars[i].measure.posangle = ShortDegree(ps1data[i].posangle); + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); stars[i].measure.pltscale = ps1data[i].pltscale; @@ -263,11 +263,11 @@ stars[i].measure.extNsigma = ps1data[i].extNsigma; - stars[i].measure.FWx = ShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ShortPixels(ps1data[i].fy); - stars[i].measure.theta = ShortDegree(ps1data[i].df); - - stars[i].measure.Mxx = ShortPixels(ps1data[i].Mxx); - stars[i].measure.Mxy = ShortPixels(ps1data[i].Mxy); - stars[i].measure.Myy = ShortPixels(ps1data[i].Myy); + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); stars[i].measure.photFlags = ps1data[i].flags; @@ -314,8 +314,8 @@ stars[i].measure.Xccd = ps1data[i].X; stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); - - stars[i].measure.posangle = ShortDegree(ps1data[i].posangle); + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); stars[i].measure.pltscale = ps1data[i].pltscale; @@ -339,11 +339,11 @@ stars[i].measure.extNsigma = ps1data[i].extNsigma; - stars[i].measure.FWx = ShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ShortPixels(ps1data[i].fy); - stars[i].measure.theta = ShortDegree(ps1data[i].df); - - stars[i].measure.Mxx = ShortPixels(ps1data[i].Mxx); - stars[i].measure.Mxy = ShortPixels(ps1data[i].Mxy); - stars[i].measure.Myy = ShortPixels(ps1data[i].Myy); + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); stars[i].measure.photFlags = ps1data[i].flags; @@ -382,8 +382,8 @@ stars[i].measure.Xccd = ps1data[i].X; stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); - - stars[i].measure.posangle = ShortDegree(ps1data[i].posangle); + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); stars[i].measure.pltscale = ps1data[i].pltscale; @@ -407,11 +407,11 @@ stars[i].measure.extNsigma = ps1data[i].extNsigma; - stars[i].measure.FWx = ShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ShortPixels(ps1data[i].fy); - stars[i].measure.theta = ShortDegree(ps1data[i].df); - - stars[i].measure.Mxx = ShortPixels(ps1data[i].Mxx); - stars[i].measure.Mxy = ShortPixels(ps1data[i].Mxy); - stars[i].measure.Myy = ShortPixels(ps1data[i].Myy); + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); stars[i].measure.photFlags = ps1data[i].flags; @@ -450,8 +450,8 @@ stars[i].measure.Xccd = ps1data[i].X; stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); - - stars[i].measure.posangle = ShortDegree(ps1data[i].posangle); + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); stars[i].measure.pltscale = ps1data[i].pltscale; @@ -475,11 +475,11 @@ stars[i].measure.extNsigma = ps1data[i].extNsigma; - stars[i].measure.FWx = ShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ShortPixels(ps1data[i].fy); - stars[i].measure.theta = ShortDegree(ps1data[i].df); - - stars[i].measure.Mxx = ShortPixels(ps1data[i].Mxx); - stars[i].measure.Mxy = ShortPixels(ps1data[i].Mxy); - stars[i].measure.Myy = ShortPixels(ps1data[i].Myy); + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); stars[i].measure.photFlags = ps1data[i].flags; Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsSDSS.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsSDSS.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsSDSS.c (revision 30631) @@ -159,6 +159,6 @@ stars[N].measure.Xccd = colc[N]; stars[N].measure.Yccd = rowc[N]; - stars[N].measure.dXccd = ShortPixels(colcErr[N]); - stars[N].measure.dYccd = ShortPixels(rowcErr[N]); + stars[N].measure.dXccd = ToShortPixels(colcErr[N]); + stars[N].measure.dYccd = ToShortPixels(rowcErr[N]); stars[N].measure.M = psfCounts[N] + ZeroPt - zeropt[j]; stars[N].measure.dM = psfCountsErr[N]; @@ -166,6 +166,6 @@ stars[N].measure.Sky = sky[N]; // adjust this to counts? stars[N].measure.dSky = skyErr[N]; - stars[N].measure.FWx = ShortPixels(seeing[j]); // reported in arcsec? - stars[N].measure.FWy = ShortPixels(seeing[j]); + stars[N].measure.FWx = ToShortPixels(seeing[j]); // reported in arcsec? + stars[N].measure.FWy = ToShortPixels(seeing[j]); if (prob_psf) { stars[N].measure.psfChisq = prob_psf[N]; // XXX not really the correct value... Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsTEXT.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsTEXT.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/ReadStarsTEXT.c (revision 30631) @@ -95,7 +95,7 @@ dparse (&df, 11, &buffer[j*BYTES_STAR]); - stars[N].measure.FWx = ShortPixels (fx); - stars[N].measure.FWy = ShortPixels (fy); - stars[N].measure.theta = ShortDegree (df); + stars[N].measure.FWx = ToShortPixels (fx); + stars[N].measure.FWy = ToShortPixels (fy); + stars[N].measure.theta = ToShortDegrees (df); } } Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/StarOps.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/StarOps.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/StarOps.c (revision 30631) @@ -9,20 +9,2 @@ return TRUE; } - -short ShortPixels (float pixels) { - - short value; - - value = 100*pixels; - - return value; -} - -short ShortDegree (float degrees) { - - short value; - - value = (0xffff/360.0)*degrees; - - return value; -} Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/calibrate.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/calibrate.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/calibrate.c (revision 30631) @@ -225,10 +225,10 @@ image[0].Mcal = M1; image[0].dMcal = M2 / sqrt (N); - image[0].Mxxxx = N; + image[0].nFitPhotom = N; } else { fprintf (stderr, "too few stars\n"); image[0].Mcal = 10.000; image[0].dMcal = 10.000; - image[0].Mxxxx = 0; - } -} + image[0].nFitPhotom = 0; + } +} Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/fakeimage.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/fakeimage.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/fakeimage.c (revision 30631) @@ -134,5 +134,4 @@ image[i+1].nstar = 0; - image[i+1].Myyyy = 0; /* check if chip hits outer bounds of mosaic */ @@ -191,5 +190,4 @@ image[0].flags = 0; image[0].nstar = 0; - image[0].Myyyy = 0; // XXX need to set the imageID here Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/get2mass_full.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/get2mass_full.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/get2mass_full.c (revision 30631) @@ -17,9 +17,9 @@ ptr = next2MASSfield (ptr); // dec (assumed to be already set) ptr = next2MASSfield (ptr); // err_maj - star[0][0].measure.FWx = ShortPixels(strtod (ptr, NULL)); + star[0][0].measure.FWx = ToShortPixels(strtod (ptr, NULL)); ptr = next2MASSfield (ptr); // err_min - star[0][0].measure.FWy = ShortPixels(strtod (ptr, NULL)); + star[0][0].measure.FWy = ToShortPixels(strtod (ptr, NULL)); ptr = next2MASSfield (ptr); // err_ang - star[0][0].measure.theta = ShortPixels(strtod (ptr, NULL)); + star[0][0].measure.theta = ToShortPixels(strtod (ptr, NULL)); star[2][0].measure.FWx = star[2][0].measure.FWx = star[0][0].measure.FWx; Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/resort_catalog.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/resort_catalog.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/resort_catalog.c (revision 30631) @@ -39,5 +39,5 @@ tmp_ave = AVE[A]; AVE[A] = AVE[B]; AVE[B] = tmp_ave; \ } -# define COMPARE(A,B)(MEAS[A] < MEAS[B]) +# define COMPARE(A,B)(AVE[A] < AVE[B]) OHANA_SORT (N, COMPARE, SWAPFUNC); # undef SWAPFUNC @@ -96,6 +96,22 @@ } + // check that averageSeq is now in order + // for (i = 1; i < Nmeasure; i++) { + // if (averageSeq[i] < averageSeq[i-1]) { + // fprintf (stderr, "%d ", (int) i); + // } + // } + // fprintf (stderr, "\n"); + SortAveMeasMatch(measureSeq, averageSeq, Nmeasure); // MARKTIME("sort : %f sec\n", dtime); + + // check that averageSeq is now in order + // for (i = 1; i < Nmeasure; i++) { + // if (averageSeq[i] < averageSeq[i-1]) { + // fprintf (stderr, "%d ", (int) i); + // } + // } + // fprintf (stderr, "\n"); // copy the measurements in the sorted order @@ -116,4 +132,5 @@ for (i = 0; i < Nmeasure; i++) { if (averageSeq[i] != currentAve) { + // we have hit the next entry in the list average[currentAve].Nmeasure = N; N = 0; @@ -127,5 +144,32 @@ // MARKTIME("update Nmeasure : %f sec\n", dtime); + int NmeasureTotal = 0; + int measureOffsetOK = TRUE; + for (i = 0; i < catalog[0].Naverage; i++) { + NmeasureTotal += catalog[0].average[i].Nmeasure; + if (VERBOSE && !(NmeasureTotal <= catalog[0].Nmeasure)) { + fprintf (stderr, "too few measurements: %d %d %d\n", (int) i, NmeasureTotal, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset < catalog[0].Nmeasure); + if (VERBOSE && !(catalog[0].average[i].measureOffset < catalog[0].Nmeasure)) { + fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure); + if (VERBOSE && !(catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure)) { + fprintf (stderr, "orrset + Nmeasure too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].measureOffset, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + } + + if (!measureOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog[0].filename); + } + + if (NmeasureTotal != catalog[0].Nmeasure) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog[0].filename); + } + // MARKTIME(" match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeasure, (long long) Naverage); + + catalog[0].sorted = TRUE; FREE (measureSeq); Index: /branches/czw_branch/20101203/Ohana/src/addstar/src/sky_tessalation.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/addstar/src/sky_tessalation.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/addstar/src/sky_tessalation.c (revision 30631) @@ -338,7 +338,8 @@ image[0].photcode = 1; // this needs to be set more sensibly - image[0].Mx = xv[0]; image[0].My = yv[0]; - image[0].Mxxx = xv[1]; image[0].Mxyy = yv[1]; - image[0].Mxxy = xv[2]; image[0].Myyy = yv[2]; + // XXX these overload these value in a silly way + image[0].dXpixSys = xv[0]; image[0].dYpixSys = yv[0]; + image[0].dMagSys = xv[1]; image[0].nFitAstrom = yv[1]; + image[0].photom_map_id = xv[2]; image[0].astrom_map_id = yv[2]; return (TRUE); Index: /branches/czw_branch/20101203/Ohana/src/dvomerge/doc/usage.txt =================================================================== --- /branches/czw_branch/20101203/Ohana/src/dvomerge/doc/usage.txt (revision 30631) +++ /branches/czw_branch/20101203/Ohana/src/dvomerge/doc/usage.txt (revision 30631) @@ -0,0 +1,121 @@ + +dvomerge + +dvomerge is used to merge dvo databases. there are two main modes of operation: + +* merge two databases into a third, previously absent, database: dvomerge (input1) and (intput2) to (output) +* merge one database into an existing database : dvomerge (input) into (output) + +Note that all dvomerge options may include the -region (RAmin) (RAmax) (DECmin) (DECmax) options + to restrict the merge to a portion of the sky. + +The second mode of operation has two minor variants: + +* dvomerge (input) into (output) continue + the 'continue' option allows the dvomerge process to operate on a + fraction of the sky, in the assumption that an initial merge was + performed (with the 'into' mode). The 'continue' is needed to tell + dvomerge that the images of the two databases have already been + merged; in this case, dvomerge will examine the image tables and + determine the mapping of image IDs from the state of the output + database. + +* dvomerge (input) into (output) from (list) + the 'from' variant is equivalent to the 'continue' variant, but in + this case the catalog tables to be merged are specified in the list + file. The file contains one catalog file per line, with the name + giving the directory and filename of the cpt files, without the + catdir component (eg, n0000/0001.01.cpt). + +Threads : threading support is available for dvomerge .. into, except +for the 'from' mode. Add the option '-threads N' to specify the +number of threads to be used. Each thread handles a separate catalog +table (read from disk, calculate the merge, write to disk). With only +a few threads, it is easy to saturate the disk I/O system, at which +point the threading does not improve the speed of the merge process. + +Some other words of warning: + +* dvomerge does not attempt to check for previous merges (successful + or not). A re-run of dvomerge will simply duplicate the data + previously merged. You must track your own results: which cpt files + were merged in a given run. + +* the region option selects the cpt files which touch the region. to + select exactly the cpt files you want, reduce the region by a small + amount to avoid overlapping neighboring regions. For example, if + you want to include only a specific cpt file with RA,DEC bounds of + 0.0 - 1.0, 2.5 - 3.5, you might use -region 0.01 0.99 2.51 3.49 + +* the cpt headers define their boundaries (RA0, RA1, DEC0, DEC1). + +* the merged dvo database is not sorted (indexed). To use it with the + dvo shell or programs like relphot or relastro, it is necessary to + re-index with addstar -resort -D CATDIR (catdir). (Note that + addstar -resort is also now able to use multiple threads: add the + -threads N option). + +dvorepair + +dvorepair can be used to fix damaged dvo databases. Repair is +possible because the measurement tables (cpm) and the image table +together contain all of the information needed to regenerate the +average (cpt) and secfilt (cps) tables. + +dvorepair has several modes of operation: + +* dvorepair -images-vs-measures (catdir) (Ntol) + + this mode scans the database and compares the number of detections + tied to a specific image (in the measure tables) with the number of + detections claimed by that image (in the average table). Images + which have too few detections are reported to standard out. The + value of Ntol specifies the number of detections which can be + dropped -- a few missing detections for a given image is not + catastrophic. + +* dvorepair -delete-image-list (catdir) (list) + + this mode accepts the list of bad images as reported by the + -images-vs-measures mode. It then scans the database and deletes + all detections associated with the deleted images. It also repairs + the average (cpt) and secfilt (cps) tables so they are consistent + with the repaired measurement tables. The repaired output tables do + not overwrite the existing tables; instead they are written with the + suffix ".fixed". It is up to the user to find and replace the + original files with the .fixed version. + + Note that the user may edit the output list from -images-vs-measures + and choose not to delete images as desired. But be careful: the + format of the lines of the input file must match the original output + format. + + Note that a repaired database is not indexed: you must run addstar + -resort to re-index the database. + +* dvorepair -fix-images (catdir) (list) + + This mode accepts the same for list file as -delete-image-list, and + in this case, deletes the images from the image table (as opposed to + the detections). + +* dvorepair -fix-cpt (images) (list) + + This mode accepts a list of corrupted cpt files and rebuilds the + average and secfilt tables from the measurements. Instead of a + (catdir), the inputs are the full path or relative path to the image + table and a list of the full or relative paths to the cpt files + which are broken, ** with the .cpt extension removed**. + + Again, repaired cpt, cps, cpm files are saved with the extension .fixed. + +dvoverify + +dvoverify will scan a dvo database or subset (by region) and verify +the database integrity. At the moment, the only check it makes is to +verify that the tables are FITS compliant. (ie, no check is made for +internal consistency of the number of average, measure, or secfilt +entries). + + + Index: /branches/czw_branch/20101203/Ohana/src/dvomerge/src/dvoverify.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/dvomerge/src/dvoverify.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/dvomerge/src/dvoverify.c (revision 30631) @@ -9,6 +9,9 @@ int VerifyTableFile (char *filename); +int CheckCatalogIndexes (char *catdir, char *filename, SkyRegion *region); # define DEBUG 0 + +int VERBOSE; int main (int argc, char **argv) { @@ -22,4 +25,10 @@ SkyRegion UserPatch; // Catalog catalog; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } // restrict to a portion of the sky @@ -83,6 +92,6 @@ if (i % 1000 == 0) fprintf (stderr, "."); - sprintf (filename, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name); - if (!VerifyTableFile (filename)) { + // sprintf (filename, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name); + if (!VerifyTableFile (inlist[0].filename[i])) { Nbad ++; } @@ -95,4 +104,8 @@ sprintf (filename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name); if (!VerifyTableFile (filename)) { + Nbad ++; + } + + if (!CheckCatalogIndexes(catdir, inlist[0].filename[i], inlist[0].regions[i])){ Nbad ++; } @@ -212,6 +225,125 @@ } - // gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); - // gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows); +// CheckCatalogIndexes(catdir, inlist[0].regions[i][0].name); + +int CheckCatalogIndexes (char *catdir, char *filename, SkyRegion *region) { + + Catalog catalog; + int i, j, m, status; + + status = TRUE; + + // set the parameters which guide catalog open/load/create + catalog.filename = filename; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = LOAD_AVES | LOAD_MEAS; + catalog.Nsecfilt = 0; + + // an error exit status here is a significant error (disk I/O or file access) + if (!dvo_catalog_open (&catalog, region, VERBOSE, "r")) { + fprintf (stderr, "ERROR: failure to open catalog file %s\n", catalog.filename); + return FALSE; + } + + // Naves_disk == 0 implies an empty catalog file, skip empty catalogs + if (catalog.Naves_disk == 0) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + return TRUE; + } + + // if the table is SORTED, then the following can be checked + // check the following: + // measure[j].averef -> average[averef] + // measure[j].objID = average[averef].objID + // measure[j].catID = average[averef].catID + // measure[j].measureOffset < Nmeasure + // \sum average[].Nmeasure = Nmeasure + + // if the table is NOT SORTED, we have a subset of checks we can make + if (!catalog.sorted) { + fprintf (stderr, "!"); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + return TRUE; + } + + int NmeasureTotal = 0; + int measureOffsetOK = TRUE; + for (i = 0; i < catalog.Naverage; i++) { + NmeasureTotal += catalog.average[i].Nmeasure; + if (VERBOSE && !(NmeasureTotal <= catalog.Nmeasure)) { + fprintf (stderr, "NmeasureTotal > catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure); + } + measureOffsetOK &= (catalog.average[i].measureOffset < catalog.Nmeasure); + if (VERBOSE && !(catalog.average[i].measureOffset < catalog.Nmeasure)) { + fprintf (stderr, "measureOffset >= catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].measureOffset, (int) catalog.Nmeasure); + } + measureOffsetOK &= (catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure); + if (VERBOSE && !(catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure)) { + fprintf (stderr, "measureOffset + Nmeasure > catalog.Nmeasure : %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure); + } + } + + if (!measureOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog.filename); + status = FALSE; + } + + if (NmeasureTotal != catalog.Nmeasure) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog.filename); + status = FALSE; + } + + if (!status) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + return (status); + } + + int objIDsOK = TRUE; + int catIDsOK = TRUE; + int averefOK = TRUE; + + for (i = 0; i < catalog.Naverage; i++) { + m = catalog.average[i].measureOffset; + for (j = 0; j < catalog.average[i].Nmeasure; j++) { + objIDsOK &= (catalog.average[i].objID == catalog.measure[m+j].objID); + catIDsOK &= (catalog.average[i].catID == catalog.measure[m+j].catID); + averefOK &= (catalog.measure[m+j].averef == i); + } + } + + if (!objIDsOK) { + fprintf (stderr, "ERROR: catalog %s has invalid obj IDs\n", catalog.filename); + status = FALSE; + } + if (!catIDsOK) { + fprintf (stderr, "ERROR: catalog %s has invalid cat IDs\n", catalog.filename); + status = FALSE; + } + if (!averefOK) { + fprintf (stderr, "ERROR: catalog %s has invalid averef values\n", catalog.filename); + status = FALSE; + } + +// for (i = 0; i < catalog.Naverage; i++) { +// m = catalog.average[i].measureOffset; +// for (j = 0; i < catalog.Nmeasure; i++) { +// objIDsOK &= (catalog.average[i].objID == catalog.measure[m+j].objID); +// catIDsOK &= (catalog.average[i].catID == catalog.measure[m+j].catID); +// averefOK &= (catalog.measure[m+j].averef = i); +// } +// } + + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + + return status; +} + +// gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow); +// gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows); Index: /branches/czw_branch/20101203/Ohana/src/dvomerge/src/help.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/dvomerge/src/help.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/dvomerge/src/help.c (revision 30631) @@ -37,5 +37,6 @@ fprintf (stderr, " dvomerge (input) into (output)\n"); fprintf (stderr, " dvomerge (input) into (output) continue\n\n"); - fprintf (stderr, " dvomerge (input) into (output) from (list)\n\n"); + fprintf (stderr, " dvomerge (input) into (output) from (list)\n"); + fprintf (stderr, " dvomerge list implies 'continue' : list contains, eg, n1500/1688.00.cpt (one cpt per line)\n\n"); fprintf (stderr, " merge DVO databases\n"); fprintf (stderr, " optional flags:\n"); Index: /branches/czw_branch/20101203/Ohana/src/getstar/src/select_by_image.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/getstar/src/select_by_image.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/getstar/src/select_by_image.c (revision 30631) @@ -38,7 +38,7 @@ stars[N].Mgal = 0.001*(catalog[0].measure[i].Mgal - catalog[0].measure[i].dt); - stars[N].fx = 0.01*catalog[0].measure[i].FWx; - stars[N].fy = stars[N].fx * 0.01*catalog[0].measure[i].fwy; - stars[N].df = (360.0/255.0)*catalog[0].measure[i].theta; + stars[N].fx = FromShortPixels(catalog[0].measure[i].FWx); + stars[N].fy = FromShortPixels(catalog[0].measure[i].FWy); + stars[N].df = FromShortDegrees(catalog[0].measure[i].theta); stars[N].found = catalog[0].measure[i].flags; Index: /branches/czw_branch/20101203/Ohana/src/imregister/imphot/dumpfits.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/imregister/imphot/dumpfits.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/imregister/imphot/dumpfits.c (revision 30631) @@ -118,5 +118,5 @@ XY_to_RD (&ra, &dec, 0.0, 0.0, &subset[0].coords); airmass = subset[0].secz; - sky = subset[0].Myyyy + 0x8000; + sky = 0.0; // subset[0].Myyyy + 0x8000; /* we should get an error here if we don't construct this line correctly */ Index: /branches/czw_branch/20101203/Ohana/src/kapa2/include/prototypes.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/kapa2/include/prototypes.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/kapa2/include/prototypes.h (revision 30631) @@ -175,5 +175,5 @@ void DrawBitmap PROTO((Graphic *graphic, int x, int y, int dx, int dy, char *bitmap, int mode)); void CrossHairs PROTO((Graphic *graphic, Picture *image)); -void hh_hms PROTO((char *line, double ra, double dec, char sep)); +void hh_hms PROTO((char *line, double ra, double dec, char sep, int Nchar)); int SetColormap PROTO((char *name)); Index: /branches/czw_branch/20101203/Ohana/src/kapa2/src/PSObjects.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/kapa2/src/PSObjects.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/kapa2/src/PSObjects.c (revision 30631) @@ -164,5 +164,5 @@ float *x, *y; double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; - double sx0, sy0, sx1, sy1, sxa; + double sx0, sy0, sx1, sy1, sxa, sya, sxo, syo; double X0, X1, Y0, Y1; @@ -189,4 +189,58 @@ for (i = 0; (i < object[0].Npts) && !(finite(x[i]) && finite(y[i])); i++); if (i >= object[0].Npts) return; + + /* first valid data point */ + sx0 = x[i]*mxi + y[i]*mxj + bx; + sy0 = x[i]*myi + y[i]*myj + by; + sx0 = MIN (MAX (sx0, X0), X1); + sy0 = MAX (MIN (sy0, Y0), Y1); + + /* find the second valid datapoint */ + for (i++; (i < object[0].Npts) && !(finite(x[i]) && finite(y[i])); i++); + if (i >= object[0].Npts) return; + + /* second valid data point */ + sx1 = x[i]*mxi + y[i]*mxj + bx; + sy1 = x[i]*myi + y[i]*myj + by; + sx1 = MIN (MAX (sx1, X0), X1); + sy1 = MAX (MIN (sy1, Y0), Y1); + + /* connect first point to second point */ + sxa = sx0 - 0.5*(sx1 - sx0); + sya = MAX (sy0, Y0); + DrawLine (sx0, sy0, sxa, sy0); + DrawLine (sxa, sy0, sxa, sya); + + /* draw segment equal distance behind first point and down to x-axis */ + sxa = 0.5*(sx0 + sx1); + DrawLine (sx0, sy0, sxa, sy0); + DrawLine (sxa, sy0, sxa, sy1); + DrawLine (sxa, sy1, sx1, sy1); + sx0 = sx1; + sy0 = sy1; + + /* continue with rest of points */ + sxo = syo = 0; + for (i++; i < object[0].Npts; i++) { + if (!(finite(x[i]) && finite(y[i]))) continue; + sx1 = x[i]*mxi + y[i]*mxj + bx; + sy1 = x[i]*myi + y[i]*myj + by; + sx1 = MIN (MAX (sx1, X0), X1); + sy1 = MAX (MIN (sy1, Y0), Y1); + sxa = 0.5*(sx0 + sx1); + DrawLine (sx0, sy0, sxa, sy0); + DrawLine (sxa, sy0, sxa, sy1); + DrawLine (sxa, sy1, sx1, sy1); + sxo = sx0; syo = sy0; + sx0 = sx1; sy0 = sy1; + } + + /* draw segment equal distance after last point and down to x-axis */ + sxa = sx1 + 0.5*(sx1 - sxo); + sya = MAX (sy1, Y0); + DrawLine (sx1, sy1, sxa, sy1); + DrawLine (sxa, sy1, sxa, sya); + +# if (0) sx0 = x[i]*mxi + y[i]*mxj + bx; sy0 = x[i]*myi + y[i]*myj + by; @@ -207,4 +261,5 @@ sx0 = sx1; sy0 = sy1; } +# endif } Index: /branches/czw_branch/20101203/Ohana/src/kapa2/src/UpdateStatusBox.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/kapa2/src/UpdateStatusBox.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/kapa2/src/UpdateStatusBox.c (revision 30631) @@ -20,5 +20,5 @@ bzero (line, 100); - sprintf (line, "(%d x %d) @ %d ", + snprintf (line, 100, "(%d x %d) @ %d ", image[0].picture.dx, image[0].picture.dy, image[0].picture.expand); XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, @@ -26,10 +26,10 @@ bzero (line, 100); - sprintf (line, "%-25s", image[0].image[0].file); + snprintf (line, 100, "%-25s", image[0].image[0].file); XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, image[0].text_x + PAD1, image[0].text_y + 5*textpad + 5*PAD1, line, strlen(line)); bzero (line, 100); - sprintf (line, "(%s) ", image[0].image[0].name); + snprintf (line, 100, "(%s) ", image[0].image[0].name); XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, image[0].text_x + PAD1, image[0].text_y + 6*textpad + 6*PAD1, line, 25); @@ -45,7 +45,7 @@ if (image[0].HexValue) { - sprintf (line, "%04x", (int) z); + snprintf (line, 100, "%04x", (int) z); } else { - sprintf (line, "%22.3f", z); + snprintf (line, 100, "%22.3f", z); } XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, @@ -53,5 +53,5 @@ bzero (line, 100); - sprintf (line, "%10.2f %10.2f", x, y); + snprintf (line, 100, "%10.2f %10.2f", x, y); XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, image[0].text_x + PAD1, image[0].text_y + 2*textpad + 2*PAD1, line, strlen(line)); @@ -59,8 +59,9 @@ bzero (line, 100); if (image[0].DecimalDegrees) { - sprintf (line, "%10.6f %10.6f", ra, dec); + snprintf (line, 100, "%10.6f %10.6f", ra, dec); } else { - hh_hms (line, ra, dec, ':'); + hh_hms (line, ra, dec, ':', 100); } + XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, image[0].text_x + PAD1, image[0].text_y + 3*textpad + 3*PAD1, line, strlen(line)); Index: /branches/czw_branch/20101203/Ohana/src/kapa2/src/bDrawObjects.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/kapa2/src/bDrawObjects.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/kapa2/src/bDrawObjects.c (revision 30631) @@ -154,5 +154,5 @@ float *x, *y; double mxi, mxj, myi, myj, bxi, bxj, byi, byj, bx, by; - double sx0, sy0, sx1, sy1, sxa; + double sx0, sy0, sx1, sy1, sxa, sya, sxo, syo; double X0, X1, Y0, Y1; @@ -179,4 +179,58 @@ for (i = 0; (i < object[0].Npts) && !(finite(x[i]) && finite(y[i])); i++); if (i >= object[0].Npts) return; + + /* first valid data point */ + sx0 = x[i]*mxi + y[i]*mxj + bx; + sy0 = x[i]*myi + y[i]*myj + by; + sx0 = MIN (MAX (sx0, X0), X1); + sy0 = MAX (MIN (sy0, Y0), Y1); + + /* find the second valid datapoint */ + for (i++; (i < object[0].Npts) && !(finite(x[i]) && finite(y[i])); i++); + if (i >= object[0].Npts) return; + + /* second valid data point */ + sx1 = x[i]*mxi + y[i]*mxj + bx; + sy1 = x[i]*myi + y[i]*myj + by; + sx1 = MIN (MAX (sx1, X0), X1); + sy1 = MAX (MIN (sy1, Y0), Y1); + + /* connect first point to second point */ + sxa = sx0 - 0.5*(sx1 - sx0); + sya = MAX (sy0, Y0); + DrawLine (buffer, sx0, sy0, sxa, sy0); + DrawLine (buffer, sxa, sy0, sxa, sya); + + /* draw segment equal distance behind first point and down to x-axis */ + sxa = 0.5*(sx0 + sx1); + DrawLine (buffer, sx0, sy0, sxa, sy0); + DrawLine (buffer, sxa, sy0, sxa, sy1); + DrawLine (buffer, sxa, sy1, sx1, sy1); + sx0 = sx1; + sy0 = sy1; + + /* continue with rest of points */ + sxo = syo = 0; + for (i++; i < object[0].Npts; i++) { + if (!(finite(x[i]) && finite(y[i]))) continue; + sx1 = x[i]*mxi + y[i]*mxj + bx; + sy1 = x[i]*myi + y[i]*myj + by; + sx1 = MIN (MAX (sx1, X0), X1); + sy1 = MAX (MIN (sy1, Y0), Y1); + sxa = 0.5*(sx0 + sx1); + DrawLine (buffer, sx0, sy0, sxa, sy0); + DrawLine (buffer, sxa, sy0, sxa, sy1); + DrawLine (buffer, sxa, sy1, sx1, sy1); + sxo = sx0; syo = sy0; + sx0 = sx1; sy0 = sy1; + } + + /* draw segment equal distance after last point and down to x-axis */ + sxa = sx1 + 0.5*(sx1 - sxo); + sya = MAX (sy1, Y0); + DrawLine (buffer, sx1, sy1, sxa, sy1); + DrawLine (buffer, sxa, sy1, sxa, sya); + +# if (0) sx0 = x[i]*mxi + y[i]*mxj + bx; sy0 = x[i]*myi + y[i]*myj + by; @@ -197,4 +251,5 @@ sx0 = sx1; sy0 = sy1; } +# endif } Index: /branches/czw_branch/20101203/Ohana/src/kapa2/src/hh_hms.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/kapa2/src/hh_hms.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/kapa2/src/hh_hms.c (revision 30631) @@ -1,5 +1,5 @@ # include "Ximage.h" -void hh_hms (char *line, double ra, double dec, char sep) { +void hh_hms (char *line, double ra, double dec, char sep, int Nchar) { int h, m, flag; @@ -13,7 +13,7 @@ s = 3600*(ra - h - m / 60.0); if (flag > 0) - sprintf (line, " %02d%c%02d%c%04.1f ", h, sep, m, sep, s); + snprintf (line, Nchar, " %02d%c%02d%c%04.1f ", h, sep, m, sep, s); else - sprintf (line, "-%02d%c%02d%c%04.1f ", h, sep, m, sep, s); + snprintf (line, Nchar, "-%02d%c%02d%c%04.1f ", h, sep, m, sep, s); flag = SIGN(dec); @@ -23,6 +23,6 @@ s = 3600*(dec - h - m / 60.0); if (flag > 0) - sprintf (&line[13], " %02d%c%02d%c%04.1f", h, sep, m, sep, s); + snprintf (&line[13], Nchar, " %02d%c%02d%c%04.1f", h, sep, m, sep, s); else - sprintf (&line[13], "-%02d%c%02d%c%04.1f", h, sep, m, sep, s); + snprintf (&line[13], Nchar, "-%02d%c%02d%c%04.1f", h, sep, m, sep, s); } Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-elixir.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-elixir.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-elixir.d (revision 30631) @@ -43,17 +43,17 @@ FIELD ccdnum, CCDNUM, unsigned char, CCD ID number FIELD dummy, DUMMY, char[20], unused -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm + +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-loneos.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-loneos.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-loneos.d (revision 30631) @@ -43,17 +43,17 @@ FIELD ccdnum, CCDNUM, unsigned char, CCD ID number FIELD dummy, DUMMY, char[20], unused -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm + +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-panstarrs-dev-0.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-panstarrs-dev-0.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-panstarrs-dev-0.d (revision 30631) @@ -53,18 +53,16 @@ FIELD dummy, DUMMY, char[10], unused -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm -# 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-panstarrs-dev-1.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-panstarrs-dev-1.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-panstarrs-dev-1.d (revision 30631) @@ -53,18 +53,16 @@ FIELD dummy, DUMMY, char[10], unused -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm -# 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-1.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-1.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-1.d (revision 30631) @@ -49,21 +49,20 @@ FIELD ccdnum, CCDNUM, unsigned char, CCD ID number FIELD imageID, IMAGE_ID, unsigned int, image ID +FIELD dummy, DUMMY, char[14], unused # 44 bytes -FIELD dummy, DUMMY, char[14], unused -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm # 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-2.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-2.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-2.d (revision 30631) @@ -53,18 +53,16 @@ # 48 bytes -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm -# 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-3.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-3.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-dev-3.d (revision 30631) @@ -54,18 +54,16 @@ # 48 bytes -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm -# 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-ref.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-ref.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-ref.d (revision 30631) @@ -55,18 +55,18 @@ # 48 bytes -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm # 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases + Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-v1.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-v1.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-v1.d (revision 30631) @@ -55,18 +55,16 @@ # 48 bytes -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm -# 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-v2.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-v2.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image-ps1-v2.d (revision 30631) @@ -60,18 +60,16 @@ # 48 bytes -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm -# 40 bytes +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/image.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/image.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/image.d (revision 30631) @@ -1,3 +1,3 @@ -STRUCT Image + STRUCT Image EXTNAME DVO_IMAGE TYPE BINTABLE @@ -6,4 +6,5 @@ # elements of the image structure +# careful of 8-byte boundaries SUBSTRUCT coords, COORDS, Coords, astrometric data @@ -60,21 +61,38 @@ # 48 bytes -FIELD order, ORDER, short, Mrel 2D polynomical order -FIELD Mx, MX, short, Mrel polyterm -FIELD My, MY, short, Mrel polyterm -FIELD Mxx, MXX, short, Mrel polyterm -FIELD Mxy, MXY, short, Mrel polyterm -FIELD Myy, MYY, short, Mrel polyterm -FIELD Mxxx, MXXX, short, Mrel polyterm -FIELD Mxxy, MXXY, short, Mrel polyterm -FIELD Mxyy, MXYY, short, Mrel polyterm -FIELD Myyy, MYYY, short, Mrel polyterm -FIELD Mxxxx, MXXXX, short, Mrel polyterm -FIELD Mxxxy, MXXXY, short, Mrel polyterm -FIELD Mxxyy, MXXYY, short, Mrel polyterm -FIELD Mxyyy, MXYYY, short, Mrel polyterm -FIELD Myyyy, MYYYY, short, Mrel polyterm +FIELD nLinkAstrom, NLINK_ASTROM, short, mean number of matched measurements for astrometry +FIELD nLinkPhotom, NLINK_PHOTOM, short, mean number of matched measurements for astrometry +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map +# nFitPhotom lands on the old location of Mxxxx, which was used to mean nFitPhotom in some cases + +# old image structure: +# FIELD order, ORDER, short, Mrel 2D polynomical order +# FIELD Mx, MX, short, Mrel polyterm +# FIELD My, MY, short, Mrel polyterm +# FIELD Mxx, MXX, short, Mrel polyterm +# FIELD Mxy, MXY, short, Mrel polyterm +# FIELD Myy, MYY, short, Mrel polyterm +# FIELD Mxxx, MXXX, short, Mrel polyterm +# FIELD Mxxy, MXXY, short, Mrel polyterm +# FIELD Mxyy, MXYY, short, Mrel polyterm +# FIELD Myyy, MYYY, short, Mrel polyterm +# FIELD Mxxxx, MXXXX, short, Mrel polyterm +# FIELD Mxxxy, MXXXY, short, Mrel polyterm +# FIELD Mxxyy, MXXYY, short, Mrel polyterm +# FIELD Mxyyy, MXYYY, short, Mrel polyterm +# FIELD Myyyy, MYYYY, short, Mrel polyterm # 40 bytes # *** 20090206 : new fields : parentID, flags (was code char), changed name to 121 bytes. -# *** 20100331 : new fields : +# *** 20100331 : new fields : RAo, DECo. Radius +# *** 20110203 : replace the old zero point polynomial terms (Mx,My,... Mxxxx,Myyyy) with dummy1 - astrom_map_id Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/measure-ps1-v2.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/measure-ps1-v2.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/measure-ps1-v2.d (revision 30631) @@ -63,6 +63,7 @@ FIELD dXccd, X_CCD_ERR, short, X coord error on chip, 1/100 of pixels FIELD dYccd, Y_CCD_ERR, short, Y coord error on chip, 1/100 of pixels +FIELD dRsys, POS_SYS_ERR, short, systematic error from astrom, 1/100 of pixels -FIELD pad, PAD, char[6], padding +FIELD pad, PAD, char[4], padding # local astrometry scales Index: /branches/czw_branch/20101203/Ohana/src/libautocode/def/measure.d =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libautocode/def/measure.d (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libautocode/def/measure.d (revision 30631) @@ -2,5 +2,5 @@ EXTNAME DVO_MEASURE TYPE BINTABLE -SIZE 146 +SIZE 148 DESCRIPTION DVO Detection Measurement Table @@ -59,4 +59,5 @@ FIELD dXccd, X_CCD_ERR, short, X coord error on chip, 1/100 of pixels FIELD dYccd, Y_CCD_ERR, short, Y coord error on chip, 1/100 of pixels +FIELD dRsys, POS_SYS_ERR, short, systematic error from astrom, 1/100 of pixels FIELD posangle, POSANGLE, short, position angle sky to chip, (0xffff/360) deg Index: /branches/czw_branch/20101203/Ohana/src/libdvo/Makefile =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/Makefile (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/Makefile (revision 30631) @@ -51,5 +51,4 @@ $(SRC)/mosaic_astrom.$(ARCH).o \ $(SRC)/fits_db.$(ARCH).o \ -$(SRC)/photfits.$(ARCH).o \ $(SRC)/dvo_image.$(ARCH).o \ $(SRC)/dvo_image_raw.$(ARCH).o \ @@ -77,6 +76,7 @@ $(SRC)/dvo_util.$(ARCH).o -# $(SRC)/dvo_convert_panstarrs.$(ARCH).o \ -# $(SRC)/dvo_convert_pmtest.$(ARCH).o \ +# $(SRC)/dvo_convert_panstarrs.$(ARCH).o +# $(SRC)/dvo_convert_pmtest.$(ARCH).o +# $(SRC)/photfits.$(ARCH).o include ../libautocode/Makefile.Targets Index: /branches/czw_branch/20101203/Ohana/src/libdvo/doc/dvo-images.txt =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/doc/dvo-images.txt (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/doc/dvo-images.txt (revision 30631) @@ -1,7 +1,90 @@ -the dvo image table is not extremely efficient. it is unsorted, and -spatial searches consist of complete scans of the table. +20110203 -a spatial index could consist of five cardinal points for each square -image (4 corners and the center) or a central point and max radius. -table would consist each would be + The Image table (PS1_V2) has a bunch of fields that are not really + used by anything / much. I would like to re-use those bytes to + store information about the quality of the astrometric and + photometric calibrations. + + The under-used fields are the 'photometric zero point variation' + polynomial terms (Mx, My, ... Mxxxx, Myyyy). + + (I list the sourceID, which does not change, because the word-boundaries matter) +FIELD sourceID, SOURCE_ID, unsigned short, analysis source ID +FIELD order, ORDER, short, Mrel 2D polynomical order +FIELD Mx, MX, short, Mrel polyterm +FIELD My, MY, short, Mrel polyterm + +FIELD Mxx, MXX, short, Mrel polyterm +FIELD Mxy, MXY, short, Mrel polyterm +FIELD Myy, MYY, short, Mrel polyterm +FIELD Mxxx, MXXX, short, Mrel polyterm + +FIELD Mxxy, MXXY, short, Mrel polyterm +FIELD Mxyy, MXYY, short, Mrel polyterm +FIELD Myyy, MYYY, short, Mrel polyterm +FIELD Mxxxx, MXXXX, short, Mrel polyterm + +FIELD Mxxxy, MXXXY, short, Mrel polyterm +FIELD Mxxyy, MXXYY, short, Mrel polyterm +FIELD Mxyyy, MXYYY, short, Mrel polyterm +FIELD Myyyy, MYYYY, short, Mrel polyterm +(30 bytes) + + I am going to replace these with: + + photom_map_id (int) : index to a 2D map of the zp variations (table and format not yet defined) + astrom_map_id (int) : index to a 2D map of the zp variations (table and format not yet defined) + dMagSys (float) : systematic photometric error (mag) + dXpixSys (float) : systematic astrometric error (pix) + dYpixSys (float) : systematic astrometric error (pix) + nFitAstrom (short) : number of stars used for astrometric calibration (saturate at 0x7fff) + nFitPhotom (short) : number of stars used for astrometric calibration (saturate at 0x7fff) +(24 bytes) + + As for the existing uses of these fields: + + The official use can just be dropped (replaced by the maps and + recalculated if needed -- no db actually uses them). There are a + few places in the code where these fields were overloaded: + + Triangular Images : in the skycell creation code, there is an option + to create images which are the triangular projection centers for a + given subdivided geometric solid. This code was overloading the Mx, + My, and Mxxx through Myyy fields. I can invalidate any existing db + with triangles (probably none are needed) and repurpose some of the + fields above for these values. (addstar/src/sky_tesslation.c, opihi/dvo/images.c) + + Mxxxx : some code was using Mxxxx to store nFitPhotom. Again, any + existing databases are unlikely to use that info (CFHT skyprobe + only). If I am clever, I can align the existing bytes so that the + old Mxxxx hits the new nFitPhotom. (addstar/src/calibrate.c, + dbExtractImages.c) + + Myyyy : dbExtractImages.c expects this value to have (sky - 0x8000), + but that value is not actually set by any code. Ignore (replace?) + +FIELD sourceID, SOURCE_ID, unsigned short, analysis source ID +FIELD dummy1, DUMMY1, short, place holder for byte boundaries +FIELD dummy2, DUMMY2, short, place holder for byte boundaries +FIELD dummy3, DUMMY3, short, place holder for byte boundaries + +FIELD dXpixSys, XPIX_SYS_ERR, float, systematic astrometry error in X +FIELD dYpixSys, YPIX_SYS_ERR, float, systematic astrometry error in Y + +FIELD dMagSys, MAG_SYS_ERR, float, systematic photometry error +FIELD nFitAstrom, N_FIT_ASTROM, short, number of stars used for astrometry cal +FIELD nFitPhotom, N_FIT_PHOTOM, short, number of stars used for photometry cal + +FIELD photom_map_id, PHOTOM_MAP_ID, unsigned int, reference to 2D zero point map +FIELD astrom_map_id, ASTROM_MAP_ID, unsigned int, reference to 2D astrometry map + +older: + + the dvo image table is not extremely efficient. it is unsorted, and + spatial searches consist of complete scans of the table. + + a spatial index could consist of five cardinal points for each + square image (4 corners and the center) or a central point and max + radius. + Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 30631) @@ -8,5 +8,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -66,5 +66,5 @@ Measure_PS1_DEV_1 *out; - ALLOCATE (out, Measure_PS1_DEV_1, Nvalues); + ALLOCATE_ZERO (out, Measure_PS1_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -113,5 +113,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -157,5 +157,5 @@ Average_PS1_DEV_1 *out; - ALLOCATE (out, Average_PS1_DEV_1, Nvalues); + ALLOCATE_ZERO (out, Average_PS1_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -191,5 +191,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -215,5 +215,5 @@ SecFilt_PS1_DEV_1 *out; - ALLOCATE (out, SecFilt_PS1_DEV_1, Nvalues); + ALLOCATE_ZERO (out, SecFilt_PS1_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -232,5 +232,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -263,19 +263,17 @@ out[i].imageID = in[i].imageID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_DEV_2 @@ -301,5 +299,5 @@ Image_PS1_DEV_1 *out; - ALLOCATE (out, Image_PS1_DEV_1, Nvalues); + ALLOCATE_ZERO (out, Image_PS1_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -332,19 +330,17 @@ out[i].imageID = in[i].imageID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V1 @@ -359,5 +355,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -398,5 +394,5 @@ PhotCode_PS1_DEV_1 *out; - ALLOCATE (out, PhotCode_PS1_DEV_1, Nvalues); + ALLOCATE_ZERO (out, PhotCode_PS1_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 30631) @@ -8,5 +8,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -63,5 +63,5 @@ Measure_PS1_DEV_2 *out; - ALLOCATE (out, Measure_PS1_DEV_2, Nvalues); + ALLOCATE_ZERO (out, Measure_PS1_DEV_2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -109,5 +109,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -151,5 +151,5 @@ Average_PS1_DEV_2 *out; - ALLOCATE (out, Average_PS1_DEV_2, Nvalues); + ALLOCATE_ZERO (out, Average_PS1_DEV_2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -185,5 +185,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -209,5 +209,5 @@ SecFilt_PS1_DEV_2 *out; - ALLOCATE (out, SecFilt_PS1_DEV_2, Nvalues); + ALLOCATE_ZERO (out, SecFilt_PS1_DEV_2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -226,5 +226,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -259,19 +259,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V1 @@ -292,5 +290,5 @@ Image_PS1_DEV_2 *out; - ALLOCATE (out, Image_PS1_DEV_2, Nvalues); + ALLOCATE_ZERO (out, Image_PS1_DEV_2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -326,19 +324,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V1 @@ -353,5 +349,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -390,5 +386,5 @@ PhotCode_PS1_DEV_2 *out; - ALLOCATE (out, PhotCode_PS1_DEV_2, Nvalues); + ALLOCATE_ZERO (out, PhotCode_PS1_DEV_2, Nvalues); for (i = 0; i < Nvalues; i++) { Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c (revision 30631) @@ -10,5 +10,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -43,19 +43,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V1 @@ -71,5 +69,5 @@ Image_PS1_DEV_3 *out; - ALLOCATE (out, Image_PS1_DEV_3, Nvalues); + ALLOCATE_ZERO (out, Image_PS1_DEV_3, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -104,19 +102,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V1 @@ -131,5 +127,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -167,5 +163,5 @@ PhotCode_PS1_DEV_3 *out; - ALLOCATE (out, PhotCode_PS1_DEV_3, Nvalues); + ALLOCATE_ZERO (out, PhotCode_PS1_DEV_3, Nvalues); for (i = 0; i < Nvalues; i++) { Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 30631) @@ -8,5 +8,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -61,5 +61,5 @@ Measure_PS1_REF *out; - ALLOCATE (out, Measure_PS1_REF, Nvalues); + ALLOCATE_ZERO (out, Measure_PS1_REF, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -84,5 +84,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -124,5 +124,5 @@ Average_PS1_REF *out; - ALLOCATE (out, Average_PS1_REF, Nvalues); + ALLOCATE_ZERO (out, Average_PS1_REF, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -147,5 +147,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -169,5 +169,5 @@ SecFilt_PS1_REF *out; - ALLOCATE (out, SecFilt_PS1_REF, Nvalues); + ALLOCATE_ZERO (out, SecFilt_PS1_REF, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -183,5 +183,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -218,19 +218,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V2 @@ -247,5 +245,5 @@ Image_PS1_REF *out; - ALLOCATE (out, Image_PS1_REF, Nvalues); + ALLOCATE_ZERO (out, Image_PS1_REF, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -282,19 +280,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; } return (out); @@ -306,5 +302,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -342,5 +338,5 @@ PhotCode_PS1_REF *out; - ALLOCATE (out, PhotCode_PS1_REF, Nvalues); + ALLOCATE_ZERO (out, PhotCode_PS1_REF, Nvalues); for (i = 0; i < Nvalues; i++) { Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 30631) @@ -8,5 +8,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -61,5 +61,5 @@ Measure_PS1_V1 *out; - ALLOCATE (out, Measure_PS1_V1, Nvalues); + ALLOCATE_ZERO (out, Measure_PS1_V1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -116,5 +116,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -156,5 +156,5 @@ Average_PS1_V1 *out; - ALLOCATE (out, Average_PS1_V1, Nvalues); + ALLOCATE_ZERO (out, Average_PS1_V1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -191,5 +191,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -213,5 +213,5 @@ SecFilt_PS1_V1 *out; - ALLOCATE (out, SecFilt_PS1_V1, Nvalues); + ALLOCATE_ZERO (out, SecFilt_PS1_V1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -232,5 +232,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -267,19 +267,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_V2 @@ -297,5 +295,5 @@ Image_PS1_V1 *out; - ALLOCATE (out, Image_PS1_V1, Nvalues); + ALLOCATE_ZERO (out, Image_PS1_V1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -333,19 +331,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; } return (out); @@ -357,5 +353,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -393,5 +389,5 @@ PhotCode_PS1_V1 *out; - ALLOCATE (out, PhotCode_PS1_V1, Nvalues); + ALLOCATE_ZERO (out, PhotCode_PS1_V1, Nvalues); for (i = 0; i < Nvalues; i++) { Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 30631) @@ -8,5 +8,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -47,4 +47,5 @@ out[i].dXccd = in[i].dXccd; out[i].dYccd = in[i].dYccd; + out[i].dRsys = in[i].dRsys; out[i].posangle = in[i].posangle; out[i].pltscale = in[i].pltscale; @@ -61,5 +62,5 @@ Measure_PS1_V2 *out; - ALLOCATE (out, Measure_PS1_V2, Nvalues); + ALLOCATE_ZERO (out, Measure_PS1_V2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -100,5 +101,5 @@ out[i].dXccd = in[i].dXccd; out[i].dYccd = in[i].dYccd; - memset(out[i].pad, 0, sizeof(out[i].pad)); + out[i].dRsys = in[i].dRsys; out[i].posangle = in[i].posangle; out[i].pltscale = in[i].pltscale; @@ -116,5 +117,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -156,5 +157,5 @@ Average_PS1_V2 *out; - ALLOCATE (out, Average_PS1_V2, Nvalues); + ALLOCATE_ZERO (out, Average_PS1_V2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -195,5 +196,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -215,5 +216,5 @@ SecFilt_PS1_V2 *out; - ALLOCATE (out, SecFilt_PS1_V2, Nvalues); + ALLOCATE_ZERO (out, SecFilt_PS1_V2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -235,5 +236,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -275,19 +276,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; } return (out); @@ -299,5 +298,5 @@ Image_PS1_V2 *out; - ALLOCATE (out, Image_PS1_V2, Nvalues); + ALLOCATE_ZERO (out, Image_PS1_V2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -339,19 +338,17 @@ out[i].sourceID = in[i].sourceID; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; } return (out); @@ -363,5 +360,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -399,5 +396,5 @@ PhotCode_PS1_V2 *out; - ALLOCATE (out, PhotCode_PS1_V2, Nvalues); + ALLOCATE_ZERO (out, PhotCode_PS1_V2, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -407,5 +404,4 @@ out[i].code = in[i].code; out[i].type = in[i].type; - memset(out[i].dummy, 0, sizeof(out[i].dummy)); out[i].C = in[i].C; out[i].dC = in[i].dC; Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_elixir.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 30631) @@ -7,5 +7,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -74,5 +74,5 @@ Measure_Elixir *out; - ALLOCATE (out, Measure_Elixir, Nvalues); + ALLOCATE_ZERO (out, Measure_Elixir, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -113,6 +113,6 @@ Average *out; - ALLOCATE (out, Average, Nvalues); - ALLOCATE (*primary, SecFilt, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); + ALLOCATE_ZERO (*primary, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -168,5 +168,5 @@ Average_Elixir *out; - ALLOCATE (out, Average_Elixir, Nvalues); + ALLOCATE_ZERO (out, Average_Elixir, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -198,5 +198,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -225,5 +225,5 @@ SecFilt_Elixir *out; - ALLOCATE (out, SecFilt_Elixir, Nvalues); + ALLOCATE_ZERO (out, SecFilt_Elixir, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -243,5 +243,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -266,19 +266,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // added or changed for PANSTARRS_DEV_0 @@ -314,5 +312,5 @@ Image_Elixir *out; - ALLOCATE (out, Image_Elixir, Nvalues); + ALLOCATE_ZERO (out, Image_Elixir, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -338,19 +336,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // added or changed for PANSTARRS_DEV_0 @@ -372,5 +368,5 @@ PhotCode *out; - ALLOCATE (out, PhotCode, Nvalues); + ALLOCATE_ZERO (out, PhotCode, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -412,5 +408,5 @@ PhotCode_Elixir *out; - ALLOCATE (out, PhotCode_Elixir, Nvalues); + ALLOCATE_ZERO (out, PhotCode_Elixir, Nvalues); for (i = 0; i < Nvalues; i++) { Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_loneos.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 30631) @@ -7,5 +7,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -76,5 +76,5 @@ Measure_Loneos *out; - ALLOCATE (out, Measure_Loneos, Nvalues); + ALLOCATE_ZERO (out, Measure_Loneos, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -104,6 +104,6 @@ Average *out; - ALLOCATE (out, Average, Nvalues); - ALLOCATE (*primary, SecFilt, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); + ALLOCATE_ZERO (*primary, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -160,5 +160,5 @@ Average_Loneos *out; - ALLOCATE (out, Average_Loneos, Nvalues); + ALLOCATE_ZERO (out, Average_Loneos, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -189,5 +189,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -218,5 +218,5 @@ SecFilt_Loneos *out; - ALLOCATE (out, SecFilt_Loneos, Nvalues); + ALLOCATE_ZERO (out, SecFilt_Loneos, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -235,5 +235,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -259,19 +259,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // added or changed for PANSTARRS_DEV_0 @@ -307,5 +305,5 @@ Image_Loneos *out; - ALLOCATE (out, Image_Loneos, Nvalues); + ALLOCATE_ZERO (out, Image_Loneos, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -331,19 +329,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // added or changed for PANSTARRS_DEV_0 Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 30631) @@ -7,5 +7,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -71,5 +71,5 @@ Measure_Panstarrs_DEV_0 *out; - ALLOCATE (out, Measure_Panstarrs_DEV_0, Nvalues); + ALLOCATE_ZERO (out, Measure_Panstarrs_DEV_0, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -120,5 +120,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -165,5 +165,5 @@ Average_Panstarrs_DEV_0 *out; - ALLOCATE (out, Average_Panstarrs_DEV_0, Nvalues); + ALLOCATE_ZERO (out, Average_Panstarrs_DEV_0, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -200,5 +200,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -225,5 +225,5 @@ SecFilt_Panstarrs_DEV_0 *out; - ALLOCATE (out, SecFilt_Panstarrs_DEV_0, Nvalues); + ALLOCATE_ZERO (out, SecFilt_Panstarrs_DEV_0, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -243,5 +243,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -273,19 +273,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_DEV_1 @@ -314,5 +312,5 @@ Image_Panstarrs_DEV_0 *out; - ALLOCATE (out, Image_Panstarrs_DEV_0, Nvalues); + ALLOCATE_ZERO (out, Image_Panstarrs_DEV_0, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -344,19 +342,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_DEV_1 Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 30631) @@ -7,5 +7,5 @@ Measure *out; - ALLOCATE (out, Measure, Nvalues); + ALLOCATE_ZERO (out, Measure, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -71,5 +71,5 @@ Measure_Panstarrs_DEV_1 *out; - ALLOCATE (out, Measure_Panstarrs_DEV_1, Nvalues); + ALLOCATE_ZERO (out, Measure_Panstarrs_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -120,5 +120,5 @@ Average *out; - ALLOCATE (out, Average, Nvalues); + ALLOCATE_ZERO (out, Average, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -165,5 +165,5 @@ Average_Panstarrs_DEV_1 *out; - ALLOCATE (out, Average_Panstarrs_DEV_1, Nvalues); + ALLOCATE_ZERO (out, Average_Panstarrs_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -200,5 +200,5 @@ SecFilt *out; - ALLOCATE (out, SecFilt, Nvalues); + ALLOCATE_ZERO (out, SecFilt, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -225,5 +225,5 @@ SecFilt_Panstarrs_DEV_1 *out; - ALLOCATE (out, SecFilt_Panstarrs_DEV_1, Nvalues); + ALLOCATE_ZERO (out, SecFilt_Panstarrs_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -243,5 +243,5 @@ Image *out; - ALLOCATE (out, Image, Nvalues); + ALLOCATE_ZERO (out, Image, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -273,19 +273,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_DEV_1 @@ -314,5 +312,5 @@ Image_Panstarrs_DEV_1 *out; - ALLOCATE (out, Image_Panstarrs_DEV_1, Nvalues); + ALLOCATE_ZERO (out, Image_Panstarrs_DEV_1, Nvalues); for (i = 0; i < Nvalues; i++) { @@ -344,19 +342,17 @@ out[i].ccdnum = in[i].ccdnum; - out[i].order = in[i].order; - out[i].Mx = in[i].Mx; - out[i].My = in[i].My; - out[i].Mxx = in[i].Mxx; - out[i].Mxy = in[i].Mxy; - out[i].Myy = in[i].Myy; - out[i].Mxxx = in[i].Mxxx; - out[i].Mxxy = in[i].Mxxy; - out[i].Mxyy = in[i].Mxyy; - out[i].Myyy = in[i].Myyy; - out[i].Mxxxx = in[i].Mxxxx; - out[i].Mxxxy = in[i].Mxxxy; - out[i].Mxxyy = in[i].Mxxyy; - out[i].Mxyyy = in[i].Mxyyy; - out[i].Myyyy = in[i].Myyyy; + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].dummy3 = in[i].dummy3; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; // changed or added for PS1_DEV_1 Index: /branches/czw_branch/20101203/Ohana/src/libdvo/src/photfits.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libdvo/src/photfits.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libdvo/src/photfits.c (revision 30631) @@ -2,11 +2,11 @@ # define MAX_ORDER 3 -/**** XXXX warning: these functions have not been corrected to handle - the change of Mcal from millimags to mags - I don't think most functions are using the Mcal polynomial terms in - any case... ****/ - -/* all terms of order > 0 are stored as 16bit floats - the zero-order term is stored as a short int (-32k,+32k) */ +/*** This file contains deprecated functions which were used to implement the 2D zero point variations per image. + This code has not been used for a long time, and is not up-to-date wrt the change from + millimag shorts to mag floats for values in the db tables. As of 2011.02.03, these + structures are no longer used. There is now an (as yet unused) index in the image table + to a zero-point map table. Someday, we may need similarly named functions to interact + with the ZP map. for now, this file is not used in the build. +***/ /* convert double to low-precision short int */ Index: /branches/czw_branch/20101203/Ohana/src/libkapa/src/DrawRotString.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libkapa/src/DrawRotString.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libkapa/src/DrawRotString.c (revision 30631) @@ -27,5 +27,5 @@ unsigned char *bitmap; char *currentname, basename[64]; - int i, dy, dx, N, X, Y, code; + int i, dy, dx, N, X, Y, code, protect; int dX, Xoff, dY, Yoff, YoffBase; int currentsize, basesize; @@ -63,4 +63,5 @@ code = FALSE; + protect = FALSE; YoffBase = Yoff; @@ -70,6 +71,10 @@ if ((N < 0) || (N >= NROTCHARS)) continue; + if (N == 39) { // single-quote + protect = protect ? FALSE : TRUE; + } + /* check for special characters */ - if (!code) { + if (!code && !protect) { if (N == 94) { SetRotFont (currentname, (int)(0.8*currentsize)); Index: /branches/czw_branch/20101203/Ohana/src/libkapa/src/PSRotFont.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libkapa/src/PSRotFont.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libkapa/src/PSRotFont.c (revision 30631) @@ -7,5 +7,5 @@ char *segment, basename[64], *currentname; - int i, N, code; + int i, N, code, protect; int dX, dY, Xoff, Yoff, X, Y, Nseg, NSEG, YoffBase; double cs, sn, fscale, currentscale; @@ -60,4 +60,6 @@ code = FALSE; + protect = FALSE; + YoffBase = 0; /* accumulate string segments with common state */ @@ -66,6 +68,10 @@ if ((N < 0) || (N >= NROTCHARS)) continue; + if (N == 39) { // single-quote + protect = protect ? FALSE : TRUE; + } + /* check for special characters */ - if (!code) { + if (!code && !protect) { /* superscript character (^) */ if (N == 94) { Index: /branches/czw_branch/20101203/Ohana/src/libkapa/src/bDrawRotFont.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libkapa/src/bDrawRotFont.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libkapa/src/bDrawRotFont.c (revision 30631) @@ -14,5 +14,5 @@ unsigned char *bitmap; char *currentname, basename[64]; - int i, dy, dx, N, X, Y, code; + int i, dy, dx, N, X, Y, code, protect; int dX, Xoff, dY, Yoff, YoffBase; int currentsize, basesize; @@ -53,4 +53,5 @@ code = FALSE; + protect = FALSE; YoffBase = Yoff; @@ -60,6 +61,10 @@ if ((N < 0) || (N >= NROTCHARS)) continue; + if (N == 39) { // single-quote + protect = protect ? FALSE : TRUE; + } + /* check for special characters */ - if (!code) { + if (!code && !protect) { if (N == 94) { SetRotFont (currentname, (int)(0.8*currentsize)); Index: /branches/czw_branch/20101203/Ohana/src/libohana/include/ohana.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libohana/include/ohana.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libohana/include/ohana.h (revision 30631) @@ -283,6 +283,12 @@ int ohana_str_to_radec PROTO((double *ra, double *dec, char *str1, char *str2)); double ohana_normalize_angle PROTO((double angle)); +double ohana_normalize_angle_to_midpoint PROTO((double angle, double Rmid)); int hstgsc_hms_to_deg PROTO((double *h0, double *h1, double *d0, double *d1, char *string)); + +short ToShortPixels PROTO((float pixels)); +short ToShortDegrees PROTO((float degrees)); +float FromShortPixels PROTO((short value)); +float FromShortDegrees PROTO((float value)); /* IO Buffer functions */ Index: /branches/czw_branch/20101203/Ohana/src/libohana/include/ohana_allocate.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libohana/include/ohana_allocate.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libohana/include/ohana_allocate.h (revision 30631) @@ -17,12 +17,14 @@ # define ALLOCATE(PTR,TYPE,SIZE) \ - PTR = (TYPE *) ohana_malloc (__FILE__, __LINE__, (SIZE), sizeof(TYPE)) + { PTR = (TYPE *) ohana_malloc (__FILE__, __LINE__, (SIZE), sizeof(TYPE)) } +# define ALLOCATE_ZERO(PTR,TYPE,SIZE) \ + { PTR = (TYPE *) ohana_malloc (__FILE__, __LINE__, (SIZE), sizeof(TYPE)); memset (PTR, 0, (SIZE)*sizeof(TYPE)); } # define REALLOCATE(PTR,TYPE,SIZE) \ - PTR = (TYPE *) ohana_realloc(__FILE__, __LINE__, PTR, (SIZE), sizeof(TYPE)); -# define CHECK_REALLOCATE(PTR,TYPE,SIZE,NCURR,DELTA) \ - if ((NCURR) >= (SIZE)) { SIZE += DELTA; \ - PTR = (TYPE *) ohana_realloc(__FILE__, __LINE__, PTR, (SIZE), sizeof(TYPE)); } -# define FREE(PTR) if (PTR != NULL) { ohana_free (__FILE__, __LINE__, PTR); } -# define free(PTR) ohana_free(__FILE__, __LINE__, PTR) + { PTR = (TYPE *) ohana_realloc(__FILE__, __LINE__, PTR, (SIZE), sizeof(TYPE)); } +# define CHECK_REALLOCATE(PTR,TYPE,SIZE,NCURR,DELTA) { \ + if ((NCURR) >= (SIZE)) { SIZE += DELTA; \ + PTR = (TYPE *) ohana_realloc(__FILE__, __LINE__, PTR, (SIZE), sizeof(TYPE)); } } +# define FREE(PTR) { if (PTR != NULL) { ohana_free (__FILE__, __LINE__, PTR); } } +# define free(PTR) { ohana_free(__FILE__, __LINE__, PTR); } # else @@ -33,22 +35,32 @@ # ifndef ALLOCATE -# define ALLOCATE(PTR,TYPE,SIZE) \ - PTR = (TYPE *) malloc ((unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ - if (PTR == NULL) { \ - fprintf(stderr,"failed malloc at %d in %s\n", __LINE__, __FILE__);\ - exit (10); } -# define REALLOCATE(PTR,TYPE,SIZE) \ +# define ALLOCATE(PTR,TYPE,SIZE) { \ + PTR = (TYPE *) malloc ((size_t)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ + if (PTR == NULL) { \ + fprintf(stderr,"failed malloc at %d in %s\n", __LINE__, __FILE__); \ + exit (10); } } + +# define ALLOCATE_ZERO(PTR,TYPE,SIZE) { \ + PTR = (TYPE *) malloc ((size_t)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ + if (PTR == NULL) { \ + fprintf(stderr,"failed malloc at %d in %s\n", __LINE__, __FILE__); \ + exit (10); \ + } memset (PTR, 0, (SIZE)*sizeof(TYPE)); } + +# define REALLOCATE(PTR,TYPE,SIZE) { \ PTR = (TYPE *) realloc(PTR,(unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ - if (PTR == NULL) { \ - fprintf(stderr,"failed realloc at %d in %s\n", __LINE__, __FILE__);\ - exit (10); } -# define CHECK_REALLOCATE(PTR,TYPE,SIZE,NCURR,DELTA) \ - if ((NCURR) >= (SIZE)) { \ - SIZE += DELTA; \ + if (PTR == NULL) { \ + fprintf(stderr,"failed realloc at %d in %s\n", __LINE__, __FILE__); \ + exit (10); } } + +# define CHECK_REALLOCATE(PTR,TYPE,SIZE,NCURR,DELTA) { \ + if ((NCURR) >= (SIZE)) { \ + SIZE += DELTA; \ PTR = (TYPE *) realloc(PTR,(unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ - if (PTR == NULL) { \ - fprintf(stderr,"failed realloc increment at %d in %s\n", __LINE__, __FILE__);\ - exit (10); } } -# define FREE(PTR) if (PTR != NULL) { free (PTR); } + if (PTR == NULL) { \ + fprintf(stderr,"failed realloc increment at %d in %s\n", __LINE__, __FILE__); \ + exit (10); } } } + +# define FREE(PTR) { if (PTR != NULL) { free (PTR); } } # endif /* ALLOCATE */ Index: /branches/czw_branch/20101203/Ohana/src/libohana/src/time.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/libohana/src/time.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/libohana/src/time.c (revision 30631) @@ -468,2 +468,73 @@ return (result); } + +double ohana_normalize_angle_to_midpoint (double angle, double Rmid) { + + double result; + + // take an input angle and force the domain to be 0.0 - 360.0 + // there are a few ways to do this: + + // option 1: This is a potentially very slow method, also subject to round off errors +# if (0) + while (angle > 360.0) angle -= 360.0; + while (angle < 0.0) angle += 360.0; +# endif + + // option 2: take sin & cos, apply atan2 (y, x) +# if (1) + double x, y; + + x = cos(angle*RAD_DEG); + y = sin(angle*RAD_DEG); + + result = DEG_RAD*atan2 (y, x); + if (result < Rmid - 180.0) result += 360.0; + if (result > Rmid + 180.0) result -= 360.0; +# endif + +# if (0) + // option 3: + int nCircle = angle / 360.0; + result -= 360.0*nCircle; + if (result < 0.0) result += 360.0; +# endif + + return (result); +} + +short ToShortPixels (float pixels) { + + short value; + + value = 100*pixels; + + return value; +} + +short ToShortDegrees (float degrees) { + + short value; + + value = ((float)0xffff/360.0) * degrees; + + return value; +} + +float FromShortPixels (short value) { + + float pixels; + + pixels = value / 100.0; + + return pixels; +} + +float FromShortDegrees (float value) { + + float degrees; + + degrees = (360.0 / (float)0xffff) * value; + + return degrees; +} Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/cgrid.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/cgrid.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/cgrid.c (revision 30631) @@ -98,5 +98,5 @@ if (argc != 1) { - gprint (GP_ERR, "USAGE: cgrid [style]\n"); + gprint (GP_ERR, "USAGE: cgrid [style] [-ra-by-hour] [-labels]\n"); return (FALSE); } Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/region.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/region.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/region.c (revision 30631) @@ -18,8 +18,28 @@ FREE (name); + NoClear = FALSE; + if ((N = get_argument (argc, argv, "-no-clear"))) { + remove_argument (N, &argc, argv); + NoClear = TRUE; + } + if ((N = get_argument (argc, argv, "-image"))) { remove_argument (N, &argc, argv); KapaGetImageCoords (kapa, &graphmode.coords); KapaGetImageRange (kapa, &graphmode.xmin, &graphmode.xmax, &graphmode.ymax, &graphmode.ymin); + + set_variable ("XMIN", graphmode.xmin); + set_variable ("XMAX", graphmode.xmax); + set_variable ("YMIN", graphmode.ymin); + set_variable ("YMAX", graphmode.ymax); + + set_variable ("RMIN", Ra + graphmode.xmin); + set_variable ("RMAX", Ra + graphmode.xmax); + set_variable ("DMIN", Dec + graphmode.ymin); + set_variable ("DMAX", Dec + graphmode.ymax); + + // if (!NoClear) KapaClearSections (kapa); + KapaSetLimits (kapa, &graphmode); + SetGraph (&graphmode); return (TRUE); @@ -45,10 +65,4 @@ remove_argument (N, &argc, argv); graphmode.flipnorth = FALSE; - } - - NoClear = FALSE; - if ((N = get_argument (argc, argv, "-no-clear"))) { - remove_argument (N, &argc, argv); - NoClear = TRUE; } Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/test/region.sh =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/test/region.sh (revision 30631) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro/test/region.sh (revision 30631) @@ -0,0 +1,185 @@ + +macro mkimage2 + if ($0 != 2) + echo "USAGE: mkimage2 (angle)" + break + end + + # create an image and set the WCS + + mcreate im 500 500 + + keyword im CTYPE1 -w "RA---TAN" + keyword im CTYPE2 -w "DEC--TAN" + + keyword im CRVAL1 -wf 10.00 + keyword im CRVAL2 -wf 45.00 + + keyword im CRPIX1 -wf 250 + keyword im CRPIX2 -wf 250 + + keyword im CDELT1 -wf {0.25/3600} + keyword im CDELT2 -wf {0.25/3600} + + keyword im PC001001 -wf {dcos($1)} + keyword im PC002001 -wf {dsin($1)} + keyword im PC001002 -wf {-1.0*dsin($1)} + keyword im PC002002 -wf {dcos($1)} + + for ix 50 500 100 + for iy 50 500 100 + im[$ix][$iy] = 1 + concat $ix Xc + concat $iy Yc + end + end + + set Rc = Xc + set Dc = Yc + + coords im -p Rc Dc + + clear -s + + tv im -0.2 1.5 + region -image -no-clear + cplot Rc Dc -pt 7 -sz 3.0 -x 2 -c red + + create r1 9.0 11.0 0.001 + create d2 44.0 46.0 0.001 + set d1 = 45.0 + zero(r1) + set r2 = 10.0 + zero(d2) + cplot r1 d1 -c blue -x 0 ; + cplot r2 d2 -c red -x 0 + + set d1 = Dc[0] + zero(r1) + set r2 = Rc[0] + zero(d2) + cplot r1 d1 -c blue -x 0 ; + cplot r2 d2 -c red -x 0 +end + +macro mkimage + + # create an image and set the WCS + + mcreate im 500 500 + + keyword im CTYPE1 -w "RA---TAN" + keyword im CTYPE2 -w "DEC--TAN" + + keyword im CRVAL1 -wf 10.00 + keyword im CRVAL2 -wf 45.00 + + keyword im CRPIX1 -wf 250 + keyword im CRPIX2 -wf 250 + + keyword im CDELT1 -wf {0.25/3600} + keyword im CDELT2 -wf {0.25/3600} + + keyword im PC001001 -wf 1.0 + keyword im PC002001 -wf 0.0 + keyword im PC001002 -wf 0.0 + keyword im PC002002 -wf 1.0 + + for ix 50 500 100 + for iy 50 500 100 + im[$ix][$iy] = 1 + concat $ix Xc + concat $iy Yc + end + end + + tv im -0.2 1.5 + + set Rc = Xc + set Dc = Yc + + coords im -p Rc Dc + + region -image + cplot Rc Dc -pt 7 -sz 3.0 -x 2 -c red +end + +macro raline + create rx 0 360 + set dx = zero(rx) + $1 + coords im -c rx dx + break -auto off + for i 0 rx[] + $ix = rx[$i] + $iy = dx[$i] + im[$ix][$iy] = 1.0 + end + break -auto on +end + +macro decline + create dx -90 90 + set rx = zero(dx) + $1 + coords im -c rx dx + break -auto off + for i 0 rx[] + $ix = rx[$i] + $iy = dx[$i] + im[$ix][$iy] = 1.0 + end + break -auto on +end + +macro mkallsky + + # create an image and set the WCS + + mcreate im 400 200 + + keyword im CTYPE1 -w "RA---AIT" + keyword im CTYPE2 -w "DEC--AIT" + + keyword im CRVAL1 -wf 0.00 + keyword im CRVAL2 -wf 0.00 + + keyword im CRPIX1 -wf 200 + keyword im CRPIX2 -wf 100 + + keyword im CDELT1 -wf 1.0 + keyword im CDELT2 -wf 1.0 + + keyword im PC001001 -wf 1.0 + keyword im PC002001 -wf 0.0 + keyword im PC001002 -wf 0.0 + keyword im PC002002 -wf 1.0 + + raline -45.0 + raline 0.0 + raline 45.0 + + decline -90.0 + decline 0.0 + decline 90.0 + + clear -s + tv im -0.2 1.5 + box + + region -image -no-clear + + skylines +end + +macro skylines + create rx 0 360 + set dx1 = zero(rx) - 45.0 + cplot rx dx1 -x 2 -pt 0 -sz 0.5 -c blue + set dx2 = zero(rx) - 0.0 + cplot rx dx2 -x 2 -pt 0 -sz 0.5 -c blue + set dx3 = zero(rx) + 45.0 + cplot rx dx3 -x 2 -pt 0 -sz 0.5 -c blue + + create dy -90 90 + set ry1 = zero(dy) - 90.0 + cplot ry1 dy -x 2 -pt 0 -sz 0.5 -c blue + set ry2 = zero(dy) - 0.0 + cplot ry2 dy -x 2 -pt 0 -sz 0.5 -c blue + set ry3 = zero(dy) + 90.0 + cplot ry3 dy -x 2 -pt 0 -sz 0.5 -c blue +end Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/limits.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/limits.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/limits.c (revision 30631) @@ -26,4 +26,19 @@ // XXX need an option to set the limits based on the current image bounds + if ((N = get_argument (argc, argv, "-image"))) { + remove_argument (N, &argc, argv); + KapaGetImageRange (kapa, &graphmode.xmin, &graphmode.xmax, &graphmode.ymax, &graphmode.ymin); + + set_variable ("XMIN", graphmode.xmin); + set_variable ("XMAX", graphmode.xmax); + set_variable ("YMIN", graphmode.ymin); + set_variable ("YMAX", graphmode.ymax); + + // if (!NoClear) KapaClearSections (kapa); + SetGraph (&graphmode); + KapaSetLimits (kapa, &graphmode); + return (TRUE); + // Set Region based on image + } if (argc == 1) { Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/match2d.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/match2d.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/match2d.c (revision 30631) @@ -1,4 +1,6 @@ # include "data.h" + int find_matches2d (Vector *X1, Vector *Y1, Vector *X2, Vector *Y2, double Radius, Vector *index1, Vector *index2); +int find_matches2d_closest (Vector *X1, Vector *Y1, Vector *X2, Vector *Y2, double Radius, Vector *index); // match2d (X1) (Y1) (X2) (Y2) (Radius) [-index1 (index1)] [-index2 (index2)] [-nomatch1 nomatch1] [-nomatch2 nomatch2] @@ -6,5 +8,5 @@ int match2d (int argc, char **argv) { - int N; + int N, CLOSEST; double Radius; char *endptr; @@ -12,4 +14,10 @@ Vector *index1, *index2; + CLOSEST = FALSE; + if ((N = get_argument (argc, argv, "-closest"))) { + remove_argument (N, &argc, argv); + CLOSEST = TRUE; + } + if ((N = get_argument (argc, argv, "-index1"))) { remove_argument (N, &argc, argv); @@ -30,9 +38,22 @@ if (argc != 6) { gprint (GP_ERR, "USAGE: match2d X1 Y1 X2 Y2 Radius [-index1 (index1)] [-index2 (index2)] [-closest]\n"); - // gprint (GP_ERR, "USAGE: match2d X1 Y1 X2 Y2 Radius [-index1 (index1)] [-index2 (index2)] [-closest]\n"); - // gprint (GP_ERR, " if -closest is provided, index1 & index2 will have the same length as X1 and X2 (respectively)\n"); - // gprint (GP_ERR, " with either the index of the match or a value of -1 for non-matches\n"); - return (FALSE); - } + gprint (GP_ERR, " if -closest is provided, index1 & index2 will have the same length as X1 and X2 (respectively)\n"); + gprint (GP_ERR, " with either the index of the match or a value of -1 for non-matches\n"); + return (FALSE); + } + + /* + we have two modes of operation: + + without -closest, we are finding all matched pairs within the match radius. in this + case, the two index vectors have the same length, one entry per matched pair. + x1[index1],y1[index1] matches to x2[index2],y2[index2]. + + with -closest selected, we are finding the closest element of set 1 to each of set 2 + and vice versa. in this case, index1 is always the same length as x1,y1, while index2 + is the same lengths as x2,y2. x2[index1],y2[index1] matches x1,y1 while + x1[index2],y1[index2] matches x2,y2 + + */ if ((X1vec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); @@ -61,5 +82,10 @@ } - find_matches2d (X1vec, Y1vec, X2vec, Y2vec, Radius, index1, index2); + if (CLOSEST) { + find_matches2d_closest (X1vec, Y1vec, X2vec, Y2vec, Radius, index1); + find_matches2d_closest (X2vec, Y2vec, X1vec, Y1vec, Radius, index2); + } else { + find_matches2d (X1vec, Y1vec, X2vec, Y2vec, Radius, index1, index2); + } return (TRUE); @@ -132,2 +158,69 @@ return (TRUE); } + +// find the elements of X2,Y2 which are closest to each element of X1,Y1 (-1 if no match) +int find_matches2d_closest (Vector *X1, Vector *Y1, Vector *X2, Vector *Y2, double Radius, Vector *index) { + + off_t i, j, Jmin, Ji, I, J, *N1, *N2, NMATCH; + double dX, dY, dR, Radius2, Rmin; + + NMATCH = X1->Nelements; + ResetVector (index, OPIHI_INT, NMATCH); + + ALLOCATE (N1, off_t, X1->Nelements); + ALLOCATE (N2, off_t, X2->Nelements); + + for (i = 0; i < X1->Nelements; i++) { N1[i] = i; } + for (i = 0; i < X2->Nelements; i++) { N2[i] = i; } + + sort_coords_indexonly (X1->elements.Flt, Y1->elements.Flt, N1, X1->Nelements); + sort_coords_indexonly (X2->elements.Flt, Y2->elements.Flt, N2, X2->Nelements); + + Radius2 = Radius*Radius; + + // find the closest entry in list 2 to the current entry in list 1: + for (i = j = 0; (i < X1->Nelements) && (j < X2->Nelements);) { + I = N1[i]; + J = N2[j]; + + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + + if (dX <= -1.02*Radius) { + // no match in list 2 to this entry + index->elements.Int[I] = -1; + i++; + continue; + } + if (dX >= +1.02*Radius) { j++; continue; } + + // look for closest matches of list2() to list1(i) + Jmin = -1; + Rmin = Radius2; + for (Ji = j; (dX > -1.02*Radius) && (Ji < X2->Nelements); Ji++) { + J = N2[Ji]; + dX = X1->elements.Flt[I] - X2->elements.Flt[J]; + dY = Y1->elements.Flt[I] - Y2->elements.Flt[J]; + dR = dX*dX + dY*dY; + if (dR > Radius2) continue; + if (dR < Rmin) { + Rmin = dR; + Jmin = J; + } + } + + // no match in list 2 to this entry + if (Jmin == -1) { + index->elements.Int[I] = -1; + i++; + continue; + } + index->elements.Int[I] = Jmin; + i++; + } + index->Nelements = NMATCH; + + free (N1); + free (N2); + + return (TRUE); +} Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/reindex.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/reindex.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/reindex.c (revision 30631) @@ -5,5 +5,5 @@ int reindex (int argc, char **argv) { - int i, Npts, Nmax, valid; + int i, Npts, Nmax; Vector *ivec, *ovec, *xvec; @@ -11,13 +11,7 @@ Npts = 0; - valid = TRUE; - valid &= (argc >= 6); - valid &= !strcmp(argv[2], "="); - valid &= !strcmp(argv[4], "using"); - if (!valid) { - gprint (GP_ERR, "USAGE: reindex (out) = (in) using (index)\n"); - gprint (GP_ERR, " creates a new vectors (out) from (in) based on sequence in (index)\n"); - return (FALSE); - } + if (argc != 6) goto usage; + if (strcmp(argv[2], "=")) goto usage; + if (strcmp(argv[4], "using")) goto usage; if ((ovec = SelectVector (argv[1], ANYVECTOR, TRUE)) == NULL) goto error; @@ -63,4 +57,9 @@ DeleteVector (ovec); return (FALSE); + +usage: + gprint (GP_ERR, "USAGE: reindex (out) = (in) using (index)\n"); + gprint (GP_ERR, " creates a new vectors (out) from (in) based on sequence in (index)\n"); + return (FALSE); } Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/subset.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/subset.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/subset.c (revision 30631) @@ -34,5 +34,5 @@ // ovec matches ivec in type - ResetVector (ovec, ivec->type, MAX (tvec[0].Nelements, 1)); + ResetVector (ovec, ivec->type, tvec[0].Nelements); // we have four cases: (ivec == flt or int) and (tvec == flt or int) @@ -75,5 +75,5 @@ // free up unused memory - ResetVector (ovec, ivec->type, MAX (Npts, 1)); + ResetVector (ovec, ivec->type, Npts); DeleteVector (tvec); Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/vstats.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/vstats.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/vstats.c (revision 30631) @@ -3,12 +3,13 @@ int vstats (int argc, char **argv) { - int i, N; - double max, min, sum, var, dvar, mean, stdev; - float IgnoreValue; + int i, iter, N, Nbin, Niter; + double max, min, pmin, pmax, sum, var, dvar, mean, stdev; + float IgnoreValue, Nsigma; int Ignore, Quiet; - int *Nval, bin, Nmode, Nmed; - double dx, mode, median; + int *Nval, bin, Nmode, Nmed, Nused; + double dx, mode, median, threshold; Vector *vec; + char *mask = NULL; IgnoreValue = 0; @@ -21,4 +22,20 @@ } + Niter = 1; + Nsigma = 3.0; + + if ((N = get_argument (argc, argv, "-sigma-clip"))) { + remove_argument (N, &argc, argv); + Nsigma = atof(argv[N]); + Niter = 3; + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-iter"))) { + remove_argument (N, &argc, argv); + Niter = atoi(argv[N]); + remove_argument (N, &argc, argv); + } + Quiet = FALSE; if ((N = get_argument (argc, argv, "-q"))) { @@ -32,5 +49,6 @@ if (argc != 2) { - gprint (GP_ERR, "USAGE: vstat (vector)\n"); + gprint (GP_ERR, "USAGE: vstat (vector) [-ignore value] [-q] [-quiet] [-iter Niter] [-sigma-clip Nsigma]\n"); + gprint (GP_ERR, " default is 1 iteration without sigma clipping or 3 with sigma clipping\n"); return (FALSE); } @@ -39,4 +57,24 @@ /* we need two passes, one for max, min, mean, sum, one for median, stdev, etc */ + + // set a good / bad mask + ALLOCATE (mask, char, vec[0].Nelements); + if (vec[0].type == OPIHI_FLT) { + opihi_flt *X = vec[0].elements.Flt; + for (i = 0; i < vec[0].Nelements; i++, X++) { + mask[i] = 1; + if (!finite (*X)) continue; + if (Ignore && (*X == IgnoreValue)) continue; + mask[i] = 0; + } + } else { + opihi_int *X = vec[0].elements.Int; + for (i = 0; i < vec[0].Nelements; i++, X++) { + mask[i] = 1; + if (!finite (*X)) continue; + if (Ignore && (*X == IgnoreValue)) continue; + mask[i] = 0; + } + } /* calculate max, min, mean, sum, npix */ @@ -47,6 +85,5 @@ opihi_flt *X = vec[0].elements.Flt; for (i = 0; i < vec[0].Nelements; i++, X++) { - if (!finite (*X)) continue; - if (Ignore && (*X == IgnoreValue)) continue; + if (mask[i]) continue; max = MAX (*X, max); min = MIN (*X, min); @@ -57,6 +94,5 @@ opihi_int *X = vec[0].elements.Int; for (i = 0; i < vec[0].Nelements; i++, X++) { - if (!finite (*X)) continue; - if (Ignore && (*X == IgnoreValue)) continue; + if (mask[i]) continue; max = MAX (*X, max); min = MIN (*X, min); @@ -67,64 +103,117 @@ mean = sum / N; - /* calculate median and mode with resolution of (max - min) / 1000 */ - dx = (max - min) / 1000; - if (dx == 0) { - median = mode = min; - stdev = 0.0; - goto skip; - } - - ALLOCATE (Nval, int, 1002); - bzero (Nval, 1000*sizeof(int)); - var = 0; - if (vec[0].type == OPIHI_FLT) { - opihi_flt *X = vec[0].elements.Flt; - for (i = 0; i < vec[0].Nelements; i++, X++) { - if (!finite (*X)) continue; - if (Ignore && (*X == IgnoreValue)) continue; - bin = MAX (0, MIN (1000, (*X - min) / dx)); - Nval[bin] ++; - dvar = (*X - mean); - var += dvar*dvar; - } - } else { - opihi_int *X = vec[0].elements.Int; - for (i = 0; i < vec[0].Nelements; i++, X++) { - if (!finite (*X)) continue; - if (Ignore && (*X == IgnoreValue)) continue; - bin = MAX (0, MIN (1000, (*X - min) / dx)); - Nval[bin] ++; - dvar = (*X - mean); - var += dvar*dvar; - } - } - stdev = sqrt (var / N); - - Nmode = 0; - mode = Nval[Nmode]; - median = 0; - Nmed = -1; - for (i = 0; i < 1001; i++) { - if (Nmed == -1) { - median += Nval[i]; - if (median >= N / 2.0) { - Nmed = i; - median = i * dx + min; + // we do Niter passes; on each pass, we exclude entries > Nsigma from the median + pmin = min; + pmax = max; + + stdev = mode = median = Nused = 0.0; + + for (iter = 0; iter < Niter; iter ++) { + + // reduce Nbin after the first iteration? + Nbin = 1000; + + /* calculate median and mode with resolution of (max - min) / 1000 */ + dx = (pmax - pmin) / Nbin; + if (dx == 0) { + median = mode = min; + stdev = 0.0; + goto skip; + } + + ALLOCATE (Nval, int, Nbin + 2); + bzero (Nval, Nbin*sizeof(int)); + var = 0; + if (vec[0].type == OPIHI_FLT) { + opihi_flt *X = vec[0].elements.Flt; + for (i = 0; i < vec[0].Nelements; i++, X++) { + if (mask[i]) continue; + bin = MAX (0, MIN (Nbin, (*X - pmin) / dx)); + Nval[bin] ++; + dvar = (*X - mean); + var += dvar*dvar; + } + } else { + opihi_int *X = vec[0].elements.Int; + for (i = 0; i < vec[0].Nelements; i++, X++) { + if (mask[i]) continue; + bin = MAX (0, MIN (Nbin, (*X - pmin) / dx)); + Nval[bin] ++; + dvar = (*X - mean); + var += dvar*dvar; + } + } + stdev = sqrt (var / N); + + Nmode = 0; + mode = Nval[Nmode]; + median = 0; + Nmed = -1; + for (i = 0; i < Nbin + 1; i++) { + if (Nmed == -1) { + median += Nval[i]; + if (median >= N / 2.0) { + Nmed = i; + median = i * dx + pmin; + } } - } - if (mode < Nval[i]) { - Nmode = i; - mode = Nval[Nmode]; - } - } - mode = Nmode * dx + min; - free (Nval); + if (mode < Nval[i]) { + Nmode = i; + mode = Nval[Nmode]; + } + } + mode = Nmode * dx + pmin; + free (Nval); + + threshold = Nsigma * stdev; + + // we are going to do another pass: mark the entries to skip + pmin = max; + pmax = min; + if (iter < Niter - 1) { + if (vec[0].type == OPIHI_FLT) { + opihi_flt *X = vec[0].elements.Flt; + for (i = 0; i < vec[0].Nelements; i++, X++) { + if (mask[i]) continue; + if (fabs(*X - median) > threshold) { + mask[i] = 1; + } else { + pmin = MIN (*X, pmin); + pmax = MAX (*X, pmax); + } + } + } else { + opihi_int *X = vec[0].elements.Int; + for (i = 0; i < vec[0].Nelements; i++, X++) { + if (mask[i]) continue; + if (fabs(*X - median) > threshold) { + mask[i] = 1; + } else { + pmin = MIN (*X, pmin); + pmax = MAX (*X, pmax); + } + } + } + } + // gprint (GP_ERR, "iter %d, mean: %g, stdev: %g, min: %g, max: %g, median: %g, mode: %g, Npts: %d\n", + // iter, mean, stdev, min, max, median, mode, N); + } + + Nused = 0; + for (i = 0; i < vec[0].Nelements; i++) { + if (mask[i]) continue; + Nused ++; + } skip: + FREE(mask); + if (!Quiet) { gprint (GP_ERR, "mean: %g, stdev: %g, min: %g, max: %g, median: %g, mode: %g, Npts: %d\n", - mean, stdev, min, max, median, mode, N); - } - + mean, stdev, min, max, median, mode, N); + } + + set_variable ("PMIN", pmin); + set_variable ("PMAX", pmax); set_variable ("MIN", min); set_variable ("MAX", max); @@ -134,4 +223,6 @@ set_variable ("TOTAL", sum); set_int_variable ("NPIX", N); + set_int_variable ("NPTS", N); + set_int_variable ("NUSED", Nused); set_variable ("SIGMA", stdev); return (TRUE); Index: /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/write_vectors.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/write_vectors.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/cmd.data/write_vectors.c (revision 30631) @@ -140,14 +140,14 @@ if (fmttype[j] == 'd') { if (vec[j][0].type == OPIHI_FLT) { - fprintf (f, fmtlist[j], (int)(vec[j][0].elements.Flt[i])); + fprintf (f, fmtlist[j], (opihi_int)(vec[j][0].elements.Flt[i])); } else { - fprintf (f, fmtlist[j], (int)(vec[j][0].elements.Int[i])); + fprintf (f, fmtlist[j], (opihi_int)(vec[j][0].elements.Int[i])); } } if (fmttype[j] == 'f') { if (vec[j][0].type == OPIHI_FLT) { - fprintf (f, fmtlist[j], (float)(vec[j][0].elements.Flt[i])); + fprintf (f, fmtlist[j], (opihi_flt)(vec[j][0].elements.Flt[i])); } else { - fprintf (f, fmtlist[j], (float)(vec[j][0].elements.Int[i])); + fprintf (f, fmtlist[j], (opihi_flt)(vec[j][0].elements.Int[i])); } } Index: /branches/czw_branch/20101203/Ohana/src/opihi/doc/image-graph-overlay.txt =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/doc/image-graph-overlay.txt (revision 30631) +++ /branches/czw_branch/20101203/Ohana/src/opihi/doc/image-graph-overlay.txt (revision 30631) @@ -0,0 +1,103 @@ + +the kapa display tool has the ability to show both images (2D grid +with grey or color scale) and graphs (plot of points and lines). + +normally, the kapa graph and image displays are used independently +(either in different windows or in different sections of the same +window). + +there are situations in which it is useful to combine these two tools +and show graphs overlayed on images. + +there are several mana / dvo functions which enable graph/image +overlays, and have use in different contexts. + +* basic points + + here are some aspects of image/graph overlays to keep in mind. + + * the image and graph are not dynamically linked. althought it is + possible to set the graph to have limits that correspond to the + image in some way (see below), this relationship is not maintained + dynamically. if you change the view of the image (eg, by + clicking, zooming, or recentering), the graph does not change + without re-issuing some of the commands given below. + + * the boundaries of the image and the graph are dependent on whether + or not they are both present and on whether or not the zoom box + tools are displayed. the status of the zoom tools is modifed with + the 'section' command as follows: + + section (name) -imtool (option) + + where (option) may be one of -x, +x, -y, +y or none + + for example, use: section default -imtool none to remove the zoom + tool from the default section of the current display. + +* set graph limits based on the current image + + the simplest way to have the image and the graph interact is to + display an image and graph in the same window, and to set the graph + limits based on the view of the displayed image: + + # create a contour and overlay in pixel coordinates + contour image xc yc 100 + tv image -50 200 + box + limits -image + plot xc yc -pt 100 -c red + +* image with WCS overlay on display 'region' + + if you have an image with WCS elements, then it is possible to + define the sky region to correspond to the displayed portion of the + image. + + # create a contour and convert to celestial coords based on the image: + contour image xc yc 100 + set rc = xc + set dc = yc + coords image -p rc dc + + # display the image, set the sky region, plot the contour + tv image -50 200 + box -ticks 0000 -labels 0000 + region -image + cplot rc dc -pt 100 -c red + +* setting the display size based on the image size + + sometimes it is convenient to ensure the whole image will fit on a + graph. it is possible to use the 'resize -by-image' and section + -image options to control this behavior: + + * resize -by-image adjusts the display size so that the image in the + currently active section completely fills that section (not that + this command assumes the image is centered, but does not center + it. + + for example, the following commands display a sample 1000x1000 + image in a section: + + section view 0.2 0.2 0.6 0.6 + section view -imtool none + box -ticks 0000 -labels 0000 + tv image -50 200 + center 500 500 -2 + resize -by-image + + * section -image gives something like the opposite behavior. It + defines the size of the section to completely contain the image, + with the lower-left corner defined. Note: this may result in a + section which extends beyond the edge of the window. + + resize 800 800 + section view -imtool none + tv image -50 200 + center 500 500 -2 + section view -image 0.1 0.1 + +NOTE: in the examples above, the 'box' command is currently necessary +to ensure a graph box is defined. Otherwise, the coordinate systems +may be inconsistent. Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/avextract.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/avextract.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/avextract.c (revision 30631) @@ -44,5 +44,8 @@ // parse skyregion options - if ((selection = SetRegionSelection (&argc, argv)) == NULL) goto escape; + if ((selection = SetRegionSelection (&argc, argv)) == NULL) { + print_error(); + goto escape; + } // command-line is of the form: avextract field,field, field [where (field op value)...] @@ -68,4 +71,8 @@ // construct the db Boolean math stack (frees cstack) stack = dbRPN (Ncstack, cstack, &Nstack); + if (Ncstack && !Nstack) { + print_error(); + goto escape; + } // add the skyregion limits to the where statement (or create) Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbExtractImages.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbExtractImages.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbExtractImages.c (revision 30631) @@ -241,8 +241,8 @@ case IMAGE_NCAL: - value.Int = image[N].Mxxxx; + value.Int = image[N].nFitPhotom; break; case IMAGE_SKY: - value.Flt = image[N].Myyyy + 0x8000; + value.Flt = NAN; break; case IMAGE_FLAGS: @@ -299,4 +299,27 @@ value.Flt = (field->ID == IMAGE_X_UR_FP) ? x : y; break; + + case IMAGE_X_ERR_SYS: + value.Flt = image[N].dXpixSys; + break; + case IMAGE_Y_ERR_SYS: + value.Flt = image[N].dXpixSys; + break; + case IMAGE_MAG_ERR_SYS: + value.Flt = image[N].dMagSys; + break; + + case IMAGE_NFIT_PHOTOM: + value.Int = image[N].nFitPhotom; + break; + case IMAGE_NFIT_ASTROM: + value.Int = image[N].nFitAstrom; + break; + case IMAGE_NLINK_PHOTOM: + value.Int = image[N].nLinkPhotom; + break; + case IMAGE_NLINK_ASTROM: + value.Int = image[N].nLinkAstrom; + break; } return (value); Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbExtractMeasures.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbExtractMeasures.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbExtractMeasures.c (revision 30631) @@ -320,23 +320,23 @@ break; case MEAS_FWHM: /* OK */ - value.Flt = 0.01*(measure[0].FWx + measure[0].FWy) / 2.0; + value.Flt = FromShortPixels(measure[0].FWx + measure[0].FWy) / 2.0; break; case MEAS_FWHM_MAJ: /* OK */ - value.Flt = 0.01*measure[0].FWx; + value.Flt = FromShortPixels(measure[0].FWx); break; case MEAS_FWHM_MIN: /* OK */ - value.Flt = 0.01*measure[0].FWy; + value.Flt = FromShortPixels(measure[0].FWy); break; case MEAS_THETA: /* OK */ - value.Flt = measure[0].theta; + value.Flt = FromShortDegrees(measure[0].theta); break; case MEAS_MXX: /* OK */ - value.Flt = measure[0].Mxx; + value.Flt = FromShortPixels(measure[0].Mxx); break; case MEAS_MXY: /* OK */ - value.Flt = measure[0].Mxy; + value.Flt = FromShortPixels(measure[0].Mxy); break; case MEAS_MYY: /* OK */ - value.Flt = measure[0].Myy; + value.Flt = FromShortPixels(measure[0].Myy); break; case MEAS_DOPHOT: /* OK */ @@ -382,8 +382,11 @@ break; case MEAS_XCCD_ERR: /* OK */ - value.Flt = measure[0].dXccd / 100.0; + value.Flt = FromShortPixels(measure[0].dXccd); break; case MEAS_YCCD_ERR: /* OK */ - value.Flt = measure[0].dYccd / 100.0; + value.Flt = FromShortPixels(measure[0].dYccd); + break; + case MEAS_POS_SYS_ERR: /* OK */ + value.Flt = FromShortPixels(measure[0].dRsys); break; case MEAS_XMOSAIC: /* OK */ @@ -425,4 +428,7 @@ value.Flt = measure[0].psfQual; break; + case MEAS_PSF_QF_PERFECT: /* OK */ + value.Flt = NAN; + break; case MEAS_PSF_CHISQ: /* OK */ value.Flt = measure[0].psfChisq; Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbFields.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbFields.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/dbFields.c (revision 30631) @@ -194,4 +194,5 @@ if (!strcasecmp (fieldName, "XCCD:ERR")) ESCAPE (MEAS_XCCD_ERR, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "YCCD:ERR")) ESCAPE (MEAS_YCCD_ERR, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "POS_SYS_ERR")) ESCAPE (MEAS_POS_SYS_ERR, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "XMOSAIC")) ESCAPE (MEAS_XMOSAIC, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "YMOSAIC")) ESCAPE (MEAS_YMOSAIC, MAG_NONE, OPIHI_FLT); @@ -205,4 +206,5 @@ if (!strcasecmp (fieldName, "IMAGEID")) ESCAPE (MEAS_IMAGE_ID, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "PSF_QF")) ESCAPE (MEAS_PSF_QF, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "PSF_QF_PERFECT")) ESCAPE (MEAS_PSF_QF_PERFECT, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "PSF_CHISQ")) ESCAPE (MEAS_PSF_CHISQ, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "PSF_NDOF")) ESCAPE (MEAS_PSF_NDOF, MAG_NONE, OPIHI_INT); @@ -398,7 +400,6 @@ if (!strcasecmp (fieldName, "trate" )) ESCAPE (IMAGE_TRATE, MAG_NONE, OPIHI_FLT); - // XXX : these are mapped to ridiculous things: Myyyy + 0x8000 and Mxxxx if (!strcasecmp (fieldName, "ncal" )) ESCAPE (IMAGE_NCAL, MAG_NONE, OPIHI_INT); - if (!strcasecmp (fieldName, "sky" )) ESCAPE (IMAGE_SKY, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "sky" )) ESCAPE (IMAGE_SKY, MAG_NONE, OPIHI_FLT); // deprecated for now if (!strcasecmp (fieldName, "imflags" )) ESCAPE (IMAGE_FLAGS, MAG_NONE, OPIHI_INT); @@ -427,4 +428,12 @@ if (!strcasecmp (fieldName, "Y_UR_FP" )) ESCAPE (IMAGE_Y_UR_FP, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "dX_SYS" )) ESCAPE (IMAGE_X_ERR_SYS, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "dY_SYS" )) ESCAPE (IMAGE_Y_ERR_SYS, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "dM_SYS" )) ESCAPE (IMAGE_MAG_ERR_SYS,MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "NFIT_PHOTOM")) ESCAPE (IMAGE_NFIT_PHOTOM,MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "NFIT_ASTROM")) ESCAPE (IMAGE_NFIT_ASTROM,MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "NLINK_PHOTOM")) ESCAPE (IMAGE_NLINK_PHOTOM,MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "NLINK_ASTROM")) ESCAPE (IMAGE_NLINK_ASTROM,MAG_NONE, OPIHI_INT); + // for words that don't parse, try a photcode gprint (GP_ERR, "unknown field '%s' for image table in DVO database\n", fieldName); Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/gimages.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/gimages.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/gimages.c (revision 30631) @@ -201,7 +201,5 @@ } - /*** XXX we need to re-introduce the use of applyMcal - Mcal = applyMcal (&image[i], 2048.0, 2048.0); - ***/ + // XXX Mcal = applyMcal (&image[i], 2048.0, 2048.0); got_spot: Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/gstar.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/gstar.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/gstar.c (revision 30631) @@ -222,6 +222,6 @@ gprint (GP_LOG, "%5d ", catalog.measure[m].photcode); gprint (GP_LOG, "%-20s ", GetPhotcodeNamebyCode (catalog.measure[m].photcode)); - gprint (GP_LOG, "%5.2f ", 0.01*catalog.measure[m].FWx); - gprint (GP_LOG, "%5.2f ", 0.01*catalog.measure[m].FWy); + gprint (GP_LOG, "%5.2f ", FromShortPixels(catalog.measure[m].FWx)); + gprint (GP_LOG, "%5.2f ", FromShortPixels(catalog.measure[m].FWy)); if (FULL_OUTPUT) { @@ -233,6 +233,6 @@ gprint (GP_LOG, "%f ", catalog.measure[m].Xccd); gprint (GP_LOG, "%f ", catalog.measure[m].Yccd); - gprint (GP_LOG, "%d ", catalog.measure[m].dXccd); - gprint (GP_LOG, "%d ", catalog.measure[m].dYccd); + gprint (GP_LOG, "%f ", FromShortPixels(catalog.measure[m].dXccd)); + gprint (GP_LOG, "%f ", FromShortPixels(catalog.measure[m].dYccd)); gprint (GP_LOG, "%f ", catalog.measure[m].Sky); gprint (GP_LOG, "%f ", catalog.measure[m].dSky); @@ -244,5 +244,5 @@ gprint (GP_LOG, "%f ", catalog.measure[m].crNsigma); gprint (GP_LOG, "%f ", catalog.measure[m].extNsigma); - gprint (GP_LOG, "%f ", (360.0/(float)0xffff)*catalog.measure[m].theta); + gprint (GP_LOG, "%f ", FromShortDegrees(catalog.measure[m].theta)); } gprint (GP_LOG, "\n"); Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/images.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/images.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/images.c (revision 30631) @@ -223,11 +223,10 @@ goto got_type; } - // For 'TrianglePts' (TRI-), we are using the Mx,My, etc terms to save the coordinates - // this means triangular images cannot carry photometric zero-point variations + // For 'TrianglePts' (TRI-), we are absurdly overloaded values otherwise not used if (typehash == TrianglePts) { Npts = 3; - x[0] = image[i].Mx; y[0] = image[i].My; - x[1] = image[i].Mxxx; y[1] = image[i].Mxyy; - x[2] = image[i].Mxxy; y[2] = image[i].Myyy; + x[0] = image[0].dXpixSys; y[0] = image[0].dYpixSys; + x[1] = image[0].dMagSys; y[1] = image[0].nFitAstrom; + x[2] = image[0].photom_map_id; y[2] = image[0].astrom_map_id; goto got_type; } Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/imextract.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/imextract.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/imextract.c (revision 30631) @@ -62,4 +62,8 @@ // construct the db Boolean math stack (frees cstack) stack = dbRPN (Ncstack, cstack, &Nstack); + if (Ncstack && !Nstack) { + print_error(); + goto escape; + } // add the skyregion limits to the where statement (or create) Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/imphot.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/imphot.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/imphot.c (revision 30631) @@ -9,5 +9,5 @@ char bufname[64]; float *p; - double fx, fy, x, y, applyMcal(); + double fx, fy, x, y; Image *image; Buffer *buf; @@ -63,5 +63,6 @@ for (y = 0; y < 200; y+=1.0) { for (x = 0; x < 100; x+=1.0, p++) { - *p = applyMcal (&image[subset[0]], (fx*x), (fy*y)); + // *p = applyMcal (&image[subset[0]], (fx*x), (fy*y)); + *p = image[subset[0]].Mcal; } } @@ -70,4 +71,8 @@ for (j = 0; j < Nsubset; j++) { i = subset[j]; + gprint (GP_ERR, "%s: %f\n", image[i].name, image[i].Mcal); + +// XXX old code when we had the option of a 2D zero point model +# if (0) switch (image[i].order) { case 0: @@ -90,4 +95,5 @@ break; } +# endif } Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/mextract.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/mextract.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/mextract.c (revision 30631) @@ -68,5 +68,5 @@ // construct the db Boolean math stack (frees cstack) stack = dbRPN (Ncstack, cstack, &Nstack); - if ((Ncstack > 0) && (Nstack < 1)) { + if (Ncstack && !Nstack) { print_error (); goto escape; Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/mmextract.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/mmextract.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/mmextract.c (revision 30631) @@ -106,5 +106,5 @@ // construct the db Boolean math stack (frees cstack) stack1 = dbRPN (Ncstack1, cstack1, &Nstack1); - if ((Ncstack1 > 0) && (Nstack1 < 1)) { + if (Ncstack1 && !Nstack1) { print_error (); goto escape; @@ -113,5 +113,5 @@ // construct the db Boolean math stack (frees cstack) stack2 = dbRPN (Ncstack2, cstack2, &Nstack2); - if ((Ncstack2 > 0) && (Nstack2 < 1)) { + if (Ncstack2 && !Nstack2) { print_error (); goto escape; Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/photometry.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/photometry.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/photometry.c (revision 30631) @@ -778,6 +778,6 @@ } else { - if (0.01 * measure[0].FWx < 3.0) return (FALSE); - if (0.01 * measure[0].FWx > 10.0) return (FALSE); + if (FromShortPixels(measure[0].FWx < 3.0)) return (FALSE); + if (FromShortPixels(measure[0].FWx > 10.0)) return (FALSE); return (TRUE); @@ -810,5 +810,5 @@ Nm = 0; for (i = 0; i < average[0].Nmeasure; i++) { - fwhm = measure[i].FWx / 100.0; + fwhm = FromShortPixels(measure[i].FWx); switch (FWHMsign) { case 0: @@ -1166,5 +1166,5 @@ break; case MEAS_FWHM: /* OK */ - value = 0.01*measure[0].FWx; + value = FromShortPixels(measure[0].FWx); break; case MEAS_DB_FLAGS: /* ? */ Index: /branches/czw_branch/20101203/Ohana/src/opihi/dvo/skycat.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/dvo/skycat.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/dvo/skycat.c (revision 30631) @@ -36,5 +36,5 @@ if (argc != 1) { - gprint (GP_ERR, "USAGE: skycat [-all]\n"); + gprint (GP_ERR, "USAGE: skycat [-all] [-depth depth] [-v]\n"); return (FALSE); } @@ -116,4 +116,6 @@ r = ohana_normalize_angle (r); + while (r < Rmin) { r += 360.0; } + while (r > Rmax) { r -= 360.0; } if (*leftside == -1) { Index: /branches/czw_branch/20101203/Ohana/src/opihi/include/dvoshell.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/include/dvoshell.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/include/dvoshell.h (revision 30631) @@ -107,4 +107,5 @@ MEAS_XCCD_ERR, MEAS_YCCD_ERR, + MEAS_POS_SYS_ERR, MEAS_XMOSAIC, MEAS_YMOSAIC, @@ -116,4 +117,5 @@ MEAS_IMAGE_ID, MEAS_PSF_QF, + MEAS_PSF_QF_PERFECT, MEAS_PSF_CHISQ, MEAS_PSF_NDOF, @@ -218,4 +220,11 @@ IMAGE_Y_UL_FP, IMAGE_Y_UR_FP, + IMAGE_X_ERR_SYS, + IMAGE_Y_ERR_SYS, + IMAGE_MAG_ERR_SYS, + IMAGE_NFIT_PHOTOM, + IMAGE_NFIT_ASTROM, + IMAGE_NLINK_PHOTOM, + IMAGE_NLINK_ASTROM }; Index: /branches/czw_branch/20101203/Ohana/src/opihi/include/shell.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/include/shell.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/include/shell.h (revision 30631) @@ -76,5 +76,5 @@ char *expand_vars PROTO((char *line)); char *expand_vectors PROTO((char *line)); -char *parse PROTO((char *line)); +char *parse PROTO((int *status, char *line)); char **parse_commands PROTO((char *, int *)); void welcome PROTO((void)); Index: /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/VectorOps.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/VectorOps.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/VectorOps.c (revision 30631) @@ -178,8 +178,8 @@ vec[0].Nelements = Nelements; if (type == OPIHI_FLT) { - REALLOCATE (vec[0].elements.Flt, opihi_flt, Nelements); + REALLOCATE (vec[0].elements.Flt, opihi_flt, MAX(1, Nelements)); vec[0].type = OPIHI_FLT; } else { - REALLOCATE (vec[0].elements.Int, opihi_int, Nelements); + REALLOCATE (vec[0].elements.Int, opihi_int, MAX(1, Nelements)); vec[0].type = OPIHI_INT; } Index: /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/command.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/command.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/command.c (revision 30631) @@ -10,6 +10,8 @@ Command *cmd; - // rawline = NULL; - rawline = strcreate (line); + // the input line is never NULL + if (!line) { fprintf (stderr, "programming error\n"); abort(); } + + rawline = strcreate (line); // used for error messages which should echo the unparsed line /* force a space between ! and first word: !ls becomes ! ls */ @@ -22,6 +24,9 @@ /* expand anything of the form $fred or $fred$sam, etc */ line = expand_vars (line); /* line is freed here, new one allocated */ + if (!line) goto escape; + /* expand anything of the form fred[N] */ line = expand_vectors (line); /* line is freed here, new one allocated */ + if (!line) goto escape; // print the line with the variables and vectors expanded, but before evaluating @@ -29,7 +34,8 @@ if (VERBOSE_SHELL == OPIHI_VERBOSE_ON) gprint (GP_ERR, "opihi: %s\n", line); - /* solve math expresions, assign variable, if needed */ - line = parse (line); /* line is freed here, new one allocated */ - /* any entry in line of the form {foo} returns value or tmp vector / buffer */ + /* solve math expresions, assign variable, if needed. any entry in line of the form {foo} + * returns value or tmp vector / buffer */ + line = parse (&status, line); /* line is freed here, new one allocated */ + if (!status) goto escape; /* we may have reallocated line, return new pointer */ @@ -39,4 +45,5 @@ if (argc == 0) { FREE (rawline); + set_int_variable ("STATUS", TRUE); return (TRUE); /* empty command or assignment */ } @@ -74,4 +81,19 @@ FREE (rawline); return (status); + + escape: + set_int_variable ("STATUS", FALSE); + + # if (DEBUG) + gprint (GP_ERR, "command: %s, status: %d\n", line, FALSE); + # endif + + FREE (rawline); + + if (VERBOSE_SHELL != OPIHI_VERBOSE_OFF) gprint (GP_ERR, "error on line: %s\n", rawline); + + // return the current value of line, in case it was modified + *outline = line; + return FALSE; } Index: /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/expand_vars.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/expand_vars.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/expand_vars.c (revision 30631) @@ -64,5 +64,14 @@ /* variable assignment (skip these variables) */ if ((L == line) && (V1 != NULL)) { - if ((*V1 == '=') || !strcmp (V1, "++") || !strcmp (V1, "--")) { + int isAssignment; + + isAssignment = FALSE; + isAssignment |= (V1[0] == '='); + isAssignment |= (V1[0] == '+') && (V1[1] == '='); + isAssignment |= (V1[0] == '-') && (V1[1] == '='); + isAssignment |= (V1[0] == '+') && (V1[1] == '+'); + isAssignment |= (V1[0] == '-') && (V1[1] == '-'); + + if (isAssignment) { *N = *L; free (V0); Index: /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/parse.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/parse.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/parse.c (revision 30631) @@ -1,13 +1,15 @@ # include "opihi.h" -char *parse (char *line) { +char *parse (int *status, char *line) { double fval; float *fptr; char *newline, *N, *L, *val, *B, *V, *V0, *V1, *c1, *c2, *p; - int Nx, Ny, Nbytes, status, size, NLINE, isBuffer, inRange; + int Nx, Ny, Nbytes, filestatus, size, NLINE, isBuffer, inRange; FILE *f; Vector *vec; Buffer *buf; + + *status = TRUE; Ny = 0; @@ -49,4 +51,48 @@ } + if (!strncmp (V1, "+=", 2)) { + V1 ++; + if (*V1 == 0) goto error; + V1 ++; + if (*V1 == 0) goto error; + + val = get_variable (V0); + if (val == NULL) { + fval = 0.0; + } else { + fval = atof (val); + } + + /* dvomath returns a new string, or NULL, with the result of the expression */ + val = dvomath (1, &V1, &size, 0); + if (val == NULL) goto error; + fval += atof(val); + // save the result + set_variable (V0, fval); + goto escape; + } + + if (!strncmp (V1, "-=", 2)) { + V1 ++; + if (*V1 == 0) goto error; + V1 ++; + if (*V1 == 0) goto error; + + val = get_variable (V0); + if (val == NULL) { + fval = 0.0; + } else { + fval = atof (val); + } + + /* dvomath returns a new string, or NULL, with the result of the expression */ + val = dvomath (1, &V1, &size, 0); + if (val == NULL) goto error; + fval -= atof(val); + // save the result + set_variable (V0, fval); + goto escape; + } + /* not an assignement, syntax error */ if (*V1 != '=') goto error; @@ -73,8 +119,8 @@ Nbytes = fread (val, 1, 1023, f); val[Nbytes] = 0; - status = pclose (f); + filestatus = pclose (f); free (B); - if (status) gprint (GP_ERR, "warning: exit status of command %d\n", status); + if (filestatus) gprint (GP_ERR, "warning: exit status of command %d\n", filestatus); /* convert all but last return to ' '. drop last return */ @@ -88,16 +134,17 @@ } } + // save the resulting value + set_str_variable (V0, val); + goto escape; /* frees temp variables */ + } + + /* simple variable assignment */ + /* dvomath returns a new string, or NULL, with the result of the expression */ + val = dvomath (1, &V1, &size, 0); + if (val == NULL) { + while (OHANA_WHITESPACE (*V1)) V1++; + val = strcreate (V1); } - - /* simple variable assignment */ - if (*V1 != '`') { - /* dvomath returns a new string, or NULL, with the result of the expression */ - val = dvomath (1, &V1, &size, 0); - if (val == NULL) { - while (OHANA_WHITESPACE (*V1)) V1++; - val = strcreate (V1); - } - } - /* both dvomath and command replacement create (char *) val */ + // save the result set_str_variable (V0, val); goto escape; /* frees temp variables */ @@ -172,5 +219,5 @@ if (!inRange) { gprint (GP_ERR, "no element %d,%d\n", Nx, Ny); - goto escape; + goto error; } if (Nx < 0) Nx += buf[0].header.Naxis[0]; @@ -188,5 +235,5 @@ if (!inRange) { gprint (GP_ERR, "no element %d\n", Nx); - goto escape; + goto error; } if (Nx < 0) Nx += vec[0].Nelements; @@ -197,5 +244,4 @@ } } - goto escape; } @@ -226,10 +272,10 @@ if (c1 == NULL) { gprint (GP_ERR, "no close brackets!\n"); - goto escape; + goto error; } c2 = strchr (L, '{'); if ((c2 != NULL) && (c2 < c1)) { gprint (GP_ERR, "can't nest brackets!\n"); - goto escape; + goto error; } *c1 = 0; @@ -240,5 +286,5 @@ if (val == NULL) { print_error (); - goto escape; + goto error; } @@ -257,13 +303,22 @@ REALLOCATE (newline, char, strlen (newline) + 1); + *status = TRUE; return (newline); error: gprint (GP_ERR, "syntax error\n"); - -escape: + // assignment or increment operation: free line and return NULL if (line != (char *) NULL) free (line); if (val != (char *) NULL) free (val); if (V0 != (char *) NULL) free (V0); + *status = FALSE; + return NULL; + +escape: + // assignment or increment operation: free line and return NULL + if (line != (char *) NULL) free (line); + if (val != (char *) NULL) free (val); + if (V0 != (char *) NULL) free (V0); + *status = TRUE; return (NULL); } Index: /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/stack_math.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/stack_math.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/opihi/lib.shell/stack_math.c (revision 30631) @@ -972,6 +972,5 @@ *out = OP; \ } \ - clear_stack (V1); \ - return (TRUE); \ + goto escape; \ } \ if ((V1->vector->type == OPIHI_INT) && (FTYPE == 'S')) { \ @@ -982,6 +981,5 @@ *out = OP; \ } \ - clear_stack (V1); \ - return (TRUE); \ + goto escape; \ } \ if ((V1->vector->type == OPIHI_INT) && (FTYPE == 's')) { \ @@ -992,6 +990,5 @@ *out = OP; \ } \ - clear_stack (V1); \ - return (TRUE); \ + goto escape; \ } } @@ -1035,4 +1032,6 @@ # undef V_FUNC +escape: + if (V1[0].type == 'v') { free (V1[0].vector[0].elements.Ptr); Index: /branches/czw_branch/20101203/Ohana/src/photdbc/include/photdbc.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/photdbc/include/photdbc.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/photdbc/include/photdbc.h (revision 30631) @@ -51,8 +51,13 @@ double DMGAIN; double CHISQ_MAX; +double SIGMA_MIN_KEEP; double SIGMA_MAX; double AVE_SIGMA_LIM; int NMEAS_MIN; +int NMEAS_MIN_FILTERED; +int NCODE_MIN; double ZERO_POINT; + +int ExcludeByMinSigma; int ExcludeByInstMag; @@ -65,4 +70,8 @@ SkyRegion REGION; PhotCodeData photcodes; + +char *PHOTCODE_DROP_LIST, *PHOTCODE_SKIP_LIST; +int NphotcodesDrop, NphotcodesSkip; +PhotCode **photcodesDrop, **photcodesSkip; # define FLAG_AREA 0X0001 Index: /branches/czw_branch/20101203/Ohana/src/photdbc/src/ConfigInit.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/photdbc/src/ConfigInit.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/photdbc/src/ConfigInit.c (revision 30631) @@ -26,7 +26,11 @@ if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); - WarnConfig (config, "PHOTDBC_JOIN_RADIUS", "%lf", 0, &JOIN_RADIUS); + // XXX join_stars (in photdbc.c) is currently disabled + // WarnConfig (config, "PHOTDBC_JOIN_RADIUS", "%lf", 0, &JOIN_RADIUS); + + // XXX unique_measures (in photdbc.c) is currently disabled // WarnConfig (config, "UNIQ_RADIUS", "%lf", 0, &UNIQ_RADIUS); + // XXX flag_measures (in photdbc.c) is currently disabled // WarnConfig (config, "XMIN", "%lf", 0, &XMIN); // WarnConfig (config, "XMAX", "%lf", 0, &XMAX); @@ -37,11 +41,14 @@ // WarnConfig (config, "MMIN", "%lf", 0, &tmp); MMIN = 1000*tmp; // WarnConfig (config, "MMAX", "%lf", 0, &tmp); MMAX = 1000*tmp; + // WarnConfig (config, "CHISQ_MAX", "%lf", 0, &CHISQ_MAX); + + // XXX get_mags (in photdbc.c) is currently disabled // WarnConfig (config, "DMSYS", "%lf", 0, &tmp); DMSYS = SQ(1000*tmp); // WarnConfig (config, "DMGAIN", "%lf", 0, &DMGAIN); - // WarnConfig (config, "CHISQ_MAX", "%lf", 0, &CHISQ_MAX); ScanConfig (config, "SIGMA_MAX", "%lf", 0, &SIGMA_MAX); ScanConfig (config, "AVE_SIGMA_LIM", "%lf", 0, &AVE_SIGMA_LIM); ScanConfig (config, "NMEAS_MIN", "%d", 0, &NMEAS_MIN); + ScanConfig (config, "NMEAS_MIN_FILTERED", "%d", 0, &NMEAS_MIN_FILTERED); WarnConfig (config, "GSCFILE", "%s", 0, GSCFILE); Index: /branches/czw_branch/20101203/Ohana/src/photdbc/src/args.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/photdbc/src/args.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/photdbc/src/args.c (revision 30631) @@ -19,5 +19,4 @@ ExcludeByInstMag = FALSE; if ((N = get_argument (argc, argv, "-instmag"))) { - remove_argument (N, &argc, argv); ExcludeByInstMag = TRUE; remove_argument (N, &argc, argv); @@ -25,12 +24,21 @@ remove_argument (N, &argc, argv); INST_MAG_MAX = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + ExcludeByMinSigma = FALSE; + if ((N = get_argument (argc, argv, "-min-sigma"))) { + ExcludeByMinSigma = TRUE; + remove_argument (N, &argc, argv); + SIGMA_MIN_KEEP = atof(argv[N]); + remove_argument (N, &argc, argv); } ExcludeByMaxMinMag = FALSE; if ((N = get_argument (argc, argv, "-maxminmag"))) { - remove_argument (N, &argc, argv); ExcludeByMaxMinMag = TRUE; remove_argument (N, &argc, argv); MAX_MIN_MAG = atof(argv[N]); + remove_argument (N, &argc, argv); } @@ -61,4 +69,18 @@ } + PHOTCODE_DROP_LIST = NULL; + if ((N = get_argument (argc, argv, "-photcode-drop"))) { + remove_argument (N, &argc, argv); + PHOTCODE_DROP_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + + PHOTCODE_SKIP_LIST = NULL; + if ((N = get_argument (argc, argv, "-photcode-skip"))) { + remove_argument (N, &argc, argv); + PHOTCODE_SKIP_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + if (argc != 2) usage(); @@ -81,20 +103,35 @@ fprintf (stderr, "USAGE: photdbc (output)\n\n"); fprintf (stderr, " this program takes an existing DVO database and makes a copy, applying a number of optional\n"); - fprintf (stderr, " filters in the process. \n"); - fprintf (stderr, "\n"); - fprintf (stderr, " photdbc (output)\n"); - fprintf (stderr, "\n"); - fprintf (stderr, " allowed filters / restrictions include:\n"); - fprintf (stderr, "\n"); + fprintf (stderr, " filters in the process. \n\n"); + + fprintf (stderr, " photdbc (output)\n\n"); + + fprintf (stderr, " allowed filters / restrictions include:\n\n"); + fprintf (stderr, " -region Rmin Rmax Dmin Dmax : limit operation to the specified region \n"); - fprintf (stderr, " -join : join measurements between stars using JOIN_RADIUS\n"); - fprintf (stderr, " -ccdregion X Y X Y : only keep detections within the specified detector region\n"); - fprintf (stderr, " (can this be limited to specific photcodes? cameras? detectors?)\n"); - fprintf (stderr, " -photcode_limits code Mmin Mmax : allow multiples of these\n"); - fprintf (stderr, "\n"); - fprintf (stderr, " SIGMA_MAX\n"); - fprintf (stderr, " NMEAS_MIN\n"); - fprintf (stderr, " INST_MAG_MAX\n"); - fprintf (stderr, " INST_MAG_MIN\n"); + + fprintf (stderr, " -photcode-drop : remove these photcodes from the output (REF or DEP only)\n"); + fprintf (stderr, " -photcode-skip : ignore these photcodes when assessing the validity (keep unless object is dropped)\n"); + + fprintf (stderr, " -instmag (min) (max) : range of valid instrumental magnitudes (or measurements are dropped)\n"); + fprintf (stderr, " -min-sigma (sigma) : object must have one measurement error less than sigma or object is dropped\n"); + + fprintf (stderr, " -maxminmag (mag) : object must have one magnitude less than this or object is dropped\n"); + + fprintf (stderr, " option options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -params : list the current parameters\n\n"); + + fprintf (stderr, "ptolemy.rc config values used by this program:\n"); + fprintf (stderr, " SIGMA_MAX : drop measurements with errors greater than this\n"); + fprintf (stderr, " NMEAS_MIN : drop objects with fewer measurements than this\n"); + fprintf (stderr, " NMEAS_MIN_FILTERED : drop objects with fewer measurements than this after filtering above\n"); + fprintf (stderr, " AVE_SIGMA_LIM : drop objects if all average mags are greater than this\n"); + exit (2); } + +// fprintf (stderr, " -join : join measurements between stars using JOIN_RADIUS\n"); +// fprintf (stderr, " -ccdregion X Y X Y : only keep detections within the specified detector region\n"); +// fprintf (stderr, " (can this be limited to specific photcodes? cameras? detectors?)\n"); +// fprintf (stderr, " -photcode_limits code Mmin Mmax : allow multiples of these\n"); Index: /branches/czw_branch/20101203/Ohana/src/photdbc/src/find_images.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/photdbc/src/find_images.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/photdbc/src/find_images.c (revision 30631) @@ -122,5 +122,6 @@ these are measurements from external sources, like USNO */ - assignMcal (&image[nimage], (double *) NULL, -1); + // assignMcal (&image[nimage], (double *) NULL, -1); + image[nimage].Mcal = 0; image[nimage].code = ID_IMAGE_NEW; Index: /branches/czw_branch/20101203/Ohana/src/photdbc/src/initialize.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/photdbc/src/initialize.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/photdbc/src/initialize.c (revision 30631) @@ -2,4 +2,7 @@ void initialize (int argc, char **argv) { + + int NPHOTCODES; + char *codename, *ptr, *list; /* are these set correctly? */ @@ -11,4 +14,46 @@ ConfigInit (&argc, argv); args (argc, argv); + + NphotcodesDrop = 0; + photcodesDrop = NULL; + if (PHOTCODE_DROP_LIST != NULL) { + NPHOTCODES = 10; + ALLOCATE (photcodesDrop, PhotCode *, NPHOTCODES); + + /* parse the comma-separated list of photcodesDrop */ + list = PHOTCODE_DROP_LIST; + while ((codename = strtok_r (list, ",", &ptr)) != NULL) { + list = NULL; // pass NULL on successive strtok_r calls + fprintf (stderr, "PHOTCODE_LIST: %s\n", PHOTCODE_DROP_LIST); + fprintf (stderr, "codename: %s\n", codename); + if ((photcodesDrop[NphotcodesDrop] = GetPhotcodebyName (codename)) == NULL) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); + } + NphotcodesDrop ++; + CHECK_REALLOCATE (photcodesDrop, PhotCode *, NPHOTCODES, NphotcodesDrop, 10); + } + } + + NphotcodesSkip = 0; + photcodesSkip = NULL; + if (PHOTCODE_SKIP_LIST != NULL) { + NPHOTCODES = 10; + ALLOCATE (photcodesSkip, PhotCode *, NPHOTCODES); + + /* parse the comma-separated list of photcodesSkip */ + list = PHOTCODE_SKIP_LIST; + while ((codename = strtok_r (list, ",", &ptr)) != NULL) { + list = NULL; // pass NULL on successive strtok_r calls + fprintf (stderr, "PHOTCODE_LIST: %s\n", PHOTCODE_SKIP_LIST); + fprintf (stderr, "codename: %s\n", codename); + if ((photcodesSkip[NphotcodesSkip] = GetPhotcodebyName (codename)) == NULL) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); + } + NphotcodesSkip ++; + CHECK_REALLOCATE (photcodesSkip, PhotCode *, NPHOTCODES, NphotcodesSkip, 10); + } + } if (SHOW_PARAMS) { Index: /branches/czw_branch/20101203/Ohana/src/photdbc/src/make_subcatalog.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/photdbc/src/make_subcatalog.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/photdbc/src/make_subcatalog.c (revision 30631) @@ -5,11 +5,24 @@ int make_subcatalog (Catalog *subcatalog, Catalog *catalog) { - off_t i, j, offset; + int found; + off_t i, j, k, offset; off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt; - double mag, minMag; - int keep; + double mag, minMag, minSigma; + int keep, *secSkip; + PhotCode *photcode; Nsecfilt = GetPhotcodeNsecfilt (); assert (catalog[0].Nsecfilt == Nsecfilt); + + // set up a list of SEC entries to ignore when evaluating a source + ALLOCATE (secSkip, int, Nsecfilt); + for (i = 0; i < Nsecfilt; i++) { + secSkip[i] = FALSE; + photcode = GetPhotcodebyNsec(i); + for (k = 0; k < NphotcodesSkip; k++) { + if (photcodesSkip[k][0].code != photcode[0].code) continue; + secSkip[i] = TRUE; + } + } /* we are moving only the subset of measurements from catalog[0] to subcatalog[0] */ @@ -38,4 +51,17 @@ } + // exclude stars with too few measurements + if (NCODE_MIN) { + // drop if all of the allowed average photcodes have ncode < NCODE_MIN values + keep = FALSE; + for (j = 0; !keep && (j < Nsecfilt); j++) { + if (secSkip[j]) continue; + if (catalog[0].secfilt[Nsecfilt*i+j].Ncode >= NCODE_MIN) { + keep = TRUE; + } + } + if (!keep) continue; + } + /* assign average and secfilt values */ subcatalog[0].average[Naverage] = catalog[0].average[i]; @@ -45,5 +71,6 @@ } - minMag = 32; + minMag = 32; + minSigma = 32; Nm = 0; for (j = 0; j < catalog[0].average[i].Nmeasure; j++) { @@ -52,15 +79,31 @@ # if 0 - if (DropPhotcode) { - ecode = GetPhotcodeEquivCodebyCode (catalog[0].measure[offset].photcode); - if (ecode == photcode[0].code) continue; - } if (DropNonStellar && catalog[0].measure[offset].dophot != 1) continue; # endif - /* exclude measurements by measurement error */ + // remove certain photcodes from the output measurements + if (NphotcodesDrop > 0) { + found = FALSE; + for (k = 0; (k < NphotcodesSkip) && !found; k++) { + if (photcodesSkip[k][0].code == catalog[0].measure[offset].photcode) found = TRUE; + if (photcodesSkip[k][0].code == GetPhotcodeEquivCodebyCode(catalog[0].measure[offset].photcode)) found = TRUE; + } + if (found) continue; + } + + // ignore certain photcodes to assess the measurements + if (NphotcodesSkip > 0) { + found = FALSE; + for (k = 0; (k < NphotcodesSkip) && !found; k++) { + if (photcodesSkip[k][0].code == catalog[0].measure[offset].photcode) found = TRUE; + if (photcodesSkip[k][0].code == GetPhotcodeEquivCodebyCode(catalog[0].measure[offset].photcode)) found = TRUE; + } + if (found) goto keep; + } + + // exclude measurements by measurement error -- drop exactly this measurement if (SIGMA_MAX && (catalog[0].measure[offset].dM > SIGMA_MAX)) continue; - /* select measurements by mag limit */ + // select measurements by mag limit -- drop exactly this measurement if (ExcludeByInstMag) { mag = PhotInst (&catalog[0].measure[offset]); @@ -69,8 +112,16 @@ } + // check measurements for this object -- drop object if no measurements pass + if (ExcludeByMinSigma) { + minSigma = MIN (minSigma, catalog[0].measure[offset].dM); + } + + // check measurements for this object -- drop object if no measurements pass if (ExcludeByMaxMinMag) { mag = PhotSys (&catalog[0].measure[offset], &catalog[0].average[i], &catalog[0].secfilt[i*Nsecfilt]); minMag = MIN (minMag, mag); } + + keep: subcatalog[0].measure[Nmeasure] = catalog[0].measure[offset]; @@ -90,9 +141,16 @@ } - // after measurement exclusion, exclude stars with too few measurements - if (Nm < NMEAS_MIN) { + // exclude faint objects + if (ExcludeByMinSigma && (minSigma > SIGMA_MIN_KEEP)) { Nmeasure -= Nm; continue; } + + // after measurement exclusion, exclude stars with too few measurements + if (NMEAS_MIN_FILTERED && (Nm < NMEAS_MIN_FILTERED)) { + Nmeasure -= Nm; + continue; + } + subcatalog[0].average[Naverage].Nmissing = 0; subcatalog[0].average[Naverage].Nmeasure = Nm; @@ -122,2 +180,24 @@ return (TRUE); } + +/** + the purpose of this function is to create a subset database. there are several ways this could be done: + + * reject all measurements from all objects that fail to meet some criteria (objects may lose measurements and/or be dropped) + + * only reject an object if all measurements fail to meet some criteria (ie, keep all measurements if any measurement passes) + + * only apply the keep / reject criteria to certain photcodes + + + *** + + * options: + + minimum value for nmeas : NMEAS_MIN + minimum value for nmeas : NMEAS_MIN_FILTERED + minimum value for ncode : NCODE_MIN (drop if all ncode < NCODE_MIN) (respect photcodes) + reject faint meas : SIGMA_MAX + reject faint source : SIGMA_MIN_KEEP (keep source source if its minimum dMag < this limit) + + **/ Index: /branches/czw_branch/20101203/Ohana/src/relastro/include/relastro.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/include/relastro.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/include/relastro.h (revision 30631) @@ -26,4 +26,5 @@ double dPos; int mask; + int Nmeas; } StarData; @@ -94,6 +95,10 @@ double SIGMA_LIM; -int SRC_MEAS_TOOFEW; //catalog objects wich fewer detections then this are ignored +int SRC_MEAS_TOOFEW; //catalog objects wich fewer detections then this are ignored double MIN_ERROR; + +int IMFIT_CLIP_NITER; // number of clipping iterations to perform in FitChip +double IMFIT_CLIP_NSIGMA; // number of sigma to clip in FitChip +double IMFIT_SYS_SIGMA_LIM; // max dMag for objects used to measure systematic scatter double RADIUS; // match radius for high-speed objects @@ -141,4 +146,5 @@ int FlagOutlier; int CLIP_THRESH; +int USE_BASIC_CHECK; FitMode FIT_MODE; @@ -296,5 +302,5 @@ void fixImageRaw (Catalog *catalog, int Ncatalog, off_t im); void FlagOutliers(Catalog *catalog); -int MeasFilterTest(Measure *measure); +int MeasFilterTest(Measure *measure, int applySigmaLim); int sun_ecliptic (double jd, double *lambda, double *beta, double *epsilon); @@ -325,10 +331,10 @@ int UpdateObjectOffsets (SkyList *skylist); -int relastroVisualPlotRawRef(StarData *raw, StarData *ref, double dRmax, int numObj); -int relastroVisualPlotScatter(double values[], double thresh, int npts); -int relastroVisualPlotOutliers(Catalog *catalog, int offset, int Nmeasure, - StatType statsR, StatType statsD, double thresh); - - +int relastroVisualPlotChipFit(StarData *raw, StarData *ref, double dRmax, int numObj); +// int relastroVisualPlotRawRef(StarData *raw, StarData *ref, double dRmax, int numObj); +// int relastroVisualPlotScatter(double values[], double thresh, int npts); +// int relastroVisualPlotOutliers(Catalog *catalog, int offset, int Nmeasure, StatType statsR, StatType statsD, double thresh); +void relastroSetVisual(int state); +int relastroGetVisual(void); int FixProblemImages (SkyList *skylist); @@ -352,2 +358,4 @@ int createStarMapPoints(); int checkStarMap(int N); + +int GetScatterRawRef(float *dLsig, float *dMsig, float *dRsig, int *nKeep, StarData *raw, StarData *ref, int Nstars, float SigmaLimit); Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/ConfigInit.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/ConfigInit.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/ConfigInit.c (revision 30631) @@ -19,6 +19,10 @@ if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); - GetConfig (config, "RELASTRO_SIGMA_LIM", "%lf", 0, &SIGMA_LIM); + GetConfig (config, "RELASTRO_SIGMA_LIM", "%lf", 0, &SIGMA_LIM); // exclude measurements on this basis GetConfig (config, "RELASTRO_SRC_MEAS_TOOFEW", "%d", 0, &SRC_MEAS_TOOFEW); + + if (!ScanConfig (config, "RELASTRO_IMFIT_CLIP_NITER", "%d", 0, &IMFIT_CLIP_NITER)) IMFIT_CLIP_NITER = 3; + if (!ScanConfig (config, "RELASTRO_IMFIT_CLIP_NSIGMA", "%lf", 0, &IMFIT_CLIP_NSIGMA)) IMFIT_CLIP_NSIGMA = 3.0; + if (!ScanConfig (config, "RELASTRO_IMFIT_SYS_SIGMA_LIM", "%lf", 0, &IMFIT_SYS_SIGMA_LIM)) IMFIT_SYS_SIGMA_LIM = 0.01; // XXX these are used in relphot to identify poor stars / images -- define Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/FitChip.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/FitChip.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/FitChip.c (revision 30631) @@ -2,44 +2,51 @@ # include "relastroVisual.h" -// XXX make these user parameters -# define FIT_CHIP_NITER 3 -# define FIT_CHIP_NSIGMA 3.0 - -// XXX save the fit[0].Npts value in the image table? - -// XXX save measurements of the fit quality (scatter, chisq) in the image table - int FitChip (StarData *raw, StarData *ref, int Nmatch, Image *image) { - int i, Nscatter, Niter, skip; - CoordFit *fit; - double dL, dM, dR, dRmax, *values; - - ALLOCATE (values, double, Nmatch); - for (Niter = 0; Niter < FIT_CHIP_NITER; Niter ++) { + int i, NstatFull, Nstat, Niter, skip; + float dLsig, dMsig, dRsig; + float dLsigFull, dMsigFull, dRsigFull; + float dL, dM, dR, dRmax; + + CoordFit *fit = NULL; + + dRmax = 0.0; + + for (Niter = 0; Niter < IMFIT_CLIP_NITER; Niter ++) { + + // measure the scatter for the unmarked (good) measurements with dM < limit + // SIGMA_LIM here is redundant with ImageOps.c:915 + GetScatterRawRef(&dLsig, &dMsig, &dRsig, &Nstat, raw, ref, Nmatch, SIGMA_LIM); + dRmax = IMFIT_CLIP_NSIGMA*dRsig; + + // (a) skip unfittable points + // (b) mark good and bad points for fit (in and outliers) // measure the scatter distribution (use only the bright end detections) - for (i = Nscatter = 0; i < Nmatch; i++) { + for (i = 0; i < Nmatch; i++) { if (raw[i].mask) continue; - if (isnan(raw[i].dMag) || raw[i].dMag > SIGMA_LIM) continue; + if (isnan(raw[i].dMag)) { + raw[i].mask |= 0x0004; + continue; + } + if (raw[i].dMag > SIGMA_LIM) { + raw[i].mask |= 0x0008; + continue; + } // is this redundant with ImageOps.c:915? dL = raw[i].L - ref[i].L; dM = raw[i].M - ref[i].M; dR = hypot (dL, dM); - - values[Nscatter] = dR; - Nscatter++; - } - - if (Nscatter > 5) { - // for a 2D Gaussian, 40% of the points are within 1 sigma; dRmax is ~ 3 sigma - // XXX this test is not sensible for Nscatter < XXX (5?) - dsort (values, Nscatter); - dRmax = FIT_CHIP_NSIGMA*values[(int)(0.40*Nscatter)]; - relastroVisualPlotScatter(values, dRmax, Nscatter); - relastroVisualPlotRawRef(raw, ref, dRmax, Nmatch); - } else { - dRmax = 0; - } + if (dR > dRmax) { + raw[i].mask |= 0x0010; + continue; + } + } + + // figures to assess the fitting process: + // x vs dx, x vs dy, y vs dx, y vs dy : + // residual vector field + // dx vs mag, dy vs mag + relastroVisualPlotChipFit(raw, ref, dRmax, Nmatch); // fit the requested order polynomial @@ -47,24 +54,10 @@ image[0].coords.Npolyterms = CHIPORDER; } + if (fit) fit_free (fit); fit = fit_init (image[0].coords.Npolyterms); // generate the fit matches for (i = 0; i < Nmatch; i++) { - if (raw[i].mask) { - continue; - } - if (isnan(raw[i].dMag) || raw[i].dMag > SIGMA_LIM) { - continue; - } - - // only keep objects within dRmax - dL = raw[i].L - ref[i].L; - dM = raw[i].M - ref[i].M; - dR = hypot (dL, dM); - - // fprintf (stderr, "fit %f %f -> %f %f : %f %f (%f vs %f) wt: %f\n", raw[i].X, raw[i].Y, ref[i].L, ref[i].M, raw[i].L, raw[i].M, dR, dRmax, raw[i].dPos); - - if ((dRmax > 0.0) && (dR > dRmax)) continue; - + if (raw[i].mask) continue; fit_add (fit, raw[i].X, raw[i].Y, ref[i].L, ref[i].M, raw[i].dPos); } @@ -88,12 +81,9 @@ } if (skip) { - if (VERBOSE) fprintf (stderr, "insufficient measurements (%d) for requested order (%d)\n", fit[0].Npts, image[0].coords.Npolyterms); + if (VERBOSE2) fprintf (stderr, "insufficient measurements (%d) for requested order (%d)\n", fit[0].Npts, image[0].coords.Npolyterms); fit_free (fit); - free (values); image[0].flags |= ID_IMAGE_ASTROM_FEW; return FALSE; } - - // fprintf (stderr, "scatter limit: %f based on %d detections; using %d of %d for fit\n", dRmax, Nscatter, fit[0].Npts, Nmatch); // measure the fit, update the coords & object coordinates @@ -110,6 +100,4 @@ } - fit_free (fit); - for (i = 0; i < Nmatch; i++) { XY_to_LM (&raw[i].L, &raw[i].M, raw[i].X, raw[i].Y, &image[0].coords); @@ -118,6 +106,85 @@ } - free (values); + // count mask classes + int nMask1 = 0; + int nMask2 = 0; + int nMask3 = 0; + int nMask4 = 0; + int nMask5 = 0; + for (i = 0; i < Nmatch; i++) { + if (!raw[i].mask) continue; + if (raw[i].mask & 0x01) nMask1 ++; + if (raw[i].mask & 0x02) nMask2 ++; + if (raw[i].mask & 0x04) nMask3 ++; + if (raw[i].mask & 0x08) nMask4 ++; + if (raw[i].mask & 0x10) nMask5 ++; + } + + GetScatterRawRef(&dLsigFull, &dMsigFull, &dRsigFull, &NstatFull, raw, ref, Nmatch, SIGMA_LIM); + GetScatterRawRef(&dLsig, &dMsig, &dRsig, &Nstat, raw, ref, Nmatch, IMFIT_SYS_SIGMA_LIM); + + int Nm = 0; + int Ns = 0; + for (i = 0; i < Nmatch; i++) { + if (raw[i].mask) continue; + Nm += raw[i].Nmeas; + Ns++; + } + image[0].nLinkAstrom = (Nm / Ns); + + if (VERBOSE) fprintf (stderr, "fit sigma: %f (%f, %f) : full: %f (%f, %f), scatter limit: %f (%d full, %d bright, %d fit, %d all) (%d %d %d %d %d)\n", dRsig, dLsig, dMsig, dRsigFull, dLsigFull, dMsigFull, dRmax, NstatFull, Nstat, fit[0].Npts, Nmatch, nMask1, nMask2, nMask3, nMask4, nMask5); + + image[0].dXpixSys = dLsig; + image[0].dYpixSys = dMsig; + image[0].nFitAstrom = fit[0].Npts; + + if (fit) fit_free (fit); + return TRUE; +} + +// measure the scatter distribution (limit selected objects by dMag) +int GetScatterRawRef(float *dLsig, float *dMsig, float *dRsig, int *nKeep, StarData *raw, StarData *ref, int Nstars, float SigmaLimit) { + + int i, Ns; + float dL, dM, dLsum, dLsum2, dMsum, dMsum2, *dR; + + ALLOCATE (dR, float, Nstars); + + Ns = 0; + dLsum = dLsum2 = dMsum = dMsum2 = 0.0; + for (i = 0; i < Nstars; i++) { + if (raw[i].mask) continue; + if (isnan(raw[i].dMag)) continue; + if ((SigmaLimit > 0.0) && (raw[i].dMag > SigmaLimit)) continue; + + dL = raw[i].L - ref[i].L; + dM = raw[i].M - ref[i].M; + + dLsum += dL; + dLsum2 += dL*dL; + dMsum += dM; + dMsum2 += dM*dM; + + dR[Ns] = hypot (dL, dM); + Ns++; + } + + *dLsig = sqrt((dLsum2 / Ns) - SQ(dLsum/Ns)); + *dMsig = sqrt((dMsum2 / Ns) - SQ(dMsum/Ns)); + *nKeep = Ns; + + if (Ns < 5) { + *dRsig = NAN; + free (dR); + return (FALSE); + } + + // for a 2D Gaussian, 40% of the points are within R = 1 sigma + fsort (dR, Ns); + *dRsig = dR[(int)(0.40*Ns)]; + + free (dR); + return (TRUE); } @@ -162,2 +229,3 @@ fclose (f); */ + Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/GetAstromError.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/GetAstromError.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/GetAstromError.c (revision 30631) @@ -1,24 +1,31 @@ # include "relastro.h" +# define WEIGHTED_ERRORS 1 float GetAstromError (Measure *measure, int mode) { - //BIG HACKXXXXXXXX - return 0.1; + PhotCode *code; - float dPobs, dPsys, dPtotal, dM, AS, MS; + float dPobs, dPsys, dPtotal, dM, AS, MS, dX, dY; + + if (!WEIGHTED_ERRORS) { + // if we don't understand the errors at all, this at least lets us get things roughly + // right: + return 0.1; + } + switch (mode) { case ERROR_MODE_RA: - dPobs = measure[0].dXccd / 100.0; // need to redefine this as RAerr + dPobs = FromShortPixels(measure[0].dXccd); // dXccd is a value in pixels break; case ERROR_MODE_DEC: - dPobs = measure[0].dYccd / 100.0; // need to redefine this as RAerr + dPobs = FromShortPixels(measure[0].dYccd); // dYccd is a value in pixels break; case ERROR_MODE_POS: - dPobs = hypot (measure[0].dXccd, measure[0].dYccd) / 100.0; // need to redefine this as RAerr + dX = FromShortPixels(measure[0].dXccd); // dXccd is a value in pixels + dY = FromShortPixels(measure[0].dYccd); // dYccd is a value in pixels + dPobs = hypot (dX, dY); break; default: abort(); } - /* the astrometric errors are not being carried yet (but should be!) */ - /* we use the photometric mag error as a weighting term */ code = GetPhotcodebyCode (measure[0].photcode); @@ -27,8 +34,5 @@ dPsys = code[0].astromErrSys; dM = measure[0].dM; - dPtotal = sqrt(SQ(dPsys) + AS*SQ(dPobs) + MS*SQ(dM)); - - //XXX dXccd, dYccd are now working correctly - //dPtotal = AS * dPobs; + dPtotal = sqrt(SQ(dPsys) + SQ(AS*dPobs) + SQ(MS*dM)); dPtotal = MAX (dPtotal, MIN_ERROR); @@ -36,2 +40,17 @@ } +/* for a long time, psphot was either not reported position errors, or was reporting + * completely wrong astrometry errors. This function lets us handle, in the + * configuration, different strategies to generating a position error + * + * astrometry systematic error : this is the minimum expected per-position error. You + * should probably measure this from you data. watch out for the chicken and egg problem! + * + * astrometry error scale : this field lets you accept position errors in (say) pixels and + * convert them with this term to arcsec. AS : pixel scale in arcsec + * + * astrometry mag scale : this field lets you define position errors based on the + * photometry error. the scale factor should be something like a typical seeing number + * (in arcsec) for the given instrument + * + */ Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/ImageOps.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/ImageOps.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/ImageOps.c (revision 30631) @@ -146,4 +146,6 @@ free (clist); free (mlist); + free (Nlist); + free (NLIST); } @@ -325,5 +327,5 @@ for (j = 0; j < average[0].Nmeasure; j++) { off = average[0].measureOffset + j; - fprintf (stderr, "%f, %f\n", measure[off].dR, measure[off].dD); + fprintf (stderr, "dR, dD, mag, dMag: %f, %f, %f, %f\n", measure[off].dR, measure[off].dD, measure[off].M, measure[off].dM); } return; @@ -336,5 +338,5 @@ off_t i, m, c, n, nPos; double X, Y, L, M, P, Q, R, D, dR, dD; - double dPos, DPOS_MAX_ASEC; + double dPos, dPosSys, DPOS_MAX_ASEC; Mosaic *mosaic; @@ -359,9 +361,32 @@ } - // accumulate the rms position offsets. if this value, or any specific entry, is too - // large, we will reset the image to the original coords at the end of the analysis - + // these are used to accumulate the rms position offsets. if this value, or any + // specific entry, is too large, we will reset the image to the original coords at the + // end of the analysis dPos = 0.0; nPos = 0; + + // convert the image systematic error in pixels to a value in arcsec + { + double dLsig, dMsig; + double Ro, Do, Rx, Dx, dP0, dP1; + Coords *coords; + + // these values are in pixels, but we to convert to arcsec + dLsig = image[0].dXpixSys; + dMsig = image[0].dYpixSys; + + if (moscoords == NULL) { + coords = imcoords; + } else { + coords = moscoords; + } + XY_to_LM (&Ro, &Do, 0.0, 0.0, coords); + XY_to_LM (&Rx, &Dx, dLsig, 0.0, coords); + dP0 = 3600.0 * hypot(Rx - Ro, Dx - Do); // convert to arcsec + XY_to_LM (&Rx, &Dx, 0.0, dLsig, coords); + dP1 = 3600.0 * hypot(Rx - Ro, Dx - Do); // convert to arcsec + dPosSys = 0.5 * (dP0 + dP1); + } for (i = 0; i < Nlist[im]; i++) { @@ -390,10 +415,10 @@ // complain if the new location is far from the average location // NOTE: This should never happen, or our StarMap tests are not working - if (fabs(dR) > 1.5*ADDSTAR_RADIUS) { + if (fabs(dR) > 3.0*ADDSTAR_RADIUS) { fprintf (stderr, "measurement is far from average location (R): %f %f (%f %f)\n", catalog[c].average[n].R, catalog[c].average[n].D, dR, dD); dump_measures (&catalog[c].average[n], catalog[c].measure); // abort (); } - if (fabs(dD) > 1.5*ADDSTAR_RADIUS) { + if (fabs(dD) > 3.0*ADDSTAR_RADIUS) { fprintf (stderr, "measurement is far from average location (D): %f %f (%f %f)\n", catalog[c].average[n].R, catalog[c].average[n].D, dR, dD); dump_measures (&catalog[c].average[n], catalog[c].measure); @@ -418,5 +443,5 @@ catalog[c].measure[m].dR = dR; catalog[c].measure[m].dD = dD; - + if (catalog[c].measure[m].dR > +180.0*3600.0) { // average on high end of boundary, move star up @@ -429,4 +454,7 @@ catalog[c].measure[m].dR = 3600.0*(catalog[c].average[n].R - R); } + + // set the systematic error for this image: + catalog[c].measure[m].dRsys = ToShortPixels(dPosSys); } @@ -544,16 +572,13 @@ // an object with only one detection provides no information about the image calibration - //XXX this is already taken care of in bcatalog - raw[i].mask = FALSE; - int mask = FALSE; + // XXX this is already taken care of in bcatalog + raw[i].mask = 0x0000; if (catalog[c].average[n].Nmeasure <= SRC_MEAS_TOOFEW) { - mask = TRUE; + raw[i].mask |= 0x0001; } if (!finite(catalog[c].measure[m].dR) || !finite(catalog[c].measure[m].dD)) { - mask = TRUE; - } - - raw[i].mask = mask; - + raw[i].mask |= 0x0002; + } + raw[i].Nmeas = catalog[c].average[n].Nmeasure; // record so we can check how well connected an image is switch (mode) { @@ -685,5 +710,5 @@ // skip measurements based on user selected criteria - if (!MeasFilterTest(&catalog[0].measure[m])) continue; + if (!MeasFilterTest(&catalog[0].measure[m], FALSE)) continue; R[N] = catalog[0].measure[m].dR; D[N] = catalog[0].measure[m].dD; @@ -711,5 +736,5 @@ // skip measurements based on user selected criteria - if (!MeasFilterTest(&catalog[0].measure[m])) continue; + if (!MeasFilterTest(&catalog[0].measure[m], FALSE)) continue; x = catalog[0].measure[m].dR - statsR.median; @@ -725,8 +750,6 @@ } - //examine results - relastroVisualPlotOutliers(catalog, catalog[0].average[j].measureOffset, - catalog[0].average[j].Nmeasure, - statsR, statsD, Ns); + // examine results + // relastroVisualPlotOutliers(catalog, catalog[0].average[j].measureOffset, catalog[0].average[j].Nmeasure, statsR, statsD, Ns); } @@ -784,5 +807,5 @@ // skip measurements based on user selected criteria - if (!MeasFilterTest(&catalog[0].measure[m])) continue; + if (!MeasFilterTest(&catalog[0].measure[m], FALSE)) continue; R[N] = catalog[0].measure[m].dR; D[N] = catalog[0].measure[m].dD; @@ -806,5 +829,5 @@ for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { //skip bad measurements - if (!MeasFilterTest(&catalog[0].measure[m])) continue; + if (!MeasFilterTest(&catalog[0].measure[m], FALSE)) continue; x = catalog[0].measure[m].dR - statsR.median; y = catalog[0].measure[m].dD - statsD.median; @@ -838,5 +861,5 @@ for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { //skip bad measurements - if (!MeasFilterTest(&catalog[0].measure[m])) continue; + if (!MeasFilterTest(&catalog[0].measure[m], FALSE)) continue; x = catalog[0].measure[m].dR - statsR.median; y = catalog[0].measure[m].dD - statsD.median; @@ -852,8 +875,6 @@ } //done rejecting outliers - //examine results - relastroVisualPlotOutliers(catalog, catalog[0].average[j].measureOffset, - catalog[0].average[j].Nmeasure, - statsR, statsD, Ns); + // examine results + // relastroVisualPlotOutliers(catalog, catalog[0].average[j].measureOffset, catalog[0].average[j].Nmeasure, statsR, statsD, Ns); } //done looping over objects @@ -871,5 +892,6 @@ /** Determine whether a measurement should be included in the analysis, based on supplied filter criteria */ -int MeasFilterTest(Measure *measure) { +// we only optionally apply the sigma limit: for object averages, this should not be used (should it?) +int MeasFilterTest(Measure *measure, int applySigmaLim) { int found, k; long mask; @@ -917,5 +939,7 @@ /* select measurements by measurement error */ - if ((SIGMA_LIM > 0) && (measure[0].dM > SIGMA_LIM)) return FALSE; + if (applySigmaLim && (SIGMA_LIM > 0) && (measure[0].dM > SIGMA_LIM)) { + return FALSE; + } /* select measurements by mag limit */ Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/StarMaps.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/StarMaps.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/StarMaps.c (revision 30631) @@ -79,5 +79,5 @@ starmap[N].stars[ybin*NX_MAP + xbin] ++; } - MARKTIME("assign stars to starmap bins: %f sec\n", dtime); + if (VERBOSE2) { MARKTIME("assign stars to starmap bins: %f sec\n", dtime); } return (TRUE); @@ -117,5 +117,5 @@ } - if (VERBOSE) fprintf (stderr, "starmap: %d points for image %s\n", starmap[i].Npoints, images[i].name); + if (VERBOSE2) fprintf (stderr, "starmap: %d points for image %s\n", starmap[i].Npoints, images[i].name); } @@ -145,8 +145,14 @@ } - if (VERBOSE) fprintf (stderr, "max deviations for %s using %d pts : %f, %f\n", images[N].name, starmap[N].Npoints, dLmax, dMmax); + if (VERBOSE2) fprintf (stderr, "max deviations for %s using %d pts : %f, %f\n", images[N].name, starmap[N].Npoints, dLmax, dMmax); - if (dLmax > DPOS_MAX) return (FALSE); - if (dMmax > DPOS_MAX) return (FALSE); + if (dLmax > DPOS_MAX) { + if (VERBOSE) fprintf (stderr, "max deviations for %s using %d pts : %f, %f\n", images[N].name, starmap[N].Npoints, dLmax, dMmax); + return (FALSE); + } + if (dMmax > DPOS_MAX) { + if (VERBOSE) fprintf (stderr, "max deviations for %s using %d pts : %f, %f\n", images[N].name, starmap[N].Npoints, dLmax, dMmax); + return (FALSE); + } return (TRUE); } Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateChips.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateChips.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateChips.c (revision 30631) @@ -1,31 +1,61 @@ # include "relastro.h" +int plotChipFits (double *Ro, double *Do, char *mode, int Nimage); +int saveCenter (Image *image, double *Ro, double *Do, int im); int UpdateChips (Catalog *catalog, int Ncatalog) { + int Nskip, Nmosaic, NnewFit, NoldFit; + /* we can measure new image parameters for each non-mosaic chip independently */ - off_t i, Nimage, Nraw, Nref; + off_t i, Nimage, Nraw, Nref, nFitAstr; Image *image; StarData *raw, *ref; Coords *oldCoords; + float dXpixSys, dYpixSys; + double *Ro, *Do; + char *mode; + + Nskip = Nmosaic = NnewFit = NoldFit = 0; image = getimages (&Nimage); + // save fit results for summary plot + ALLOCATE (Ro, double, Nimage); + ALLOCATE (Do, double, Nimage); + ALLOCATE (mode, char, Nimage); + for (i = 0; i < Nimage; i++) { /* skip all except WRP images */ - if (strcmp(&image[i].coords.ctype[4], "-WRP")) continue; + if (strcmp(&image[i].coords.ctype[4], "-WRP")) { + Nmosaic ++; + mode[i] = 0; + continue; + } /* convert measure coordinates to raw entries */ raw = getImageRaw (catalog, Ncatalog, i, &Nraw, MODE_MOSAIC); - if (!raw) continue; + if (!raw) { + Nskip ++; + mode[i] = 0; + continue; + } /* convert average coordinates to ref entries */ ref = getImageRef (catalog, Ncatalog, i, &Nref, MODE_MOSAIC); - if (!ref) continue; + if (!ref) { + Nskip ++; + mode[i] = 0; + continue; + } // note that Nraw & Nref must be equal: if not, we made a programming error in one of these two functions. assert (Nraw == Nref); + // save these in case of failure saveCoords (&image[i].coords, i); + dXpixSys = image[i].dXpixSys; + dYpixSys = image[i].dYpixSys; + nFitAstr = image[i].nFitAstrom; // FitChip does iterative, clipped fitting @@ -33,6 +63,15 @@ if (!FitChip (raw, ref, Nraw, &image[i])) { if (VERBOSE) fprintf (stderr, "reject fit for image %s ("OFF_T_FMT") : Nstars: "OFF_T_FMT"\n", image[i].name, i, Nraw); + + // restore status quo ante oldCoords = getCoords (i); memcpy (&image[i].coords, oldCoords, sizeof(Coords)); + image[i].dXpixSys = dXpixSys; + image[i].dYpixSys = dYpixSys; + image[i].nFitAstrom = nFitAstr; + + saveCenter (image, &Ro[i], &Do[i], i); + mode[i] = 1; + NoldFit ++; free (raw); free (ref); @@ -42,14 +81,178 @@ if (!checkStarMap (i)) { if (VERBOSE) fprintf (stderr, "fit diverges too much for image %s ("OFF_T_FMT") : Nstars: "OFF_T_FMT"\n", image[i].name, i, Nraw); + // restore status quo ante oldCoords = getCoords (i); memcpy (&image[i].coords, oldCoords, sizeof(Coords)); + image[i].dXpixSys = dXpixSys; + image[i].dYpixSys = dYpixSys; + image[i].nFitAstrom = nFitAstr; + + saveCenter (image, &Ro[i], &Do[i], i); + mode[i] = 2; image[i].flags |= ID_IMAGE_ASTROM_POOR; - } - + NoldFit ++; + free (raw); + free (ref); + continue; + } + + saveCenter (image, &Ro[i], &Do[i], i); + mode[i] = 3; + NnewFit ++; free (raw); free (ref); } + plotChipFits (Ro, Do, mode, Nimage); + + fprintf (stderr, "UpdateChips: %d fitted, %d keep old, %d skipped, %d mosaic (skipped)\n", NnewFit, NoldFit, Nskip, Nmosaic); return (TRUE); } +int saveCenter (Image *image, double *Ro, double *Do, int im) { + + Mosaic *mosaic; + Coords *moscoords, *imcoords; + double X, Y, L, M, P, Q, R, D; + + moscoords = NULL; + if (!strcmp(&image[im].coords.ctype[4], "-WRP")) { + mosaic = getMosaicForImage (im); + if (mosaic == NULL) return FALSE; // if we cannot find the associated image, skip it + moscoords = &mosaic[0].coords; + } + imcoords = &image[im].coords; + + if (!strcmp(&image[im].coords.ctype[4], "-WRP")) { + X = 0.5*image[im].NX; + Y = 0.5*image[im].NY; + } else { + X = 0.0; + Y = 0.0; + } + + if (moscoords == NULL) { + // this is a Simple image (not a mosaic) + // note that for a Simple image, L,M = P,Q + XY_to_LM (&L, &M, X, Y, imcoords); + LM_to_RD (&R, &D, L, M, imcoords); + } else { + XY_to_LM (&L, &M, X, Y, imcoords); + XY_to_LM (&P, &Q, L, M, moscoords); + LM_to_RD (&R, &D, P, Q, moscoords); + } + + double Rmid; + if (UserCatalog) { + Rmid = UserCatalogRA; + } else { + Rmid = 0.5*(UserPatch.Rmin + UserPatch.Rmax); + } + + R = ohana_normalize_angle_to_midpoint (R, Rmid); + + *Ro = R; + *Do = D; + + return (TRUE); +} + +int plotChipFits (double *Ro, double *Do, char *mode, int Nimage) { + + static int kapa = -1; + + int i, N; + double Rmin, Rmax, Dmin, Dmax; + float *xvec, *yvec; + Graphdata graphdata; + + if (!relastroGetVisual()) return (TRUE); + + if (kapa == -1) { + kapa = KapaOpenNamedSocket("kapa", "relastro"); + if (kapa == -1) { + fprintf (stderr, "can't open kapa window\n"); + return FALSE; + } + } + + Rmin = +720; + Rmax = -720; + Dmin = +90; + Dmax = -90; + + // find the R, D range + for (i = 0; i < Nimage; i++) { + if (!mode[i]) continue; + + Rmin = MIN(Rmin, Ro[i]); + Rmax = MAX(Rmax, Ro[i]); + Dmin = MIN(Dmin, Do[i]); + Dmax = MAX(Dmax, Do[i]); + } + + ALLOCATE (xvec, float, Nimage); + ALLOCATE (yvec, float, Nimage); + + bzero (&graphdata, sizeof(Graphdata)); + plot_defaults (&graphdata); + graphdata.xmin = Rmin; + graphdata.xmax = Rmax; + graphdata.ymin = Dmin; + graphdata.ymax = Dmax; + graphdata.style = 2; + graphdata.size = 1; + + KapaSetFont (kapa, "helvetica", 14); + KapaSetLimits (kapa, &graphdata); + KapaBox (kapa, &graphdata); + + // *** good images *** + N = 0; + for (i = 0; i < Nimage; i++) { + if (mode[i] != 3) continue; + xvec[N] = Ro[i]; + yvec[N] = Do[i]; + N++; + } + graphdata.ptype = 7; + graphdata.color = KapaColorByName("black"); + KapaPrepPlot (kapa, N, &graphdata); + KapaPlotVector (kapa, N, xvec, "x"); + KapaPlotVector (kapa, N, yvec, "y"); + + // *** reject fit *** + N = 0; + for (i = 0; i < Nimage; i++) { + if (mode[i] != 1) continue; + xvec[N] = Ro[i]; + yvec[N] = Do[i]; + N++; + } + graphdata.ptype = 3; + graphdata.color = KapaColorByName("red"); + KapaPrepPlot (kapa, N, &graphdata); + KapaPlotVector (kapa, N, xvec, "x"); + KapaPlotVector (kapa, N, yvec, "y"); + + // *** divergent fit *** + N = 0; + for (i = 0; i < Nimage; i++) { + if (mode[i] != 2) continue; + xvec[N] = Ro[i]; + yvec[N] = Do[i]; + N++; + } + graphdata.ptype = 2; + graphdata.color = KapaColorByName("blue"); + KapaPrepPlot (kapa, N, &graphdata); + KapaPlotVector (kapa, N, xvec, "x"); + KapaPlotVector (kapa, N, yvec, "y"); + + free (xvec); + free (yvec); + + return (TRUE); +} + +// XXX if (!FindMosaicForImage (image, Nimage, i)) { } Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateObjectOffsets.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 30631) @@ -40,4 +40,6 @@ UpdateMeasures (&catalog, 1); + UpdateObjects (&catalog, 1); + freeImageBins (1); Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateObjects.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateObjects.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/UpdateObjects.c (revision 30631) @@ -103,5 +103,5 @@ //does the measurement pass the supplied filtering constraints? - if (!MeasFilterTest(&catalog[i].measure[m])) { + if (!MeasFilterTest(&catalog[i].measure[m], FALSE)) { catalog[i].measure[m].dbFlags &= ~ID_MEAS_USED_OBJ; continue; @@ -134,9 +134,18 @@ // dX, dY : error in arcsec -- - // dX[N] = GetAstromError (&catalog[i].measure[m], ERROR_MODE_RA); - // dY[N] = GetAstromError (&catalog[i].measure[m], ERROR_MODE_DEC); - - dX[N] = 0.1; - dY[N] = 0.1; + dX[N] = GetAstromError (&catalog[i].measure[m], ERROR_MODE_RA); + dY[N] = GetAstromError (&catalog[i].measure[m], ERROR_MODE_DEC); + + // add systematic error in quadrature, if desired + // only do this after the fit has converged (or you will never improve the poor images) + // if (INCLUDE_SYS_ERR) { + // float dRsys = FromShortPixels(catalog[i].measure[m].dRsys); + // dX[N] = hypot(dX[N], dRsys); + // dY[N] = hypot(dY[N], dRsys); + // } + + // dX[N] = 0.1; + // dY[N] = 0.1; + dT[N] = catalog[i].measure[m].dt; Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/args.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/args.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/args.c (revision 30631) @@ -13,19 +13,20 @@ remove_argument (N, &argc, argv); FIT_TARGET = TARGET_OBJECTS; - - // check for object fitting modes (not valid for images) - if ((N = get_argument (argc, argv, "-pm"))) { - remove_argument (N, &argc, argv); - FIT_MODE = FIT_PM_ONLY; - } - if ((N = get_argument (argc, argv, "-par"))) { - remove_argument (N, &argc, argv); - FIT_MODE = FIT_PAR_ONLY; - } - if ((N = get_argument (argc, argv, "-pmpar"))) { - remove_argument (N, &argc, argv); - FIT_MODE = FIT_PM_AND_PAR; - } - } + } + + // check for object fitting modes + if ((N = get_argument (argc, argv, "-pm"))) { + remove_argument (N, &argc, argv); + FIT_MODE = FIT_PM_ONLY; + } + if ((N = get_argument (argc, argv, "-par"))) { + remove_argument (N, &argc, argv); + FIT_MODE = FIT_PAR_ONLY; + } + if ((N = get_argument (argc, argv, "-pmpar"))) { + remove_argument (N, &argc, argv); + FIT_MODE = FIT_PM_AND_PAR; + } + if ((N = get_argument (argc, argv, "-high-speed"))) { // XXX include a parallax / no-parallax option @@ -93,4 +94,10 @@ usage (); } + } + + USE_BASIC_CHECK = FALSE; + if ((N = get_argument (argc, argv, "-basic-image-search"))) { + remove_argument (N, &argc, argv); + USE_BASIC_CHECK = TRUE; } @@ -144,4 +151,9 @@ VERBOSE = VERBOSE2 = TRUE; remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-visual"))) { + remove_argument (N, &argc, argv); + relastroSetVisual(TRUE); } Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/bcatalog.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/bcatalog.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/bcatalog.c (revision 30631) @@ -45,6 +45,6 @@ offset = catalog[0].average[i].measureOffset + j; - // filter objects based on user supplied criteria - if (!MeasFilterTest(&catalog[0].measure[offset])) { + // filter objects based on user supplied criteria, including SIGMA_LIM + if (!MeasFilterTest(&catalog[0].measure[offset], TRUE)) { catalog[0].measure[offset].dbFlags &= ~ID_MEAS_USED_CHIP; continue; @@ -78,5 +78,5 @@ subcatalog[0].measure[Nmeasure].dbFlags &= ~ID_MEAS_AREA; } - + Nmeasure ++; Nm ++; Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/load_catalogs.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/load_catalogs.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/load_catalogs.c (revision 30631) @@ -26,9 +26,9 @@ pcatalog[0].Nsecfilt = GetPhotcodeNsecfilt (); - if (!dvo_catalog_open (pcatalog, skylist[0].regions[i], VERBOSE, "w")) { + if (!dvo_catalog_open (pcatalog, skylist[0].regions[i], VERBOSE2, "w")) { fprintf (stderr, "ERROR: failure reading catalog %s\n", pcatalog[0].filename); exit (1); } - if (VERBOSE && !pcatalog[0].Naves_disk) fprintf (stderr, "no data in %s, skipping\n", pcatalog[0].filename); + if (VERBOSE2 && !pcatalog[0].Naves_disk) fprintf (stderr, "no data in %s, skipping\n", pcatalog[0].filename); //outlier rejection Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/plotstuff.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/plotstuff.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/plotstuff.c (revision 30631) @@ -146,3 +146,12 @@ graphdata[0].ymax = dUNDEF; + graphdata[0].ticktextPad = NAN; + graphdata[0].labelPadXm = NAN; + graphdata[0].labelPadXp = NAN; + graphdata[0].labelPadYm = NAN; + graphdata[0].labelPadYp = NAN; + graphdata[0].padXm = NAN; + graphdata[0].padXp = NAN; + graphdata[0].padYm = NAN; + graphdata[0].padYp = NAN; } Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/relastro.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/relastro.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/relastro.c (revision 30631) @@ -78,4 +78,6 @@ MARKTIME("update chips: %f sec\n", dtime); } + // create summary plots of the process + // relastroVisualSummaryChips(); break; Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/relastroVisual.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/relastroVisual.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/relastroVisual.c (revision 30631) @@ -11,13 +11,14 @@ #define KAPAY 700 -static int kapa = -1; +static int kapa1 = -1; static int kapa2 = -1; static int kapa3 = -1; +static int kapa4 = -1; static int isVisual = FALSE; -static int plotRawRef = FALSE; -static int plotScatter = FALSE; -static int plotResid = FALSE; -static int plotVector = FALSE; +static int plotRawRef = TRUE; +// static int plotScatter = TRUE; +// static int plotResid = TRUE; +// static int plotVector = TRUE; static int plotOutliers = TRUE; @@ -29,9 +30,8 @@ fprintf(stderr, "Failure to open kapa.\n"); isVisual = 0; - return 0; + return FALSE; } - // KapaResize (*kapid, KAPAX, KAPAY); - } - return 1; + } + return TRUE; } @@ -49,5 +49,13 @@ isVisual = 0; } - return 1; + return TRUE; +} + +void relastroSetVisual(int state) { + isVisual = state; +} + +int relastroGetVisual(void) { + return isVisual; } @@ -70,11 +78,10 @@ graphdata->xmax = xhi; graphdata->ymax = yhi; - return 1; -} - -static int residPlot(float x[], float y[], - float xVec[], float yVec[], - int npts, int *kapaID) { - if (!isVisual || !plotResid) return TRUE; + return TRUE; +} + +/** 4-panel plot of x vs dx, y vs dx, x vs dy, y vs dy **/ +int relastroVisualRawRef(int *kapaID, float *x, float *y, float *dx, float *dy, float *dPos, int npts) { + Graphdata graphdata; KapaSection section; @@ -83,5 +90,5 @@ KapaInitGraph(&graphdata); - KapaClearPlots(*kapaID); + KapaClearSections(*kapaID); KapaSetFont(*kapaID, "helvetica", 14); @@ -89,9 +96,11 @@ section.x = 0.0; section.y = 0.0; section.dx = .45, section.dy = .45; + section.bg = KapaColorByName("white"); graphdata.ptype = 7; graphdata.style = 2; + graphdata.etype |= 0x01; KapaSetSection(*kapaID, §ion); - if(!scaleGraphdata(x, xVec, &graphdata, npts)) return 0; + if(!scaleGraphdata(x, dx, &graphdata, npts)) return 0; KapaSetLimits(*kapaID, &graphdata); KapaBox(*kapaID, &graphdata); @@ -100,9 +109,11 @@ KapaPrepPlot(*kapaID, npts, &graphdata); KapaPlotVector(*kapaID, npts, x, "x"); - KapaPlotVector(*kapaID, npts, xVec, "y"); + KapaPlotVector(*kapaID, npts, dx, "y"); + KapaPlotVector(*kapaID, npts, dPos, "dym"); + KapaPlotVector(*kapaID, npts, dPos, "dyp"); section.x = .5; section.y = 0; section.name="1"; KapaSetSection(*kapaID, §ion); - if(!scaleGraphdata(x, yVec, &graphdata, npts)) return 0; + if(!scaleGraphdata(x, dy, &graphdata, npts)) return 0; KapaSetLimits(*kapaID, &graphdata); KapaBox(*kapaID, &graphdata); @@ -111,9 +122,11 @@ KapaPrepPlot(*kapaID, npts, &graphdata); KapaPlotVector(*kapaID, npts, x, "x"); - KapaPlotVector(*kapaID, npts, yVec, "y"); + KapaPlotVector(*kapaID, npts, dy, "y"); + KapaPlotVector(*kapaID, npts, dPos, "dym"); + KapaPlotVector(*kapaID, npts, dPos, "dyp"); section.x = .0; section.y = .5; section.name="2"; KapaSetSection(*kapaID, §ion); - if(!scaleGraphdata(y, xVec, &graphdata, npts)) return 0;; + if(!scaleGraphdata(y, dx, &graphdata, npts)) return 0;; KapaSetLimits(*kapaID, &graphdata); KapaBox(*kapaID, &graphdata); @@ -122,9 +135,11 @@ KapaPrepPlot(*kapaID, npts, &graphdata); KapaPlotVector(*kapaID, npts, y, "x"); - KapaPlotVector(*kapaID, npts, xVec, "y"); + KapaPlotVector(*kapaID, npts, dx, "y"); + KapaPlotVector(*kapaID, npts, dPos, "dym"); + KapaPlotVector(*kapaID, npts, dPos, "dyp"); section.x = .5; section.y = .5; section.name="3"; KapaSetSection(*kapaID, §ion); - if(!scaleGraphdata(y, yVec, &graphdata, npts)) return 0; + if(!scaleGraphdata(y, dy, &graphdata, npts)) return 0; KapaSetLimits(*kapaID, &graphdata); KapaBox(*kapaID, &graphdata); @@ -133,30 +148,32 @@ KapaPrepPlot(*kapaID, npts, &graphdata); KapaPlotVector(*kapaID, npts, y, "x"); - KapaPlotVector(*kapaID, npts, yVec, "y"); - - return 1; -} - - -/**Plot a vector field*/ -static int plotVectorField(float x[], float y[], - float xVec[], float yVec[], - int npts, int *kapaID, double maxVecLength) { - - if(!plotVector) return 1; + KapaPlotVector(*kapaID, npts, dy, "y"); + KapaPlotVector(*kapaID, npts, dPos, "dym"); + KapaPlotVector(*kapaID, npts, dPos, "dyp"); + + return TRUE; +} + + +/** Plot a vector field (circles at vector origin, scaled lines giving vector directions */ +int relastroVisualVectorField(int *kapaID, float *x, float *y, float *dx, float *dy, int npts, double maxVecLength) { Graphdata graphdata; - float singleX[2], singleY[2]; + float *xVec, *yVec; float vecScaleFactor; float graphSize; int i; char plotTitle[50]; - sprintf(plotTitle, "Maximum Vector Size = %5.1e", maxVecLength); - + + if (!npts) return FALSE; if (!initWindow(kapaID)) return 0; + snprintf(plotTitle, 50, "Maximum Vector Size = %5.1e", maxVecLength); + KapaInitGraph(&graphdata); - KapaClearPlots(*kapaID); - if(!scaleGraphdata(x, y, &graphdata, npts)) return 0; + KapaClearSections(*kapaID); + KapaSetFont(*kapaID, "helvetica", 14); + + if (!scaleGraphdata(x, y, &graphdata, npts)) return 0; graphSize = graphdata.xmax - graphdata.xmin; @@ -164,11 +181,7 @@ graphSize = graphdata.ymax - graphdata.ymin; } - vecScaleFactor = graphSize * 0.02 / (float)maxVecLength; -#ifdef TESTING - fprintf(stderr, "GraphSize: %e\n maxVecLength: %e\n vecScaleFactor: %e\n", - graphSize, maxVecLength, vecScaleFactor); -#endif - + + // fprintf(stderr, "GraphSize: %e\n maxVecLength: %e\n vecScaleFactor: %e\n", graphSize, maxVecLength, vecScaleFactor); KapaSetFont (*kapaID, "helvetica", 14); @@ -179,93 +192,142 @@ graphdata.ptype = 7; graphdata.style = 2; + graphdata.color = KapaColorByName("black"); KapaPrepPlot(*kapaID, npts, &graphdata); KapaPlotVector(*kapaID, npts, x, "x"); KapaPlotVector(*kapaID, npts, y, "y"); - //plot each vector individually - graphdata.ptype = 0; - graphdata.style = 0; + ALLOCATE (xVec, float, 2*npts); + ALLOCATE (yVec, float, 2*npts); + for (i = 0; i < npts; i++) { + xVec[2*i + 0] = x[i]; + yVec[2*i + 0] = y[i]; + xVec[2*i + 1] = x[i] + dx[i] * vecScaleFactor; + yVec[2*i + 1] = y[i] + dy[i] * vecScaleFactor; + } + + graphdata.ptype = 100; // line segements by point pair + graphdata.style = 2; graphdata.color = KapaColorByName("blue"); - for(i = 0; i < npts; i++) { - singleX[0] = x[i]; - singleY[0] = y[i]; - singleX[1] = x[i] + xVec[i] * vecScaleFactor; - singleY[1] = y[i] + yVec[i] * vecScaleFactor; - KapaPrepPlot(*kapaID, 2, &graphdata); - KapaPlotVector(*kapaID, 2, singleX, "x"); - KapaPlotVector(*kapaID, 2, singleY, "y"); - } - return 1; -} - -int relastroVisualPlotScatter(double values[], double thresh, int npts) { - float *x, *data; - int i; - float xline[2], yline[2]; - if (!isVisual || !plotScatter) return 1; - if (!initWindow(&kapa2)) return 0; - - ALLOCATE(x, float, npts); - ALLOCATE(data, float, npts); - - for(i = 0; i < npts; i++) { - x[i] = i; - data[i] = (float) values[i]; - } + KapaPrepPlot (*kapaID, 2*npts, &graphdata); + KapaPlotVector(*kapaID, 2*npts, xVec, "x"); + KapaPlotVector(*kapaID, 2*npts, yVec, "y"); + + free (xVec); + free (yVec); + return TRUE; +} + +int relastroVisualPlotScatter(int *kapaID, float *dXfit, float *dYfit, float *mag, int npts) { Graphdata graphdata; KapaSection section; - section.x = 0; section.y = 0; section.dx = 1; section.dy = 1; - section.name = "junk"; + + if (!initWindow(kapaID)) return 0; KapaInitGraph(&graphdata); - KapaClearPlots(kapa2); - KapaSetSection(kapa2, §ion); - - graphdata.ptype = 0; - graphdata.style = 0; - graphdata.xmin = 0; - graphdata.xmax = npts; - graphdata.ymin = 0; - graphdata.ymax = data[npts-1]; - - KapaSetFont(kapa2, "helvetica", 14); - KapaSetLimits(kapa2, &graphdata); - KapaBox(kapa2, &graphdata); - KapaSendLabel( kapa2, "Object", KAPA_LABEL_XM); - KapaSendLabel( kapa2, "Offset(pixels)", KAPA_LABEL_YM); - KapaSendLabel( kapa2, "Astrometric Offset with fit cutoff", - KAPA_LABEL_XP); - KapaPrepPlot(kapa2, npts, &graphdata); - KapaPlotVector(kapa2, npts, x, "x"); - KapaPlotVector(kapa2, npts, data, "y"); - - graphdata.color = KapaColorByName("red"); - KapaPrepPlot(kapa2, 2, &graphdata); - yline[0] = (float) thresh; - yline[1] = (float) thresh; - xline[0] = graphdata.xmin; - xline[1] = graphdata.xmax; - KapaPlotVector(kapa2, 2, xline, "x"); - KapaPlotVector(kapa2, 2, yline, "y"); - - askUser(&plotScatter); - return 1; -} - - + KapaClearSections(*kapaID); + KapaSetFont(*kapaID, "helvetica", 14); + + section.name = "a"; + section.x = 0.0; section.y = 0.0; section.dx = 1.0; section.dy = 0.5; + section.bg = KapaColorByName("white"); + KapaSetSection(*kapaID, §ion); + + graphdata.ptype = 2; + graphdata.style = 2; + + if(!scaleGraphdata(mag, dXfit, &graphdata, npts)) return 0; + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, "mag", KAPA_LABEL_XM); + KapaSendLabel(*kapaID, "dXfit", KAPA_LABEL_YM); + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, mag, "x"); + KapaPlotVector(*kapaID, npts, dXfit, "y"); + + section.name = "b"; + section.x = 0.0; section.y = 0.5; section.dx = 1.0; section.dy = 0.5; + section.bg = KapaColorByName("white"); + KapaSetSection(*kapaID, §ion); + + graphdata.ptype = 2; + graphdata.style = 2; + + if(!scaleGraphdata(mag, dYfit, &graphdata, npts)) return 0; + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, "mag", KAPA_LABEL_XM); + KapaSendLabel(*kapaID, "dYfit", KAPA_LABEL_YM); + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, mag, "x"); + KapaPlotVector(*kapaID, npts, dYfit, "y"); + + return TRUE; +} /** plot raw vs ref (L, M). Only those whose distance is < drMax are used in fit*/ -int relastroVisualPlotRawRef(StarData *raw, StarData *ref, double dRmax, int numObj) { - - if( !isVisual || !plotRawRef) return 1; - if( !initWindow(&kapa)) return 0; +int relastroVisualPlotFittedStars(int *kapaID, float *rawX, float *rawY, float *refX, float *refY, int numNoFit, float *rawXfit, float *rawYfit, float *refXfit, float *refYfit, int numFit) { + + Graphdata graphdata; + + if (!initWindow(kapaID)) return 0; + + KapaInitGraph(&graphdata); + KapaClearPlots(*kapaID); + + graphdata.ptype = 7; + graphdata.style = 2; + + if(!scaleGraphdata(rawXfit, rawYfit, &graphdata, numFit)) { + fprintf(stderr, "Not enough finite points for plotting"); + return 0; + } + + KapaSetFont(*kapaID, "helvetica", 14); + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel( *kapaID, "X", KAPA_LABEL_XM); + KapaSendLabel( *kapaID, "Y", KAPA_LABEL_YM); + KapaSendLabel( *kapaID, "orange, red, green, blue: (raw, ref), (nofit, fit)", + KAPA_LABEL_XP); + + graphdata.color = KapaColorByName("orange"); + graphdata.size = 1; + KapaPrepPlot(*kapaID, numNoFit, &graphdata); + KapaPlotVector(*kapaID, numNoFit, rawX, "x"); + KapaPlotVector(*kapaID, numNoFit, rawY, "y"); + + graphdata.color = KapaColorByName("red"); + graphdata.size = 2; + KapaPrepPlot(*kapaID, numNoFit, &graphdata); + KapaPlotVector(*kapaID, numNoFit, refX, "x"); + KapaPlotVector(*kapaID, numNoFit, refY, "y"); + + graphdata.color = KapaColorByName("green"); + graphdata.size = 1; + KapaPrepPlot(*kapaID, numFit, &graphdata); + KapaPlotVector(*kapaID, numFit, rawXfit, "x"); + KapaPlotVector(*kapaID, numFit, rawYfit, "y"); + + graphdata.color = KapaColorByName("blue"); + graphdata.size = 2; + KapaPrepPlot(*kapaID, numFit, &graphdata); + KapaPlotVector(*kapaID, numFit, refXfit, "x"); + KapaPlotVector(*kapaID, numFit, refYfit, "y"); + + return TRUE; +} + +/** create various plots using the data in raw and ref **/ +int relastroVisualPlotChipFit(StarData *raw, StarData *ref, double dRmax, int numObj) { float *rawX, *rawY, *refX, *refY; float *rawXfit, *rawYfit, *refXfit, *refYfit; float *magRaw, *magRef, *magRawfit, *magReffit; - float *xVec, *yVec; + float *dXfit, *dYfit, *dPos; int numFit = 0, numNoFit = 0; double dL, dM, dR; + + if (!isVisual) return TRUE; ALLOCATE(rawX, float, numObj); @@ -273,21 +335,28 @@ ALLOCATE(refX, float, numObj); ALLOCATE(refY, float, numObj); + ALLOCATE(magRaw, float, numObj); + ALLOCATE(magRef, float, numObj); + ALLOCATE(rawXfit, float, numObj); ALLOCATE(rawYfit, float, numObj); ALLOCATE(refXfit, float, numObj); ALLOCATE(refYfit, float, numObj); - ALLOCATE(magRaw, float, numObj); - ALLOCATE(magRef, float, numObj); ALLOCATE(magRawfit, float, numObj); ALLOCATE(magReffit, float, numObj); + ALLOCATE(dXfit, float, numObj); + ALLOCATE(dYfit, float, numObj); + ALLOCATE(dPos, float, numObj); int i; for(i = 0; i < numObj; i++) { - if (raw[i].mask) continue; + if (raw[i].mask) continue; // XXX dL = raw[i].L - ref[i].L; dM = raw[i].M - ref[i].M; dR = hypot (dL, dM); + + // XXX change the selection to a mask-based thing if (dR > dRmax) { + // UNFITTED values rawX[numNoFit] = raw[i].X; rawY[numNoFit] = raw[i].Y; @@ -298,10 +367,14 @@ numNoFit++; } else { - rawXfit[numFit] = raw[i].X; - rawYfit[numFit] = raw[i].Y; - refXfit[numFit] = ref[i].X; - refYfit[numFit] = ref[i].Y; - magRaw[numFit] = raw[i].Mag; - magRef[numFit] = ref[i].Mag; + // FITTED values + rawXfit[numFit] = raw[i].X; + rawYfit[numFit] = raw[i].Y; + refXfit[numFit] = ref[i].X; + refYfit[numFit] = ref[i].Y; + magRawfit[numFit] = raw[i].Mag; + magReffit[numFit] = ref[i].Mag; + dPos[numFit] = ref[i].dPos; + dXfit[numFit] = raw[i].X - ref[i].X; + dYfit[numFit] = raw[i].Y - ref[i].Y; numFit++; } @@ -310,69 +383,21 @@ if (numFit == 0) return 0; - Graphdata graphdata; - - KapaInitGraph(&graphdata); - KapaClearPlots(kapa); - - graphdata.ptype = 7; - graphdata.style = 2; - - if(!scaleGraphdata(rawXfit, rawYfit, &graphdata, numFit)) { - fprintf(stderr, "Not enough finite points for plotting"); - return 0; - } - - KapaSetFont(kapa, "helvetica", 14); - KapaSetLimits(kapa, &graphdata); - KapaBox(kapa, &graphdata); - KapaSendLabel( kapa, "X", KAPA_LABEL_XM); - KapaSendLabel( kapa, "Y", KAPA_LABEL_YM); - KapaSendLabel( kapa, "orange, red, green, blue: (raw, ref), (nofit, fit)", - KAPA_LABEL_XP); - - graphdata.color = KapaColorByName("orange"); - graphdata.size = 1; - KapaPrepPlot(kapa, numNoFit, &graphdata); - KapaPlotVector(kapa, numNoFit, rawX, "x"); - KapaPlotVector(kapa, numNoFit, rawY, "y"); - - graphdata.color = KapaColorByName("red"); - graphdata.size = 2; - KapaPrepPlot(kapa, numNoFit, &graphdata); - KapaPlotVector(kapa, numNoFit, refX, "x"); - KapaPlotVector(kapa, numNoFit, refY, "y"); - - graphdata.color = KapaColorByName("green"); - graphdata.size = 1; - KapaPrepPlot(kapa, numFit, &graphdata); - KapaPlotVector(kapa, numFit, rawXfit, "x"); - KapaPlotVector(kapa, numFit, rawYfit, "y"); - - graphdata.color = KapaColorByName("blue"); - graphdata.size = 2; - KapaPrepPlot(kapa, numFit, &graphdata); - KapaPlotVector(kapa, numFit, refXfit, "x"); - KapaPlotVector(kapa, numFit, refYfit, "y"); - - ALLOCATE(xVec, float, numFit); - ALLOCATE(yVec, float, numFit); - - //plot the fitted objects as vectors - for(i = 0; i < numFit; i++) { - xVec[i] = rawXfit[i] - refXfit[i]; - yVec[i] = rawYfit[i] - refYfit[i]; - } - - plotVectorField(rawXfit, rawYfit, xVec, yVec, numFit, &kapa3, dRmax); - if(!residPlot(rawXfit, rawYfit, xVec, yVec, numFit, &kapa2)) { - fprintf(stderr, "Unable to plot residuals"); - return 0; - } - - FREE(xVec); - FREE(yVec); + // 4-panel plot of x vs dx, y vs dx, x vs dy, y vs dy + relastroVisualRawRef(&kapa1, rawXfit, rawYfit, dXfit, dYfit, dPos, numFit); + + // vector line plot for the fit + relastroVisualVectorField(&kapa2, rawXfit, rawYfit, dXfit, dYfit, numFit, dRmax); + + // dXfit, dYfit vs mag + relastroVisualPlotScatter(&kapa3, dXfit, dYfit, magRawfit, numFit); + + // plot the positions of the fitted stars on the chip + relastroVisualPlotFittedStars(&kapa4, rawX, rawY, refX, refY, numNoFit, rawXfit, rawYfit, refXfit, refYfit, numFit); askUser(&plotRawRef); + FREE(dXfit); + FREE(dYfit); + FREE(dPos); FREE(rawX); FREE(rawY); @@ -388,5 +413,5 @@ FREE(magReffit); - return 1; + return TRUE; } @@ -403,6 +428,6 @@ KapaSection section; - if (!isVisual || !plotOutliers) return 1; - if (!initWindow(&kapa)) return 0; + if (!isVisual || !plotOutliers) return TRUE; + if (!initWindow(&kapa1)) return 0; // populate vectors @@ -426,5 +451,5 @@ for(i = 0; i < Nmeasure; i++, m++) { meas = catalog[0].measure[m]; - if (!MeasFilterTest(&meas)) continue; + if (!MeasFilterTest(&meas, FALSE)) continue; xmin = MIN(xmin, meas.dR); xmax = MAX(xmax, meas.dR); @@ -459,8 +484,9 @@ section.x = 0; section.y = 0; section.dx = 1; section.dy = 1; section.name = "junk"; + section.bg = KapaColorByName("white"); KapaInitGraph(&graphdata); - KapaClearPlots(kapa); - KapaSetFont(kapa, "helvetica", 14); + KapaClearPlots(kapa1); + KapaSetFont(kapa1, "helvetica", 14); graphdata.ptype = 7; @@ -472,32 +498,30 @@ graphdata.ymax = ymax; - KapaSetSection(kapa, §ion); - KapaSetLimits(kapa, &graphdata); - KapaBox(kapa, &graphdata); - - KapaSendLabel( kapa, "RA (arcsec)", KAPA_LABEL_XM); - KapaSendLabel( kapa, "Dec (arcsec)", KAPA_LABEL_YM); - KapaSendLabel( kapa, "Points flagged as outliers (red)", + KapaSetSection(kapa1, §ion); + KapaSetLimits(kapa1, &graphdata); + KapaBox(kapa1, &graphdata); + + KapaSendLabel( kapa1, "RA (arcsec)", KAPA_LABEL_XM); + KapaSendLabel( kapa1, "Dec (arcsec)", KAPA_LABEL_YM); + KapaSendLabel( kapa1, "Points flagged as outliers (red)", KAPA_LABEL_XP); graphdata.color = KapaColorByName("green"); - KapaPrepPlot(kapa, Nin, &graphdata); - KapaPlotVector(kapa, Nin, Rin, "x"); - KapaPlotVector(kapa, Nin, Din, "y"); + KapaPrepPlot(kapa1, Nin, &graphdata); + KapaPlotVector(kapa1, Nin, Rin, "x"); + KapaPlotVector(kapa1, Nin, Din, "y"); graphdata.color = KapaColorByName("red"); - KapaPrepPlot(kapa, Nout, &graphdata); - KapaPlotVector(kapa, Nout, Rout, "x"); - KapaPlotVector(kapa, Nout, Dout, "y"); + KapaPrepPlot(kapa1, Nout, &graphdata); + KapaPlotVector(kapa1, Nout, Rout, "x"); + KapaPlotVector(kapa1, Nout, Dout, "y"); graphdata.color = KapaColorByName("black"); graphdata.ptype = 0; graphdata.style = 0; - KapaPrepPlot(kapa, 100, &graphdata); - KapaPlotVector(kapa, 100, xCirc, "x"); - KapaPlotVector(kapa, 100, yCirc, "y"); - - - + KapaPrepPlot(kapa1, 100, &graphdata); + KapaPlotVector(kapa1, 100, xCirc, "x"); + KapaPlotVector(kapa1, 100, yCirc, "y"); + askUser(&plotOutliers); @@ -510,3 +534,14 @@ } - +# if (0) +int relastroVisualSummaryChips() { + + // plot the dXsys, dYsys histograms + + // plot x vs dx, y vs dy, etc for all mosaics + + // plot a map of median star scatter + +} +# endif + Index: /branches/czw_branch/20101203/Ohana/src/relastro/src/select_images.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relastro/src/select_images.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relastro/src/select_images.c (revision 30631) @@ -15,4 +15,10 @@ void dsortindex (double *X, off_t *Y, int N); off_t getRegionStartByRA (double R, double *Rref, off_t Nregions); + +# define MARKTIME(MSG,...) { \ + float dtime; \ + gettimeofday (&stop, (void *) NULL); \ + dtime = DTIME (stop, start); \ + fprintf (stderr, MSG, __VA_ARGS__); } Image *select_images (SkyList *skylist, Image *timage, off_t Ntimage, off_t **LineNumber, off_t *Nimage) { @@ -25,5 +31,8 @@ Coords tcoords; SkyRegionCoords *skycoords; - + struct timeval start, stop; + + double RmaxSkyRegion, RminSkyRegion, RmidSkyRegion, DminSkyRegion, DmaxSkyRegion; + double *RmaxSky; off_t *index; @@ -40,4 +49,6 @@ return NULL; } + + gettimeofday (&start, (void *) NULL); // the comparison is made in the catalog local projection. below we set crval1,2 @@ -52,4 +63,9 @@ ALLOCATE (RmaxSky, double, skylist[0].Nregions); ALLOCATE (index, off_t, skylist[0].Nregions); + + RminSkyRegion = +360.0; + RmaxSkyRegion = -360.0; + DminSkyRegion = +90.0; + DmaxSkyRegion = -90.0; /* compare with each region file */ @@ -80,10 +96,19 @@ skycoords[i].Xc[3] -= dx; skycoords[i].Yc[3] += dy; skycoords[i].Xc[4] -= dx; skycoords[i].Yc[4] -= dy; - } + + RminSkyRegion = MIN(RminSkyRegion, skylist[0].regions[i][0].Rmin); + RmaxSkyRegion = MAX(RmaxSkyRegion, skylist[0].regions[i][0].Rmax); + DminSkyRegion = MIN(DminSkyRegion, skylist[0].regions[i][0].Dmin); + DmaxSkyRegion = MAX(DmaxSkyRegion, skylist[0].regions[i][0].Dmax); + } + RmidSkyRegion = 0.5*(RminSkyRegion + RmaxSkyRegion); + MARKTIME("create sky region coords: %f sec\n", dtime); dsortindex (RmaxSky, index, skylist[0].Nregions); + MARKTIME("sort sky coords: %f sec\n", dtime); if (VERBOSE) fprintf (stderr, "finding images\n"); BuildChipMatch (timage, Ntimage); + MARKTIME("build chip match: %f sec\n", dtime); nimage = 0; @@ -124,18 +149,51 @@ } - /* define image corners */ - Xi[0] = 0; Yi[0] = 0; - Xi[1] = timage[i].NX; Yi[1] = 0; - Xi[2] = timage[i].NX; Yi[2] = timage[i].NY; - Xi[3] = 0; Yi[3] = timage[i].NY; - Xi[4] = 0; Yi[4] = 0; + /* define image corners - note the DIS images (mosaic phu) are special */ + if (!strcmp(&timage[i].coords.ctype[4], "-DIS")) { + Xi[0] = -0.5*timage[i].NX; Yi[0] = -0.5*timage[i].NY; + Xi[1] = +0.5*timage[i].NX; Yi[1] = -0.5*timage[i].NY; + Xi[2] = +0.5*timage[i].NX; Yi[2] = +0.5*timage[i].NY; + Xi[3] = -0.5*timage[i].NX; Yi[3] = +0.5*timage[i].NY; + Xi[4] = -0.5*timage[i].NX; Yi[4] = -0.5*timage[i].NY; + } else { + Xi[0] = 0; Yi[0] = 0; + Xi[1] = timage[i].NX; Yi[1] = 0; + Xi[2] = timage[i].NX; Yi[2] = timage[i].NY; + Xi[3] = 0; Yi[3] = timage[i].NY; + Xi[4] = 0; Yi[4] = 0; + } found = FALSE; /* transform corners to ra,dec */ double RminImage = 360.0; + double RmaxImage = 0.0; + double DminImage = +90.0; + double DmaxImage = -90.0; + // int leftside = FALSE; for (j = 0; j < 5; j++) { XY_to_RD (&Ri[j], &Di[j], Xi[j], Yi[j], &timage[i].coords); + Ri[j] = ohana_normalize_angle_to_midpoint (Ri[j], RmidSkyRegion); + RminImage = MIN(RminImage, Ri[j]); - } + RmaxImage = MAX(RmaxImage, Ri[j]); + DminImage = MIN(DminImage, Di[j]); + DmaxImage = MAX(DmaxImage, Di[j]); + } + if (RmaxImage - RminImage > 180.0) { + double tmp = RminImage; + RmaxImage = RminImage; + RminImage = tmp - 360.0; + } + + // check that this image is even in range of the searched region + if (DminImage > DmaxSkyRegion) continue; + if (DmaxImage < DminSkyRegion) continue; + + // the sky region RA is defined to be 0 - 360.0 + if (RminImage > RmaxSkyRegion) continue; + if (RmaxImage < RminSkyRegion) continue; + + // image overlaps region, keep it + if (USE_BASIC_CHECK) goto found_it; // RA(nStart) is guaranteed to be < RminImage: @@ -161,8 +219,10 @@ for (j = 0; (j < 4) && !found; j++) { found = corner_check (&Xi[j], &Yi[j], &skycoords[m].Xc[0], &skycoords[m].Yc[0]); + if (found) goto found_it; } /* check if catalog corner inside image */ for (j = 0; (j < 4) && !found; j++) { found = corner_check (&skycoords[m].Xc[j], &skycoords[m].Yc[j], &Xi[0], &Yi[0]); + if (found) goto found_it; } /* check if edges cross */ @@ -170,29 +230,33 @@ for (k = 0; (k < 4) && !found; k++) { found = edge_check (&Xi[j], &Yi[j], &skycoords[m].Xc[k], &skycoords[m].Yc[k]); + if (found) goto found_it; } } - if (!found) continue; - - image[nimage] = timage[i]; - /* always allow 'few' images to succeed, if possible */ - if (image[nimage].flags & ID_IMAGE_ASTROM_FEW) { - image[nimage].flags &= ~ID_IMAGE_ASTROM_FEW; - } - if (RESET) { - // XXX do we need / want to do this in relastro? - assignMcal (&image[nimage], (double *) NULL, -1); - image[nimage].dMcal = NAN; - image[nimage].flags &= ~badImage; - } - line_number[nimage] = i; - nimage ++; - if (nimage == NIMAGE) { - NIMAGE += 100; - REALLOCATE (image, Image, NIMAGE); - REALLOCATE (line_number, off_t, NIMAGE); - } - } - } - + } + if (!found) continue; + + found_it: + image[nimage] = timage[i]; + /* always allow 'few' images to succeed, if possible */ + if (image[nimage].flags & ID_IMAGE_ASTROM_FEW) { + image[nimage].flags &= ~ID_IMAGE_ASTROM_FEW; + } + if (RESET) { + // XXX do we need / want to do this in relastro? + // assignMcal (&image[nimage], (double *) NULL, -1); + // image[nimage].Mcal = NAN; + // image[nimage].dMcal = NAN; + image[nimage].flags &= ~badImage; + } + line_number[nimage] = i; + nimage ++; + if (nimage == NIMAGE) { + NIMAGE += 100; + REALLOCATE (image, Image, NIMAGE); + REALLOCATE (line_number, off_t, NIMAGE); + } + } + MARKTIME("finish image selection: %f sec\n", dtime); + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" images\n", nimage); Index: /branches/czw_branch/20101203/Ohana/src/relphot/Makefile =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/Makefile (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/Makefile (revision 30631) @@ -29,4 +29,5 @@ $(SRC)/StarOps.$(ARCH).o \ $(SRC)/args.$(ARCH).o \ +$(SRC)/help.$(ARCH).o \ $(SRC)/bcatalog.$(ARCH).o \ $(SRC)/global_stats.$(ARCH).o \ Index: /branches/czw_branch/20101203/Ohana/src/relphot/include/relphot.h =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/include/relphot.h (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/include/relphot.h (revision 30631) @@ -13,4 +13,6 @@ float Mcal; float dMcal; + float dMsys; + short nFitPhotom; short Xm; float secz; @@ -50,4 +52,5 @@ double STAR_CHISQ; double MIN_ERROR; +double IMFIT_SYS_SIGMA_LIM; int VERBOSE; @@ -103,4 +106,6 @@ SkyRegion UserPatch; int UserPatchSelect; + +int USE_BASIC_CHECK; # ifdef GRID_V1 @@ -215,2 +220,5 @@ void write_coords PROTO((Header *header, Coords *coords)); int relphot_objects (void); + +void relphot_usage (void); +void relphot_help (int argc, char **argv); Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/ConfigInit.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/ConfigInit.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/ConfigInit.c (revision 30631) @@ -20,4 +20,7 @@ GetConfig (config, "MAG_LIM", "%lf", 0, &MAG_LIM); GetConfig (config, "SIGMA_LIM", "%lf", 0, &SIGMA_LIM); + + if (!ScanConfig (config, "RELPHOT_IMFIT_SYS_SIGMA_LIM", "%lf", 0, &IMFIT_SYS_SIGMA_LIM)) IMFIT_SYS_SIGMA_LIM = 0.01; + GetConfig (config, "STAR_SCATTER", "%lf", 0, &STAR_SCATTER); GetConfig (config, "IMAGE_SCATTER", "%lf", 0, &IMAGE_SCATTER); Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/ImageOps.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/ImageOps.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/ImageOps.c (revision 30631) @@ -124,4 +124,6 @@ free (clist); free (mlist); + free (Nlist); + free (NLIST); } @@ -299,6 +301,6 @@ off_t i, j, m, c, n, N, Nmax, mark, bad; - int Nfew, Nbad, Nmos, Ngrid, Nrel, Nsys; - float Msys, Mrel, Mmos, Mgrid; + int Nfew, Nbad, Nmos, Ngrid, Nrel, Nsys, Nbright; + float Msys, Mrel, Mmos, Mgrid, McalBright, McalBright2; double *list, *dlist; StatType stats; @@ -318,4 +320,8 @@ Nfew = Nbad = Nmos = Ngrid = Nrel = Nsys = 0; + + // counters to measure the bright-end scatter + McalBright = McalBright2 = 0.0; + Nbright = 0; for (i = 0; i < Nimage; i++) { @@ -361,4 +367,9 @@ list[N] = Msys - Mrel - Mmos - Mgrid; dlist[N] = MAX (catalog[c].measure[m].dM, MIN_ERROR); + if (catalog[c].measure[m].dM < IMFIT_SYS_SIGMA_LIM) { + McalBright += list[N]; + McalBright2 += SQ(list[N]); + Nbright ++; + } N++; } @@ -378,5 +389,7 @@ liststats (list, dlist, N, &stats); image[i].Mcal = stats.mean; - image[i].dMcal = stats.sigma; + image[i].dMcal = stats.error; + image[i].dMagSys = stats.sigma; + image[i].nFitPhotom = N; image[i].Xm = 100.0*log10(stats.chisq); } Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/MosaicOps.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/MosaicOps.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/MosaicOps.c (revision 30631) @@ -73,4 +73,5 @@ mosaic[Nmosaic].Mcal = 0.0; mosaic[Nmosaic].dMcal = 0.0; + mosaic[Nmosaic].dMsys = 0.0; mosaic[Nmosaic].Xm = 0.0; mosaic[Nmosaic].flags = image[i].flags; @@ -189,4 +190,6 @@ image[im].dMcal = mosaic[i].dMcal; image[im].Xm = mosaic[i].Xm; + image[im].dMagSys = mosaic[i].dMsys; + image[im].nFitPhotom = mosaic[i].nFitPhotom; image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_FEW); image[im].flags |= (mosaic[i].flags & ID_IMAGE_PHOTOM_POOR); @@ -400,5 +403,5 @@ mark = (N < IMAGE_TOOFEW) || (N < IMAGE_GOOD_FRACTION*Nlist[i]); if (mark) { - fprintf (stderr, "marked mosaic %s ("OFF_T_FMT"), ("OFF_T_FMT" < %d) || ("OFF_T_FMT" < %f*"OFF_T_FMT")\n", image[imlist[i][0]].name, i, N, IMAGE_TOOFEW, N, IMAGE_GOOD_FRACTION, Nlist[i]); + if (VERBOSE2) { fprintf (stderr, "marked mosaic %s ("OFF_T_FMT"), ("OFF_T_FMT" < %d) || ("OFF_T_FMT" < %f*"OFF_T_FMT")\n", image[imlist[i][0]].name, i, N, IMAGE_TOOFEW, N, IMAGE_GOOD_FRACTION, Nlist[i]); } mosaic[i].flags |= ID_IMAGE_PHOTOM_FEW; Nfew ++; @@ -410,5 +413,7 @@ if (PoorImages) fprintf (stderr, "Mmos: %f %f %d "OFF_T_FMT"\n", stats.mean, stats.sigma, stats.Nmeas, N); mosaic[i].Mcal = stats.mean; - mosaic[i].dMcal = stats.sigma; + mosaic[i].dMcal = stats.error; + mosaic[i].dMsys = stats.sigma; + mosaic[i].nFitPhotom = N; mosaic[i].Xm = 100.0*log10(stats.chisq); } Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/args.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/args.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/args.c (revision 30631) @@ -1,4 +1,3 @@ # include "relphot.h" -void usage (void); int args (int argc, char **argv) { @@ -53,4 +52,10 @@ } + USE_BASIC_CHECK = FALSE; + if ((N = get_argument (argc, argv, "-basic-image-search"))) { + remove_argument (N, &argc, argv); + USE_BASIC_CHECK = TRUE; + } + VERBOSE = VERBOSE2 = FALSE; if ((N = get_argument (argc, argv, "-v"))) { @@ -228,32 +233,6 @@ if (UpdateAverages && (argc == 1)) return TRUE; if (UserPatchSelect && (argc == 2)) return TRUE; - if (argc != 3) usage (); + if (argc != 3) relphot_usage (); return TRUE; } - -void usage () { - fprintf (stderr, "ERROR: USAGE: relphot (region) (photcode)\n"); - fprintf (stderr, " or: relphot (photcode) -region RA RA DEC DEC\n"); - fprintf (stderr, " or: relphot -averages -region RA RA DEC DEC\n"); - fprintf (stderr, " options: \n"); - fprintf (stderr, " -time (start) (stop)\n"); - fprintf (stderr, " -v : verbose output\n"); - fprintf (stderr, " -vv : more verbose output\n"); - fprintf (stderr, " -outroot (outroot)\n"); - fprintf (stderr, " -plot\n"); - fprintf (stderr, " -plotdelay (seconds)\n"); - fprintf (stderr, " -statmode (mode)\n"); - fprintf (stderr, " -refcode (name) : give extra weight to this photcode\n"); - fprintf (stderr, " -n (nloop)\n"); - fprintf (stderr, " -reset\n"); - fprintf (stderr, " -update\n"); - fprintf (stderr, " -params\n"); - fprintf (stderr, " -mosaic (mosaic)\n"); - fprintf (stderr, " -imfreeze\n"); - fprintf (stderr, " -grid\n"); - fprintf (stderr, " -area Xmin Xmax Ymin Ymax\n"); - fprintf (stderr, " -instmag min max\n"); - fprintf (stderr, " \n"); - exit (2); -} Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/bcatalog.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/bcatalog.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/bcatalog.c (revision 30631) @@ -119,6 +119,6 @@ if (VERBOSE) { - fprintf (stderr, "using "OFF_T_FMT" stars ("OFF_T_FMT" measures) of "OFF_T_FMT" for catalog\n", - subcatalog[0].Naverage, subcatalog[0].Nmeasure, i); + fprintf (stderr, "using "OFF_T_FMT" stars ("OFF_T_FMT" measures) of "OFF_T_FMT" for catalog %s\n", + subcatalog[0].Naverage, subcatalog[0].Nmeasure, i, catalog[0].filename); fprintf (stderr, "rejections: %d code, %d time, %d dophot, %d mag, %d sigma, %d imag, %d few\n", Ncode, Ntime, Ndophot, Nmag, Nsigma, Nimag, Nfew); Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/help.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/help.c (revision 30631) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/help.c (revision 30631) @@ -0,0 +1,44 @@ +# include "relphot.h" + +void relphot_usage (void) { + fprintf (stderr, "ERROR: USAGE: relphot (region) (photcode)\n"); + fprintf (stderr, " or: relphot (photcode) -region RA RA DEC DEC\n"); + fprintf (stderr, " or: relphot -averages -region RA RA DEC DEC\n"); + fprintf (stderr, " use -h for more usage information\n"); + exit (2); +} + +void relphot_help (int argc, char **argv) { + + /* check for help request */ + if (get_argument (argc, argv, "-help")) goto show_help; + if (get_argument (argc, argv, "-h")) goto show_help; + if (argc == 1) relphot_usage(); + return; + +show_help: + fprintf (stderr, "ERROR: USAGE: relphot (region) (photcode)\n"); + fprintf (stderr, " or: relphot (photcode) -region RA RA DEC DEC\n"); + fprintf (stderr, " or: relphot -averages -region RA RA DEC DEC\n"); + fprintf (stderr, " options: \n"); + fprintf (stderr, " -time (start) (stop)\n"); + fprintf (stderr, " -v : verbose output\n"); + fprintf (stderr, " -vv : more verbose output\n"); + fprintf (stderr, " -outroot (outroot)\n"); + fprintf (stderr, " -plot\n"); + fprintf (stderr, " -plotdelay (seconds)\n"); + fprintf (stderr, " -statmode (mode)\n"); + fprintf (stderr, " -refcode (name) : give extra weight to this photcode\n"); + fprintf (stderr, " -n (nloop)\n"); + fprintf (stderr, " -reset\n"); + fprintf (stderr, " -update\n"); + fprintf (stderr, " -params\n"); + fprintf (stderr, " -mosaic (mosaic)\n"); + fprintf (stderr, " -imfreeze\n"); + fprintf (stderr, " -grid\n"); + fprintf (stderr, " -area Xmin Xmax Ymin Ymax\n"); + fprintf (stderr, " -instmag min max\n"); + fprintf (stderr, " \n"); + exit (2); +} + Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/initialize.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/initialize.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/initialize.c (revision 30631) @@ -5,4 +5,5 @@ int N; + relphot_help (argc, argv); ConfigInit (&argc, argv); args (argc, argv); Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/load_catalogs.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/load_catalogs.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/load_catalogs.c (revision 30631) @@ -6,5 +6,5 @@ Catalog *catalog, tcatalog; - if (VERBOSE) fprintf (stderr, "loading catalog data\n"); + if (VERBOSE2) fprintf (stderr, "loading catalog data\n"); ALLOCATE (catalog, Catalog, skylist[0].Nregions); @@ -22,9 +22,15 @@ tcatalog.Nsecfilt = GetPhotcodeNsecfilt (); // set the desired number in case we need to create the catalog - if (!dvo_catalog_open (&tcatalog, skylist[0].regions[i], VERBOSE, "r")) { + if (!dvo_catalog_open (&tcatalog, skylist[0].regions[i], VERBOSE2, "r")) { fprintf (stderr, "ERROR: failure reading catalog %s\n", tcatalog.filename); exit (1); } - if (VERBOSE && !tcatalog.Naves_disk) fprintf (stderr, "no data in %s, skipping\n", tcatalog.filename); + if (!tcatalog.Naves_disk) { + if (VERBOSE2) { fprintf (stderr, "no data in %s, skipping\n", tcatalog.filename); } + dvo_catalog_unlock (&tcatalog); + dvo_catalog_free (&tcatalog); + continue; + } + Nstar_total += tcatalog.Naverage; Nmeas_total += tcatalog.Nmeasure; Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/load_images.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/load_images.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/load_images.c (revision 30631) @@ -1,3 +1,9 @@ # include "relphot.h" + +# define MARKTIME(MSG,...) { \ + float dtime; \ + gettimeofday (&stop, (void *) NULL); \ + dtime = DTIME (stop, start); \ + fprintf (stderr, MSG, __VA_ARGS__); } SkyList *load_images (FITS_DB *db, char *regionName, SkyRegion *region, int RegionSelect) { @@ -6,7 +12,10 @@ off_t Nimage, Nsubset, Nchar; off_t *LineNumber; + struct timeval start, stop; SkyTable *sky = NULL; SkyList *skylist = NULL; + + gettimeofday (&start, (void *) NULL); // load the current sky table (layout of all SkyRegions) @@ -29,12 +38,18 @@ exit (2); } + MARKTIME("read image table: %f sec\n", dtime); // select the images which overlap the selected sky regions subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset); + MARKTIME("selected images: %f sec\n", dtime); gfits_vtable_from_ftable (&db[0].ftable, &db[0].vtable, LineNumber, Nsubset); + MARKTIME("converted ftable to vtable: %f sec\n", dtime); initImages (subset, Nsubset); + MARKTIME("init images: %f sec\n", dtime); + initMosaics (subset, Nsubset); + MARKTIME("init mosaics: %f sec\n", dtime); return (skylist); Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/relphot.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/relphot.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/relphot.c (revision 30631) @@ -85,5 +85,5 @@ if (PLOTSTUFF) { plot_star_coords (catalog, Ncatalog); - plot_mosaic_fields (catalog); + // plot_mosaic_fields (catalog); } Index: /branches/czw_branch/20101203/Ohana/src/relphot/src/select_images.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/relphot/src/select_images.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/relphot/src/select_images.c (revision 30631) @@ -15,4 +15,10 @@ void dsortindex (double *X, off_t *Y, int N); off_t getRegionStartByRA (double R, double *Rref, off_t Nregions); + +# define MARKTIME(MSG,...) { \ + float dtime; \ + gettimeofday (&stop, (void *) NULL); \ + dtime = DTIME (stop, start); \ + fprintf (stderr, MSG, __VA_ARGS__); } Image *select_images (SkyList *skylist, Image *timage, off_t Ntimage, off_t **LineNumber, off_t *Nimage) { @@ -25,5 +31,8 @@ Coords tcoords; SkyRegionCoords *skycoords; - + struct timeval start, stop; + + double RmaxSkyRegion, RminSkyRegion, DminSkyRegion, DmaxSkyRegion, RmidSkyRegion; + double *RmaxSky; off_t *index; @@ -35,4 +44,6 @@ return NULL; } + + gettimeofday (&start, (void *) NULL); // the comparison is made in the catalog local projection. below we set crval1,2 @@ -47,4 +58,9 @@ ALLOCATE (RmaxSky, double, skylist[0].Nregions); ALLOCATE (index, off_t, skylist[0].Nregions); + + RminSkyRegion = +360.0; + RmaxSkyRegion = -360.0; + DminSkyRegion = +90.0; + DmaxSkyRegion = -90.0; /* compare with each region file */ @@ -75,10 +91,19 @@ skycoords[i].Xc[3] -= dx; skycoords[i].Yc[3] += dy; skycoords[i].Xc[4] -= dx; skycoords[i].Yc[4] -= dy; - } + + RminSkyRegion = MIN(RminSkyRegion, skylist[0].regions[i][0].Rmin); + RmaxSkyRegion = MAX(RmaxSkyRegion, skylist[0].regions[i][0].Rmax); + DminSkyRegion = MIN(DminSkyRegion, skylist[0].regions[i][0].Dmin); + DmaxSkyRegion = MAX(DmaxSkyRegion, skylist[0].regions[i][0].Dmax); + } + RmidSkyRegion = 0.5*(RminSkyRegion + RmaxSkyRegion); + MARKTIME("create sky region coords: %f sec\n", dtime); dsortindex (RmaxSky, index, skylist[0].Nregions); + MARKTIME("sort sky coords: %f sec\n", dtime); if (VERBOSE) fprintf (stderr, "finding images\n"); BuildChipMatch (timage, Ntimage); + MARKTIME("build chip match: %f sec\n", dtime); nimage = 0; @@ -100,4 +125,5 @@ } + // this adds 1.3 sec for 3M images if (!FindMosaicForImage (timage, Ntimage, i)) { fprintf (stderr, "cannot find mosaic for "OFF_T_FMT"\n", i); @@ -113,16 +139,40 @@ found = FALSE; - /* transform corners to ra,dec */ + /* transform corners to ra,dec -- costs ~3sec for 3M images */ double RminImage = 360.0; + double RmaxImage = 0.0; + double DminImage = +90.0; + double DmaxImage = -90.0; for (j = 0; j < 5; j++) { XY_to_RD (&Ri[j], &Di[j], Xi[j], Yi[j], &timage[i].coords); + Ri[j] = ohana_normalize_angle_to_midpoint (Ri[j], RmidSkyRegion); + RminImage = MIN(RminImage, Ri[j]); - } - - // RA(nStart) is guaranteed to be < RminImage: + RmaxImage = MAX(RmaxImage, Ri[j]); + DminImage = MIN(DminImage, Di[j]); + DmaxImage = MAX(DmaxImage, Di[j]); + } + if (RmaxImage - RminImage > 180.0) { + double tmp = RminImage; + RmaxImage = RminImage; + RminImage = tmp - 360.0; + } + + // check that this image is even in range of the searched region + if (DminImage > DmaxSkyRegion) continue; + if (DmaxImage < DminSkyRegion) continue; + + // the sky region RA is defined to be 0 - 360.0 + if (RminImage > RmaxSkyRegion) continue; + if (RmaxImage < RminSkyRegion) continue; + + // image overlaps region, keep it + if (USE_BASIC_CHECK) goto found_it; + + // RA(nStart) is guaranteed to be < RminImage: -- costs 0.5sec for 3M images nStart = getRegionStartByRA (RminImage, RmaxSky, skylist[0].Nregions); /* compare with each region file */ - for (iSky = 0; (iSky < skylist[0].Nregions) && !found; iSky++) { + for (iSky = nStart; (iSky < skylist[0].Nregions) && !found; iSky++) { m = index[iSky]; @@ -142,8 +192,10 @@ for (j = 0; (j < 4) && !found; j++) { found = corner_check (&Xi[j], &Yi[j], &skycoords[m].Xc[0], &skycoords[m].Yc[0]); + if (found) goto found_it; } /* check if catalog corner inside image */ for (j = 0; (j < 4) && !found; j++) { found = corner_check (&skycoords[m].Xc[j], &skycoords[m].Yc[j], &Xi[0], &Yi[0]); + if (found) goto found_it; } /* check if edges cross */ @@ -151,28 +203,33 @@ for (k = 0; (k < 4) && !found; k++) { found = edge_check (&Xi[j], &Yi[j], &skycoords[m].Xc[k], &skycoords[m].Yc[k]); + if (found) goto found_it; } } - if (!found) continue; - - image[nimage] = timage[i]; - /* always allow 'few' images to succeed, if possible */ - if (image[nimage].flags & ID_IMAGE_PHOTOM_FEW) { - image[nimage].flags &= ~(ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_POOR); - } - if (RESET) { - assignMcal (&image[nimage], (double *) NULL, -1); - image[nimage].dMcal = NAN; - image[nimage].flags &= ~ID_IMAGE_PHOTOM_POOR; - } - line_number[nimage] = i; - nimage ++; - if (nimage == NIMAGE) { - NIMAGE += 100; - REALLOCATE (image, Image, NIMAGE); - REALLOCATE (line_number, off_t, NIMAGE); - } - } - } - + } + if (!found) continue; + + found_it: + image[nimage] = timage[i]; + /* always allow 'few' images to succeed, if possible */ + if (image[nimage].flags & ID_IMAGE_PHOTOM_FEW) { + image[nimage].flags &= ~(ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_POOR); + } + if (RESET) { + // XXX assignMcal (&image[nimage], (double *) NULL, -1); + // XXX this needs to be thought through a bit more + image[nimage].Mcal = 0.0; + image[nimage].dMcal = NAN; + image[nimage].flags &= ~ID_IMAGE_PHOTOM_POOR; + } + line_number[nimage] = i; + nimage ++; + if (nimage == NIMAGE) { + NIMAGE += 100; + REALLOCATE (image, Image, NIMAGE); + REALLOCATE (line_number, off_t, NIMAGE); + } + } + MARKTIME("finish image selection: %f sec\n", dtime); + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" images\n", nimage); @@ -295,2 +352,21 @@ return (Nlo); } + +off_t getRegionStopByRA (double R, double *Rref, off_t Nregions) { + + // use bisection to find the overlapping mosaic + + off_t Nlo, Nhi, N; + + // find the last mosaic before start + Nlo = 0; Nhi = Nregions; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (Rref[N] < R) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N, Nregions); + } + } + return (Nlo); +} Index: /branches/czw_branch/20101203/Ohana/src/skycalc/Makefile =================================================================== --- /branches/czw_branch/20101203/Ohana/src/skycalc/Makefile (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/skycalc/Makefile (revision 30631) @@ -20,5 +20,5 @@ libskycalc: $(DESTLIB)/libskycalc.a $(DESTLIB)/libskycalc.$(DLLTYPE) -all: libskycalc dusktime moondata sundata +all: libskycalc dusktime moondata sundata sunriseset programs: all @@ -43,4 +43,5 @@ dusktime : $(BIN)/dusktime.$(ARCH) +sunriseset : $(BIN)/sunriseset.$(ARCH) moondata : $(BIN)/moondata.$(ARCH) sundata : $(BIN)/sundata.$(ARCH) @@ -48,4 +49,5 @@ DUSKTIME = $(SRC)/dusktime.$(ARCH).o +SUNRISESET = $(SRC)/sunriseset.$(ARCH).o MOONDATA = $(SRC)/moondata.$(ARCH).o SUNDATA = $(SRC)/sundata.$(ARCH).o @@ -53,4 +55,5 @@ $(DUSKTIME) : $(MYINCS) +$(SUNRISESET) : $(MYINCS) $(MOONDATA) : $(MYINCS) $(SUNDATA) : $(MYINCS) @@ -58,9 +61,10 @@ $(BIN)/dusktime.$(ARCH) : $(DUSKTIME) +$(BIN)/sunriseset.$(ARCH) : $(SUNRISESET) $(BIN)/moondata.$(ARCH) : $(MOONDATA) $(BIN)/sundata.$(ARCH) : $(SUNDATA) $(BIN)/sunmoon.$(ARCH) : $(SUNMOON) -INSTALL = dusktime moondata sundata sunmoon +INSTALL = dusktime sunriseset moondata sundata sunmoon # dependancy rules for binary code ######################### Index: /branches/czw_branch/20101203/Ohana/src/skycalc/src/sunriseset.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/skycalc/src/sunriseset.c (revision 30631) +++ /branches/czw_branch/20101203/Ohana/src/skycalc/src/sunriseset.c (revision 30631) @@ -0,0 +1,52 @@ +# include +# include +# define VERBOSE 0 + +// XXX is this set for MKO?? +void SC_set_site (double *longit, double *lat, double *elevsea, double *elev) { + + *longit = 10.36478; /* W longitude in decimal hours */ + *lat = 19.8267; /* N latitude in decimal degrees */ + *elevsea = 4215.; /* elevation above sea level (for absolute location) */ + *elev = 4215.; /* observatory elevation above horizon, meters */ + +} + +int main (int argc, char **argv) { + + struct SC_date_time date; + double longit, lat, elevsea, elev; + double jdnow, jdset, jdrise; + time_t tzero; + struct tm *stm; + + if (argc != 2) { + fprintf (stderr, "USAGE: sunriseset (date)\n"); + exit (1); + } + + if (!ohana_str_to_time (argv[1], &tzero)) { + fprintf (stderr, "syntax error\n"); + exit (1); + } + stm = gmtime (&tzero); + date.y = (short) (stm->tm_year + 1900); + date.mo = (short) (stm->tm_mon + 1); + date.d = (short) (stm->tm_mday); + date.h = (short) (stm->tm_hour); + date.mn = (short) (stm->tm_min); + date.s = (float) (stm->tm_sec); + + if (VERBOSE) fprintf (stderr, "%4d/%02d/%02d %02d:%02d:%02f\n", date.y, date.mo, date.d, date.h, date.mn, date.s); + + SC_set_site (&longit, &lat, &elevsea, &elev); + + jdnow = (int)(SC_date_to_jd (date)); + jdset = SC_sunset_tonight (date, lat, longit, elev); + jdrise = SC_sunrise_tonight (date, lat, longit, elev); + + fprintf (stdout, "jd %f rise %f set %f day length %f\n", jdnow, 24*60*(jdrise - jdnow), 24*60*(jdset - jdnow), 24*60*(jdset - jdrise)); + exit (0); +} + + /* set_zenith (date, lat, longit, objepoch, &objra, &objdec); */ Index: /branches/czw_branch/20101203/Ohana/src/tools/src/roc.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/tools/src/roc.c (revision 30630) +++ /branches/czw_branch/20101203/Ohana/src/tools/src/roc.c (revision 30631) @@ -4,6 +4,5 @@ # include -// # define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); abort(); } } -# define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); exit(1); } } +# define myAssert(LOGIC,...) { if (!(LOGIC)) { fprintf (stderr, __VA_ARGS__); abort(); } } # define ROC_HEADER_SIZE 0x1000 @@ -15,4 +14,6 @@ int roc_insert (int argc, char **argv); int roc_delete (int argc, char **argv); +int roc_validate (int argc, char **argv); + int usage (void); int print_block (char *header, int *header_size, int *Noff, char *format,...); @@ -29,4 +30,5 @@ if (!strcasecmp(argv[1], "-insert")) roc_insert (argc, argv); if (!strcasecmp(argv[1], "-delete")) roc_delete (argc, argv); + if (!strcasecmp(argv[1], "-validate")) roc_validate (argc, argv); usage(); @@ -302,4 +304,111 @@ } +int roc_validate (int argc, char **argv) { + + int i, j, n, Ninput, Nblocks, header_size, Nbytes, Nread; + off_t *sizes, *bytes_read; + char value; + char *header, *line, *ptr; + char **inputName, *targetName; + char **inputData; + char *targetData; + FILE **input, *target; + + if (argc != 3) usage(); + + + /* find the md5 sum for each input file (ADD LATER) + * find the size of the input files + * define the output file size = MAX(sizes) + * create the target file header + */ + + // the output file + targetName = argv[2]; + + target = fopen(targetName, "r"); + myAssert (target, "failed to open roc datafile %s\n",targetName); + + ALLOCATE (line, char, 1024); + + scan_line (target, line); + myAssert (!strncmp(line, "ROC Version 0", strlen("ROC Verion 0")), "invalid ROC file or format in %s",targetName); + + scan_line (target, line); + sscanf (line, "HEADER SIZE %x", &header_size); + + ALLOCATE (header, char, header_size); + + // read the full header + fseeko (target, 0, SEEK_SET); + Nbytes = fread (header, 1, header_size, target); + myAssert (Nbytes == header_size, "failed to read header data\n"); + + ptr = header; + ptr = strchr (ptr, '\n'); ptr ++; // ROC Version + ptr = strchr (ptr, '\n'); ptr ++; // HEADER SIZE + + sscanf (ptr, "N_FILES %d", &Ninput); ptr = strchr (ptr, '\n'); ptr ++; + sscanf (ptr, "N_BLOCKS %d", &Nblocks); ptr = strchr (ptr, '\n'); ptr ++; + + // the input files + ALLOCATE (inputName, char *, Ninput); + ALLOCATE (sizes, off_t, Ninput); + ALLOCATE (bytes_read, off_t, Ninput); + for (i = 0; i < Ninput; i++) { + ALLOCATE (inputName[i], char, 1024); + sscanf (ptr, "%*s : %s", inputName[i]); ptr = strchr (ptr, '\n'); ptr ++; + ptr = strchr (ptr, '\n'); ptr ++; // BASE + sscanf (ptr, "%*s : "OFF_T_FMT, &sizes[i]); ptr = strchr (ptr, '\n'); ptr ++; + ptr = strchr (ptr, '\n'); ptr ++; // MD5 + bytes_read[i] = 0; + } + + ALLOCATE (input, FILE *, Ninput); + for (i = 0; i < Ninput; i++) { + input[i] = fopen(inputName[i], "r"); + myAssert (input[i], "failed to open file %s in %s\n",inputName[i],targetName); + } + + ALLOCATE (inputData, char *, Ninput); + ALLOCATE (targetData, char, ROC_BLOCKSIZE); + for (i = 0; i < Ninput; i++) { + ALLOCATE (inputData[i], char, ROC_BLOCKSIZE); + } + + for (n = 0; n < Nblocks; n++) { + for (i = 0; i < Ninput; i++) { + Nread = MIN (ROC_BLOCKSIZE, sizes[i] - bytes_read[i]); + Nbytes = fread (inputData[i], 1, Nread, input[i]); + myAssert (Nbytes == Nread, "failed to read data file: %s read: %d expect: %d prev_read: %d block: %d in %s\n",inputName[i],Nbytes,Nread,(int) bytes_read[i],n,targetName); + if (Nread < ROC_BLOCKSIZE) { + // if we have reached the end of the file, fill in the rest with NULLs + memset (&inputData[i][Nread], 0, ROC_BLOCKSIZE - Nread); + } + bytes_read[i] += Nread; + } + Nbytes = fread (targetData, 1, ROC_BLOCKSIZE, target); + myAssert (Nbytes == ROC_BLOCKSIZE, "failed to read roc file: %s read: %d expect %d\n", targetName, Nbytes, ROC_BLOCKSIZE); + + for (j = 0; j < ROC_BLOCKSIZE; j++) { + value = targetData[j]; + for (i = 0; i < Ninput; i++) { + value = value ^ inputData[i][j]; + } + myAssert(value == 0, "validation failed on block %d and byte %d in %s\n",n,j,targetName); + } + + } + + for (i = 0; i < Ninput; i++) { + fclose(input[i]); + } + + fclose (target); + + fprintf (stderr, "validation: %s appears correct.\n",targetName); + exit (0); +} + int roc_insert (int argc, char **argv) { @@ -334,6 +443,8 @@ fprintf (stderr, " -create (target) (input) (input) (input) ... [options]\n"); fprintf (stderr, " -repair (target) (file #) (output\n"); + fprintf (stderr, " -validate (target)\n"); fprintf (stderr, " -insert (target) (input)\n"); fprintf (stderr, " -delete (target) (file)\n"); + exit (2); } Index: /branches/czw_branch/20101203/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- /branches/czw_branch/20101203/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 30630) +++ /branches/czw_branch/20101203/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 30631) @@ -1148,43 +1148,10 @@ if ($neb->storage_object_exists($output)) { if ($delete_existing) { - # avoid dead instances by moving the file before deleting it. - # append current time to form new name - my $todelete; - eval { - # parse the key so that we can compute the new name - require Nebulous::Key; - }; - if ($@) { - carp "Can't find Nebulous::Key"; - $$r_error = $PS_EXIT_CONFIG_ERROR; - return undef; - } - eval { - # parse the key so that we can compute the new name - my $neb_key = Nebulous::Key::parse_neb_key($output); - die "parse_neb_key failed" if !$neb_key; - my $path = $neb_key->path; - die "neb_key has no path" if !$path; - my $ticks = time(); - $todelete = "ipp_trash/$path.$ticks"; - $neb->move($output, $todelete); - }; - if ($@) { - carp "nebulous move failed for $output"; + $$r_error = $self->_kill_nebulous_file($output); + if ($$r_error) { $output = undef; - $$r_error = $PS_EXIT_SYS_ERROR; - } - if ($todelete) { - eval { - $neb->delete($todelete); - }; - if ($@) { - carp "nebulous delete for $todelete failed. Ignoring.\n"; - $$r_error = $PS_EXIT_SYS_ERROR; - } } } else { # Make sure that there is only 1 instance. - eval { $neb->there_can_be_only_one($output); @@ -1198,4 +1165,77 @@ } return $output; +} + +# _kill_nebulous_file: reliably get a nebulous file out of way. +# First move it to the trash then attempt to delete it. No failure if delete fails. +# The file is in the trash. +# Assumes that file is a nebulous file with a storage object that exists +# Users should call kill_file() +# Returns 0 on success otherwise and a PS_EXIT error code on failure +sub _kill_nebulous_file { + my $self = shift; + my $filename = shift; + my $neb = $self->nebulous; + + # avoid dead instances by moving the file before deleting it. + # append current time to form new name + my $todelete; + eval { + # parse the key so that we can compute the new name + require Nebulous::Key; + }; + if ($@) { + carp "Can't find Nebulous::Key"; + return $PS_EXIT_CONFIG_ERROR; + } + eval { + # parse the key so that we can compute the new name + my $neb_key = Nebulous::Key::parse_neb_key($filename); + die "parse_neb_key failed" if !$neb_key; + my $path = $neb_key->path; + die "neb_key has no path" if !$path; + my $ticks = time(); + $todelete = "ipp_trash/$path.$ticks"; + $neb->move($filename, $todelete); + }; + if ($@) { + carp "nebulous move failed for $filename"; + return $PS_EXIT_SYS_ERROR; + } + if ($todelete) { + eval { + $neb->delete($todelete); + }; + if ($@) { + carp "nebulous delete for $todelete failed. Ignoring.\n"; + return $PS_EXIT_SYS_ERROR; + } + } + return 0; +} + +# user level interface to kill_file reliably get a file out of way. +# +# If a nebulous file, move it to the trash then attempt to delete it. +# if non nebulous file just delete it. +# Returns 0 on success otherwise and a PS_EXIT error code on failure + +sub kill_file { + my $self = shift; + my $filename = shift; + my $neb = $self->nebulous; + + my $scheme = file_scheme($filename); + if (!$scheme or ($scheme ne 'neb')) { + if ($self->file_exists($filename)) { + if (!$self->file_delete($filename)) { + carp "failed to delete $filename"; + return $PS_EXIT_SYS_ERROR; + } + } + } elsif ($neb->storage_object_exists($filename)) { + $self->_kill_nebulous_file($filename); + } + return 0; } Index: /branches/czw_branch/20101203/doc/psphot/psphot.tex =================================================================== --- /branches/czw_branch/20101203/doc/psphot/psphot.tex (revision 30630) +++ /branches/czw_branch/20101203/doc/psphot/psphot.tex (revision 30631) @@ -159,4 +159,6 @@ Python). +\note{discuss the psphot program varients} + \section{PSPhot Design Goals} @@ -276,15 +278,15 @@ \end{itemize} -Note that a given run of PSPhot \note{should} allow the user to -perform any of these stages as an option. For example, the PSF model -may already be available from external information, in which case the -PSF modeling stage can be skipped. Or, when used as a library -function, the image may have already been loaded and the mask and -weight images constructed. In some implementations, it may be -possible to skip the initial object detection stage because only -supplied sources are measured. These are only some of the possible -configurations. The use of these different configurations depends on -the source of the image, the desired detail and speed of the -processing, and the level of accuracy desired from the analysis. +Note that a given run of PSPhot allows the user to perform many of +these stages only if needed. For example, the PSF model may already be +available from external information, in which case the PSF modeling +stage can be skipped. Or, when used as a library function, the image +may have already been loaded and the mask and weight images +constructed. In some implementations, it may be possible to skip the +initial object detection stage because only supplied sources are +measured. These are only some of the possible configurations. The +use of these different configurations depends on the source of the +image, the desired detail and speed of the processing, and the level +of accuracy desired from the analysis. \subsection{Image Preparation} @@ -292,8 +294,8 @@ The first step is to prepare the image for detection of the astronomical objects. We need three separate images: the measured -flux, the corresponding noise level, and a mask defining which pixels -are valid and which should be ignored. For the stand-alone program, -the input flux image is a required program argument. When it is -loaded, it is converted by default to 32-bit floating point +flux, the corresponding variance image, and a mask defining which +pixels are valid and which should be ignored. For the stand-alone +program, the input flux image is a required program argument. When it +is loaded, it is converted by default to 32-bit floating point representation. In the function-call form of PSPhot, the image must be supplied by the user in 32-bit floating point format. The noise @@ -307,7 +309,10 @@ automatically by PSPhot. -For the mask, we use an 8-bit image in which a value of 0 represents a -valid pixel. We use each of the 8 bits to define different reasons a -pixel should be ignored. This allows use to optionally respect or +\note{describe the use of the covariance image} +\note{describe the difference between 'bad' and 'suspect' pixels} + +For the mask, we use a 16-bit image in which a value of 0 represents a +valid pixel. We use each of the 16 bits to define different reasons a +pixel should be ignored. This allows us to optionally respect or ignore the mask depending on the circumstance. For example, in some cases, we ignore saturated pixels completely while in other @@ -325,7 +330,5 @@ \code{XMIN}, \code{XMAX}, \code{YMIN}, \code{YMAX}. -\note{Mask values are currently hard-wired numbers. We need a method - for user-defined mask values to be supplied. PSLib needs to have a - mask registration system.} +\note{discuss the mask.config file, in which the mask meanings are assigned to bit values} The noise image, if not supplied is constructed by default from the @@ -337,5 +340,5 @@ valid. For example, if the input flux image is the result of an image stack with significantly variable number of input measurements per -pixel, it will necessary to supply a noise image which accurately +pixel, it will be necessary to supply a noise image which accurately represents the noise as a function of position in the image. @@ -343,13 +346,13 @@ The objects are initially detected by finding the location of local -peaks in the image. The flux image is smoothed with a very small -circularly symmetric kernel using a two-pass 1D Gaussian. At this +peaks in the image. The flux and variance images are smoothed with a +small circularly symmetric kernel using a two-pass 1D Gaussian +(\note{KEYWORD?}). The smoothed flux and variance images are combined +to generate a significance image in signal-to-noise units +\note{including correction for the covariance, if known}. At this stage, the goal is only to detect the brighter sources, above a user defined S/N limit (configuration keyword: \code{PEAK_NSIGMA}). The detection efficiency for the brighter sources is not strongly dependent on the form of this smoothing function. - -\note{Is this smoothing needed? we could save time here by skipping -it.} The local peaks in the smoothed image are found by first detecting @@ -364,8 +367,15 @@ the maximum $X$ and $Y$ corners of the region. -\note{The current implementation ignores the S/N map in making the -peak detection. This code must be modified (a la Kaiser) to be used -for a peak-detection pass in a difference image or to re-find peaks in -the image after the modeled objects have been subtracted}. +\subsection{Footprints} + +\note{need to describe the process of generating the source footprints + and then culling the insignificant peaks} + +\subsubsection{Moments and related} + +\note{disucss the Kron mags} + +\note{this section is wrong: we no longer use S/N clipping, but a + Gaussian window function, chosed based on the measured moment} Once a collection of peaks have been identified, basic properties of @@ -391,4 +401,7 @@ \subsubsection{Determination of the Peak Coordinates and Errors} + +\note{this section is wrong: it is a poor estimator of the source + position errors. we gave up a reverted to using the FWHM / (S/N)} We use the 9 pixels which include the source peak to fit for the @@ -605,8 +618,9 @@ the minimization values. PSPhot uses the first and second moments to make a good guess for the centroid and shape parameters for the PSF -models. In order to minimize the impact of close neighbors, the noise -values used in the fit are enhanced by a fraction of the deviation of -the particular pixel value from the model guess. Any objects which -fail to converge in the fit are flagged as invalid. +models. \note{still true? In order to minimize the impact of close + neighbors, the noise values used in the fit are enhanced by a + fraction of the deviation of the particular pixel value from the + model guess.} Any objects which fail to converge in the fit are +flagged as invalid. \note{does the noise enhancement introduce too much bias?} @@ -1044,7 +1058,4 @@ \subsection{Difference Images} - -\note{much of this discussion is theoretical: PSPhot can incorporate - these modifications, but it currently does not.} The noise map for a difference image must be generated from the two Index: /branches/czw_branch/20101203/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/diff_skycell.pl (revision 30630) +++ /branches/czw_branch/20101203/ippScripts/scripts/diff_skycell.pl (revision 30631) @@ -117,7 +117,12 @@ } +# ppSub does (input) - (template) after PSF-match convolution. +# one of the files needs to be assigned to 'input' and the other to 'template'. +# if either of these is a WARP, then we can perform magic on it. If we are going to +# run magic, then we need to save the convolved version of the OTHER image +# (see magic_process.pl:155) + foreach my $file (@$files) { if (defined $file->{template} and $file->{template}) { -# $template = $file->{uri}; $templatePath = $file->{path_base}; if ($file->{warp_id} == 0) { @@ -142,8 +147,7 @@ $templateSources = "PSWARP.OUTPUT.SOURCES"; $templateMagic = $file->{magicked}; - $saveRefConv = 1; - } - } else { -# $input = $file->{uri}; + $saveInConv = 1; + } + } else { $inputPath = $file->{path_base}; $inputMagic = $file->{magicked}; # if input is a stack the output can't be "magicked" @@ -165,5 +169,5 @@ $inputVariance = "PSWARP.OUTPUT.VARIANCE"; $inputSources = "PSWARP.OUTPUT.SOURCES"; - $saveInConv = 1; + $saveRefConv = 1; } } Index: /branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl (revision 30630) +++ /branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl (revision 30631) @@ -1900,36 +1900,12 @@ { my $files = shift; # reference to a list of files to unlink -# my $test_verbose = 1; - -# if ($test_verbose == 1) { -# open(TMPLOG,">>/tmp/czw.cleanup.log"); -# flock(TMPLOG,2); -# } - - # this script is, of course, very dangerous. + foreach my $file (@$files) { print STDERR "unlinking $stage $stage_id $file"; - unless ($ipprc->file_exists($file)) { - print STDERR "\t File not found\n"; - } - else { - print STDERR "\n"; - } - -# if ($test_verbose == 1) { -# print TMPLOG "$stage $stage_id $file"; - -# else { -# print TMPLOG "\n"; -# } -# } - - $ipprc->file_delete($file); - } - -# if ($test_verbose == 1) { -# flock(TMPLOG,8); -# close(TMPLOG); -# } + + my $error_code = $ipprc->kill_file($file); + + &my_die("failed to kill $file", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if $error_code; + } return 1; Index: /branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30630) +++ /branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30631) @@ -44,5 +44,5 @@ 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier 'camera=s' => \$camera, # camera for evaluating file rules - 'stage=s' => \$stage, # camera for evaluating file rules + 'stage=s' => \$stage, # ipp stage for this magicDSRun 'save-temps' => \$save_temps, # Save temporary files? 'dbname=s' => \$dbname, # Database name @@ -160,5 +160,5 @@ my $backup_path_base = $comp->{backup_path_base}; my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); - my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom); +# my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom); if ($stage eq "chip") { @@ -219,5 +219,5 @@ } - delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask); + delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask); if ($stage eq "diff" and $warp_warp) { @@ -252,14 +252,6 @@ foreach my $file (@_) { if ($file) { - if ($ipprc->file_exists($file)) { - if (!$no_update) { - print STDERR "deleting $file\n" if $verbose; - $ipprc->file_delete($file, 1) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR); - } else { - print STDERR "skipping delete $file\n"; - } - } else { - print STDERR "$file not found\n" if $verbose; - } + my $error_code = $ipprc->kill_file($file); + my_die("Failed to delete $file", $magic_ds_id, $error_code) if $error_code; } } Index: /branches/czw_branch/20101203/ippScripts/scripts/magic_process.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/magic_process.pl (revision 30630) +++ /branches/czw_branch/20101203/ippScripts/scripts/magic_process.pl (revision 30631) @@ -151,4 +151,8 @@ &my_die("failed to resolve inputs", $magic_id, $node, $PS_EXIT_DATA_ERROR); } + + # to run magic on a diff image, we need the convolved version + # of the corresponding TEMPLATE image + # (see diff_skycell.pl:124) my $diff_base = $innode->{diff_path_base}; # Base name for diff Index: /branches/czw_branch/20101203/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/stack_skycell.pl (revision 30630) +++ /branches/czw_branch/20101203/ippScripts/scripts/stack_skycell.pl (revision 30631) @@ -270,4 +270,9 @@ $command .= " -image_id $image_id" if defined $image_id; $command .= " -source_id $source_id" if defined $source_id; + + $command .= " -stack_id $stack_id" if defined $stack_id; + $command .= " -skycell_id $skycell_id" if defined $skycell_id; + $command .= " -tess_id $tess_base" if defined $tess_base; + if ($run_state eq 'new') { $command .= " -dumpconfig $configuration"; Index: /branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl (revision 30630) +++ /branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl (revision 30631) @@ -159,5 +159,5 @@ my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1); my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1); -my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1); +my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, $skycell_id, 1); my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1); my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1); Index: /branches/czw_branch/20101203/ippTasks/Makefile.am =================================================================== --- /branches/czw_branch/20101203/ippTasks/Makefile.am (revision 30630) +++ /branches/czw_branch/20101203/ippTasks/Makefile.am (revision 30631) @@ -48,5 +48,4 @@ site.manoa.pro \ site.mhpcc.pro \ - site.mhpcc_testing.pro \ surveys.mhpcc.config \ simtest.pro \ Index: /branches/czw_branch/20101203/ippTools/src/difftool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/difftool.c (revision 30630) +++ /branches/czw_branch/20101203/ippTools/src/difftool.c (revision 30631) @@ -2095,5 +2095,5 @@ } - query = pxDataGet("difftool_definestackstack_part1.sql"); + if (pretend) { // negative simple so the default is true @@ -2109,4 +2109,7 @@ psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print for (long i = 0; i < output->n; i++) { + query = pxDataGet("difftool_definestackstack_part1.sql"); + + psMetadata *row = output->data[i]; // Output row from query bool mdok; // Status of MD lookup Index: /branches/czw_branch/20101203/ippTools/src/regtool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30630) +++ /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30631) @@ -320,5 +320,5 @@ psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@"); - fprintf(stderr,"%s",query); + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { @@ -345,5 +345,5 @@ } - fprintf (stderr, "found %ld rows\n", output->n); + // fprintf (stderr, "found %ld rows\n", output->n); if (!psArrayLength(output)) { psTrace("regtool", PS_LOG_INFO, "no rows found"); Index: /branches/czw_branch/20101203/ippconfig/gpc1/dvo.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/gpc1/dvo.config (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/gpc1/dvo.config (revision 30631) @@ -52,5 +52,5 @@ IMAGE_OFFSET 0.100 # mark images POOR if abs(delta(Mcal)) > IMAGE_OFFSET STAR_CHISQ 10.0 # mark stars POOR if Xm > STAR_CHISQ -STAR_TOOFEW 3 # mark star FEW if N(good) < STAR_TOOFEW +STAR_TOOFEW 2 # mark star FEW if N(good) < STAR_TOOFEW GRID_TOOFEW 10 # keep grid FROZEN if N(good) < GRID_TOOFEW IMAGE_TOOFEW 10 # mark image FEW if N(good) < IMAGE_TOOFEW Index: /branches/czw_branch/20101203/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/czw_branch/20101203/ippconfig/recipes/filerules-simple.mdc (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/recipes/filerules-simple.mdc (revision 30631) @@ -195,10 +195,10 @@ # outputs for psphotStack: -PSPHOT.CHISQ.IMAGE OUTPUT {OUTPUT}.chisq.im.fits IMAGE NONE FPA TRUE SIMPLE -PSPHOT.CHISQ.MASK OUTPUT {OUTPUT}.chisq.mk.fits MASK NONE FPA TRUE SIMPLE -PSPHOT.CHISQ.VARIANCE OUTPUT {OUTPUT}.chisq.wt.fits VARIANCE NONE FPA TRUE SIMPLE -PSPHOT.STACK.OUTPUT.IMAGE OUTPUT {OUTPUT}.{FILE.ID}.im.fits IMAGE NONE FPA TRUE SIMPLE -PSPHOT.STACK.OUTPUT.MASK OUTPUT {OUTPUT}.{FILE.ID}.mk.fits MASK NONE FPA TRUE SIMPLE -PSPHOT.STACK.OUTPUT.VARIANCE OUTPUT {OUTPUT}.{FILE.ID}.wt.fits VARIANCE NONE FPA TRUE SIMPLE +PSPHOT.CHISQ.IMAGE OUTPUT {OUTPUT}.chisq.im.fits IMAGE NONE FPA TRUE NONE +PSPHOT.CHISQ.MASK OUTPUT {OUTPUT}.chisq.mk.fits MASK NONE FPA TRUE NONE +PSPHOT.CHISQ.VARIANCE OUTPUT {OUTPUT}.chisq.wt.fits VARIANCE NONE FPA TRUE NONE +PSPHOT.STACK.OUTPUT.IMAGE OUTPUT {OUTPUT}.{FILE.ID}.im.fits IMAGE NONE FPA TRUE NONE +PSPHOT.STACK.OUTPUT.MASK OUTPUT {OUTPUT}.{FILE.ID}.mk.fits MASK NONE FPA TRUE NONE +PSPHOT.STACK.OUTPUT.VARIANCE OUTPUT {OUTPUT}.{FILE.ID}.wt.fits VARIANCE NONE FPA TRUE NONE PSPHOT.STACK.OUTPUT OUTPUT {OUTPUT}.{FILE.ID}.cmf CMF NONE FPA TRUE NONE Index: /branches/czw_branch/20101203/ippconfig/recipes/ppSim.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/recipes/ppSim.config (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/recipes/ppSim.config (revision 30631) @@ -128,4 +128,7 @@ ZEROPOINT F32 NAN # exposure time (seconds) SKY.MAGS F32 NAN # exposure time (seconds) + +SEEING.MAX F32 NAN # exposure time (seconds) +SEEING.RAMP BOOL F # exposure time (seconds) FILTER STR NONE # exposure time (seconds) Index: /branches/czw_branch/20101203/ippconfig/recipes/ppSub.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/recipes/ppSub.config (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/recipes/ppSub.config (revision 30631) @@ -2,10 +2,10 @@ KERNEL.TYPE STR ISIS # Kernel type to use (POIS|ISIS|SPAM|FRIES|GUNK|RINGS) -KERNEL.SIZE S32 15 # Kernel half-size (pixels) +KERNEL.SIZE S32 20 # Kernel half-size (pixels) SPATIAL.ORDER S32 1 # Spatial polynomial order REGION.SIZE F32 0 # Iso-kernel region size (pixels) SOURCE.RADIUS F32 3.0 # Source matching radius (pixels) -STAMP.SPACING F32 300 # Typical spacing between stamps (pixels) -STAMP.FOOTPRINT S32 15 # Size of stamps (pixels) +STAMP.SPACING F32 200 # Typical spacing between stamps (pixels) +STAMP.FOOTPRINT S32 20 # Size of stamps (pixels) STAMP.THRESHOLD F32 5 # Flux threshold for stamps (stdev above background) STRIDE S32 128 # Size of convolution patches (pixels) @@ -32,4 +32,6 @@ INPUT.MATCH.MIN.SN F32 10.0 +ADD.NOT.SUBTRACT BOOL FALSE # add the images instead of subtracting (ie, image + ref, not image - ref) + LOW.THRESHOLD F32 20.0 # Mask pixels below this threshold (std dev) POOR.FRACTION F32 0.20 # Maximum fraction of bad weight for poor pixels @@ -40,7 +42,10 @@ @ISIS.ORDERS S32 6 4 2 # Polynomial orders for ISIS kernels +#@ISIS.WIDTHS F32 2.4 5.0 10.0 # Gaussian kernel FWHM values +#@ISIS.ORDERS S32 6 4 2 # Polynomial orders for ISIS kernels + # for autoscaling, the above kernel widths are multiplied by the factor max(FWHM_1,FWHM_2) / SCALE.REF SCALE BOOL TRUE # Scale kernel parameters by the seeing? -SCALE.REF F32 6.0 # FWHM reference for kernel parameter scaling +SCALE.REF F32 5.0 # FWHM reference for kernel parameter scaling SCALE.MIN F32 0.5 # Minimum scale value SCALE.MAX F32 2.0 # Maximum scale value @@ -93,10 +98,13 @@ # DUAL convolution is more sensitive to the number of kernels # do not provide as many orders as for SINGLE - @ISIS.WIDTHS F32 2.5 4.0 7.0 # Gaussian kernel FWHM values + # @ISIS.WIDTHS F32 2.4 5.0 10.0 # Gaussian kernel FWHM values + @ISIS.WIDTHS F32 1.5 3.0 6.0 # Gaussian kernel FWHM values + # @ISIS.WIDTHS F32 2.0 3.0 5.0 # Gaussian kernel FWHM values + # @ISIS.WIDTHS F32 2.1 4.2 8.4 # Gaussian kernel FWHM values @ISIS.ORDERS S32 2 2 2 # Polynomial orders for ISIS kernels - SCALE.REF F32 12.0 # FWHM reference for kernel parameter scaling - KERNEL.SIZE S32 24 # Kernel half-size (pixels) - STAMP.FOOTPRINT S32 24 # Size of stamps (pixels) + SCALE.REF F32 5.0 # FWHM reference for kernel parameter scaling + KERNEL.SIZE S32 15 # Kernel half-size (pixels) + STAMP.FOOTPRINT S32 15 # Size of stamps (pixels) END Index: /branches/czw_branch/20101203/ippconfig/recipes/psphot.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/recipes/psphot.config (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/recipes/psphot.config (revision 30631) @@ -199,7 +199,15 @@ # @RADIAL.ANNULAR.BINS.UPPER F32 5.0 10.0 20.0 40.0 80.0 160.0 -# SDSS values: -@RADIAL.ANNULAR.BINS.LOWER F32 0.00 0.56 1.69 2.59 4.41 7.51 11.58 18.58 28.55 45.50 70.51 110.53 172.49 269.52 420.51 -@RADIAL.ANNULAR.BINS.UPPER F32 0.56 1.69 2.59 4.41 7.51 11.58 18.58 28.55 45.50 70.51 110.53 172.49 269.52 420.51 652.50 +# SDSS values (in SDSS pixels) +#@RADIAL.ANNULAR.BINS.LOWER F32 0.00 0.56 1.69 2.59 4.41 7.51 11.58 18.58 28.55 45.50 70.51 110.53 172.49 269.52 420.51 +#@RADIAL.ANNULAR.BINS.UPPER F32 0.56 1.69 2.59 4.41 7.51 11.58 18.58 28.55 45.50 70.51 110.53 172.49 269.52 420.51 652.50 + +# PS1 values? (all SDSS @ PS1 0.2 arcsec pixel scales) +# @RADIAL.ANNULAR.BINS.LOWER F32 0.0 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 +# @RADIAL.ANNULAR.BINS.UPPER F32 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 1305.0 + +# PS1 values? +@RADIAL.ANNULAR.BINS.LOWER F32 0.0 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 +@RADIAL.ANNULAR.BINS.UPPER F32 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 # Extended source fit parameters @@ -311,6 +319,7 @@ PSPHOT.STACK.MATCH.PSF BOOL T # convolved input images to common target PSF PSPHOT.STACK.MATCH.PSF.SOURCE STR AUTO # which inputs to convolve? (RAW, CNV, AUTO) -PSPHOT.STACK.TARGET.PSF.AUTO BOOL T # automatically determine target PSF size? -PSPHOT.STACK.TARGET.PSF.FWHM F32 5.0 # FWHM of target PSF (if NOT AUTO sized; pixels) +PSPHOT.STACK.TARGET.PSF.AUTO BOOL F # automatically determine target PSF size? +@PSPHOT.STACK.TARGET.PSF.FWHM F32 6.5 9.0 # FWHM of target PSF (if NOT AUTO sized; pixels) +PSPHOT.STACK.USE.RAW BOOL T RADIAL_APERTURES BOOL F # calculate flux in circular radial apertures? @@ -338,6 +347,6 @@ EXTENDED_SOURCE_ANNULI BOOL TRUE PSPHOT.STACK.MATCH.PSF.SOURCE STR AUTO # which inputs to convolve? (RAW, CNV, AUTO) - PSPHOT.STACK.TARGET.PSF.AUTO BOOL T # automatically determine target PSF size? - PSPHOT.STACK.TARGET.PSF.FWHM F32 7.5 # FWHM of target PSF (if NOT AUTO sized; pixels) + PSPHOT.STACK.TARGET.PSF.AUTO BOOL F # automatically determine target PSF size? + @PSPHOT.STACK.TARGET.PSF.FWHM F32 5.0 6.5 8.0 # FWHM of target PSF (if NOT AUTO sized; pixels) RADIAL_APERTURES BOOL T # calculate flux in circular radial apertures? RADIAL_APERTURES_SN_LIM F32 0.0 # S/N limit for radial aperture calculation Index: /branches/czw_branch/20101203/ippconfig/simtest/camera.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/simtest/camera.config (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/simtest/camera.config (revision 30631) @@ -64,4 +64,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/czw_branch/20101203/ippconfig/simtest/ppSub.config =================================================================== --- /branches/czw_branch/20101203/ippconfig/simtest/ppSub.config (revision 30630) +++ /branches/czw_branch/20101203/ippconfig/simtest/ppSub.config (revision 30631) @@ -1,9 +1,5 @@ ### Recipe file for ppSub -STAMP.SPACING F32 100 # Typical spacing between stamps (pixels) - -RENORM BOOL FALSE # Renormalize weight maps? - -DUAL BOOL FALSE # Dual convolution? +STAMP.SPACING F32 25 # Typical spacing between stamps (pixels) TEST.RADIAL METADATA @@ -50,6 +46,2 @@ END -SPATIAL.ORDER S32 0 # Spatial polynomial order - -#STAMP.FOOTPRINT S32 25 # Size of stamps (pixels) -#KERNEL.SIZE S32 25 # Kernel half-size (pixels) Index: /branches/czw_branch/20101203/ppSim/src/Makefile.am =================================================================== --- /branches/czw_branch/20101203/ppSim/src/Makefile.am (revision 30630) +++ /branches/czw_branch/20101203/ppSim/src/Makefile.am (revision 30631) @@ -47,8 +47,4 @@ ppSimUtils.c \ ppSimLoop.c \ - ppSimLoadSpots.c \ - ppSimPhotom.c \ - ppSimPhotomFiles.c \ - ppSimLoadForceSources.c \ ppSimMergeReadouts.c \ ppSimDetections.c \ @@ -59,8 +55,4 @@ ppSimBadCTE.c \ ppSimVersion.c - -# these functions have been broken by the API change in psphot (Jan 2010) -# ppSimPhotomReadoutFake.c -# ppSimPhotomReadoutForce.c ppSimSequence_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSASTRO_CFLAGS) $(ppSim_CFLAGS) Index: /branches/czw_branch/20101203/ppSim/src/ppSimArguments.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimArguments.c (revision 30630) +++ /branches/czw_branch/20101203/ppSim/src/ppSimArguments.c (revision 30631) @@ -49,4 +49,6 @@ psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Photometric zero point", NAN); psMetadataAddF32(arguments, PS_LIST_TAIL, "-seeing", 0, "Seeing FWHM (arcsec)", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-seeing-max", 0, "Seeing FWHM (arcsec) at end of ramp", NAN); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-seeing-ramp", 0, "Use a seeing ramp", false); psMetadataAddF32(arguments, PS_LIST_TAIL, "-starslum", 0, "Fake star luminosity function slope", NAN); psMetadataAddF32(arguments, PS_LIST_TAIL, "-starsmag", 0, "Brightest magnitude for fake stars", NAN); @@ -130,4 +132,6 @@ // 'seeing' is not required: we can load a psf-model instead; but if not, it is allowed ppSimArgToRecipeF32(&status, options, "SEEING", arguments, "-seeing"); // seeing (FWHM in arcsec) + ppSimArgToRecipeF32(&status, options, "SEEING.MAX", arguments, "-seeing-max"); // seeing (FWHM in arcsec) + ppSimArgToRecipeBool(&status, options, "SEEING.RAMP", arguments, "-seeing-ramp"); // seeing (FWHM in arcsec) // 'scale' is not required: we can use the WCS instead; but if not, it is allowed @@ -227,4 +231,10 @@ psMetadataAddF32(options, PS_LIST_TAIL, "PA", 0, "Boresight position angle (radians)",pa * M_PI / 180.0); + if (psMetadataLookupBool(NULL, arguments, "-seeing-ramp")) { + float seeingMax = psMetadataLookupF32(NULL, arguments, "-seeing-max"); // seeing at ramp end + psMetadataAddF32(options, PS_LIST_TAIL, "SEEING.MAX", 0, "Seeing FWHM (arcsec)", seeingMax); + psMetadataAddBool(options, PS_LIST_TAIL, "SEEING.RAMP", 0, "use seeing ramp", true); + } + ppSimArgToRecipeF32(&status, options, "ZEROPOINT", arguments, "-zp"); // Zero point } Index: /branches/czw_branch/20101203/ppSim/src/ppSimCreate.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimCreate.c (revision 30630) +++ /branches/czw_branch/20101203/ppSim/src/ppSimCreate.c (revision 30631) @@ -103,4 +103,7 @@ psMetadataAddF32(recipe, PS_LIST_TAIL, "SEEING", PS_META_REPLACE, "Seeing SIGMA (pixels)", seeing / 2.0 / sqrt(2.0 * log(2.0)) / scale); + float seeingMax = psMetadataLookupF32(&status, recipe, "SEEING.MAX"); + psMetadataAddF32(recipe, PS_LIST_TAIL, "SEEING.MAX", PS_META_REPLACE, "Seeing SIGMA (pixels)", seeingMax / 2.0 / sqrt(2.0 * log(2.0)) / scale); + // if we have been supplied an input image, but no ra & dec, use the input image values if (input) { @@ -132,65 +135,19 @@ bool doPhotom = psMetadataLookupBool(&status, recipe, "PHOTOM"); // Density of fakes if (doPhotom) { - - // XXX at the moment, we can perform photometry on the fake sources and the forced - // photometry positions, but not one or the other. Also, we only support photometry in - // the context of an image previously analysed by psphot. Add support for: - // * photometry of a pure fake image (requires peak detection and psf creation) - // * photometry of forced source positions without fake image (this might work, it just - // requires not generating any fake features). - - if (!input) { - psError(PS_ERR_UNKNOWN, false, "input image not found; currently required for photometry"); - return NULL; - } - - // we need a chip image if we perform photometry (is it necessary to build it if we don't use it?) - pmFPAfile *fakeImage = pmFPAfileDefineChipMosaic(config, output->fpa, "PPSIM.FAKE.CHIP"); - if (!fakeImage) { - psError(PS_ERR_IO, false, _("Unable to generate new file from PPSIM.FAKE.CHIP")); - psFree(fpa); - return NULL; - } - if (fakeImage->type != PM_FPA_FILE_IMAGE) { - psError(PS_ERR_IO, true, "PPSIM.FAKE.CHIP is not of type IMAGE"); - psFree(fpa); - return NULL; - } - - // we need a chip image if we perform photometry (is it necessary to build it if we don't use it?) - pmFPAfile *forceImage = NULL; - if (input) { - forceImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PPSIM.FORCE.CHIP"); - if (!forceImage) { - psError(PS_ERR_IO, false, _("Unable to generate new file from PPSIM.FORCE.CHIP")); - psFree(fpa); - return NULL; - } - if (forceImage->type != PM_FPA_FILE_IMAGE) { - psError(PS_ERR_IO, true, "PPSIM.FORCE.CHIP is not of type IMAGE"); - psFree(fpa); - return NULL; - } - } - - // define associated psphot input/output files - if (!ppSimPhotomFiles (config, fakeImage, forceImage)) { - psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files for psphot"); - psFree(fpa); - return NULL; - } - } else { - // have we supplied a psf model? this happens in ppSimPhotomFiles if we request a photometry - // analysis. however, even if we do not, a psf model may be used to generate the fake - // sources. - if (psMetadataLookupPtr(NULL, config->arguments, "PSPHOT.PSF")) { - // tie the psf file to the chipMosaic - pmFPAfileBindFromArgs(&status, output, config, "PSPHOT.PSF.LOAD", "PSPHOT.PSF"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "Failed to find/build PSPHOT.PSF.LOAD"); - psFree(fpa); - return NULL; - } - } + psError(PS_ERR_UNKNOWN, false, "in-line photometry in ppSim had been deprecated"); + return NULL; + } + + // have we supplied a psf model? this happens in ppSimPhotomFiles if we request a photometry + // analysis. however, even if we do not, a psf model may be used to generate the fake + // sources. + if (psMetadataLookupPtr(NULL, config->arguments, "PSPHOT.PSF")) { + // tie the psf file to the chipMosaic + pmFPAfileBindFromArgs(&status, output, config, "PSPHOT.PSF.LOAD", "PSPHOT.PSF"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Failed to find/build PSPHOT.PSF.LOAD"); + psFree(fpa); + return NULL; + } } @@ -257,4 +214,17 @@ } } + // XXX this is a hack, but I don't have a better way at the moment: assumes a single cell per chip + if (nx) { + psMetadataAddS32(chip->concepts, PS_LIST_TAIL, "CHIP.XSIZE", PS_META_REPLACE, "", nx); + } else { + nx = psMetadataLookupF32(&status, cell->concepts, "CELL.XSIZE"); + psMetadataAddS32(chip->concepts, PS_LIST_TAIL, "CHIP.XSIZE", PS_META_REPLACE, "", nx); + } + if (ny) { + psMetadataAddS32(chip->concepts, PS_LIST_TAIL, "CHIP.YSIZE", PS_META_REPLACE, "", ny); + } else { + ny = psMetadataLookupF32(&status, cell->concepts, "CELL.YSIZE"); + psMetadataAddS32(chip->concepts, PS_LIST_TAIL, "CHIP.YSIZE", PS_META_REPLACE, "", ny); + } } } Index: anches/czw_branch/20101203/ppSim/src/ppSimLoadForceSources.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimLoadForceSources.c (revision 30630) +++ (revision ) @@ -1,113 +1,0 @@ -# include "ppSim.h" - -// Reset a pointer: free and set to NULL -#define RESET(PTR) \ - psFree(PTR); \ - PTR = NULL; - -psArray *ppSimLoadForceSources(pmConfig *config, const pmFPAview *view) { - - bool status; - - pmFPAfile *outFile = psMetadataLookupPtr(NULL, config->files, "PPSIM.OUTPUT"); - psAssert(outFile, "missing PPSIM.OUTPUT"); - psAssert(outFile->fpa, "missing fpr for PPSIM.OUTPUT"); - - pmChip *outChip = pmFPAviewThisChip (view, outFile->fpa); - pmPSF *psf = psMetadataLookupPtr (&status, outChip->analysis, "PSPHOT.PSF"); - assert (psf); - - psArray *spots = psMetadataLookupPtr(&status, outFile->fpa->analysis, "FORCED.SPOTS"); - PS_ASSERT_PTR_NON_NULL (spots, NULL); - - // in this program, we are looping over the output image, rather than the input as in ppImage - pmFPAfile *srcFile = psMetadataLookupPtr(NULL, config->files, "PPSIM.REAL.SOURCES"); - psAssert(srcFile, "missing PPSIM.REAL.SOURCES"); - - // select the fpa and chip from PPSIM.REAL.SOURCES, which carries the astrometry in the headers - pmFPA *fpa = srcFile->fpa; - pmChip *chip = pmFPAviewThisChip (view, fpa); - - // we use the readout for the PPSIM.FORCE.CHIP, which contains the chip-mosaicked image(s) - pmReadout *readout = pmFPAfileThisReadout (config->files, view, "PPSIM.FORCE.CHIP"); - - // XXX we should have only one cell and readout per chip, right? - - // read WCS from existing output from psphot & psastro - // XXX can we be more flexible? ie, use the header of PPSIM.FORCE.CHIP? - bool bilevelAstrometry = false; - status = psastroAstromGuessSetFPA (fpa, &bilevelAstrometry); - psAssert (status, "failed to set astrometry"); - - // the pixel scale is only used to set the focal-plane coordinate system - // setting this value to 1 makes this system have the units of pixels - float pixelScale = 1.0; - status = psastroAstromGuessSetChip (fpa, chip, view, pixelScale, bilevelAstrometry); - psAssert (status, "failed to set astrometry"); - - float minX = 0.0; - float maxX = readout->image->numCols; - float minY = 0.0; - float maxY = readout->image->numRows; - - // the sources is a subset within range of this chip - psArray *sources = psArrayAllocEmpty (100); - - // Select the spots within range of this readout. Project the spots to this chip - for (int i = 0; i < spots->n; i++) { - pmAstromObj *obj = spots->data[i]; - - psProject (obj->TP, obj->sky, fpa->toSky); - psPlaneTransformApply (obj->FP, fpa->fromTPA, obj->TP); - psPlaneTransformApply (obj->chip, chip->fromFPA, obj->FP); - - // limit the X,Y range of the objs to the selected chip - if (obj->chip->x < minX) continue; - if (obj->chip->x > maxX) continue; - if (obj->chip->y < minY) continue; - if (obj->chip->y > maxY) continue; - - // convert the pmAstromObj to pmSource - - // instantiate a model for the PSF at this location, Io = 1.0 - pmModel *model = pmModelFromPSFforXY (psf, obj->chip->x, obj->chip->y, 1.0); - - // XXX let the flux limit be a user-defined number of sky sigmas (not just 1.0) - // XXX use a fixed radius?? - // float radius = model->modelRadius (model->params, roughNoise); - // radius = PS_MAX (radius, 1.0); - float radius = 5.0; - - // construct a source, with model flux pixels set, based on the model - pmSource *source = pmSourceFromModel (model, readout, radius, PM_SOURCE_TYPE_STAR); - - // XXX set the mag & err values (should this be done in pmSourceFromModel?) - // XXX i should be applying the gain and the correct effective area - // psEllipseAxes axes = pmPSF_ModelToAxes (model->params->data.F32, 20.0); - // psF64 Area = 2.0 * M_PI * axes.major * axes.minor; - - // these are not really needed since we will be fitting for them - source->psfMag = NAN; - source->errMag = NAN; - - // insert the source flux in the image - // XXX not sure the offset is really 0,0 - pmSourceAddWithOffset (source, PM_MODEL_OP_FULL, 0xff, 0.0, 0.0); - pmSourceAddWithOffset (source, PM_MODEL_OP_FULL | PM_MODEL_OP_NOISE, 0xff, 0.0, 0.0); - - // Blow away the image parts of the source, which makes the memory explode - RESET(source->pixels); - RESET(source->variance); - RESET(source->maskObj); - RESET(source->maskView); - RESET(source->modelFlux); - RESET(source->psfImage); - RESET(source->blends); - - psArrayAdd (sources, 100, source); - psFree(source); // Drop local reference - } - psTrace ("psastro", 4, "Added %ld sources\n", sources->n); - - return sources; -} Index: anches/czw_branch/20101203/ppSim/src/ppSimLoadSpots.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimLoadSpots.c (revision 30630) +++ (revision ) @@ -1,71 +1,0 @@ -# include "ppSim.h" - -// Load the relevant forced-photometry positions for this field. This function does not determine -// the pixel coordinates, merely the celestial coordinates for sources in the general vicinity. The -// sources are saved on the fpa->analysis metadata as FORCED.SPOTS. We always create an entry; it -// will be empty if no sources were selected or forced photometry is not desired. -bool ppSimLoadSpots (pmFPA *fpa, pmConfig *config) { - - bool status; - - - psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, PPSIM_RECIPE); // Recipe - - bool forcedPhot = psMetadataLookupBool(&status, recipe, "FORCED.PHOT"); // Density of fakes - if (!forcedPhot) { - psArray *spots = psArrayAllocEmpty (1); - psMetadataAddArray (fpa->analysis, PS_LIST_TAIL, "FORCED.SPOTS", PS_META_REPLACE, "forced photometry positions", spots); - psFree (spots); // free the extra (local) reference; a copy remains on fpa->analysis - return true; - } - - // We read the catalogue stars using psastroLoadRefstars - psMetadata *astroRecipe = psMetadataLookupPtr(NULL, config->recipes, PSASTRO_RECIPE); - if (!astroRecipe) { - psError(PSASTRO_ERR_CONFIG, false, "Can't find recipe %s", PSASTRO_RECIPE); - return false; - } - - float ra0 = psMetadataLookupF32(NULL, recipe, "RA"); // Boresight RA (radians) - float dec0 = psMetadataLookupF32(NULL, recipe, "DEC"); // Boresight Dec (radians) - float scale = psMetadataLookupF32(NULL, recipe, "PIXEL.SCALE") * M_PI / 3600.0 / 180.0; // Plate scale (radians/pixel) - - if (isnan(ra0) || isnan(dec0)) { - psError(PS_ERR_UNKNOWN, false, "image boresite coords not defined."); - return false; - } - - char *catdir = psMetadataLookupStr(NULL, recipe, "FORCED.CATDIR"); - - // Size of FPA - psRegion *bounds = ppSimFPABounds (fpa); - float radius = 0.5 * PS_MAX(bounds->x1 - bounds->x0, bounds->y1 - bounds->y0) * scale; - psFree(bounds); - - // modify the PSASTRO recipe to use the values desired for FORCED.PHOT. we can use a view on - // the PSASTRO recipe because we are not calling psastro elsewhere in this program. XXX need to - // use the WCS to define the overlap region - psMetadataAddStr(astroRecipe, PS_LIST_TAIL, "PSASTRO.CATDIR", PS_META_REPLACE, "", catdir); - psMetadataAddF32(astroRecipe, PS_LIST_TAIL, "RA_MIN", PS_META_REPLACE, "", ra0 - radius); - psMetadataAddF32(astroRecipe, PS_LIST_TAIL, "RA_MAX", PS_META_REPLACE, "", ra0 + radius); - psMetadataAddF32(astroRecipe, PS_LIST_TAIL, "DEC_MIN", PS_META_REPLACE, "", dec0 - radius); - psMetadataAddF32(astroRecipe, PS_LIST_TAIL, "DEC_MAX", PS_META_REPLACE, "", dec0 + radius); - - // tell psastroLoadRefstars to ignore photcode and maglim - psMetadataAddStr(astroRecipe, PS_LIST_TAIL, "DVO.GETSTAR.PHOTCODE", PS_META_REPLACE, "", "NONE"); - psMetadataAddF32(astroRecipe, PS_LIST_TAIL, "DVO.GETSTAR.MAG.MAX", PS_META_REPLACE, "", NAN); - - // load refstars from the catalog - psArray *spots = psastroLoadRefstars(config, "PSASTRO.INPUT"); - if (!spots || spots->n == 0) { - psError(PS_ERR_UNKNOWN, false, "Unable to find reference stars."); - psFree(spots); - return false; - } - psLogMsg("ppSim", PS_LOG_INFO, "Adding %ld reference stars", spots->n); - - psMetadataAddArray (fpa->analysis, PS_LIST_TAIL, "FORCED.SPOTS", PS_META_REPLACE, "forced photometry positions", spots); - psFree (spots); // free the extra (local) reference; a copy remains on fpa->analysis - - return true; -} Index: /branches/czw_branch/20101203/ppSim/src/ppSimLoop.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimLoop.c (revision 30630) +++ /branches/czw_branch/20101203/ppSim/src/ppSimLoop.c (revision 30631) @@ -42,5 +42,5 @@ if (type == PPSIM_TYPE_OBJECT) { // Load forced-photometry positions (these are placed on fpa->analysis for use in ppSimPhotomReadout) - if (!ppSimLoadSpots (fpa, config)) ESCAPE (PS_ERR_UNKNOWN, "failed to load forced-photometry spots"); + // if (!ppSimLoadSpots (fpa, config)) ESCAPE (PS_ERR_UNKNOWN, "failed to load forced-photometry spots"); // Load catalogue stars @@ -58,7 +58,4 @@ pmFPAview *view = pmFPAviewAlloc(0);// View for iterating over FPA - - // XXX if we include the psphot analysis, we will need to activate the correct pmFPAfiles - // at the correct times. ppSim operates on PPSIM.OUTPUT and PPSIM.SOURCES // load any needed files (eg, input image, PSF) @@ -209,5 +206,5 @@ // we perform photometry on the readouts of this chip in the output - if (!ppSimPhotom (config, view)) ESCAPE (PS_ERR_UNKNOWN, "problem performing photometry"); + // if (!ppSimPhotom (config, view)) ESCAPE (PS_ERR_UNKNOWN, "problem performing photometry"); if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { Index: /branches/czw_branch/20101203/ppSim/src/ppSimMakeGalaxies.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimMakeGalaxies.c (revision 30630) +++ /branches/czw_branch/20101203/ppSim/src/ppSimMakeGalaxies.c (revision 30631) @@ -28,4 +28,6 @@ float galaxyThetaMax = psMetadataLookupF32(&mdok, recipe, "GALAXY.THETA.MAX"); // Density of fakes float galaxyThetaMin = psMetadataLookupF32(&mdok, recipe, "GALAXY.THETA.MIN"); // Density of fakes + galaxyThetaMax *= PS_RAD_DEG; + galaxyThetaMin *= PS_RAD_DEG; float galaxyIndexMin = psMetadataLookupF32(&mdok, recipe, "GALAXY.INDEX.MIN"); // Density of fakes @@ -95,4 +97,6 @@ int i = 0; + float refNorm = 1.0; + float ourNorm = 1.0; for (long iy = 0.5*galaxyGridDY; iy < ySize; iy += galaxyGridDY) { @@ -104,5 +108,7 @@ galaxy->y = iy; - galaxy->peak = 1000; + // galaxy->peak = 1000; + galaxy->peak = bright / 100.0; + // fprintf (stderr, "Io: %f\n", bright); // galaxyIndex from user should be for function of this form: exp(-r^(1/n)) @@ -111,9 +117,9 @@ float rndValue; - rndValue = galaxyGridRandom ? drand48() : i / num; + rndValue = galaxyGridRandom ? drand48() : i / (float) num; float index = (galaxyIndexMin + rndValue * galaxyIndexSlope); galaxy->index = 0.5/index; // factor of 0.5 because the Sersic model creates exp(-z^n), not exp(-r^n) - rndValue = galaxyGridRandom ? drand48() : i / num; + rndValue = galaxyGridRandom ? drand48() : i / (float) num; float scale = (galaxyRmajorMin + rndValue * galaxyRmajorSlope); @@ -125,14 +131,18 @@ galaxy->Rmaj = scale; - rndValue = galaxyGridRandom ? drand48() : i / num; + rndValue = galaxyGridRandom ? drand48() : i / (float) num; galaxy->Rmin = (galaxyARatioMin + rndValue * galaxyARatioSlope) * galaxy->Rmaj; - rndValue = galaxyGridRandom ? drand48() : i / num; + rndValue = galaxyGridRandom ? drand48() : i / (float) num; galaxy->theta = (galaxyThetaMin + rndValue * galaxyThetaSlope); - - // galaxy->peak *= Io; + + if (i == 0) { + refNorm = Io*scale*scale; + } + ourNorm = refNorm / (Io*scale*scale); + galaxy->peak *= ourNorm; if (0) { - fprintf (stderr, "Rmaj: %f, scale: %f, index: %f, bn: %f, Ro: %f, Io: %f\n", galaxy->Rmaj, scale, index, bn, fR, Io); + fprintf (stderr, "Rmaj: %f, scale: %f, index: %f, bn: %f, Ro: %f, Io: %f, theta: %f\n", galaxy->Rmaj, scale, index, bn, fR, Io, galaxy->theta); } Index: anches/czw_branch/20101203/ppSim/src/ppSimPhotom.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimPhotom.c (revision 30630) +++ (revision ) @@ -1,65 +1,0 @@ -# include "ppSim.h" - -// In this function, we perform the psphot analysis routine for the chip-mosaicked images -bool ppSimPhotom (pmConfig *config, pmFPAview *view) { - - bool status; - pmCell *cell; - pmReadout *readout; - - psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, PPSIM_RECIPE); // Recipe - - bool doPhotom = psMetadataLookupBool(&status, recipe, "PHOTOM"); // Density of fakes - if (!doPhotom) return true; - - psphotInit (); - - psImageMaskType blankMask = 1; // XXX not sure what this should be set to... - ppSimMosaicChip(config, blankMask, view, "PPSIM.FORCE.CHIP", "PPSIM.INPUT"); - ppSimMosaicChip(config, blankMask, view, "PPSIM.FAKE.CHIP", "PPSIM.OUTPUT"); - - // use PPSIM.FAKE.CHIP as the base since it is guaranteed to exist (derived from PPSIM.OUTPUT) - pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PPSIM.FAKE.CHIP"); - if (!status) { - psError(PSPHOT_ERR_CONFIG, false, "PPSIM.FAKE.CHIP I/O file is not defined"); - return false; - } - - // XXX If we want to be able to write out both the positive and residual images, we will - // need to define an additional pmFPAfile container, and then make a copy as is done for - // PPIMAGE.CHIP -> PSPHOT.INPUT in ppImage. At the moment, PPSIM.CHIP will write out the - // mosaicked image with the sources subtracted. - - // XXX if input -> PSPHOT.INPUT, which is tied to PPSIM.CHIP with pmFPAfileDefineFromFile, - // then the following code creates the chip copy: - // pmChip *oldChip = pmFPAviewThisChip (view, input->src); - // pmChip *newChip = pmFPAviewThisChip (view, input->fpa); - // pmChipCopy (newChip, oldChip); - - // iterate over the cells and readout for this chip - while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { - psLogMsg ("ppSimPhotom", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); - if (! cell->process || ! cell->file_exists) { continue; } - - // process each of the readouts - while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { - if (! readout->data_exists) { continue; } - -// these are broken by the Jan 2010 API changes to psphot -#if 0 - // run the actual photometry analysis - if (!ppSimPhotomReadoutFake (config, view)) { - psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); - return false; - } - if (!ppSimPhotomReadoutForce (config, view)) { - psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); - return false; - } - // ppSimDetectionLimits (config, view); -#endif - } - } - - return true; -} Index: anches/czw_branch/20101203/ppSim/src/ppSimPhotomFiles.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimPhotomFiles.c (revision 30630) +++ (revision ) @@ -1,83 +1,0 @@ -# include "ppSim.h" - -// define the needed / desired I/O files -bool ppSimPhotomFiles (pmConfig *config, pmFPAfile *fakeFile, pmFPAfile *forceFile) { - - bool status = false; - - // select recipe options supplied on command line - psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); - - // optionally save the background model (small FITS image) - if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKMDL")) { - int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); - int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); - pmFPAfile *output = pmFPAfileDefineFromFile (config, fakeFile, DX, DY, "PSPHOT.BACKMDL"); - if (!output) { - psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKMDL"); - return false; - } - output->save = true; - } - // optionally save the background model's standard deviation (small FITS image) - if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKMDL.STDEV")) { - int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); - int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); - pmFPAfile *output = pmFPAfileDefineFromFile (config, fakeFile, DX, DY, "PSPHOT.BACKMDL.STDEV"); - if (!output) { - psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKMDL.STDEV"); - return false; - } - output->save = true; - } - // optionally save the full background image - if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKGND")) { - pmFPAfile *output = pmFPAfileDefineFromFile (config, fakeFile, 1, 1, "PSPHOT.BACKGND"); - if (!output) { - psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKGND"); - return false; - } - output->save = true; - } - // optionally save the background-subtracted image - if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKSUB")) { - pmFPAfile *output = pmFPAfileDefineFromFile (config, fakeFile, 1, 1, "PSPHOT.BACKSUB"); - if (!output) { - psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKSUB"); - return false; - } - output->save = true; - } - - // the input sources are required for sensible measurements of the fake or force sources - pmFPAfile *cmffile = pmFPAfileDefineFromArgs (&status, config, "PPSIM.REAL.SOURCES", "INPUT.SOURCES"); - if (!cmffile) { - psError(PS_ERR_UNKNOWN, false, "input sources not found; required for photometry (or error with PPSIM.REAL.SOURCES filerule)"); - return false; - } - - // the fake sources are carried on the fakeFile->fpa structures (PPSIM.FAKE.CHIP) - pmFPAfile *outsources = pmFPAfileDefineOutput (config, fakeFile->fpa, "PPSIM.FAKE.SOURCES"); - if (!outsources) { - psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PPSIM.FAKE.SOURCES"); - return false; - } - outsources->save = true; - - // the forced sources are carried on the forceFile->fpa structures (PPSIM.FORCE.CHIP) - pmFPAfile *outforced = pmFPAfileDefineOutput (config, forceFile->fpa, "PPSIM.FORCE.SOURCES"); - if (!outforced) { - psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PPSIM.FORCE.SOURCES"); - return false; - } - outforced->save = true; - - // we bind the psffile to the chip-mosaicked image so we have a reference to the image - pmFPAfile *psffile = pmFPAfileBindFromArgs(&status, fakeFile, config, "PSPHOT.PSF.LOAD", "PSPHOT.PSF"); - if (!psffile) { - psError(PSPHOT_ERR_CONFIG, false, "Failed to find/build PSPHOT.PSF.LOAD"); - return false; - } - - return true; -} Index: anches/czw_branch/20101203/ppSim/src/ppSimPhotomReadout.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimPhotomReadout.c (revision 30630) +++ (revision ) @@ -1,202 +1,0 @@ -# include "ppSim.h" - -psArray *ppSimSelectSources (pmConfig *config, const pmFPAview *view, const char *filename) { - - pmReadout *readout = pmFPAfileThisReadout (config->files, view, filename); - PS_ASSERT_PTR_NON_NULL (readout, NULL); - - pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); - psAssert (detections, "missing detections?"); - - psArray *sources = detections->allSources; - return sources; -} - -bool ppSimDefinePixels (psArray *sources, pmReadout *readout, psMetadata *recipe) { - - bool status; - - float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); - if (!status) return NULL; - - for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - - // allocate image, weight, mask arrays for each peak (square of radius OUTER) - pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); - } - return true; -} - -bool ppSimPhotomReadout(pmConfig *config, const pmFPAview *view) { - - psTimerStart ("psphotReadout"); - - // select the current recipe - psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); - if (!recipe) { - psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); - return false; - } - -# if 0 - // set the photcode for this image - if (!psphotAddPhotcodeReadout (recipe, config, view, "PPSIM.CHIP", 0)) { - psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); - return false; - } -# endif - - // find the currently selected readout. - // we always perform photometry on the mosaiced chip - pmReadout *readout = pmFPAfileThisReadout (config->files, view, "PPSIM.CHIP"); - PS_ASSERT_PTR_NON_NULL (readout, false); - - // the previously measured real sources are loaded into the PPSIM.REAL.SOURCES pmFPAfile - psArray *realSources = ppSimSelectSources (config, view, "PPSIM.REAL.SOURCES"); - PS_ASSERT_PTR_NON_NULL (realSources, false); - - // the newly injected fake sources are saved on the PPSIM.OUTPUT pmFPAfile - psArray *injectedSources = ppSimSelectSources (config, view, "PPSIM.OUTPUT"); - PS_ASSERT_PTR_NON_NULL (injectedSources, false); - - // XXX need to define the source pixels - ppSimDefinePixels (realSources, readout, recipe); - ppSimDefinePixels (injectedSources, readout, recipe); - - // make a copy of the fake-source parameters so these can be saved independently from the - // measured fake-source parameters - psArray *fakeSources = psArrayAlloc (injectedSources->n); - for (int i = 0; i < injectedSources->n; i++) { - fakeSources->data[i] = pmSourceCopyData (injectedSources->data[i]); - } - - // load the forced source lists - psArray *forceSources = ppSimLoadForceSources (config, view); - psAssert (forceSources, "failed to load force photometry sources"); - - // Generate the mask and weight images, including the user-defined analysis region of interest - psphotSetMaskAndWeight (config, view, recipe); - - // load the psf model, if suppled. FWHM_X,FWHM_Y,etc are saved in the recipe - // this function uses PSPHOT.PSF.LOAD as the pmFPAfile - pmPSF *psf = psphotLoadPSF (config, view, recipe); - assert (psf); - - // remove all sources - psphotRemoveAllSources (realSources, recipe); - - // generate a background model (median, smoothed image) - if (!psphotModelBackgroundReadoutFileIndex (config, view, "PPSIM.CHIP", 0)) { - return psphotReadoutCleanup (config, readout, recipe, NULL, NULL, NULL); - } - if (!psphotSubtractBackgroundReadout (config, view, "PPSIM.CHIP", 0)) { - return psphotReadoutCleanup (config, readout, recipe, NULL, NULL, NULL); - } - - // add noise for real (subtracted) objects - psphotAddNoise (readout, realSources, recipe); - - // XXX fake sources should measure peak->x,y, force sources should not - psImageMaskType maskVal = 0xff; - psImage *significance = psphotSignificanceImage (readout, recipe, 1, maskVal); - ppSimDetections (significance, recipe, fakeSources); - ppSimDetections (significance, recipe, forceSources); - psFree (significance); - - // remove noise for subtracted objects (ie, return to normal noise level) - psphotSubNoise (readout, realSources, recipe); - - // replace all sources - psphotReplaceAllSources (realSources, recipe); - - // construct an initial model for each object - psphotGuessModels (config, readout, realSources, psf); - psphotGuessModels (config, readout, fakeSources, psf); - psphotGuessModels (config, readout, forceSources, psf); - - psArray *sources = NULL; - - // linear fit to real + fake sources - sources = ppSimMergeSources (realSources, fakeSources); - psphotFitSourcesLinear (readout, sources, recipe, psf, FALSE); // XXX option to NOT subtract - psphotReplaceAllSources (sources, recipe); - psFree (sources); // only frees the merged references - - // linear fit to real + forced sources (if any were selected) - if (forceSources->n) { - sources = ppSimMergeSources (realSources, forceSources); - psphotFitSourcesLinear (readout, sources, recipe, psf, FALSE); // XXX option to NOT subtract - psphotReplaceAllSources (sources, recipe); - psFree (sources); // only frees the merged references - } - - // XXX do we need to measure aperture photometry corrections? - // XXX do we store the pre and post correction magnitudes? - // XXX for the fake sources, these must be identically zero - // XXX for the force sources, need the apresid to put the mags on the correct system - if (0) { - if (!psphotApResid (readout, forceSources, recipe, psf)) { - psLogMsg ("psphot", 3, "failed on psphotApResid"); - return psphotReadoutCleanup (config, readout, recipe, NULL, psf, NULL); - } - } - - // calculate source magnitudes (for which set??) - pmReadout *background = psphotSelectBackground (config, view, false); - psphotMagnitudes(config, background, view, fakeSources, psf); - psphotMagnitudes(config, background, view, forceSources, psf); - - // drop the references to the image pixels held by each source - psphotSourceFreePixels (realSources); - psphotSourceFreePixels (fakeSources); - psphotSourceFreePixels (forceSources); - - // create the exported-metadata and free local data - psphotReadoutCleanup(config, readout, recipe, NULL, psf, NULL); - - // add forceSources to the readout (real and fake already there) - // these are outputs: we need to generate the fpa structure - - pmCell *forceCell = pmFPAfileThisCell (config->files, view, "PPSIM.FORCE.SOURCES"); psAssert (forceCell, "no cell?"); - pmChip *forceChip = pmFPAfileThisChip (config->files, view, "PPSIM.FORCE.SOURCES"); psAssert (forceChip, "no chip?"); - pmReadout *forceReadout = pmFPAfileThisReadout (config->files, view, "PPSIM.FORCE.SOURCES"); - if (!forceReadout) { - forceReadout = pmReadoutAlloc (forceCell); - psFree (forceReadout); // there is a copy on 'cell' as well - } - psAssert (forceReadout, "no forceReadout?"); - pmChipSetDataStatus (forceChip, true); - - pmDetections *detectionsForce = pmDetectionsAlloc(); - detectionForce->allSources = forceSources; - - // save detections on the readout->analysis - if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "force sources", detectionsForce)) { - psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); - return false; - } - psFree(detectionsForce); - - pmCell *fakeCell = pmFPAfileThisCell (config->files, view, "PPSIM.FAKE.SOURCES"); psAssert (fakeCell, "no cell?"); - pmChip *fakeChip = pmFPAfileThisChip (config->files, view, "PPSIM.FAKE.SOURCES"); psAssert (fakeChip, "no chip?"); - pmReadout *fakeReadout = pmFPAfileThisReadout (config->files, view, "PPSIM.FAKE.SOURCES"); - if (!fakeReadout) { - fakeReadout = pmReadoutAlloc (fakeCell); - psFree (fakeReadout); // there is a copy on 'cell' as well - } - psAssert (fakeReadout, "no fakeReadout?"); - pmChipSetDataStatus (fakeChip, true); - - pmDetections *detectionsFake = pmDetectionsAlloc(); - detectionFake->allSources = fakeSources; - - // save detections on the readout->analysis - if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "fake sources", detectionsFake)) { - psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); - return false; - } - psFree(detectionsFake); - - return true; -} Index: anches/czw_branch/20101203/ppSim/src/ppSimPhotomReadoutFake.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimPhotomReadoutFake.c (revision 30630) +++ (revision ) @@ -1,121 +1,0 @@ -# include "ppSim.h" - -// XXX this function is currently disabled -- need to update code below to distinguish the fake -// and force positions, etc -bool ppSimPhotomReadoutFake(pmConfig *config, const pmFPAview *view) { - - psTimerStart ("psphotReadout"); - - // select the current recipe - psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); - if (!recipe) { - psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); - return false; - } - - // *** in this section, perform the photometry for real + fake sources on PPSIM.FAKE.CHIP *** - - // find the currently selected readout. - // we always perform photometry on the mosaiced chip - pmReadout *readout = pmFPAfileThisReadout (config->files, view, "PPSIM.FAKE.CHIP"); - PS_ASSERT_PTR_NON_NULL (readout, false); - - // the previously measured real sources are loaded into the PPSIM.REAL.SOURCES pmFPAfile - psArray *realMeasuredSources = ppSimSelectSources (config, view, "PPSIM.REAL.SOURCES"); - PS_ASSERT_PTR_NON_NULL (realMeasuredSources, false); - - // the newly injected fake sources are saved on the PPSIM.OUTPUT pmFPAfile - psArray *injectedSources = ppSimSelectSources (config, view, "PPSIM.OUTPUT"); - PS_ASSERT_PTR_NON_NULL (injectedSources, false); - - // Generate the mask and variance images, including the user-defined analysis region of interest - psphotSetMaskAndVariance (config, view); - - // XXX need to define the source pixels - ppSimDefinePixels (realMeasuredSources, readout, recipe); - ppSimDefinePixels (injectedSources, readout, recipe); - - // make a copy of the injected fake-source parameters so these can be saved independently - // from the measured fake-source parameters - psArray *fakeSources = psArrayAlloc (injectedSources->n); - for (int i = 0; i < injectedSources->n; i++) { - fakeSources->data[i] = pmSourceCopyData (injectedSources->data[i]); - } - - // make a copy of the measured real-source parameters so these can be fitted here and in - // ppSimPhotomReadoutForce, if desired. - psArray *realSources = psArrayAlloc (realMeasuredSources->n); - for (int i = 0; i < realMeasuredSources->n; i++) { - realSources->data[i] = pmSourceCopyData (realMeasuredSources->data[i]); - } - - // load the psf model, if suppled. FWHM_X,FWHM_Y,etc are saved in the recipe - // this function uses PSPHOT.PSF.LOAD as the pmFPAfile - psphotLoadPSF (config, view); - - // remove all sources - psphotRemoveAllSources (realSources, recipe); - - // generate a background model (median, smoothed image) - if (!psphotModelBackgroundReadoutFileIndex (config, view, "PPSIM.FAKE.CHIP", 0)) { - return psphotReadoutCleanup (config, readout, recipe, NULL, NULL, NULL); - } - if (!psphotSubtractBackgroundReadout (config, view, "PPSIM.FAKE.CHIP", 0, recipe)) { - return psphotReadoutCleanup (config, readout, recipe, NULL, NULL, NULL); - } - - // add noise for real (subtracted) objects - psphotAddNoise (readout, realSources, recipe); - - // XXX fake sources should measure peak->x,y, force sources should not - psImageMaskType maskVal = 0xff; - psImage *significance = psphotSignificanceImage (readout, recipe, 1, maskVal); - ppSimDetections (significance, recipe, fakeSources); - psFree (significance); - - // remove noise for subtracted objects (ie, return to normal noise level) - psphotSubNoise (readout, realSources, recipe); - - // replace all sources - psphotReplaceAllSources (realSources, recipe); - - // construct an initial model for each object - psphotGuessModels (config, readout, realSources, psf); - psphotGuessModels (config, readout, fakeSources, psf); - - // linear fit to real + fake sources - psArray *sources = ppSimMergeSources (realSources, fakeSources); - psphotFitSourcesLinear (readout, sources, recipe, psf, FALSE); // XXX option to NOT subtract - psphotReplaceAllSources (sources, recipe); - psFree (sources); // only frees the merged references - - // calculate source magnitudes (for which set??) - psphotMagnitudes(config, readout, view, fakeSources, psf); - - // drop the references to the image pixels held by each source - psphotSourceFreePixels (realSources); - psphotSourceFreePixels (fakeSources); - - // create the exported-metadata and free local data - // XXX this places the sources on readout->analysis as PSPHOT.SOURCES. modify? - // (or don't supply the sources, and do this with a different function) - psphotReadoutCleanup(config, readout, recipe, NULL, psf, NULL); - - pmCell *fakeCell = pmFPAfileThisCell (config->files, view, "PPSIM.FAKE.SOURCES"); psAssert (fakeCell, "no cell?"); - pmChip *fakeChip = pmFPAfileThisChip (config->files, view, "PPSIM.FAKE.SOURCES"); psAssert (fakeChip, "no chip?"); - pmReadout *fakeReadout = pmFPAfileThisReadout (config->files, view, "PPSIM.FAKE.SOURCES"); - if (!fakeReadout) { - fakeReadout = pmReadoutAlloc (fakeCell); - psFree (fakeReadout); // there is a copy on 'cell' as well - } - psAssert (fakeReadout, "no fakeReadout?"); - pmChipSetDataStatus (fakeChip, true); - - pmDetections *detections = pmDetectionsAlloc(); - detection->allSources = fakeSources; - - psMetadataAddArray (fakeReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE, "fake photometry ", detections); - psFree(detections); - - return true; -} Index: anches/czw_branch/20101203/ppSim/src/ppSimPhotomReadoutForce.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimPhotomReadoutForce.c (revision 30630) +++ (revision ) @@ -1,118 +1,0 @@ -# include "ppSim.h" - -bool ppSimPhotomReadoutForce(pmConfig *config, const pmFPAview *view) { - - psTimerStart ("psphotReadout"); - - // select the current recipe - psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); - if (!recipe) { - psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); - return false; - } - -# if 0 - // set the photcode for this image - if (!psphotAddPhotcodeReadout (recipe, config, view, "PPSIM.FAKE.CHIP", 0)) { - psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); - return false; - } -# endif - - // *** in this section, perform the photometry for real + force sources on PPSIM.FORCE.CHIP *** - - // find the currently selected readout. - // we always perform photometry on the mosaiced chip - pmReadout *readout = pmFPAfileThisReadout (config->files, view, "PPSIM.FORCE.CHIP"); - PS_ASSERT_PTR_NON_NULL (readout, false); - - // the previously measured real sources are loaded into the PPSIM.REAL.SOURCES pmFPAfile - psArray *realMeasuredSources = ppSimSelectSources (config, view, "PPSIM.REAL.SOURCES"); - PS_ASSERT_PTR_NON_NULL (realMeasuredSources, false); - - // load the forced source lists - psArray *forceSources = ppSimLoadForceSources (config, view); - psAssert (forceSources, "failed to load force photometry sources"); - - // Generate the mask and variance images, including the user-defined analysis region of interest - psphotSetMaskAndVariance (config, view, recipe); - - // XXX need to define the source pixels - ppSimDefinePixels (realMeasuredSources, readout, recipe); - ppSimDefinePixels (forceSources, readout, recipe); - - // make a copy of the measured real-source parameters so these can be fitted here and in - // ppSimPhotomReadoutForce, if desired. - psArray *realSources = psArrayAlloc (realMeasuredSources->n); - for (int i = 0; i < realMeasuredSources->n; i++) { - realSources->data[i] = pmSourceCopyData (realMeasuredSources->data[i]); - } - - // load the psf model, if suppled. FWHM_X,FWHM_Y,etc are saved in the recipe - // this function uses PSPHOT.PSF.LOAD as the pmFPAfile - pmPSF *psf = psphotLoadPSF (config, view, recipe); - assert (psf); - - // remove all sources - psphotRemoveAllSources (realSources, recipe); - - // generate a background model (median, smoothed image) - if (!psphotModelBackgroundReadoutFileIndex (config, view, "PPSIM.FORCE.CHIP", 0)) { - return psphotReadoutCleanup (config, readout, recipe, NULL, NULL, NULL); - } - if (!psphotSubtractBackgroundReadout (config, view, "PPSIM.FORCE.CHIP", 0)) { - return psphotReadoutCleanup (config, readout, recipe, NULL, NULL, NULL); - } - - // add noise for real (subtracted) objects - psphotAddNoise (readout, realSources, recipe); - - // XXX fake sources should measure peak->x,y, force sources should not - psImageMaskType maskVal = 0xff; - psImage *significance = psphotSignificanceImage (readout, recipe, 1, maskVal); - ppSimDetections (significance, recipe, forceSources); - psFree (significance); - - // remove noise for subtracted objects (ie, return to normal noise level) - psphotSubNoise (readout, realSources, recipe); - - // replace all sources - psphotReplaceAllSources (realSources, recipe); - - // construct an initial model for each object - psphotGuessModels (config, readout, realSources, psf); - psphotGuessModels (config, readout, forceSources, psf); - - // linear fit to real + force sources - psArray *sources = ppSimMergeSources (realSources, forceSources); - psphotFitSourcesLinear (readout, sources, recipe, psf, FALSE); // XXX option to NOT subtract - psphotReplaceAllSources (sources, recipe); - psFree (sources); // only frees the merged references - - // calculate source magnitudes (for which set??) - psphotMagnitudes(config, readout, view, forceSources, psf); - - // drop the references to the image pixels held by each source - psphotSourceFreePixels (realSources); - psphotSourceFreePixels (forceSources); - - // create the exported-metadata and free local data - psphotReadoutCleanup(config, readout, recipe, NULL, psf, NULL); - - pmCell *forceCell = pmFPAfileThisCell (config->files, view, "PPSIM.FORCE.SOURCES"); psAssert (forceCell, "no cell?"); - pmChip *forceChip = pmFPAfileThisChip (config->files, view, "PPSIM.FORCE.SOURCES"); psAssert (forceChip, "no chip?"); - pmReadout *forceReadout = pmFPAfileThisReadout (config->files, view, "PPSIM.FORCE.SOURCES"); - if (!forceReadout) { - forceReadout = pmReadoutAlloc (forceCell); - psFree (forceReadout); // there is a copy on 'cell' as well - } - psAssert (forceReadout, "no forceReadout?"); - pmChipSetDataStatus (forceChip, true); - - pmDetections *detections = pmDetectionsAlloc(); - detection->allSources = forceSources; - psMetadataAddArray (forceReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE, "force photometry ", detections); - psFree(detections); - - return true; -} Index: /branches/czw_branch/20101203/ppSim/src/ppSimSetPSF.c =================================================================== --- /branches/czw_branch/20101203/ppSim/src/ppSimSetPSF.c (revision 30630) +++ /branches/czw_branch/20101203/ppSim/src/ppSimSetPSF.c (revision 30631) @@ -23,4 +23,7 @@ float theta = psMetadataLookupF32(&status, recipe, "PSF.THETA"); // Seeing SIGMA (pixels) + float seeingMax = psMetadataLookupF32(&status, recipe, "SEEING.MAX"); // Seeing SIGMA (pixels) + bool seeingRamp = psMetadataLookupBool(&status, recipe, "SEEING.RAMP"); // Seeing SIGMA (pixels) + char *psfModelName = psMetadataLookupStr(&status, recipe, "PSF.MODEL"); // Name of PSF model if (psfModelName == NULL) { @@ -39,44 +42,99 @@ int xSize = psMetadataLookupS32(NULL, chip->concepts, "CHIP.XSIZE"); int ySize = psMetadataLookupS32(NULL, chip->concepts, "CHIP.YSIZE"); - xSize = 1000; - ySize = 1000; - // no spatial variation - options->psfTrendMode = PM_TREND_POLY_ORD; - options->psfTrendNx = 0; - options->psfTrendNy = 0; - options->psfFieldNx = xSize; - options->psfFieldNy = ySize; + if (seeingRamp) { + psEllipseAxes axes; + psEllipsePol pol; + psEllipsePol polMax; - // generate the psf - psf = pmPSFAlloc (options); + // try spatial variation + options->psfTrendMode = PM_TREND_POLY_ORD; + options->psfTrendNx = 1; + options->psfTrendNy = 1; + options->psfFieldNx = xSize; + options->psfFieldNy = ySize; - psEllipseAxes axes; - psEllipsePol pol; + // generate the psf + psf = pmPSFAlloc (options); - // supply the semi-major axis (these are SIGMA values in PIXELS) - axes.major = seeing; - axes.minor = aRatio * seeing; - axes.theta = theta * PS_RAD_DEG; + // supply the semi-major axis (these are SIGMA values in PIXELS) + axes.major = seeing; + axes.minor = aRatio * seeing; + axes.theta = theta * PS_RAD_DEG; - pol = psEllipseAxesToPol (axes); + pol = psEllipseAxesToPol (axes); - param = psf->params->data[PM_PAR_E0]; - param->poly->coeff[0][0] = pol.e0; + axes.major = seeingMax; + axes.minor = aRatio * seeingMax; + polMax = psEllipseAxesToPol (axes); - param = psf->params->data[PM_PAR_E1]; - param->poly->coeff[0][0] = pol.e1; + param = psf->params->data[PM_PAR_E0]; + param->poly->coeff[0][0] = pol.e0; + param->poly->coeff[1][0] = (polMax.e0 - pol.e0) / xSize; + param->poly->coeff[0][1] = (polMax.e0 - pol.e0) / ySize; - param = psf->params->data[PM_PAR_E2]; - param->poly->coeff[0][0] = pol.e2; + param = psf->params->data[PM_PAR_E1]; + param->poly->coeff[0][0] = pol.e1; + param->poly->coeff[1][0] = 0.0; + param->poly->coeff[0][1] = 0.0; - if (!strcasecmp (psfModelName, "PS_MODEL_QGAUSS")) { - param = psf->params->data[PM_PAR_7]; - param->poly->coeff[0][0] = 1.0; - } + param = psf->params->data[PM_PAR_E2]; + param->poly->coeff[0][0] = pol.e2; + param->poly->coeff[1][0] = 0.0; + param->poly->coeff[0][1] = 0.0; - if (!strcasecmp (psfModelName, "PS_MODEL_RGAUSS")) { - param = psf->params->data[PM_PAR_7]; - param->poly->coeff[0][0] = 1.0; + if (!strcasecmp (psfModelName, "PS_MODEL_QGAUSS")) { + param = psf->params->data[PM_PAR_7]; + param->poly->coeff[0][0] = 1.0; + param->poly->coeff[1][0] = 0.0; + param->poly->coeff[0][1] = 0.0; + } + + if (!strcasecmp (psfModelName, "PS_MODEL_RGAUSS")) { + param = psf->params->data[PM_PAR_7]; + param->poly->coeff[0][0] = 1.0; + param->poly->coeff[1][0] = 0.0; + param->poly->coeff[0][1] = 0.0; + } + + } else { + psEllipseAxes axes; + psEllipsePol pol; + + // no spatial variation + options->psfTrendMode = PM_TREND_POLY_ORD; + options->psfTrendNx = 0; + options->psfTrendNy = 0; + options->psfFieldNx = xSize; + options->psfFieldNy = ySize; + + // generate the psf + psf = pmPSFAlloc (options); + + // supply the semi-major axis (these are SIGMA values in PIXELS) + axes.major = seeing; + axes.minor = aRatio * seeing; + axes.theta = theta * PS_RAD_DEG; + + pol = psEllipseAxesToPol (axes); + + param = psf->params->data[PM_PAR_E0]; + param->poly->coeff[0][0] = pol.e0; + + param = psf->params->data[PM_PAR_E1]; + param->poly->coeff[0][0] = pol.e1; + + param = psf->params->data[PM_PAR_E2]; + param->poly->coeff[0][0] = pol.e2; + + if (!strcasecmp (psfModelName, "PS_MODEL_QGAUSS")) { + param = psf->params->data[PM_PAR_7]; + param->poly->coeff[0][0] = 1.0; + } + + if (!strcasecmp (psfModelName, "PS_MODEL_RGAUSS")) { + param = psf->params->data[PM_PAR_7]; + param->poly->coeff[0][0] = 1.0; + } } Index: /branches/czw_branch/20101203/ppStack/src/Makefile.am =================================================================== --- /branches/czw_branch/20101203/ppStack/src/Makefile.am (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/Makefile.am (revision 30631) @@ -49,4 +49,7 @@ ppStackFinish.c \ ppStackTarget.c \ + ppStackUpdateHeader.c \ + ppStackJPEGs.c \ + ppStackStats.c \ ppStackErrorCodes.c Index: /branches/czw_branch/20101203/ppStack/src/ppStack.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStack.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStack.c (revision 30631) @@ -1,20 +1,8 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" - -#define TIMER_NAME "PPSTACK" // Name of timer int main(int argc, char *argv[]) { psLibInit(NULL); - psTimerStart(TIMER_NAME); + psTimerStart("PPSTACK"); psTimerStart("PPSTACK_STEPS"); @@ -23,8 +11,9 @@ psphotErrorRegister(); + ppStackOptions *options = NULL; // Options for stacking + pmConfig *config = pmConfigRead(&argc, argv, PPSTACK_RECIPE); // Configuration - ppStackOptions *options = NULL; // Options for stacking if (!config) { - goto die; + ppStackCleanup(config, options); } @@ -33,100 +22,30 @@ if (!pmModelClassInit()) { psError(PPSTACK_ERR_PROG, false, "Unable to initialise model classes."); - goto die; + ppStackCleanup(config, options); } if (!psphotInit()) { psError(PPSTACK_ERR_PROG, false, "Error initialising psphot."); - goto die; + ppStackCleanup(config, options); } - (void)psTraceSetLevel("ppStack", 5); - if (!ppStackArgumentsSetup(argc, argv, config)) { - goto die; + ppStackCleanup(config, options); } if (!ppStackCamera(config)) { - goto die; + ppStackCleanup(config, options); } if (!ppStackArgumentsParse(config)) { - goto die; + ppStackCleanup(config, options); } options = ppStackOptionsAlloc(); if (!ppStackLoop(config, options)) { - goto die; + ppStackCleanup(config, options); } - - die: - // Common code for the death. - { - psExit exitValue = ppStackExitCode(PS_EXIT_SUCCESS); // Exit code - - // Ensure everything closes - if (config) { - ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true); - ppStackFileActivation(config, PPSTACK_FILES_CONVOLVE, true); - ppStackFileActivation(config, PPSTACK_FILES_STACK, true); - ppStackFileActivation(config, PPSTACK_FILES_UNCONV, true); - ppStackFileActivation(config, PPSTACK_FILES_PHOT, true); - if (!ppStackFilesIterateUp(config)) { - psError(psErrorCodeLast(), false, "Unable to close files."); - exitValue = ppStackExitCode(exitValue); - pmFPAfileFreeSetStrict(false); - } - } - - // Write out summary statistics - if (options && options->stats) { - - psMetadataAddS32(options->stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, - "Bad data quality flag", options->quality); - psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_STACK", 0, - "Total time", psTimerClear("PPSTACK_TOTAL")); - - const char *statsMDC = psMetadataConfigFormat(options->stats); - if (!statsMDC || strlen(statsMDC) == 0) { - psError(PS_ERR_IO, false, "Unable to get statistics MDC file."); - return false; - } - if (fprintf(options->statsFile, "%s", statsMDC) != strlen(statsMDC)) { - psError(PS_ERR_IO, false, "Unable to write statistics MDC file."); - return false; - } - psFree(statsMDC); - if (fclose(options->statsFile) == EOF) { - psError(PS_ERR_IO, false, "Unable to close statistics MDC file."); - return false; - } - options->statsFile = NULL; - pmConfigRunFilenameAddWrite(config, "STATS", - psMetadataLookupStr(NULL, config->arguments, "STATS")); - } - psFree(options); - - // Dump configuration - bool mdok; // Status of MD lookup - psString dump = psMetadataLookupStr(&mdok, config->arguments, "DUMP_CONFIG"); // File for config - if (dump && !pmConfigDump(config, dump)) { - psError(psErrorCodeLast(), false, "Unable to dump configuration."); - exitValue = ppStackExitCode(exitValue); - } - - psTrace("ppStack", 1, "Finished at %f sec\n", psTimerMark(TIMER_NAME)); - psTimerStop(); - - psFree(config); - pmModelClassCleanup(); - pmConfigDone(); - psLibFinalize(); - pmVisualClose(); - pmVisualCleanup (); - - exitValue = ppStackExitCode(exitValue); - exit(exitValue); - } + ppStackCleanup(config, options); } Index: /branches/czw_branch/20101203/ppStack/src/ppStack.h =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStack.h (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStack.h (revision 30631) @@ -1,2 +1,12 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + #ifndef PPSTACK_H #define PPSTACK_H @@ -7,7 +17,4 @@ #include #include - -#include "ppStackOptions.h" -#include "ppStackErrorCodes.h" // Mask values for inputs @@ -25,4 +32,5 @@ // List of files typedef enum { + PPSTACK_FILES_NONE, // NOP list PPSTACK_FILES_PREPARE, // Files for preparation PPSTACK_FILES_TARGET, // Files for target generation @@ -33,4 +41,8 @@ } ppStackFileList; +#include "ppStackOptions.h" +#include "ppStackThread.h" +#include "ppStackLoop.h" +#include "ppStackErrorCodes.h" // Setup command-line arguments @@ -180,6 +192,15 @@ ); +bool ppStackWriteVariance(const char *name, // Name of image + psMetadata *header, // Header + const psImage *variance, // Variance + const psImage *covariance, // Variance + pmConfig *config // Configuration + ); + /// Return an appropriate exit code based on the error code psExit ppStackExitCode(psExit exitValue); +bool ppStackCleanup(pmConfig *config, ppStackOptions *options) PS_ATTR_NORETURN; + #endif Index: /branches/czw_branch/20101203/ppStack/src/ppStackArguments.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackArguments.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackArguments.c (revision 30631) @@ -1,11 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" @@ -189,11 +180,31 @@ psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-mask", 0, "Suffix for temporary masks", NULL); psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-variance", 0, "Suffix for temporary variance maps", NULL); - psMetadataAddBool(arguments, PS_LIST_TAIL, "-temp-delete", 0, - "Delete temporary files on completion?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-temp-delete", 0, "Delete temporary files on completion?", false); psMetadataAddS32(arguments, PS_LIST_TAIL, "-threads", 0, "Number of threads to use", 0); psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "visualisation", false); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stack_id", 0, "stack ID", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-tess_id", 0, "tessellation ID", NULL); + if (argc == 1) { usage(argv[0], arguments, config); + } + + // stack-type : used to define the stack for PSPS + if ((argNum = psArgumentGet (argc, argv, "-stack-type"))) { + if (argc <= argNum+1) { + psErrorStackPrint(stderr, "Expected to see an argument for -stack-type"); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (argNum, &argc, argv); + if (strcasecmp(argv[argNum], "NIGHTLY") && strcasecmp(argv[argNum], "DEEP") && strcasecmp(argv[argNum], "BEST_IQ")) { + psErrorStackPrint(stderr, "Invalid option for -stack-type %s (must be one of NIGHTLY_STACK, DEEP_STACK, IQ_STACK)", argv[argNum]); + exit(PS_EXIT_CONFIG_ERROR); + } + psMetadataAddStr (arguments, PS_LIST_TAIL, "STACK_TYPE", PS_META_REPLACE, "Stack Type", argv[argNum]); + psArgumentRemove (argNum, &argc, argv); + } else { + psMetadataAddStr (arguments, PS_LIST_TAIL, "STACK_TYPE", PS_META_REPLACE, "Stack Type", "DEEP_STACK"); } Index: /branches/czw_branch/20101203/ppStack/src/ppStackCamera.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackCamera.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackCamera.c (revision 30631) @@ -1,12 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include -#include - #include "ppStack.h" Index: /branches/czw_branch/20101203/ppStack/src/ppStackCleanup.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackCleanup.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackCleanup.c (revision 30631) @@ -1,16 +1,6 @@ -#ifdef HAVE_CONFIG_H -#include -#endif +#include "ppStack.h" -#include -#include -#include -#include - -#include "ppStack.h" -#include "ppStackLoop.h" - - -bool ppStackCleanup(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +// ppStackCleanupFiles +bool ppStackCleanupFiles(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config, ppStackFileList stackFiles, ppStackFileList photFiles, bool closeJPEGs) { psAssert(stack, "Require stack"); @@ -18,78 +8,30 @@ psAssert(config, "Require configuration"); - psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe - psAssert(recipe, "We've thrown an error on this before."); - -#if 0 - // Ensure masked regions really look masked - { - psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad - psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels - if (!pmReadoutMaskApply(options->outRO, maskBad)) { - psWarning("Unable to apply mask"); - } - } -#endif - - // Generate binned JPEGs - { - int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level - int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level - - // Target cells - pmFPAview *view = pmFPAviewAlloc(0); // View to cells of interest - view->chip = view->cell = 0; - pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG1"); - pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG2"); - psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask - psFree(view); - - pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts - if (!pmReadoutRebin(ro1, options->outRO, maskValue, bin1, bin1) || - !pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) { - psError(PPSTACK_ERR_DATA, false, "Unable to bin output."); - psFree(ro1); - psFree(ro2); - return false; - } - psFree(ro1); - psFree(ro2); - } - - // Statistics on output - if (options->stats) { - psTrace("ppStack", 1, "Gathering statistics on stacked image....\n"); - psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad - psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels - - pmFPAview *view = pmFPAviewAlloc(0); // View to readout - view->chip = view->cell = view->readout = 0; - - ppStatsFPA(options->stats, options->outRO->parent->parent->parent, view, maskBad, config); - - psFree(view); - } - if (!ppStackFilesIterateUp(config)) { psError(psErrorCodeLast(), false, "Unable to close files."); return false; } - ppStackFileActivation(config, PPSTACK_FILES_STACK, false); - ppStackFileActivation(config, PPSTACK_FILES_PHOT, false); + ppStackFileActivation(config, stackFiles, false); + ppStackFileActivation(config, photFiles, false); // Ensure files are freed - { - options->outRO->data_exists = false; - options->outRO->parent->data_exists = false; - options->outRO->parent->parent->data_exists = false; - psFree(options->outRO); - options->outRO = NULL; + options->outRO->data_exists = false; + options->outRO->parent->data_exists = false; + options->outRO->parent->parent->data_exists = false; + psFree(options->outRO); + options->outRO = NULL; - options->expRO->data_exists = false; - options->expRO->parent->data_exists = false; - options->expRO->parent->parent->data_exists = false; - psFree(options->expRO); - options->expRO = NULL; + options->expRO->data_exists = false; + options->expRO->parent->data_exists = false; + options->expRO->parent->parent->data_exists = false; + psFree(options->expRO); + options->expRO = NULL; + for (int i = 0; i < options->num; i++) { + pmCellFreeData(options->cells->data[i]); + } + + if (closeJPEGs) { + // XXX move these close / free operations to the jpeg creation function pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy view->chip = view->cell = 0; // pmFPAviewFreeData doesn't want to deal with readouts @@ -118,2 +60,70 @@ return true; } + +bool ppStackCleanup (pmConfig *config, ppStackOptions *options) { + + psExit exitValue = ppStackExitCode(PS_EXIT_SUCCESS); // Exit code + + // Ensure everything closes + if (config) { + ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true); + ppStackFileActivation(config, PPSTACK_FILES_CONVOLVE, true); + ppStackFileActivation(config, PPSTACK_FILES_STACK, true); + ppStackFileActivation(config, PPSTACK_FILES_UNCONV, true); + ppStackFileActivation(config, PPSTACK_FILES_PHOT, true); + if (!ppStackFilesIterateUp(config)) { + psError(psErrorCodeLast(), false, "Unable to close files."); + exitValue = ppStackExitCode(exitValue); + pmFPAfileFreeSetStrict(false); + } + } + + // Write out summary statistics + if (options && options->stats) { + + psMetadataAddS32(options->stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "Bad data quality flag", options->quality); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_STACK", 0, "Total time", psTimerClear("PPSTACK_TOTAL")); + + const char *statsMDC = psMetadataConfigFormat(options->stats); + if (!statsMDC || strlen(statsMDC) == 0) { + psError(PS_ERR_IO, false, "Unable to get statistics MDC file."); + exitValue = ppStackExitCode(exitValue); + exit(exitValue); + } + if (fprintf(options->statsFile, "%s", statsMDC) != strlen(statsMDC)) { + psError(PS_ERR_IO, false, "Unable to write statistics MDC file."); + exitValue = ppStackExitCode(exitValue); + exit(exitValue); + } + psFree(statsMDC); + if (fclose(options->statsFile) == EOF) { + psError(PS_ERR_IO, false, "Unable to close statistics MDC file."); + exitValue = ppStackExitCode(exitValue); + exit(exitValue); + } + options->statsFile = NULL; + pmConfigRunFilenameAddWrite(config, "STATS", psMetadataLookupStr(NULL, config->arguments, "STATS")); + } + psFree(options); + + // Dump configuration + bool mdok; // Status of MD lookup + psString dump = psMetadataLookupStr(&mdok, config->arguments, "DUMP_CONFIG"); // File for config + if (dump && !pmConfigDump(config, dump)) { + psError(psErrorCodeLast(), false, "Unable to dump configuration."); + exitValue = ppStackExitCode(exitValue); + } + + psTrace("ppStack", 1, "Finished at %f sec\n", psTimerMark("PPSTACK")); + psTimerStop(); + + psFree(config); + pmModelClassCleanup(); + pmConfigDone(); + psLibFinalize(); + pmVisualClose(); + pmVisualCleanup (); + + exitValue = ppStackExitCode(exitValue); + exit(exitValue); +} Index: /branches/czw_branch/20101203/ppStack/src/ppStackCombineFinal.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackCombineFinal.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackCombineFinal.c (revision 30631) @@ -1,14 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" - -#define WCS_TOLERANCE 0.001 // Tolerance for WCS //#define TESTING // Enable test output @@ -107,53 +96,4 @@ } - // Propagate WCS - bool wcsDone = false; // Have we done the WCS? - for (int i = 0; i < options->num && !wcsDone; i++) { - if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { - continue; - } - - ppStackThread *thread = stack->threads->data[0]; // Representative stack - pmReadout *inRO = thread->readouts->data[i]; // Template readout - if (inRO && !wcsDone) { - // Copy astrometry over - wcsDone = true; - pmHDU *inHDU = pmHDUFromCell(inRO->parent); // Template HDU - pmHDU *outHDU = pmHDUFromCell(outRO->parent); // Output HDU - pmChip *outChip = outRO->parent->parent; // Output chip - pmFPA *outFPA = outChip->parent; // Output FPA - if (!outHDU || !inHDU) { - psWarning("Unable to find HDU at FPA level to copy astrometry."); - } else { - if (!pmAstromReadWCS(outFPA, outChip, inHDU->header, 1.0)) { - psErrorClear(); - psWarning("Unable to read WCS astrometry from input FPA."); - wcsDone = false; - } else { - if (!outHDU->header) { - outHDU->header = psMetadataAlloc(); - } - if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_TOLERANCE)) { - psErrorClear(); - psWarning("Unable to write WCS astrometry to output FPA."); - wcsDone = false; - } - } - } - } - } - - // Put version information into the header - pmHDU *hdu = pmHDUFromCell(outRO->parent); - if (!hdu) { - psError(PPSTACK_ERR_PROG, false, "Unable to find HDU for output."); - return false; - } - if (!hdu->header) { - hdu->header = psMetadataAlloc(); - } - ppStackVersionHeader(hdu->header); - - #ifdef TESTING static int pass = 0; // Pass through Index: /branches/czw_branch/20101203/ppStack/src/ppStackCombineInitial.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackCombineInitial.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackCombineInitial.c (revision 30631) @@ -1,12 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" //#define TESTING // Enable test output Index: /branches/czw_branch/20101203/ppStack/src/ppStackCombinePrepare.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackCombinePrepare.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackCombinePrepare.c (revision 30631) @@ -1,12 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" bool ppStackCombinePrepare(const char *outName, const char *expName, Index: /branches/czw_branch/20101203/ppStack/src/ppStackConvolve.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackConvolve.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackConvolve.c (revision 30631) @@ -1,15 +1,5 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" //#define TESTING - // Update the value of a concept @@ -20,6 +10,4 @@ item->data.F32 = VALUE; \ } - - bool ppStackConvolve(ppStackOptions *options, pmConfig *config) @@ -69,6 +57,5 @@ ppStackFileActivationSingle(config, PPSTACK_FILES_CONVOLVE, true, i); if (options->convolve) { - // XXX PPSTACK.CONV.KERNEL not defined unless convolve - // pmFPAfileActivate(config->files, true, "PPSTACK.CONV.KERNEL"); + // PPSTACK.CONV.KERNEL not defined unless convolve pmFPAfileActivateSingle(config->files, true, "PPSTACK.CONV.KERNEL", i); // Activated file } @@ -191,5 +178,5 @@ } psFree(maskHeader); - if (!ppStackWriteImage(options->convVariances->data[i], hdu->header, readout->variance, config)) { + if (!ppStackWriteVariance(options->convVariances->data[i], hdu->header, readout->variance, readout->covariance->image, config)) { psError(PPSTACK_ERR_IO, false, "Unable to write convolved variance %d", i); psFree(fpaList); Index: /branches/czw_branch/20101203/ppStack/src/ppStackErrorCodes.c.in =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackErrorCodes.c.in (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackErrorCodes.c.in (revision 30631) @@ -1,2 +1,4 @@ +#include "ppStack.h" + /** @file ppStackErrorCodes.c.in * @@ -10,7 +12,4 @@ * Copyright 2009 Institute for Astronomy, University of Hawaii */ - -#include "pslib.h" -#include "ppStackErrorCodes.h" /* Index: /branches/czw_branch/20101203/ppStack/src/ppStackFiles.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackFiles.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackFiles.c (revision 30631) @@ -1,15 +1,8 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" - // Here follows lists of files for activation/deactivation at various stages. Each must be NULL-terminated. + +/// NOP list +static char *filesNOP[] = { NULL }; /// Files required in preparation for convolution @@ -41,4 +34,5 @@ { switch (list) { + case PPSTACK_FILES_NONE: return filesNOP; case PPSTACK_FILES_PREPARE: return filesPrepare; case PPSTACK_FILES_TARGET: return filesTarget; @@ -206,2 +200,46 @@ return true; } + +// Write an image to a FITS file +bool ppStackWriteVariance(const char *name, // Name of image + psMetadata *header, // Header + const psImage *variance, // Variance + const psImage *covariance, // Variance + pmConfig *config // Configuration + ) +{ + assert(name); + assert(variance); + + psString resolved = pmConfigConvertFilename(name, config, true, true); // Resolved file name + psFits *fits = psFitsOpen(resolved, "w"); + if (!fits) { + psError(PPSTACK_ERR_IO, false, "Unable to open FITS file %s to write image.", resolved); + psFree(resolved); + return false; + } + if (!psFitsWriteImage(fits, header, variance, 0, NULL)) { + psError(PPSTACK_ERR_IO, false, "Unable to write FITS image %s.", resolved); + psFitsClose(fits); + psFree(resolved); + return false; + } + if (covariance) { + psMetadata *tmphead = psMetadataAlloc(); + psMetadataAddS32(tmphead, PS_LIST_TAIL, "COVARIANCE.CENTRE.X", PS_META_REPLACE, "center", (int)(covariance->numCols / 2)); + psMetadataAddS32(tmphead, PS_LIST_TAIL, "COVARIANCE.CENTRE.Y", PS_META_REPLACE, "center", (int)(covariance->numRows / 2)); + if (!psFitsWriteImage(fits, tmphead, covariance, 0, "COVAR_SkyChip_SkyCell")) { + psError(PPSTACK_ERR_IO, false, "Unable to write FITS image %s.", resolved); + psFitsClose(fits); + psFree(resolved); + return false; + } + } + if (!psFitsClose(fits)) { + psError(PPSTACK_ERR_IO, false, "Unable to close FITS image %s.", resolved); + psFree(resolved); + return false; + } + psFree(resolved); + return true; +} Index: /branches/czw_branch/20101203/ppStack/src/ppStackFinish.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackFinish.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackFinish.c (revision 30631) @@ -1,15 +1,5 @@ -#ifdef HAVE_CONFIG_H -#include -#endif +#include "ppStack.h" -#include -#include -#include -#include -#include - -#include "ppStack.h" -#include "ppStackLoop.h" - +// this function unlinks the temporary files (if desired) bool ppStackFinish(ppStackOptions *options, pmConfig *config) { @@ -32,10 +22,8 @@ } - psString imageResolved = pmConfigConvertFilename(options->convImages->data[i], - config, false, false); - psString maskResolved = pmConfigConvertFilename(options->convMasks->data[i], - config, false, false); - psString varianceResolved = pmConfigConvertFilename(options->convVariances->data[i], - config, false, false); + // XXX careful about repeatative resolution of nebulous names (though these are probably not neb names) + psString imageResolved = pmConfigConvertFilename(options->convImages->data[i], config, false, false); + psString maskResolved = pmConfigConvertFilename(options->convMasks->data[i], config, false, false); + psString varianceResolved = pmConfigConvertFilename(options->convVariances->data[i], config, false, false); if (unlink(imageResolved) == -1 || unlink(maskResolved) == -1 || unlink(varianceResolved) == -1) { @@ -47,6 +35,4 @@ } } - - return true; } @@ -60,54 +46,56 @@ psErrorCode errorCode = psErrorCodeLast(); // Error code - if (errorCode != PS_ERR_NONE) { - psErrorStackPrint(stderr, "Unable to perform stack."); - pmFPAfileFreeSetStrict(false); - switch (errorCode) { - case PPSTACK_ERR_UNKNOWN: - case PS_ERR_UNKNOWN: - psLogMsg("ppStack", PS_LOG_WARN, "Unknown error code: %x", errorCode); - exitValue = PS_EXIT_UNKNOWN_ERROR; - break; - case PS_ERR_IO: - case PS_ERR_DB_CLIENT: - case PS_ERR_DB_SERVER: - case PS_ERR_BAD_FITS: - case PS_ERR_OS_CALL_FAILED: - case PM_ERR_SYS: - case PPSTACK_ERR_IO: - psLogMsg("ppStack", PS_LOG_WARN, "I/O error code: %x", errorCode); - exitValue = PS_EXIT_SYS_ERROR; - break; - case PS_ERR_BAD_PARAMETER_VALUE: - case PS_ERR_BAD_PARAMETER_TYPE: - case PS_ERR_BAD_PARAMETER_NULL: - case PS_ERR_BAD_PARAMETER_SIZE: - case PPSTACK_ERR_ARGUMENTS: - case PPSTACK_ERR_CONFIG: - psLogMsg("ppStack", PS_LOG_WARN, "Configuration error code: %x", errorCode); - exitValue = PS_EXIT_CONFIG_ERROR; - break; - case PPSTACK_ERR_PSF: - case PSPHOT_ERR_PSF: - case PM_ERR_STAMPS: - case PM_ERR_SMALL_AREA: - case PPSTACK_ERR_REJECTED: - case PPSTACK_ERR_DATA: - psLogMsg("ppStack", PS_LOG_WARN, "Data error code: %x", errorCode); - exitValue = PS_EXIT_DATA_ERROR; - break; - case PS_ERR_UNEXPECTED_NULL: - case PS_ERR_PROGRAMMING: - case PPSTACK_ERR_NOT_IMPLEMENTED: - case PPSTACK_ERR_PROG: - psLogMsg("ppStack", PS_LOG_WARN, "Programming error code: %x", errorCode); - exitValue = PS_EXIT_PROG_ERROR; - break; - default: - // It's a programming error if we're not dealing with the error correctly - psLogMsg("ppStack", PS_LOG_WARN, "Unrecognised error code: %x", errorCode); - exitValue = PS_EXIT_PROG_ERROR; - break; - } + if (errorCode == PS_ERR_NONE) { + return exitValue; + } + + psErrorStackPrint(stderr, "Unable to perform stack."); + pmFPAfileFreeSetStrict(false); + switch (errorCode) { + case PPSTACK_ERR_UNKNOWN: + case PS_ERR_UNKNOWN: + psLogMsg("ppStack", PS_LOG_WARN, "Unknown error code: %x", errorCode); + exitValue = PS_EXIT_UNKNOWN_ERROR; + break; + case PS_ERR_IO: + case PS_ERR_DB_CLIENT: + case PS_ERR_DB_SERVER: + case PS_ERR_BAD_FITS: + case PS_ERR_OS_CALL_FAILED: + case PM_ERR_SYS: + case PPSTACK_ERR_IO: + psLogMsg("ppStack", PS_LOG_WARN, "I/O error code: %x", errorCode); + exitValue = PS_EXIT_SYS_ERROR; + break; + case PS_ERR_BAD_PARAMETER_VALUE: + case PS_ERR_BAD_PARAMETER_TYPE: + case PS_ERR_BAD_PARAMETER_NULL: + case PS_ERR_BAD_PARAMETER_SIZE: + case PPSTACK_ERR_ARGUMENTS: + case PPSTACK_ERR_CONFIG: + psLogMsg("ppStack", PS_LOG_WARN, "Configuration error code: %x", errorCode); + exitValue = PS_EXIT_CONFIG_ERROR; + break; + case PPSTACK_ERR_PSF: + case PSPHOT_ERR_PSF: + case PM_ERR_STAMPS: + case PM_ERR_SMALL_AREA: + case PPSTACK_ERR_REJECTED: + case PPSTACK_ERR_DATA: + psLogMsg("ppStack", PS_LOG_WARN, "Data error code: %x", errorCode); + exitValue = PS_EXIT_DATA_ERROR; + break; + case PS_ERR_UNEXPECTED_NULL: + case PS_ERR_PROGRAMMING: + case PPSTACK_ERR_NOT_IMPLEMENTED: + case PPSTACK_ERR_PROG: + psLogMsg("ppStack", PS_LOG_WARN, "Programming error code: %x", errorCode); + exitValue = PS_EXIT_PROG_ERROR; + break; + default: + // It's a programming error if we're not dealing with the error correctly + psLogMsg("ppStack", PS_LOG_WARN, "Unrecognised error code: %x", errorCode); + exitValue = PS_EXIT_PROG_ERROR; + break; } return exitValue; Index: /branches/czw_branch/20101203/ppStack/src/ppStackJPEGs.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackJPEGs.c (revision 30631) +++ /branches/czw_branch/20101203/ppStack/src/ppStackJPEGs.c (revision 30631) @@ -0,0 +1,36 @@ +#include "ppStack.h" + +bool ppStackJPEGs(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + // Generate binned JPEGs + int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level + int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level + + // Target cells + pmFPAview *view = pmFPAviewAlloc(0); // View to cells of interest + view->chip = view->cell = 0; + pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG1"); + pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG2"); + psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask + psFree(view); + + pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts + if (!pmReadoutRebin(ro1, options->outRO, maskValue, bin1, bin1) || + !pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) { + psError(PPSTACK_ERR_DATA, false, "Unable to bin output."); + psFree(ro1); + psFree(ro2); + return false; + } + psFree(ro1); + psFree(ro2); + + return true; +} Index: /branches/czw_branch/20101203/ppStack/src/ppStackLoop.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackLoop.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackLoop.c (revision 30631) @@ -1,17 +1,219 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" + +// static functions are defined below +static int stackSummary(const ppStackOptions *options, const char *place); + +bool ppStackLoop(pmConfig *config, ppStackOptions *options) +{ + assert(config); + + psTimerStart("PPSTACK_TOTAL"); + psTimerStart("PPSTACK_STEPS"); + + // Setup + psTrace("ppStack", 1, "Setup....\n"); + if (!ppStackSetup(options, config)) { + psError(psErrorCodeLast(), false, "Unable to setup."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 0: Setup: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("setup"); + + // Preparation for stacking + psTrace("ppStack", 1, "Preparation for stacking: merging sources, determining target PSF....\n"); + + if (!ppStackPrepare(options, config)) { + psError(psErrorCodeLast(), false, "Unable to prepare for stacking."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 1: Load Sources and Generate Target PSF: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("prepare"); + if (options->quality) return true; // Can't do anything else + + // Convolve inputs + psTrace("ppStack", 1, "Convolving inputs to target PSF....\n"); + if (!ppStackConvolve(options, config)) { + psError(psErrorCodeLast(), false, "Unable to convolve images."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 2: Generate Convolutions and Save: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("convolve"); + if (options->quality) return true; // Can't do anything else + + // Ensure sufficient inputs + { + int numGood = stackSummary(options, "initial combination"); + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + bool safe = psMetadataLookupBool(NULL, recipe, "SAFE"); // Be safe when combining + if (safe && numGood <= 1) { + options->quality = PPSTACK_ERR_REJECTED; + psErrorStackPrint(stderr, "Insufficient inputs for combination with safety on"); + psErrorClear(); + psWarning("Insufficient inputs for combination with safety on"); + return true; + } + } + + // Start threading + ppStackThreadInit(); + ppStackThreadData *stack = ppStackThreadDataSetup(options, config, true); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // Prepare for combination + if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", PPSTACK_FILES_STACK, stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to prepare for combination."); + psFree(stack); + return false; + } + + // Initial combination + psTrace("ppStack", 1, "Initial stack of convolved images....\n"); + if (!ppStackCombineInitial(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to perform initial combination."); + psFree(stack); + return false; + } + ppStackMemDump("initial"); + psLogMsg("ppStack", PS_LOG_INFO, "Stage 3: Make Initial Stack: %f sec", psTimerClear("PPSTACK_STEPS")); + + // Done with stack inputs for now + // XXX is this where we are leaking?? + for (int i = 0; i < options->num; i++) { + pmCellFreeData(options->cells->data[i]); + } + psFree(stack); + + // Pixel rejection + psTrace("ppStack", 1, "Reject pixels....\n"); + if (!ppStackReject(options, config)) { + psError(psErrorCodeLast(), false, "Unable to reject pixels."); + psFree(stack); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 4: Pixel Rejection: %f sec", psTimerClear("PPSTACK_STEPS")); + + // Check inputs + { + int numGood = stackSummary(options, "final combination"); + if (numGood <= 0) { + options->quality = PPSTACK_ERR_REJECTED; + psErrorStackPrint(stderr, "Insufficient inputs survived rejection stage"); + psErrorClear(); + psWarning("Insufficient inputs survived rejection stage"); + return true; + } + } + + stack = ppStackThreadDataSetup(options, config, true); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // Final combination + psTrace("ppStack", 2, "Final stack of convolved images....\n"); + if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, false, true)) { + psError(psErrorCodeLast(), false, "Unable to perform final combination."); + psFree(stack); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 5: Final Stack: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("final"); + + // Photometry + psTrace("ppStack", 1, "Photometering stacked image....\n"); + if (!ppStackPhotometry(options, config)) { + psError(psErrorCodeLast(), false, "Unable to perform photometry."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("photometry"); + + // Update Header + if (!ppStackUpdateHeader(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to update header."); + psFree(stack); + return false; + } + // Create JPEGS + if (!ppStackJPEGs(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to make jpegs."); + psFree(stack); + return false; + } + // Assemble Stats + if (!ppStackStats(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to assemble statistics."); + psFree(stack); + return false; + } + + // Clean up + psTrace("ppStack", 2, "Cleaning up after combination....\n"); + if (!ppStackCleanupFiles(stack, options, config, PPSTACK_FILES_STACK, PPSTACK_FILES_PHOT, true)) { + psError(psErrorCodeLast(), false, "Unable to clean up."); + psFree(stack); + return false; + } + psFree(stack); + psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: Cleanup, WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("cleanup"); + + // Unconvolved stack --- it's cheap to calculate, compared to everything else! + // XXX unconvolved stack is currently using the convolved mask! oops! + if (options->convolve) { + // Start threading + ppStackThreadData *stack = ppStackThreadDataSetup(options, config, false); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // Prepare for combination + if (!ppStackCombinePrepare("PPSTACK.UNCONV", "PPSTACK.UNCONV.EXP", PPSTACK_FILES_UNCONV, + stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to prepare for combination."); + psFree(stack); + return false; + } + + // generate the unconvolved stack + psTrace("ppStack", 2, "Stack of unconvolved images....\n"); + if (!ppStackCombineFinal(stack, options->origCovars, options, config, + false, true, false)) { + psError(psErrorCodeLast(), false, "Unable to perform unconvolved combination."); + psFree(stack); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Unconvolved Stack: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("unconv"); + + // Clean up unconvolved stack + psTrace("ppStack", 2, "Cleaning up after unconvolved stack....\n"); + if (!ppStackCleanupFiles(stack, options, config, PPSTACK_FILES_UNCONV, PPSTACK_FILES_NONE, false)) { + psError(psErrorCodeLast(), false, "Unable to clean up."); + psFree(stack); + return false; + } + psFree(stack); + } + psFree(options->cells); options->cells = NULL; + + // Finish up + psTrace("ppStack", 1, "Finishing up....\n"); + if (!ppStackFinish(options, config)) { + psError(psErrorCodeLast(), false, "Unable to finish up."); + return false; + } + ppStackMemDump("finish"); + + return true; +} /// Print a summary of the inputs, and return the number of good inputs -static int stackSummary(const ppStackOptions *options, // Stack options, with input mask - const char *place // Place in code - ) +static int stackSummary(const ppStackOptions *options, const char *place) { int numGood = 0; // Number of good inputs @@ -47,221 +249,2 @@ return numGood; } - - - -bool ppStackLoop(pmConfig *config, ppStackOptions *options) -{ - assert(config); - - psTimerStart("PPSTACK_TOTAL"); - psTimerStart("PPSTACK_STEPS"); - - // Setup - psTrace("ppStack", 1, "Setup....\n"); - if (!ppStackSetup(options, config)) { - psError(psErrorCodeLast(), false, "Unable to setup."); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 0: Setup: %f sec", psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("setup"); - - - // Preparation for stacking - psTrace("ppStack", 1, "Preparation for stacking: merging sources, determining target PSF....\n"); - if (!ppStackPrepare(options, config)) { - psError(psErrorCodeLast(), false, "Unable to prepare for stacking."); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 1: Load Sources and Generate Target PSF: %f sec", - psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("prepare"); - if (options->quality) { - // Can't do anything else - return true; - } - - // Convolve inputs - psTrace("ppStack", 1, "Convolving inputs to target PSF....\n"); - if (!ppStackConvolve(options, config)) { - psError(psErrorCodeLast(), false, "Unable to convolve images."); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 2: Generate Convolutions and Save: %f sec", - psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("convolve"); - - if (options->quality) { - // Can't do anything else - return true; - } - - // Ensure sufficient inputs - { - int numGood = stackSummary(options, "initial combination"); - psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe - bool safe = psMetadataLookupBool(NULL, recipe, "SAFE"); // Be safe when combining - if (safe && numGood <= 1) { - options->quality = PPSTACK_ERR_REJECTED; - psErrorStackPrint(stderr, "Insufficient inputs for combination with safety on"); - psErrorClear(); - psWarning("Insufficient inputs for combination with safety on"); - return true; - } - } - - // Start threading - ppStackThreadInit(); - ppStackThreadData *stack = ppStackThreadDataSetup(options, config, true); - if (!stack) { - psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); - return false; - } - - // Prepare for combination - if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", PPSTACK_FILES_STACK, - stack, options, config)) { - psError(psErrorCodeLast(), false, "Unable to prepare for combination."); - psFree(stack); - return false; - } - - // Initial combination - psTrace("ppStack", 1, "Initial stack of convolved images....\n"); - if (!ppStackCombineInitial(stack, options, config)) { - psError(psErrorCodeLast(), false, "Unable to perform initial combination."); - psFree(stack); - return false; - } - ppStackMemDump("initial"); - psLogMsg("ppStack", PS_LOG_INFO, "Stage 3: Make Initial Stack: %f sec", psTimerClear("PPSTACK_STEPS")); - - // Done with stack inputs for now - for (int i = 0; i < options->num; i++) { - pmCellFreeData(options->cells->data[i]); - } - psFree(stack); - - // Pixel rejection - psTrace("ppStack", 1, "Reject pixels....\n"); - if (!ppStackReject(options, config)) { - psError(psErrorCodeLast(), false, "Unable to reject pixels."); - psFree(stack); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 4: Pixel Rejection: %f sec", psTimerClear("PPSTACK_STEPS")); - - // Check inputs - { - int numGood = stackSummary(options, "final combination"); - if (numGood <= 0) { - options->quality = PPSTACK_ERR_REJECTED; - psErrorStackPrint(stderr, "Insufficient inputs survived rejection stage"); - psErrorClear(); - psWarning("Insufficient inputs survived rejection stage"); - return true; - } - } - - stack = ppStackThreadDataSetup(options, config, true); - if (!stack) { - psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); - return false; - } - - // Final combination - psTrace("ppStack", 2, "Final stack of convolved images....\n"); - if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, false, true)) { - psError(psErrorCodeLast(), false, "Unable to perform final combination."); - psFree(stack); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 5: Final Stack: %f sec", psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("final"); - - // Photometry - psTrace("ppStack", 1, "Photometering stacked image....\n"); - if (!ppStackPhotometry(options, config)) { - psError(psErrorCodeLast(), false, "Unable to perform photometry."); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("photometry"); - - // Clean up - psTrace("ppStack", 2, "Cleaning up after combination....\n"); - if (!ppStackCleanup(stack, options, config)) { - psError(psErrorCodeLast(), false, "Unable to clean up."); - psFree(stack); - return false; - } - for (int i = 0; i < options->num; i++) { - pmCellFreeData(options->cells->data[i]); - } - psFree(stack); - psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: Cleanup, WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("cleanup"); - -#if 1 - // Unconvolved stack --- it's cheap to calculate, compared to everything else! - // XXX unconvolved stack is currently using the convolved mask! oops! - if (options->convolve) { - // Start threading - ppStackThreadData *stack = ppStackThreadDataSetup(options, config, false); - if (!stack) { - psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); - return false; - } - - // Prepare for combination - if (!ppStackCombinePrepare("PPSTACK.UNCONV", "PPSTACK.UNCONV.EXP", PPSTACK_FILES_UNCONV, - stack, options, config)) { - psError(psErrorCodeLast(), false, "Unable to prepare for combination."); - psFree(stack); - return false; - } - - psTrace("ppStack", 2, "Stack of unconvolved images....\n"); - if (!ppStackCombineFinal(stack, options->origCovars, options, config, - false, true, false)) { - psError(psErrorCodeLast(), false, "Unable to perform unconvolved combination."); - psFree(stack); - return false; - } - psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Unconvolved Stack: %f sec", psTimerClear("PPSTACK_STEPS")); - ppStackMemDump("unconv"); - - if (!ppStackFilesIterateUp(config)) { - psError(psErrorCodeLast(), false, "Unable to close files."); - psFree(stack); - return false; - } - ppStackFileActivation(config, PPSTACK_FILES_UNCONV, false); - options->outRO->data_exists = false; - options->outRO->parent->data_exists = false; - options->outRO->parent->parent->data_exists = false; - psFree(options->outRO); - options->outRO = NULL; - options->expRO->data_exists = false; - options->expRO->parent->data_exists = false; - options->expRO->parent->parent->data_exists = false; - psFree(options->expRO); - options->expRO = NULL; - - for (int i = 0; i < options->num; i++) { - pmCellFreeData(options->cells->data[i]); - } - psFree(stack); - } - psFree(options->cells); options->cells = NULL; -#endif - - // Finish up - psTrace("ppStack", 1, "Finishing up....\n"); - if (!ppStackFinish(options, config)) { - psError(psErrorCodeLast(), false, "Unable to finish up."); - return false; - } - ppStackMemDump("finish"); - - return true; -} Index: /branches/czw_branch/20101203/ppStack/src/ppStackLoop.h =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackLoop.h (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackLoop.h (revision 30631) @@ -1,12 +1,4 @@ #ifndef PPSTACK_LOOP_H #define PPSTACK_LOOP_H - -#include -#include - -#include "ppStackOptions.h" -#include "ppStackThread.h" -#include "ppStack.h" - // Loop over the inputs, doing the combination @@ -71,9 +63,12 @@ // Cleanup following combination -bool ppStackCleanup( +bool ppStackCleanupFiles( ppStackThreadData *stack, // Stack ppStackOptions *options, // Options - pmConfig *config // Configuration - ); + pmConfig *config, // Configuration + ppStackFileList stackFiles, // cleanup these stack files + ppStackFileList photFiles, // cleanup these phot files (PHOT or NOP) + bool closeJPEGs // close the jpeg files? +); // Photometry @@ -89,3 +84,8 @@ ); + +bool ppStackUpdateHeader(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); +bool ppStackJPEGs(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); +bool ppStackStats(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); + #endif Index: /branches/czw_branch/20101203/ppStack/src/ppStackMatch.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackMatch.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackMatch.c (revision 30631) @@ -1,11 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" @@ -325,15 +316,21 @@ // Scale the input parameters psVector *widthsCopy = psVectorCopy(NULL, widths, PS_TYPE_F32); // Copy of kernel widths - if (scale && !pmSubtractionParamsScale(&size, &footprint, widthsCopy, scaleRef, scaleMin, scaleMax)) { - psError(psErrorCodeLast(), false, "Unable to scale kernel parameters"); - psFree(fake); - psFree(optWidths); - psFree(conv); - psFree(widthsCopy); - if (threads > 0) { - pmSubtractionThreadsFinalize(); - } - return false; - } + + pmSubtractionParamScaleOptions(scale, scaleRef, scaleMin, scaleMax); + + // XXX EAM : the kernel scaling process has changed: the scale is now set + // inside pmSubtractionMatch after the normalization window is measured + + // if (scale && !pmSubtractionParamsScale(&size, &footprint, widthsCopy, scaleRef, scaleMin, scaleMax)) { + // psError(psErrorCodeLast(), false, "Unable to scale kernel parameters"); + // psFree(fake); + // psFree(optWidths); + // psFree(conv); + // psFree(widthsCopy); + // if (threads > 0) { + // pmSubtractionThreadsFinalize(); + // } + // return false; + // } if (!pmSubtractionMatch(NULL, conv, fake, readout, footprint, stride, regionSize, spacing, Index: /branches/czw_branch/20101203/ppStack/src/ppStackOptions.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackOptions.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackOptions.c (revision 30631) @@ -1,8 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include "ppStackLoop.h" +#include "ppStack.h" static void stackOptionsFree(ppStackOptions *options) @@ -36,4 +31,5 @@ psFree(options->inspect); psFree(options->rejected); + return; } Index: /branches/czw_branch/20101203/ppStack/src/ppStackOptions.h =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackOptions.h (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackOptions.h (revision 30631) @@ -1,7 +1,4 @@ #ifndef PPSTACK_OPTIONS_H #define PPSTACK_OPTIONS_H - -#include -#include /// Options for stacking process Index: /branches/czw_branch/20101203/ppStack/src/ppStackPSF.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackPSF.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackPSF.c (revision 30631) @@ -1,10 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" Index: /branches/czw_branch/20101203/ppStack/src/ppStackPhotometry.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackPhotometry.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackPhotometry.c (revision 30631) @@ -1,13 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" bool ppStackPhotometry(ppStackOptions *options, pmConfig *config) Index: /branches/czw_branch/20101203/ppStack/src/ppStackPrepare.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackPrepare.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackPrepare.c (revision 30631) @@ -1,12 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" #define PHOT_SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_SATSTAR | PM_SOURCE_MODE_BLEND | \ Index: /branches/czw_branch/20101203/ppStack/src/ppStackReadout.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackReadout.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackReadout.c (revision 30631) @@ -1,12 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - -#include "ppStackThread.h" #include "ppStack.h" Index: /branches/czw_branch/20101203/ppStack/src/ppStackReject.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackReject.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackReject.c (revision 30631) @@ -1,12 +1,3 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" // #define TESTING Index: /branches/czw_branch/20101203/ppStack/src/ppStackSetup.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackSetup.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackSetup.c (revision 30631) @@ -1,16 +1,5 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" -#include "ppStackLoop.h" #define BUFFER 16 // Buffer for name array - bool ppStackSetup(ppStackOptions *options, pmConfig *config) Index: /branches/czw_branch/20101203/ppStack/src/ppStackSources.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackSources.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackSources.c (revision 30631) @@ -1,8 +1,2 @@ -#include -#include -#include -#include -#include - #include "ppStack.h" Index: /branches/czw_branch/20101203/ppStack/src/ppStackStats.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackStats.c (revision 30631) +++ /branches/czw_branch/20101203/ppStack/src/ppStackStats.c (revision 30631) @@ -0,0 +1,27 @@ +#include "ppStack.h" + +bool ppStackStats(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + // Statistics on output + if (options->stats) { + psTrace("ppStack", 1, "Gathering statistics on stacked image....\n"); + psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + pmFPAview *view = pmFPAviewAlloc(0); // View to readout + view->chip = view->cell = view->readout = 0; + + ppStatsFPA(options->stats, options->outRO->parent->parent->parent, view, maskBad, config); + + psFree(view); + } + + return true; +} Index: /branches/czw_branch/20101203/ppStack/src/ppStackTarget.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackTarget.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackTarget.c (revision 30631) @@ -1,6 +1,2 @@ -#include -#include -#include - #include "ppStack.h" @@ -37,4 +33,5 @@ return psMemIncrRefCounter(sources); } + exclusion = 2; int num = sources->n; // Number of sources Index: /branches/czw_branch/20101203/ppStack/src/ppStackThread.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackThread.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackThread.c (revision 30631) @@ -1,18 +1,5 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - #include "ppStack.h" -#include "ppStackOptions.h" -#include "ppStackThread.h" - #define THREAD_WAIT 10000 // Microseconds to wait if thread is not available - static void stackThreadFree(ppStackThread *thread) Index: /branches/czw_branch/20101203/ppStack/src/ppStackThread.h =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackThread.h (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackThread.h (revision 30631) @@ -1,9 +1,4 @@ #ifndef PPSTACK_THREAD_H #define PPSTACK_THREAD_H - -#include -#include - -#include "ppStackOptions.h" // Thread for stacking chunks Index: /branches/czw_branch/20101203/ppStack/src/ppStackUpdateHeader.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackUpdateHeader.c (revision 30631) +++ /branches/czw_branch/20101203/ppStack/src/ppStackUpdateHeader.c (revision 30631) @@ -0,0 +1,73 @@ +#include "ppStack.h" + +#define WCS_TOLERANCE 0.001 // Tolerance for WCS + +bool ppStackUpdateHeader(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) { + + pmReadout *outRO = options->outRO; // Output readout + + // Propagate WCS + bool wcsDone = false; // Have we done the WCS? + for (int i = 0; i < options->num && !wcsDone; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + + ppStackThread *thread = stack->threads->data[0]; // Representative stack + pmReadout *inRO = thread->readouts->data[i]; // Template readout + if (inRO && !wcsDone) { + // Copy astrometry over + wcsDone = true; + pmHDU *inHDU = pmHDUFromCell(inRO->parent); // Template HDU + pmHDU *outHDU = pmHDUFromCell(outRO->parent); // Output HDU + pmChip *outChip = outRO->parent->parent; // Output chip + pmFPA *outFPA = outChip->parent; // Output FPA + if (!outHDU || !inHDU) { + psWarning("Unable to find HDU at FPA level to copy astrometry."); + } else { + if (!pmAstromReadWCS(outFPA, outChip, inHDU->header, 1.0)) { + psErrorClear(); + psWarning("Unable to read WCS astrometry from input FPA."); + wcsDone = false; + } else { + if (!outHDU->header) { + outHDU->header = psMetadataAlloc(); + } + if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_TOLERANCE)) { + psErrorClear(); + psWarning("Unable to write WCS astrometry to output FPA."); + wcsDone = false; + } + } + } + } + } + + // Put version information into the header + pmHDU *hdu = pmHDUFromCell(outRO->parent); + if (!hdu) { + psError(PPSTACK_ERR_PROG, false, "Unable to find HDU for output."); + return false; + } + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppStackVersionHeader(hdu->header); + + // other interesting header info + psMetadataAddS32(hdu->header, PS_LIST_TAIL, "NINPUTS", PS_META_REPLACE, "Number of input images", options->num); + + psString stacktype = psMetadataLookupStr(NULL, config->arguments, "STACK_TYPE"); // NIGHTLY, DEEP, BEST_IQ + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "STK_TYPE", PS_META_REPLACE, "type of stack", stacktype); + + psString stackID = psMetadataLookupStr(NULL, config->arguments, "-stack_id"); // stack ID (eg, 123222) + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "STK_ID", PS_META_REPLACE, "type of stack", stackID); + + psString skycellID = psMetadataLookupStr(NULL, config->arguments, "-skycell_id"); // skycell ID (eg, skycell.101.00) + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "SKYCELL", PS_META_REPLACE, "type of stack", skycellID); + + psString tessID = psMetadataLookupStr(NULL, config->arguments, "-tess_id"); // tessellation ID (eg, RINGS.V0) + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "TESS_ID", PS_META_REPLACE, "type of stack", tessID); + + return true; +} Index: /branches/czw_branch/20101203/ppStack/src/ppStackVersion.c =================================================================== --- /branches/czw_branch/20101203/ppStack/src/ppStackVersion.c (revision 30630) +++ /branches/czw_branch/20101203/ppStack/src/ppStackVersion.c (revision 30631) @@ -1,12 +1,2 @@ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include -#include - #include "ppStack.h" #include "ppStackVersionDefinitions.h" Index: /branches/czw_branch/20101203/ppSub/src/ppSub.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSub.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSub.c (revision 30631) @@ -112,4 +112,5 @@ psLibFinalize(); + fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, stdout, false), "ppSub"); exitValue = ppSubExitCode(exitValue); exit(exitValue); Index: /branches/czw_branch/20101203/ppSub/src/ppSubFiles.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubFiles.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubFiles.c (revision 30631) @@ -125,4 +125,5 @@ // FPA if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psFree (view); return false; } @@ -131,4 +132,5 @@ view->chip = 0; if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psFree (view); return false; } @@ -137,4 +139,5 @@ view->cell = 0; if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psFree (view); return false; } @@ -143,9 +146,11 @@ view->readout = 0; if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { - return false; - } - - ppSubFilesActivate(config, PPSUB_FILES_ALL, false); - + psFree (view); + return false; + } + + ppSubFilesActivate(config, PPSUB_FILES_ALL, false); + + psFree (view); return true; } @@ -162,4 +167,5 @@ // Readout if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psFree (view); return false; } @@ -168,4 +174,5 @@ view->readout = -1; if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psFree (view); return false; } @@ -174,4 +181,5 @@ view->cell = -1; if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psFree (view); return false; } @@ -180,4 +188,5 @@ view->chip = -1; if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psFree (view); return false; } @@ -222,4 +231,5 @@ ppSubFilesActivate(config, PPSUB_FILES_ALL, false); + psFree (view); return true; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubInputDetections.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubInputDetections.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubInputDetections.c (revision 30631) @@ -118,4 +118,5 @@ if (!psMetadataCopy(sourcesHDU->header, imageHDU->header)) { psError(PPSUB_ERR_PROG, false, "Unable to copy header"); + psFree(view); return false; } @@ -130,8 +131,10 @@ if (!psphotCopyResults (foundDetections, sourcesFile, photFile, view)) { psError(PPSUB_ERR_PROG, false, "Unable to copy psphot outputs"); + psFree(view); return false; } // if no sources were found here, we report that back and let them handle it + psFree(view); return true; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubLoop.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubLoop.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubLoop.c (revision 30631) @@ -26,4 +26,5 @@ pmReadout *out = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); psphotSaveImage (NULL, out->image, name); + psFree(view); return true; } @@ -111,4 +112,5 @@ goto ESCAPE; } + if (data->quality) { // Can't do anything at all @@ -257,6 +259,8 @@ psError(psErrorCodeLast(), false, "Unable to generate mask statistics"); success = false; + psFree(view); goto ESCAPE; } + psFree(view); } // dumpout(config, "diff.4.fits"); Index: /branches/czw_branch/20101203/ppSub/src/ppSubMakePSF.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubMakePSF.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubMakePSF.c (revision 30631) @@ -79,4 +79,5 @@ if (!detections || !detections->allSources) { psError(PPSUB_ERR_CONFIG, true, "No sources from which to determine PSF."); + psFree(view); return false; } @@ -86,4 +87,5 @@ // use flags to toss totally bogus entries? psArray *goodSources = ppSubSelectPSFSources (sources); + if (!psphotReadoutFindPSF(config, view, "PSPHOT.INPUT", goodSources)) { // This is likely a data quality issue @@ -126,4 +128,5 @@ outputRO = pmReadoutAlloc(outputCell); outputRO->image = psMemIncrRefCounter(inputRO->image); + psFree(outputRO); // I have a copy on the outputCell } Index: /branches/czw_branch/20101203/ppSub/src/ppSubMatchPSFs.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubMatchPSFs.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubMatchPSFs.c (revision 30631) @@ -78,5 +78,4 @@ return true; } - psFree(view); @@ -127,4 +126,5 @@ if (!inSourceRO || !refSourceRO) { psError(PPSUB_ERR_DATA, false, "Unable to scale kernel, since no sources were provided."); + psFree(view); return false; } @@ -164,24 +164,22 @@ // is auto-scaling needed? - if (!psMetadataLookupBool(NULL, recipe, "SCALE")) { - // No scaling requested - psFree(view); - return true; - } - + bool scale = psMetadataLookupBool(NULL, recipe, "SCALE"); float scaleRef = psMetadataLookupF32(NULL, recipe, "SCALE.REF"); // Reference for scaling float scaleMin = psMetadataLookupF32(NULL, recipe, "SCALE.MIN"); // Minimum for scaling float scaleMax = psMetadataLookupF32(NULL, recipe, "SCALE.MAX"); // Maximum for scaling - if (!isfinite(scaleRef) || !isfinite(scaleMin) || !isfinite(scaleMax)) { + + if (scale && (!isfinite(scaleRef) || !isfinite(scaleMin) || !isfinite(scaleMax))) { psError(PPSUB_ERR_ARGUMENTS, false, - "Scale parameters (SCALE.REF=%f, SCALE.MIN=%f, SCALE.MAX=%f) not set in recipe.", + "auto-scale selected but scale parameters (SCALE.REF=%f, SCALE.MIN=%f, SCALE.MAX=%f) not set in recipe.", scaleRef, scaleMin, scaleMax); return false; } - if (!pmSubtractionParamsScale(kernelSize, stampSize, kernelWidths, scaleRef, scaleMin, scaleMax)) { - psError(PPSUB_ERR_DATA, false, "Unable to scale parameters."); - return false; - } + pmSubtractionParamScaleOptions(scale, scaleRef, scaleMin, scaleMax); + + // if (!pmSubtractionParamsScale(kernelSize, stampSize, kernelWidths)) { + // psError(PPSUB_ERR_DATA, false, "Unable to scale parameters."); + // return false; + // } return true; @@ -221,4 +219,5 @@ pmCell *cell = pmFPAfileThisCell(config->files, view, "PPSUB.INPUT.CONV"); // Cell for convolved input inConv = pmReadoutAlloc(cell); + psFree(inConv); } pmReadout *refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference convolved @@ -226,4 +225,5 @@ pmCell *cell = pmFPAfileThisCell(config->files, view, "PPSUB.REF.CONV"); // Cell for convolved ref. refConv = pmReadoutAlloc(cell); + psFree(refConv); } @@ -380,8 +380,12 @@ if (inRO->covariance) { - psImageCovarianceTruncate(inRO->covariance, COVAR_FRAC); + psKernel *truncated = psImageCovarianceTruncate(inRO->covariance, COVAR_FRAC); + psFree(inRO->covariance); + inRO->covariance = truncated; } if (refRO->covariance) { - psImageCovarianceTruncate(refRO->covariance, COVAR_FRAC); + psKernel *truncated = psImageCovarianceTruncate(refRO->covariance, COVAR_FRAC); + psFree(refRO->covariance); + refRO->covariance = truncated; } @@ -494,8 +498,12 @@ if (inConv->covariance) { - psImageCovarianceTruncate(inConv->covariance, COVAR_FRAC); + psKernel *truncated = psImageCovarianceTruncate(inConv->covariance, COVAR_FRAC); + psFree(inConv->covariance); + inConv->covariance = truncated; } if (refConv->covariance) { - psImageCovarianceTruncate(refConv->covariance, COVAR_FRAC); + psKernel *truncated = psImageCovarianceTruncate(refConv->covariance, COVAR_FRAC); + psFree(refConv->covariance); + refConv->covariance = truncated; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubReadoutForcedPhot.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubReadoutForcedPhot.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubReadoutForcedPhot.c (revision 30631) @@ -61,4 +61,5 @@ // XXX remove the pixels from photFile? // XXX other cleanup operations? + psFree(view); return true; } @@ -89,4 +90,5 @@ if (!psphotCopyResults (&foundDetections, outputFile, photFile, view)) { psError(PPSUB_ERR_PROG, false, "Unable to copy psphot outputs"); + psFree(view); return false; } @@ -97,4 +99,5 @@ psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_PHOT", PS_META_REPLACE, "Time to do photometry", elapsed); + psFree(view); return true; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubReadoutInverse.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubReadoutInverse.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubReadoutInverse.c (revision 30631) @@ -36,4 +36,5 @@ if (!pmAstromWriteWCS(invHDU->header, outFPA, outChip, WCS_TOLERANCE)) { psError(psErrorCodeLast(), false, "Unable to write WCS astrometry to PPSUB.INVERSE."); + psFree(view); return false; } @@ -41,7 +42,9 @@ if (!pmAstromReadWCS(invFPA, invChip, invHDU->header, 1.0)) { psError(psErrorCodeLast(), false, "Unable to read WCS astrometry."); + psFree(view); return false; } + psFree(view); return true; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubReadoutJpeg.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubReadoutJpeg.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubReadoutJpeg.c (revision 30631) @@ -139,4 +139,5 @@ } + psFree(view); return true; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubReadoutPhotometry.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubReadoutPhotometry.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubReadoutPhotometry.c (revision 30631) @@ -109,12 +109,15 @@ if (!psMetadataCopySingle(inRO->analysis, photRO->analysis, "PSPHOT.DETECTIONS")) { psError(PPSUB_ERR_PROG, false, "Unable to copy PSPHOT.DETECTIONS"); + psFree(view); return false; } if (!psMetadataCopySingle(inRO->analysis, photRO->analysis, "PSPHOT.HEADER")) { psError(PPSUB_ERR_PROG, false, "Unable to copy PSPHOT.HEADER"); + psFree(view); return false; } if (!psMetadataCopySingle(inRO->analysis, photRO->analysis, PM_DETEFF_ANALYSIS)) { psError(PPSUB_ERR_PROG, false, "Unable to copy Detection Efficiency"); + psFree(view); return false; } @@ -128,9 +131,10 @@ } + psFree(view); return true; } #ifdef TESTING - // Record data about sources: not everything gets into the output CMF files +// Record data about sources: not everything gets into the output CMF files { pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with the sources Index: /branches/czw_branch/20101203/ppSub/src/ppSubReadoutSubtract.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubReadoutSubtract.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubReadoutSubtract.c (revision 30631) @@ -31,4 +31,5 @@ bool reverse = psMetadataLookupBool(&mdok, config->arguments, "REVERSE"); // Reverse sense of subtraction? + bool addPair = psMetadataLookupBool(&mdok, recipe, "ADD.NOT.SUBTRACT"); // add instead of subtracting pmFPAview *view = ppSubViewReadout(); // View to readout @@ -47,5 +48,10 @@ // Do the actual subtraction pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); - outRO->image = (psImage*)psBinaryOp(outRO->image, minuend->image, "-", subtrahend->image); + + if (addPair) { + outRO->image = (psImage*)psBinaryOp(outRO->image, minuend->image, "+", subtrahend->image); + } else { + outRO->image = (psImage*)psBinaryOp(outRO->image, minuend->image, "-", subtrahend->image); + } outRO->mask = (psImage*)psBinaryOp(outRO->mask, minuend->mask, "|", subtrahend->mask); outRO->variance = (psImage*)psBinaryOp(outRO->variance, minuend->variance, "+", subtrahend->variance); @@ -70,4 +76,9 @@ psFree(rng); + // XXX EAM : I suspect that the weighted averaging is giving the wrong answer for + // single-direction PSF matching (and maybe DUAL as well). I am testing the difference by + // generating A+B images instead of A-B images and then checking the significance of the + // sources in the image + // Combine the covariances // These are weighted by the appropriate mean variance. This is probably not perfectly correct, but it @@ -79,5 +90,9 @@ covarWeights->data.F32[0] = minuendVar; covarWeights->data.F32[1] = subtrahendVar; +# if (0) outRO->covariance = psImageCovarianceAverageWeighted(covars, covarWeights); +# else + outRO->covariance = psImageCovarianceAverage(covars); +# endif psFree(covars); psFree(covarWeights); @@ -109,4 +124,5 @@ pmChip *outChip = pmFPAfileThisChip(config->files, view, "PPSUB.OUTPUT"); // Output chip psFree(view); + if (!outHDU || !inHDU) { psError(PPSUB_ERR_PROG, true, "Unable to find HDU at FPA level to copy astrometry."); Index: /branches/czw_branch/20101203/ppSub/src/ppSubSetMasks.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubSetMasks.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubSetMasks.c (revision 30631) @@ -80,8 +80,10 @@ if (!pmReadoutMaskInvalid(inRO, maskValue, satValue)) { psError(PPSUB_ERR_DATA, false, "Unable to mask non-finite pixels in input."); + psFree(view); return false; } if (!pmReadoutMaskInvalid(refRO, maskValue, satValue)) { psError(PPSUB_ERR_DATA, false, "Unable to mask non-finite pixels in reference."); + psFree(view); return false; } @@ -94,4 +96,5 @@ if (interpMode == PS_INTERPOLATE_NONE) { psError(PPSUB_ERR_CONFIG, false, "Unknown interpolation mode: %s", interpModeStr); + psFree(view); return false; } @@ -104,8 +107,10 @@ if (!pmReadoutInterpolateBadPixels(inRO, maskVal, interpMode, poorFrac, maskPoor, maskBad)) { psError(PPSUB_ERR_DATA, false, "Unable to interpolate bad pixels for input image."); + psFree(view); return false; } if (!pmReadoutInterpolateBadPixels(refRO, maskVal, interpMode, poorFrac, maskPoor, maskBad)) { psError(PPSUB_ERR_DATA, false, "Unable to interpolate bad pixels for reference image."); + psFree(view); return false; } @@ -113,4 +118,5 @@ #endif + psFree(view); return true; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubThreshold.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubThreshold.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubThreshold.c (revision 30631) @@ -98,4 +98,5 @@ if (!in) { psError(PPSUB_ERR_UNKNOWN, false, "Unable to find readout."); + psFree(view); return false; } @@ -104,4 +105,5 @@ if (!ref) { psError(PPSUB_ERR_UNKNOWN, false, "Unable to find readout."); + psFree(view); return false; } @@ -115,8 +117,10 @@ if (!lowThreshold(in, thresh, maskVal, maskThresh, region, "input convolved image")) { psError(psErrorCodeLast(), false, "Unable to threshold input image."); + psFree(view); return false; } if (!lowThreshold(ref, thresh, maskVal, maskThresh, region, "reference convolved image")) { psError(psErrorCodeLast(), false, "Unable to threshold input image."); + psFree(view); return false; } Index: /branches/czw_branch/20101203/ppSub/src/ppSubVarianceRescale.c =================================================================== --- /branches/czw_branch/20101203/ppSub/src/ppSubVarianceRescale.c (revision 30630) +++ /branches/czw_branch/20101203/ppSub/src/ppSubVarianceRescale.c (revision 30631) @@ -57,4 +57,5 @@ // Nothing to renormalise psWarning("Renormalisation of the variance requested, but no variance provided."); + psFree(view); return true; } @@ -69,4 +70,5 @@ } + psFree(view); return true; } Index: /branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c =================================================================== --- /branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c (revision 30630) +++ /branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c (revision 30631) @@ -165,5 +165,5 @@ if (num == 0) { //All detections were NULL?! - psTrace("ppMops.merge", 3, "All %d detections were NULL\n", detections->n); + psTrace("ppMops.merge", 3, "All %ld detections were NULL\n", detections->n); return NULL; } Index: /branches/czw_branch/20101203/psLib/src/imageops/psImageConvolve.c =================================================================== --- /branches/czw_branch/20101203/psLib/src/imageops/psImageConvolve.c (revision 30630) +++ /branches/czw_branch/20101203/psLib/src/imageops/psImageConvolve.c (revision 30631) @@ -67,5 +67,8 @@ } -psKernel *psKernelAlloc(int xMin, int xMax, int yMin, int yMax) +psKernel *p_psKernelAlloc(const char *file, + unsigned int lineno, + const char *func, + int xMin, int xMax, int yMin, int yMax) { // Check the inputs to make sure max > min; if not, switch. @@ -91,5 +94,5 @@ int numCols = xMax - xMin + 1; // Number of columns for kernel image - psKernel *kernel = psAlloc(sizeof(psKernel)); // The kernel, to be returned + psKernel *kernel = p_psAlloc(file, lineno, func, sizeof(psKernel)); // The kernel, to be returned psMemSetDeallocator(kernel,(psFreeFunc)kernelFree); @@ -272,7 +275,9 @@ float threshold = sumKernel * (1.0 - frac); // Threshold for truncation + int truncateRadius = maxSize; // Truncation radius + bool truncate = false; + // Find truncation size - int truncate = 0; // Truncation radius - for (int radius = 1; truncate == 0 && radius < maxSize; radius++) { + for (int radius = 1; !truncate && (radius < maxSize); radius++) { int uMin = PS_MAX(-radius, xMin); int uMax = PS_MIN(radius, xMax); @@ -290,11 +295,13 @@ } } + // This is the truncation radius if (sum > threshold) { - // This is the truncation radius - truncate = radius; - } - } - if (truncate == maxSize) { - // No truncation possible + truncate = true; + truncateRadius = radius; + } + } + + // Do nothing if no truncation is possible + if (!truncate) { return true; } @@ -302,9 +309,9 @@ // Truncate the kernel { - int uMin = PS_MAX(-truncate, xMin); - int uMax = PS_MIN(truncate, xMax); - int vMin = PS_MAX(-truncate, yMin); - int vMax = PS_MIN(truncate, yMax); - int r2 = PS_SQR(truncate); + int uMin = PS_MAX(-truncateRadius, xMin); + int uMax = PS_MIN(truncateRadius, xMax); + int vMin = PS_MAX(-truncateRadius, yMin); + int vMax = PS_MIN(truncateRadius, yMax); + int r2 = PS_SQR(truncateRadius); for (int v = vMin; v <= vMax; v++) { int v2 = PS_SQR(v); @@ -317,8 +324,8 @@ } } - kernel->xMin = PS_MAX(-truncate, kernel->xMin); - kernel->xMax = PS_MIN(truncate, kernel->xMax); - kernel->yMin = PS_MAX(-truncate, kernel->yMin); - kernel->yMax = PS_MIN(truncate, kernel->yMax); + kernel->xMin = PS_MAX(-truncateRadius, kernel->xMin); + kernel->xMax = PS_MIN(truncateRadius, kernel->xMax); + kernel->yMin = PS_MAX(-truncateRadius, kernel->yMin); + kernel->yMax = PS_MIN(truncateRadius, kernel->yMax); return true; Index: /branches/czw_branch/20101203/psLib/src/imageops/psImageConvolve.h =================================================================== --- /branches/czw_branch/20101203/psLib/src/imageops/psImageConvolve.h (revision 30630) +++ /branches/czw_branch/20101203/psLib/src/imageops/psImageConvolve.h (revision 30631) @@ -84,4 +84,5 @@ /// @return psKernel* A new kernel object /// +#ifdef DOXYGEN psKernel *psKernelAlloc( int xMin, ///< Most negative x index @@ -89,5 +90,18 @@ int yMin, ///< Most negative y index int yMax ///< Most positive y index +); +#else // ifdef DOXYGEN +psKernel *p_psKernelAlloc( + const char *file, ///< File of caller + unsigned int lineno, ///< Line number of caller + const char *func, ///< Function name of caller + int xMin, ///< Most negative x index + int xMax, ///< Most positive x index + int yMin, ///< Most negative y index + int yMax ///< Most positive y index ) PS_ATTR_MALLOC; +#define psKernelAlloc(xMin, xMax, yMin, yMax) \ + p_psKernelAlloc(__FILE__, __LINE__, __func__, (xMin), (xMax), (yMin), (yMax)) +#endif // ifdef DOXYGEN /// Allocate a convolution kernel from a provided image Index: /branches/czw_branch/20101203/psLib/src/imageops/psImageCovariance.c =================================================================== --- /branches/czw_branch/20101203/psLib/src/imageops/psImageCovariance.c (revision 30630) +++ /branches/czw_branch/20101203/psLib/src/imageops/psImageCovariance.c (revision 30631) @@ -30,4 +30,15 @@ return covar; } + +/** + + * changes from 28666 + ** add scale to imageCovarianceCalculate (& multiply at return) + ** add scale to imageCovarianceCalculateThread + + ** accumulate scale (sum kernel^2) in psImageCovarianceCalculate and pass to imageCovarianceCalculate + ** accumulate scale (sum kernel^2) in psImageCovarianceCalculateFactor and pass to imageCovarianceCalculate + + **/ /// Calculation of covariance matrix element when convolving Index: /branches/czw_branch/20101203/psLib/src/sys/psLogMsg.c =================================================================== --- /branches/czw_branch/20101203/psLib/src/sys/psLogMsg.c (revision 30630) +++ /branches/czw_branch/20101203/psLib/src/sys/psLogMsg.c (revision 30631) @@ -291,10 +291,11 @@ head_ptr += strlen(head_ptr); } - // Hostname should be 20 characters. + + // Hostname should be 10 characters. if (logHost) { if (head_ptr > head) { *head_ptr++ = '|'; } - maxLength -= snprintf(head_ptr, maxLength, "%-20s", hostname); + maxLength -= snprintf(head_ptr, maxLength, " %s ", hostname); head_ptr += strlen(head_ptr); } @@ -310,12 +311,15 @@ *head_ptr++ = '|'; } - maxLength -= snprintf(head_ptr, maxLength, "%s", name); + maxLength -= snprintf(head_ptr, maxLength, "%s|\n", name); head_ptr += strlen(head_ptr); - } - - if (head_ptr > head) { - *head_ptr++ = '\n'; - } else if (!logMsg) { // no output desired + } else { + if (head_ptr > head) { + *head_ptr++ = '|'; + } + } + + // rather than putting in a return for the message, let's only put in the return if we asked for the function name + if ((head_ptr == head) && !logMsg) { // no output desired return; } @@ -332,5 +336,5 @@ char *line = strtok_r(msg, "\n", &msgPtr); while (line) { - if(write(logFD, " ", 4)) {;} // ignore return value + if(write(logFD, " ", PS_MIN(2*level, 10))) {;} // ignore return value if(write(logFD, line, strlen(line))) {;} // ignore return value if(write(logFD, "\n", 1)) {;} // ignore return value Index: /branches/czw_branch/20101203/psLib/src/sys/psMemory.c =================================================================== --- /branches/czw_branch/20101203/psLib/src/sys/psMemory.c (revision 30630) +++ /branches/czw_branch/20101203/psLib/src/sys/psMemory.c (revision 30631) @@ -626,5 +626,6 @@ nleak++; - if (fd != NULL) { + // only print a max of 500 leaks (make this an argument) + if ((nleak < 500) && (fd != NULL)) { if (nleak == 1) { fprintf(fd, "# func at (file:line) ID: X Ref: X\n"); Index: /branches/czw_branch/20101203/psLib/test/imageops/Makefile.am =================================================================== --- /branches/czw_branch/20101203/psLib/test/imageops/Makefile.am (revision 30630) +++ /branches/czw_branch/20101203/psLib/test/imageops/Makefile.am (revision 30631) @@ -26,5 +26,6 @@ tap_psImageMapFit \ tap_psImageMapFit2 \ - tap_psImageMaskOps + tap_psImageMaskOps \ + tap_psImageCovariance # tap_psImageShiftKernel Index: /branches/czw_branch/20101203/psLib/test/imageops/samples/conv.sample1.001.fits =================================================================== --- /branches/czw_branch/20101203/psLib/test/imageops/samples/conv.sample1.001.fits (revision 30631) +++ /branches/czw_branch/20101203/psLib/test/imageops/samples/conv.sample1.001.fits (revision 30631) @@ -0,0 +1,140 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = -32 / number of bits per data pixel NAXIS = 2 / number of data axes NAXIS1 = 37 / length of data axis 1 NAXIS2 = 37 / length of data axis 2 EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H BZERO = 0.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK BSCALE = 1.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK END .I­.ÍpÇ/‹:œ01/r0ÓÀÓ1m²–1ú¬ô2xmŒ2çmR3J¶?3§î4uP4<çJ4ÃÑ4§êA4̶Ç4ë6K4þÔ5ù4ú’4ã–4Âó³4}Q4oïš4,P@3éH +3”ÌZ32ϵ2ÊXä2Wœ1ج1KÄp0ŽŽD0§/l¯.­ÑB-ðkˆ.Ëô/’Ôð0Fqî0ûÉx1•üŒ2'Ò¬2°ge3.<ð3¡ÄA4 +7ô4gëš4³3.5X52‘¬5fz +5Œ.¬5 ÆL5­æ÷5±r5ªÕ·5›2H5 + +š5W9–5$<4ìjî4 rØ4MGü3÷xÂ3Œ~Ó30Œ2—92ó1~Œ¶0Ôù?0'tª/wf€.«§ˆ/‰.r0Dñ÷1­º1§Í/2G>’2Þ)®3h¯3äû†4SÌŸ4ž4B5²;5h Q5š0æ5å¬X6È«63Pa6M9i6]¢6aìÒ6YmM6E“G6) +Á6 S­5Ò,5— +5NRû5n•4 ;=46¡Ø3Äü3Eü2Œ!L2(10É0ÞÝ*0%:/fï0-8Ÿ0÷ë¢1ŠyÊ2Qґ2ø=ž3‰àQ4×.4Œû5Ý5`ñž5·B +6 +‚N6Jä'6ŠÖ6±6Ö$·6ô}Ž7ÇT7-C7t6ê©$6ɏk6£’È6zÆ +65¿5÷ù‘5ŸÉO5B"Ü4Þ>P4o•à3ó©3h<2Ð82/ŸÜ1‹+à0Ï(0Ãx0Í\1’{£2D +«2ö093‘Þ4 }É4Šžv5"®£5•+T6—%6P‰Ó6Ÿ&06äÅ7ô7Eõ7nšZ7‡Ú7’" +7”˜ +7Žàè7îÊ7_ +75Æ +7 +Ÿé6Ër6‹@µ64E­5Ü€5}2¶5 *{4‹Þ­4( +3qñ82Í32#)í1sà¢0«¥1d¢Š2"ˆ‘2ØŒô3‡64Ò4¯>.553æ5¯ô¶6 †œ6‰©m6Þ&7(”è7qÿ7¢zÚ7Ο“7øK8 +À€8É8SB8Y 8Ç7ç7Œ\Î7‘Qð7T7!ž6Ÿ-,6iuì65“*5çÉ4‘‡4è¬3`Š2³¢³2Ö·1=÷ +1ï +\2©cZ3a¢4 +*w4£É,53Žv5žŒ62oš6¡â'7 +·7]T77§?7ígG888I9†8pvª8‡à8‘Zœ8“Kÿ8xå8€Ñ*8^c~85ëâ8 +ò7ίÄ7Fè7;/6çÊÈ6†ö`6B5˜µD5%"4‡ +®3ç539Í:2Œ +Ð1Æ=°2k.3%Þ +3ÛzF4ˆ Î5 +Tç5¬µŽ60ŒK6©ºU7 +è7¥\7΂8’8Z|8‘nL8¶Ë{8Ù5*8ôO959šy8üº€8æT8ÇO>8£®_8~ð¶8< +8C(7­ !7WyÙ6üÆ?6‹—Ü6\5›ž5ß4_žQ3Ž€î3á†2B†Ã2ÙP3˜«­4I2o4øyX5Í66 +Z6žÁ +7ƒ~7‡µ?7äJ84tŽ8†0 +8»óÐ8øH +9å³96ÛÔ9L%9Y'9[ 9RX9?”©9&@“9 + +8Öø8Ÿ¿ó8`Ìd8€Œ7»©47]ûØ6÷$6Y·5þcB5jÒÎ4Ë_î4%0\3{•ì2³“º3< œ4&4­oe5U7N5õŽŽ6„|Ÿ7ûU7~ + +7áç-8<†â8“É8Ùß +9:õ9F +w9u09Æk9 9©þ5 +ª5¬"u6E;€6Ӟž7Tš7È3Œ80Š:8’-?8ã9%Ë{9dŽ9”m9¶M_9ÕLè9ítÐ9ûJ 9üq›9ðÇP9ڋì9œdÏ9Â9xö®9<'È9 ·8·gþ8ká8 +Æ~7 Æž7++$6ªíÇ6ÿ75ŒQN4æ +ƒ41Rx3nÀ3íKà4¥“5VÐ,6Öu6•%À7m7žÚ8w48ïÚ8Õ +:9#ð +9m9¡Â‡9ђ :ÈÎ:3–:/:<$Ž:=¶Š:3 +º:I±:æž9ÛåØ9­.Z9ƒN¬9><®9¶=8©„ô8NÖ7ìøš7~œ€7*R6qªŒ5ÕIŒ50-ß4ˆ03Å©4+÷È4î[?5š†Ô6;g£6ԏ„7aw +7ߊ8O}J8ŽŸ9Qº9^÷9Ÿý·9ÚAX: ê:8Ø>:h4=:‹Y:›t:žXü:’΄:z$Q:GæG:÷T9é\¡9®M39}Ö9/ŠÈ8曚8Ž[{8$Öó7²ÅD75rŒ6¬GÒ6ü5~²4ÅTç4S‹4jÝÊ5"> 5їð6} Ò7އ7–·*8p8ˆÕ’8둩9=¯9]h9ÍpÁ:Šp:E÷t:‹#É:ÀzC:øƒ;};êÖ;œM:ÛŸé:£-0:d £: +²p9àÌ>9 yè9_£9i8¹aÌ8Xïê7ínV7rád6è(ì6OXŠ5­ +W5õB4DÄÈ4—5n5PL÷6'6¡Wè75Uß7ŸT‘8:„48ªŸ®9»×9hÓø9¯(î9ýŠœ:8gŸ:ŒÎ::Þr2;(r;dÒª;ˆxÔ;td;|Çþ;GJ`; +l:µuÎ:`¶:eo9đ‰9‡Âæ95xž8åe8‡\Å8i‚7™ðb7È6„ãb5ÞÄ +5.bB4%4·ŠT5|]F6"p6ÂH™7Yˆ17ãQA8]t8Ƀ9*òÇ9‡³ì9Ì;8:bg:lšì:Éñê;/»x;0ò;ÑÐ<â<,ª;< +ÎU;¶¯·;k;€€:¢.::&F9êÓž9ž/$9S³Ô9ž|8 ’É82‰ž7¹r72ÞÒ6¡?}6¢ 5Tüê4œ(²4Ò|Þ5V‚68æø6Ý(²7v߈8ˆ.8y‘ø8áҖ9>‰9–œû9ãš~:.6þ:’Ú8; œ;|ñÖ;ÝDeV&:©TÄ;&L©;ž~< I”<რ+œ1bz<Öa8< +F";†)¯; +1‡:ŠŠ†:» +9¿EP9}Zn9"ƒS8õš8[“Û7åLœ7^öT6Éþ6*šÕ5†i4Å© +4è³65Ÿ;z6K}c6ò·7‡08 +8‡e‚8óċ9L +È9¡ö9ôð>:BHÐ:¯ÅA;.Å~;šù<7G°= +5œF +£>6lPœD–Ø= ™¿<($n;“ Ë;ÒG:“ºQ: +bV9Å839‚A—9'8”8ɟª8brß7ìª47fCá6е360Vv5Š÷ +4Ìf’4àe5™š6D[&6ê3t7‚KÂ8+â8‚ª–8ëI»9Emò9›Y]9ë]Þ:8>Î:¢Ÿ; +î;”¹v<1«<эHœL[ <ÐóA< +oi;‡ý; +Ó°:Œc}:Ó\9Áø¯9€|.9$Ò8Æoå8^Ž~7èOÚ7aÊV6ÌvŒ6,5‡ï 4ÇÑn4̲þ5Œ',63<Š6ÕúŽ7n[’7÷Ñ8oÄÖ8ØQ!95é„9A;9׈†:#Ëð:ˆjÒ:ú;a\Î;ÁAõ[±8ð +¶8Ž$b8 +р7¡mÐ7É6‹5èÀÎ55õ›4„ä»4[þJ5}r5à >6j­ï7íõ7Š”8ù8y@8ÕrÔ9*öb9€ƒ„9·„9üzâ:-[”:p +:£¡ˆ:Ñâ1:õ§4;Ř:ö” :Ò—:¡Šl:gÊÙ:#RÒ9é²b9šW9kk§9 +ŸH8Ä#f8e­³7ûYª7€z„6õ_Þ6Zâ}5¶lá5­4Nك4`4Ü%æ5Ž3u6+ÈÎ6 7Léš7ÊT#8:Т8¡[œ9{_9Eñ†9uõ9ÀB·9ü +t:!n:Jq:rÆL:‰:ŽêP:ˆÍ":pé:F :Éd9ïÞb9µPp9„Ѓ99“’8ôŽ+8—vŠ8/ 7ŸŽN7A]ž6·x"6"Á~5†þ4Ñbõ4Ü¡3ÙÖ~4–õ95CĀ5í‰ë6†ÕŸ7:l7ŽaV8~$8fðª8ŒŠ×9€E9P¬ 9Ž5€9žI9ä« :fé: +Ê:*:.ð :)9¯:Mx:²E9Þ3j9±Ðn9ˆ’9GÝ`9 +]I8ŽgÐ8\çŒ7ý£ò7ˆi67 _š6z>5äb5Ÿ3‰Ÿ3*|\3îŸ4›­“5>¶V5ÚÞr6k]€6í>ð7`C67Æíj8%¹8Îß8¿x9929/,89ZKô90ë9‘„€9œ)º9Ÿ¿q9›Éo9ÑÕ9€F÷9XOP9-I$9–þ8ŒûÊ8€ +„8#g67Äâ7]^6éã^6hr5×Úú5<#"4™ŸA3êÿ3(WÍ2À3ˆ¶[43—Š4Ýü5! 6 +6Œ%7dï7n˜F7ȋ*8 +•y8lA 8Šþ8Üz9 +–?9%7~9;™9Ip¡9Nl9IZ89:ê9%H9 +[n8Ü +8¥Š8k®®8 +- +7È7mñx7¥6‹­ì6 £Á5~o†4ÜÎ43 +3ˆ[š2‰Ü2R +Ê3Ӎ3à +s4qMX5 +ùú5˜X†6›ø6•>P7}p7cìÁ7µÇô8šÐ8A°ß8·ó8€]m8ÅHZ8à“Ü8ò±8ùž8òõ-8áŒ8Åæƒ8¥/8‚Vs8Bœ48 j +7¶îq7eoI7h”6–H&6 +³µ5™jä5 +õ +4rúh3Äd3ÓB2SpÕ1ÔÛÚ2–g<3GQÉ3÷Â84x~5 + ¢5¢€ê6 +áS6Ž\J6ó €7CO7“ço7ÓP*8¡85Õ8[Ò8zþ:8ˆÄ8‹Ôû8ˆU +8|8\çð87U¥8÷7ÕŽ +7•À97EîÇ6ö{6t6?§5€þô5 ’k4’µ$3û”Ú3J_$2˜®w1Ø +81JáH2éÐ2¿‡‚3o+æ4 +("4š9œ5f +5šÐq6 +`â6ró@6č7Ø +7W|17’C•7»šˆ7㉠8’98 +àŸ88O+8[ +7å‹Ã7œ¿œ7”G7Zåð7{6ÈL6wÞ6^Š5ž2*5"õ +4·<4YÂ3t¡02Ãã 2*Ž1Ol +0µÊ˜1qš2,ú92Øô03wþ4 +B;4’Äš5Uô5ƒ¥ž5ãŽp69/q6Žš6ÍZ’7 + +T74o7[ש7}7‰Ãß7å7ŠD 7~äA7^,c77 +¹7t06ÑP¬6‘'6=ˆ‡5éCj5‡ +‚5BÀ4–åó4OD3ƒpù2ßIó22 1 +8-0»(0@0ÚÍ.1’ÃŽ28כ2ڝ’3râx3ý•84xæD4åÈf5G¡]5£M¥5û·66áñ6z«¿6¢#Ð6Æò6ä‘Â6ùA¥7sÂ6úD×6æi +6ÈlV6€š66f×6:ã]6óÈ5§³'5Mf§4ìÍÐ4€k%4÷Ã3{²2â +¶2??x1—á0âoÌ0 +ià/r4g0-²00éá¬1“ÛT2/—@2Ãë§3Mu™3ʕÁ4;äJ4£þM5Õ5P£ô5˜7<5Ñm6èì6&zF6@ƒ76R7 +6Xȧ6S" +6B.•6(Z6 +1.5Õ·"5›Øô5V@Ž5 +À +4©9·4BAô3ÑÈ#3UU2ËY%26cF1™ª&0ó'à04žd/{Þï.³òü/zO0.óL0Þ1„TŸ2<2œ3ˆ°3ë¶3üC×4P"Ÿ4¡Œ}4ìÖÊ5#t®5TÀ(5‚™Í5—I>5¥a +5ªŸK5Š#5˜©Ú5„]-5X„D5&þª4òÔø4Š]Ë4V¶ª4s 3•,N3 xÊ2¢T²2\ô1‰ðF0瘈06š2/‡0.»ìŠ-û.µ5‚/uŠõ0 +eØ0» 1RX²1ÞLõ2\êj2΂<35¥«3–fð3êˆ.4,@Ï4nmV4›Š4¿UÑ4Ýö˜4òÝ +4úŠä4ôž4à +š4Ây4žd/4sɊ40Ê¥3ñ‹r3›b,3%;ð= +"œœ@þ`=Z<'ڒ;”J};%Z:¯9ùöT9jX9š4Ÿ:Ì:‰Y;Û$;ô#<èÞ<Ó + +=*êMœ<¿"=+šM<ÔŒd<âU;‘8ç; +üo:~Öó9ùÁ9Oâ9’±˜9ðgÆ:]GS:àò®;^ ;Ì,Ê9ê+8:&z:b$Š:Œï«:›jX:•s‚:|Y±:@žò: +°9ȎŒ9¥9Jô[9ü²8—ÛŽ8ãÕc9"p59^YÜ9“²„9¿%™9îEï: +¹ú:°:Ú2:ÙÏ9ß$9³0N9‹+‚9PVh9²¶8ȋˆ8IˆŸ8˜˜m8ÚŸ99AÚ:9qp¹9²”9¢[59¬ªH9¬U9 ö‚9Ž>+9nŸÜ9>q]9ôú8Í`¥8‰É¡ Index: /branches/czw_branch/20101203/psLib/test/imageops/samples/incovar.sample1.001.fits =================================================================== --- /branches/czw_branch/20101203/psLib/test/imageops/samples/incovar.sample1.001.fits (revision 30631) +++ /branches/czw_branch/20101203/psLib/test/imageops/samples/incovar.sample1.001.fits (revision 30631) @@ -0,0 +1,19 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = -32 / number of bits per data pixel NAXIS = 2 / number of data axes NAXIS1 = 11 / length of data axis 1 NAXIS2 = 11 / length of data axis 2 EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H BZERO = 0.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK BSCALE = 1.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK END 4„¶×“&7=ò¶µÛ›Qž­æ9ºvÌž­æ:µÛ›”7=òŠ¶×“4„Ÿ¶Õ©9.mṙ±ç81¯; +Ä?<ÇÐ; +Ä?81¯¹™±Û9.mÙ¶Õ©ó7<-G¹™ :]$žœ}L»wõ­œ/üg»wõ­žœ}L:]¹™ 7<.µÙ·81»žœš€75Ù:n;˗ +:n75žœšp81ºðµÙžžªØ; +Á|»tƒ™: +YN<àA +>Ÿ@A<àA: +YW»tƒ +; +Áwžªºp`<ÃÇ,œ,~ø;Çph>žKÂ?€>žKÂ;Çphœ,~ê<ÃÇ$ºpdžªØ; +Á|»tƒ˜: +YW<àA>Ÿ@A<àA +: +YN»tƒ +; +ÁwžªµÙ·81»žœšq75:n;˗ +:n75Ùžœšx81ºðµÙž7<-8¹™Ÿú:]žœ}@»wõ™œ/üY»wõ™žœ}@:]¹™Ÿô7<.¶Õ©9.mÙ¹™±á81®ù; +Ä9<ÇÏù; +Ä981®ù¹™±Õ9.mÒ¶Õ©ï4„¢¶×”7=óƒµÛœ€ž­æöºvÍËž­æôµÛœ>7=ór¶×”4„& Index: /branches/czw_branch/20101203/psLib/test/imageops/samples/incovar.sample1.002.fits =================================================================== --- /branches/czw_branch/20101203/psLib/test/imageops/samples/incovar.sample1.002.fits (revision 30631) +++ /branches/czw_branch/20101203/psLib/test/imageops/samples/incovar.sample1.002.fits (revision 30631) @@ -0,0 +1,8 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = -32 / number of bits per data pixel NAXIS = 2 / number of data axes NAXIS1 = 11 / length of data axis 1 NAXIS2 = 11 / length of data axis 2 EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H BZERO = 0.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK BSCALE = 1.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK END 4š«¶ìƒŸ7P-ѵëå8žŒO³ºƒ}$žŒO²µëä'7P-Ѷ샧4š««¶ì'ƒ94ù¹žì^84â;Ì<ÈÖ2;Ì84 +¹žì^94û¶ì(r7O×깞é: +ßBžž‚»}.œ0ÃÊ»}0žž€û: +ßB¹žé +7OØ¿µë‘+84ïžž‡j73¥a:p!;ÈU£:p!73€Xžž‡l84ìµë’Xž»Qä;6n»|î:ي<ä>à>Ÿnv<ä>à:Øø»|î;6qž»R¥º‚–><ǬDœ/À ;Ç+>Ÿ'c?€>Ÿ'c;Ç*0œ/À <ǬIº‚–Äž»Qç;6m»|î:ٍ<ä>à>Ÿnv<ä>à:Øõ»|î;6qž»R€µë˜84&žž†È73€:o;ÈTÔ:o73£Æžž†Æ849µë‘$7O×깞é: +ßBžžš»}0œ0ÃÊ»}.žž€ä: +ßB¹žé +7OØ¿¶ì'‰94ü¹žìa84ç;Ì <ÈÖ6;Ì 84"¹žìa94ÿ¶ì(w4š« ¶ì„7P.€µëå垌Pqºƒ}©žŒPrµëåZ7P.€¶ì„˜4š¬H Index: /branches/czw_branch/20101203/psLib/test/imageops/tap_psImageCovariance.c =================================================================== --- /branches/czw_branch/20101203/psLib/test/imageops/tap_psImageCovariance.c (revision 30631) +++ /branches/czw_branch/20101203/psLib/test/imageops/tap_psImageCovariance.c (revision 30631) @@ -0,0 +1,70 @@ +/** @file tap_psImageCovariance.c + * + * @brief Contains the tests for psImageCovariance.[ch] + * + * @version $Revision: 1.2 $ + * @date $Date: 2007-05-02 04:14:33 $ + * + * Copyright 2011 Institute for Astronomy, University of Hawaii + */ + +#include +#include +#include +#include "tap.h" +#include "pstap.h" + +psKernel *psKernelReadSimple(char *filename) { + + psRegion region = {0, 0, 0, 0}; + + psFits *fits = psFitsOpen(filename, "r"); + psFitsDumpErrors(0, "foo"); + ok(fits, "opened FITS file"); + + psImage *image = psFitsReadImage(fits, region, 0); + psFitsDumpErrors(0, "foo"); + ok(image, "read image from FITS file"); + + psFitsClose(fits); + + int x0 = image->numCols / 2; + int y0 = image->numRows / 2; + psKernel *kernel = psKernelAllocFromImage(image, x0, y0); + ok(kernel, "allocated kernel from image"); + + psFree(image); + + return kernel; +} + +int main(int argc, char **argv) { + + plan_tests(4); + + // return null for null input image + { + psMemId id = psMemGetId(); + + // read in the kernel and the covariance matrix + psKernel *kernel = psKernelReadSimple("kernel.fits"); + ok(kernel, "read kernel"); + + psKernel *inputCov = psKernelReadSimple("cov.input.fits"); + ok(inputCov, "read input covariance"); + + psKernel *outputCov = psImageCovarianceCalculate(kernel, inputCov); + ok(outputCov, "calculated covariance"); + + psFits *fits = psFitsOpen("cov.output.fits", "w"); + psFitsWriteImage(fits, NULL, outputCov->image, 0, NULL); + psFitsClose(fits); + + psFree(kernel); + psFree(inputCov); + psFree(outputCov); + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } +} + + Index: /branches/czw_branch/20101203/psModules/src/extras/pmVisual.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/extras/pmVisual.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/extras/pmVisual.c (revision 30631) @@ -16,34 +16,37 @@ #include -#include "pmHDU.h" -#include "pmFPA.h" -#include "pmFPAfile.h" -#include "pmAstrometryObjects.h" -#include "pmSubtractionStamps.h" - -#include "pmTrend2D.h" -#include "pmResiduals.h" -#include "pmGrowthCurve.h" -#include "pmSpan.h" -#include "pmFootprintSpans.h" -#include "pmFootprint.h" -#include "pmPeaks.h" -#include "pmMoments.h" -#include "pmModelFuncs.h" -#include "pmModel.h" -#include "pmSourceMasks.h" -#include "pmSourceExtendedPars.h" -#include "pmSourceDiffStats.h" -#include "pmSource.h" -#include "pmSourceFitModel.h" -#include "pmPSF.h" -#include "pmPSFtry.h" - -#include "pmFPAExtent.h" - -#include "pmAstrometryVisual.h" -#include "pmSubtractionVisual.h" -#include "pmStackVisual.h" -#include "pmSourceVisual.h" +bool pmSubtractionVisualClose(void); +bool pmAstromVisualClose(void); +bool pmSubtractionVisualClose(void); +bool pmStackVisualClose(void); +bool pmSourceVisualClose(void); + +// #include "pmHDU.h" +// #include "pmFPA.h" +// #include "pmFPAfile.h" +// #include "pmAstrometryObjects.h" +// #include "pmSubtractionStamps.h" +// #include "pmTrend2D.h" +// #include "pmResiduals.h" +// #include "pmGrowthCurve.h" +// #include "pmSpan.h" +// #include "pmFootprintSpans.h" +// #include "pmFootprint.h" +// #include "pmPeaks.h" +// #include "pmMoments.h" +// #include "pmModelFuncs.h" +// #include "pmModel.h" +// #include "pmSourceMasks.h" +// #include "pmSourceExtendedPars.h" +// #include "pmSourceDiffStats.h" +// #include "pmSource.h" +// #include "pmSourceFitModel.h" +// #include "pmPSF.h" +// #include "pmPSFtry.h" +// #include "pmFPAExtent.h" +// #include "pmAstrometryVisual.h" +// #include "pmSubtractionVisual.h" +// #include "pmStackVisual.h" +// #include "pmSourceVisual.h" # if (HAVE_KAPA) @@ -306,4 +309,27 @@ } +bool pmVisualRangeImage (int kapaFD, psImage *inImage, const char *name, int channel, float min, float max) { + + KiiImage image; + KapaImageData data; + Coords coords; + + strcpy (coords.ctype, "RA---TAN"); + + image.data2d = inImage->data.F32; + image.Nx = inImage->numCols; + image.Ny = inImage->numRows; + + strcpy (data.name, name); + strcpy (data.file, name); + data.zero = min; + data.range = max - min; + data.logflux = 0; + + KiiSetChannel (kapaFD, channel); + KiiNewPicture2D (kapaFD, &image, &data, &coords); + + return true; +} psImage* pmVisualImageToFloat(psImage *image) { Index: /branches/czw_branch/20101203/psModules/src/extras/pmVisual.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/extras/pmVisual.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/extras/pmVisual.h (revision 30631) @@ -64,4 +64,5 @@ const char *name, int channel, bool clip); +bool pmVisualRangeImage (int kapaFD, psImage *inImage, const char *name, int channel, float min, float max); /** Calculate statistics on an image. Index: /branches/czw_branch/20101203/psModules/src/imcombine/Makefile.am =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/Makefile.am (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/Makefile.am (revision 30631) @@ -30,4 +30,5 @@ pmStackReject.h \ pmSubtraction.h \ + pmSubtractionTypes.h \ pmSubtractionAnalysis.h \ pmSubtractionEquation.h \ Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmReadoutCombine.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmReadoutCombine.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmReadoutCombine.c (revision 30631) @@ -33,5 +33,5 @@ params->blank = 0; params->nKeep = 0; - params->fracHigh = 0.0; + params->fracLow = 0.0; params->fracHigh = 0.0; params->iter = 1; Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmStackReject.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmStackReject.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmStackReject.c (revision 30631) @@ -7,4 +7,6 @@ #include +#include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionThreads.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtraction.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtraction.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtraction.c (revision 30631) @@ -20,8 +20,11 @@ #include "pmHDU.h" // Required for pmFPA.h #include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtractionStamps.h" #include "pmSubtractionEquation.h" #include "pmSubtractionVisual.h" #include "pmSubtractionThreads.h" + +bool psFitsWriteImageSimple (char *filename, psImage *image, psMetadata *header); #include "pmSubtraction.h" @@ -773,5 +776,4 @@ if (convolutions) { - // Already done return convolutions; } @@ -787,4 +789,15 @@ } + +bool pmSubtractionConvolveStampThread(psThreadJob *job) +{ + PS_ASSERT_THREAD_JOB_NON_NULL(job, false); + + pmSubtractionStamp *stamp = job->args->data[0]; // List of stamps + pmSubtractionKernels *kernels = job->args->data[1]; // Kernels + int footprint = PS_SCALAR_VALUE(job->args->data[2], S32); // Stamp index + + return pmSubtractionConvolveStamp(stamp, kernels, footprint); +} bool pmSubtractionConvolveStamp (pmSubtractionStamp *stamp, pmSubtractionKernels *kernels, int footprint) @@ -818,21 +831,111 @@ } +#ifdef TESTING + for (int j = 0; j < kernels->num; j++) { + if (stamp->convolutions1) { + psString convName = NULL; + psStringAppend(&convName, "conv1_%03d_%03d.fits", index, j); + psFits *fits = psFitsOpen(convName, "w"); + psFree(convName); + psKernel *conv = stamp->convolutions1->data[j]; + psFitsWriteImage(fits, NULL, conv->image, 0, NULL); + psFitsClose(fits); + } + + if (stamp->convolutions2) { + psString convName = NULL; + psStringAppend(&convName, "conv2_%03d_%03d.fits", index, j); + psFits *fits = psFitsOpen(convName, "w"); + psFree(convName); + psKernel *conv = stamp->convolutions2->data[j]; + psFitsWriteImage(fits, NULL, conv->image, 0, NULL); + psFitsClose(fits); + } + } +#endif + return true; } +bool pmSubtractionConvolveStamps(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); + + psTimerStart("pmSubtractionConvolveStamps"); + + int footprint = stamps->footprint; // Half-size of stamps + + // We iterate over each stamp and generate the convolution if needed. We do NOT need the + // convolution if (a) it has already been calculated or (b) the stamp is not available for + // use (available = USED or CALCULATE) + + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + + bool keep = false; + keep |= (stamp->status == PM_SUBTRACTION_STAMP_USED); + keep |= (stamp->status == PM_SUBTRACTION_STAMP_CALCULATE); + if (!keep) continue; + + bool haveConvolutions = false; + if (kernels->mode == PM_SUBTRACTION_MODE_1) { + haveConvolutions = (stamp->convolutions1 != NULL); + } + if (kernels->mode == PM_SUBTRACTION_MODE_2) { + haveConvolutions = (stamp->convolutions2 != NULL); + } + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + haveConvolutions = (stamp->convolutions1 != NULL) && (stamp->convolutions2 != NULL); + } + if (haveConvolutions) { + continue; + } + + if (pmSubtractionThreaded()) { + psThreadJob *job = psThreadJobAlloc("PSMODULES_SUBTRACTION_CONVOLVE_STAMP"); + psArrayAdd(job->args, 1, stamp); + psArrayAdd(job->args, 1, kernels); + PS_ARRAY_ADD_SCALAR(job->args, footprint, PS_TYPE_S32); + if (!psThreadJobAddPending(job)) { + return false; + } + } else { + pmSubtractionConvolveStamp(stamp, kernels, footprint); + } + } + if (!psThreadPoolWait(true)) { + psError(psErrorCodeLast(), false, "Error waiting for threads."); + return false; + } + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolve stamps: %f sec", psTimerClear("pmSubtractionConvolveStamps")); + return true; +} int pmSubtractionRejectStamps(pmSubtractionKernels *kernels, pmSubtractionStampList *stamps, - const psVector *deviations, psImage *subMask, float sigmaRej) + pmSubtractionQuality *match, psImage *subMask, float sigmaRej) { PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, -1); - PS_ASSERT_VECTOR_NON_NULL(deviations, -1); - PS_ASSERT_VECTOR_TYPE(deviations, PS_TYPE_F32, -1); PS_ASSERT_IMAGE_NON_EMPTY(subMask, -1); PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_IMAGE_MASK, -1); - // I used to measure the rms deviation about zero, and use that as the sigma against which to clip, but - // the distribution is actually something like a chi^2 or Student's t, both of which become Gaussian-like - // with large N. Therefore, let's just treat this as a Gaussian distribution. + // Comment from PAP (r18287): I used to measure the rms deviation about zero, and use that as the + // sigma against which to clip, but the distribution is actually something like a chi^2 or + // Student's t, both of which become Gaussian-like with large N. Therefore, let's just + // treat this as a Gaussian distribution. + + // Comment from EAM (r29777): The residual distribution is only chisq-like if the model is + // a good fit to the data. In the (likely) case that there is a systematic difference + // between the model and the data, the squared-residual distribution grows quadratically + // with increasing flux: the systematic residual flux is a constant factor times the source + // flux; the squared-residual is then of the form (k0 + k1*flux)^2, where k0 comes from the + // Gaussian distributed residual and k1*flux is the systematic residual error. + + // By rejecting sources with the largest squared-residuals, the rejection biases against + // the brighter sources; in severe cases, this pushes the measurement to the weakest + // sources with the most noise. To account for this, let's fit a 2nd order polynomial to + // the distribution of flux vs squared-residual, subtract that fit, and reject sources + // which are significantly deviant from that distribution. kernels->mean = NAN; @@ -840,67 +943,35 @@ kernels->numStamps = -1; - int numStamps = 0; // Number of used stamps - psVector *mask = psVectorAlloc(stamps->num, PS_TYPE_VECTOR_MASK); // Mask, for statistics - psVectorInit(mask, 0); - for (int i = 0; i < stamps->num; i++) { - pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest - if (stamp->status != PM_SUBTRACTION_STAMP_USED) { - mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xff; - continue; - } - numStamps++; - } - psTrace("psModules.imcombine", 1, "Number of good stamps: %d\n", numStamps); - - if (numStamps == 0) { - psError(PM_ERR_STAMPS, true, "No good stamps found."); - psFree(mask); - return -1; - } - - psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | - PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_QUARTILE); // Statistics for deviatns - if (!psVectorStats(stats, deviations, NULL, mask, 0xff)) { - psError(PM_ERR_DATA, false, "Unable to measure statistics for deviations."); + psTrace("psModules.imcombine", 1, "Number of good stamps: %d\n", match->nGood); + + // the chisq & flux vectors are calculated by pmSubtractionCalculateChisqAndMoments + + // use 3hi/3lo sigma clipping on the chisq fit + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); + stats->clipSigma = 5.0; + stats->clipIter = 2; + psPolynomial1D *model = psPolynomial1DAlloc (PS_POLYNOMIAL_ORD, 2); + + bool result = psVectorClipFitPolynomial1D(model, stats, match->stampMask, 0xff, match->chisq, NULL, match->fluxes); + if (!result) { + psError(PM_ERR_DATA, false, "Unable to measure statistics for deviations."); + psFree(model); psFree(stats); - psFree(mask); - return -1; - } - psFree(mask); - - // XXX raise an error? + return -1; + } if (isnan(stats->sampleMean)) { + psError(PM_ERR_DATA, false, "Unable to measure statistics for deviations."); + psFree(model); psFree(stats); return -1; } - double mean, rms; // Mean and RMS of deviations - if (numStamps < MIN_SAMPLE_STATS) { - mean = stats->sampleMean; - rms = stats->sampleStdev; - } else { - mean = stats->sampleMedian; - rms = 0.74 * (stats->sampleUQ - stats->sampleLQ); - } - psFree(stats); - - psTrace("psModules.imcombine", 1, "Mean: %f\n", mean); - psTrace("psModules.imcombine", 1, "RMS deviation: %f\n", rms); - - kernels->mean = mean; - kernels->rms = rms; - kernels->numStamps = numStamps; - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "Mean deviation from %d stamps: %lf +/- %lf", - numStamps, mean, rms); - - if (!isfinite(sigmaRej) || sigmaRej <= 0.0) { - // User just wanted to calculate and record the deviation for posterity - return 0; - } - - float limit = sigmaRej * rms; // Limit on maximum deviation - psTrace("psModules.imcombine", 1, "Deviation limit: %f\n", limit); - + kernels->mean = stats->sampleMean; + kernels->rms = stats->sampleStdev; + kernels->numStamps = stats->clippedNvalues; + + psLogMsg ("pmPSFtry", 4, "chisq vs flux model: %e + %e flux + %e flux^2\n", model->coeff[0], model->coeff[1], model->coeff[2]); + psLogMsg ("pmPSFtry", 4, "chisq vs flux resid: %f +/- %f\n", stats->sampleMean, stats->sampleStdev); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Mean deviation from %d stamps: %lf +/- %lf", kernels->numStamps, kernels->mean, kernels->rms); psString ds9name = NULL; // Filename for ds9 region file @@ -914,10 +985,11 @@ int numRejected = 0; // Number of stamps rejected int numGood = 0; // Number of good stamps - double newMean = 0.0; // New mean psString log = NULL; // Log message - psStringAppend(&log, "Rejecting stamps, mean = %f, threshold = %f\n", mean, limit); + + // save DS9 region files for the stamps and mark for rejection and replacement for (int i = 0; i < stamps->num; i++) { pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest - if (stamp->status == PM_SUBTRACTION_STAMP_USED) { + if (stamp->status != PM_SUBTRACTION_STAMP_USED) { continue; } + if (match->stampMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { // Should we reject stars with low deviation? Well, if this is really a Gaussian-like // distribution and they're low, then we have the right to ask why. Isn't it suspicious that @@ -926,45 +998,40 @@ // subtract well, in which case very few (if any) stars will be legitimately rejected for being // low. - if (fabsf(deviations->data.F32[i] - mean) > limit) { - // Mask out the stamp in the image so you it's not found again - psTrace("psModules.imcombine", 3, "Rejecting stamp %d (%d,%d)\n", i, - (int)(stamp->x - 0.5), (int)(stamp->y - 0.5)); - psStringAppend(&log, "Stamp %d (%d,%d): %f\n", i, - (int)(stamp->x - 0.5), (int)(stamp->y - 0.5), - fabsf(deviations->data.F32[i] - mean)); - numRejected++; - for (int y = stamp->y - footprint; y <= stamp->y + footprint; y++) { - for (int x = stamp->x - footprint; x <= stamp->x + footprint; x++) { - subMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= PM_SUBTRACTION_MASK_REJ; - } - } - pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "red"); - - // Set stamp for replacement - stamp->x = 0; - stamp->y = 0; - stamp->xNorm = NAN; - stamp->yNorm = NAN; - stamp->status = PM_SUBTRACTION_STAMP_REJECTED; - // Recalculate convolutions - psFree(stamp->convolutions1); - psFree(stamp->convolutions2); - stamp->convolutions1 = stamp->convolutions2 = NULL; - psFree(stamp->image1); - psFree(stamp->image2); - psFree(stamp->weight); - stamp->image1 = stamp->image2 = stamp->weight = NULL; - psFree(stamp->matrix); - stamp->matrix = NULL; - psFree(stamp->vector); - stamp->vector = NULL; - } else { - numGood++; - newMean += deviations->data.F32[i]; - pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "green"); - } - } - } - newMean /= numGood; + psTrace("psModules.imcombine", 3, "Rejecting stamp %d (%d,%d)\n", i, + (int)(stamp->x - 0.5), (int)(stamp->y - 0.5)); + psStringAppend(&log, "Stamp %d (%d,%d): %f : %f : %f\n", + i, (int)(stamp->x - 0.5), (int)(stamp->y - 0.5), + match->chisq->data.F32[i], match->fluxes->data.F32[i], match->chisq->data.F32[i] - psPolynomial1DEval(model, match->fluxes->data.F32[i])); + numRejected++; + for (int y = stamp->y - footprint; y <= stamp->y + footprint; y++) { + for (int x = stamp->x - footprint; x <= stamp->x + footprint; x++) { + subMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= PM_SUBTRACTION_MASK_REJ; + } + } + pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "red"); + + // Set stamp for replacement + stamp->x = 0; + stamp->y = 0; + stamp->xNorm = NAN; + stamp->yNorm = NAN; + stamp->status = PM_SUBTRACTION_STAMP_REJECTED; + // Recalculate convolutions + psFree(stamp->convolutions1); + psFree(stamp->convolutions2); + stamp->convolutions1 = stamp->convolutions2 = NULL; + psFree(stamp->image1); + psFree(stamp->image2); + psFree(stamp->weight); + stamp->image1 = stamp->image2 = stamp->weight = NULL; + psFree(stamp->matrix); + stamp->matrix = NULL; + psFree(stamp->vector); + stamp->vector = NULL; + } else { + numGood++; + pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "green"); + } + } if (numRejected == 0) { @@ -978,12 +1045,11 @@ } + psFree(model); + psFree(stats); + if (numRejected > 0) { - psLogMsg("psModules.imcombine", PS_LOG_INFO, - "%d good stamps; %d rejected.\nMean deviation: %lf --> %lf\n", - numGood, numRejected, mean, newMean); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "%d good stamps; %d rejected.\n", numGood, numRejected); } else { - psLogMsg("psModules.imcombine", PS_LOG_INFO, - "%d good stamps; 0 rejected.\nMean deviation: %lf\n", - numGood, mean); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "%d good stamps; 0 rejected.\n", numGood); } @@ -1373,4 +1439,7 @@ psFree(kernelErr2); + static int nOut1 = 0; + static int nOut2 = 0; + // Calculate covariances // This can be fairly involved, so we only do it for a small number of instances @@ -1386,4 +1455,23 @@ psKernelTruncate(kernel, covarFrac); covars->data[i] = psImageCovarianceCalculate(kernel, ro1->covariance); + if (0) { + char name[128]; + snprintf (name, 128, "covar.sample1.%03d.fits", nOut1); + psKernel *cov = covars->data[i]; + psFitsWriteImageSimple (name, cov->image, NULL); + + snprintf (name, 128, "incovar.sample1.%03d.fits", nOut1); + psFitsWriteImageSimple (name, ro1->covariance->image, NULL); + + snprintf (name, 128, "conv.sample1.%03d.fits", nOut1); + psFitsWriteImageSimple (name, kernel->image, NULL); + + fprintf (stderr, "incov: %d,%d; kern: %d,%d, outcov: %d,%d\n", + ro1->covariance->image->numCols, ro1->covariance->image->numRows, + kernel->image->numCols, kernel->image->numRows, + cov->image->numCols, cov->image->numRows); + + nOut1 ++; + } psFree(kernel); } @@ -1407,4 +1495,23 @@ psKernelTruncate(kernel, covarFrac); covars->data[i] = psImageCovarianceCalculate(kernel, ro2->covariance); + if (0) { + char name[128]; + snprintf (name, 128, "covar.sample2.%03d.fits", nOut2); + psKernel *cov = covars->data[i]; + psFitsWriteImageSimple (name, cov->image, NULL); + + snprintf (name, 128, "incovar.sample2.%03d.fits", nOut2); + psFitsWriteImageSimple (name, ro2->covariance->image, NULL); + + snprintf (name, 128, "conv.sample2.%03d.fits", nOut2); + psFitsWriteImageSimple (name, kernel->image, NULL); + + fprintf (stderr, "incov: %d,%d; kern: %d,%d, outcov: %d,%d\n", + ro2->covariance->image->numCols, ro2->covariance->image->numRows, + kernel->image->numCols, kernel->image->numRows, + cov->image->numCols, cov->image->numRows); + + nOut2 ++; + } psFree(kernel); } @@ -1422,8 +1529,12 @@ psImageCovarianceSetThreads(oldThreads); - // Copy anything that wasn't convolved + // Copy anything that wasn't convolved (they may have been allocated though, so free them) switch (kernels->mode) { case PM_SUBTRACTION_MODE_1: if (out2) { + psFree(out2->image); + psFree(out2->variance); + psFree(out2->mask); + psFree(out2->covariance); out2->image = psMemIncrRefCounter(ro2->image); out2->variance = psMemIncrRefCounter(ro2->variance); @@ -1434,4 +1545,8 @@ case PM_SUBTRACTION_MODE_2: if (out1) { + psFree(out1->image); + psFree(out1->variance); + psFree(out1->mask); + psFree(out1->covariance); out1->image = psMemIncrRefCounter(ro1->image); out1->variance = psMemIncrRefCounter(ro1->variance); @@ -1479,2 +1594,28 @@ return true; } + +static void pmSubtractionQualityFree(pmSubtractionQuality *quality) { + + psFree (quality->fluxes); + psFree (quality->chisq); + psFree (quality->moments); + psFree (quality->stampMask); +} + +pmSubtractionQuality *pmSubtractionQualityAlloc() { + + pmSubtractionQuality *quality = psAlloc(sizeof(pmSubtractionQuality)); // Stamp list to return + psMemSetDeallocator(quality, (psFreeFunc)pmSubtractionQualityFree); + + quality->fluxes = NULL; + quality->chisq = NULL; + quality->moments = NULL; + quality->stampMask = NULL; + + quality->score = NAN; + quality->mode = PM_SUBTRACTION_MODE_ERR; + quality->spatialOrder = -1; + quality->nGood = 0; + + return quality; +} Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtraction.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtraction.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtraction.h (revision 30631) @@ -14,10 +14,9 @@ #define PM_SUBTRACTION_H -#include - -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include // if we use the original ppSub implementation, we subtract a central delta-function for all @@ -30,18 +29,4 @@ /// @addtogroup imcombine Image Combinations /// @{ - -/// Mask values for the subtraction mask -typedef enum { - PM_SUBTRACTION_MASK_CLEAR = 0x00, // No masking - PM_SUBTRACTION_MASK_BAD_1 = 0x01, // Image 1 is bad - PM_SUBTRACTION_MASK_BAD_2 = 0x02, // Image 2 is bad - PM_SUBTRACTION_MASK_CONVOLVE_1 = 0x04, // If image 1 is convolved, would be poor or bad - PM_SUBTRACTION_MASK_CONVOLVE_2 = 0x08, // If image 2 is convolved, would be poor or bad - PM_SUBTRACTION_MASK_CONVOLVE_BAD_1 = 0x10, // If image 1 is convolved, would be bad - PM_SUBTRACTION_MASK_CONVOLVE_BAD_2 = 0x20, // If image 2 is convolved, would be bad - PM_SUBTRACTION_MASK_BORDER = 0x40, // Image border - PM_SUBTRACTION_MASK_REJ = 0x80, // Previously tried as a stamp, and rejected -} pmSubtractionMasks; - /// Number of terms in a polynomial @@ -70,8 +55,12 @@ ); +bool pmSubtractionConvolveStamps(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels); + +bool pmSubtractionConvolveStampThread(psThreadJob *job); + /// Reject stamps int pmSubtractionRejectStamps(pmSubtractionKernels *kernels, ///< Kernel parameters to update pmSubtractionStampList *stamps, ///< Stamps - const psVector *deviations, ///< Deviations for each stamp + pmSubtractionQuality *match, ///< data on the subtraction quality psImage *subMask, ///< Subtraction mask float sigmaRej ///< Number of RMS deviations above zero at which to reject @@ -167,4 +156,6 @@ bool pmSubtractionSetFWHMs(float fwhm1, float fwhm2); +pmSubtractionQuality *pmSubtractionQualityAlloc(); + /// @} #endif Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionAnalysis.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionAnalysis.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionAnalysis.c (revision 30631) @@ -7,4 +7,5 @@ #include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionKernels.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionDeconvolve.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionDeconvolve.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionDeconvolve.c (revision 30631) @@ -10,4 +10,5 @@ #include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtractionKernels.h" #include "pmSubtractionDeconvolve.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.c (revision 30631) @@ -8,4 +8,7 @@ #include "pmErrorCodes.h" +#include "pmVisual.h" +#include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionKernels.h" @@ -16,14 +19,16 @@ #include "pmSubtractionVisual.h" -//#define TESTING // TESTING output for debugging; may not work with threads! - -//#define USE_WEIGHT // Include weight (1/variance) in equation? - -// XXX TEST: -//# define USE_WINDOW // window to avoid neighbor contamination +//# define TESTING // TESTING output for debugging; may not work with threads! +# define USE_WEIGHT // Include weight (1/variance) in equation? +# define USE_WINDOW // window to avoid neighbor contamination + +/* I believe we want to apply the WEIGHT to the chisq portions of the calculation (but not the WINDOW), + * and the WINDOW to the moments portiosn of the calculations (but not the WEIGHT) + * + */ # define PENALTY false # define MOMENTS (!PENALTY) -# define MOMENTS_PENALTY_SCALE 2e-5 // XXX this value is not completely arbitrary, but I don't understand why it needs to be this value... +# define MOMENTS_PENALTY_SCALE 20 // up-weight the moments somewhat ////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -107,5 +112,6 @@ cc *= weight->kernel[y][x]; } - if (window) { + // XXX NOTE: do NOT apply the window to the chisq portions of the calculation + if (false && window) { cc *= window->kernel[y][x]; } @@ -138,5 +144,6 @@ rc *= wtVal; } - if (window) { + // XXX NOTE: do NOT apply the window to the chisq portions of the calculation + if (false && window) { float winVal = window->kernel[y][x]; ic *= winVal; @@ -173,4 +180,5 @@ } +# define RENORM_BY_FLUX 0 // Calculate the least-squares matrix and vector for dual convolution @@ -268,32 +276,34 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { + + // XXX NOTE: clipping low S/N pixels does not seem to work very well + if (false && weight) { + float i1 = image1->kernel[y][x]; + float i2 = image2->kernel[y][x]; + float sn = (i1 + i2) / sqrt (weight->kernel[y][x]); + if (sn < 0.5) continue; + } + double aa = iConv1->kernel[y][x] * jConv1->kernel[y][x] * PS_SQR(normValue); double bb = iConv2->kernel[y][x] * jConv2->kernel[y][x]; double ab = iConv1->kernel[y][x] * jConv2->kernel[y][x] * normValue; - if (weight) { - float wtVal = weight->kernel[y][x]; - aa *= wtVal; - bb *= wtVal; - ab *= wtVal; - } - if (window) { - float wtVal = window->kernel[y][x]; - aa *= wtVal; - bb *= wtVal; - ab *= wtVal; - } - sumAA += aa; - sumBB += bb; - sumAB += ab; + + float wtVal = (weight) ? weight->kernel[y][x] : 1.0; + sumAA += wtVal*aa; + sumBB += wtVal*bb; + sumAB += wtVal*ab; if (MOMENTS) { - MxxAA += x*x*aa; - MyyAA += y*y*aa; - MxxBB += x*x*bb; - MyyBB += y*y*bb; + float winVal = (window) ? window->kernel[y][x] : 1.0; + MxxAA += winVal*x*x*aa; + MyyAA += winVal*y*y*aa; + MxxBB += winVal*x*x*bb; + MyyBB += winVal*y*y*bb; } } } + // XXX does normSquare1,2 mess up the relative scaling? + // XXX no: normSquare1,2 is the sum of the flux^2 for the source if (MOMENTS) { MxxAA /= stamp->normSquare1 * PS_SQR(normValue); @@ -305,7 +315,9 @@ // XXX this makes the Chisq portion independent of the normalization and star flux // but may be mis-scaling between stars of different fluxes +# if (RENORM_BY_FLUX) sumAA /= PS_SQR(stamp->normI2); sumAB /= PS_SQR(stamp->normI2); sumBB /= PS_SQR(stamp->normI2); +# endif // fprintf (stderr, "i,j : %d %d : M(xx,yy)(AA,BB) : %f %f %f %f\n", i, j, MxxAA, MyyAA, MxxBB, MyyBB); @@ -328,5 +340,4 @@ matrix->data.F64[iIndex][jIndex] += kernels->penalty * MxxAA * MOMENTS_PENALTY_SCALE; matrix->data.F64[iIndex][jIndex] += kernels->penalty * MyyAA * MOMENTS_PENALTY_SCALE; - matrix->data.F64[jIndex][iIndex] += kernels->penalty * MxxAA * MOMENTS_PENALTY_SCALE; matrix->data.F64[jIndex][iIndex] += kernels->penalty * MyyAA * MOMENTS_PENALTY_SCALE; @@ -334,5 +345,4 @@ matrix->data.F64[iIndex + numParams][jIndex + numParams] += kernels->penalty * MxxBB * MOMENTS_PENALTY_SCALE; matrix->data.F64[iIndex + numParams][jIndex + numParams] += kernels->penalty * MyyBB * MOMENTS_PENALTY_SCALE; - matrix->data.F64[jIndex + numParams][iIndex + numParams] += kernels->penalty * MxxBB * MOMENTS_PENALTY_SCALE; matrix->data.F64[jIndex + numParams][iIndex + numParams] += kernels->penalty * MyyBB * MOMENTS_PENALTY_SCALE; @@ -354,5 +364,6 @@ ab *= weight->kernel[y][x]; } - if (window) { + // XXX NOTE: do NOT apply the window to the chisq portions of the calculation + if (false && window) { ab *= window->kernel[y][x]; } @@ -363,5 +374,7 @@ // XXX this makes the Chisq portion independent of the normalization and star flux // but may be mis-scaling between stars of different fluxes +# if (RENORM_BY_FLUX) sumAB /= PS_SQR(stamp->normI2); +# endif // Spatial variation of kernel coeffs @@ -391,4 +404,10 @@ float i2 = image2->kernel[y][x]; + // XXX NOTE: clipping low S/N pixels does not seem to work very well + if (false && weight) { + float sn = (i1 + i2) / sqrt (weight->kernel[y][x]); + if (sn < 0.5) continue; + } + double ai2 = a * i2 * normValue; double bi2 = b * i2; @@ -396,28 +415,16 @@ double bi1 = b * i1 * normValue; - if (weight) { - float wtVal = weight->kernel[y][x]; - ai2 *= wtVal; - bi2 *= wtVal; - ai1 *= wtVal; - bi1 *= wtVal; - } - if (window) { - float wtVal = window->kernel[y][x]; - ai2 *= wtVal; - bi2 *= wtVal; - ai1 *= wtVal; - bi1 *= wtVal; - } - sumAI2 += ai2; - sumBI2 += bi2; - sumAI1 += ai1; - sumBI1 += bi1; + float wtVal = (weight) ? weight->kernel[y][x] : 1.0; + sumAI2 += wtVal*ai2; + sumBI2 += wtVal*bi2; + sumAI1 += wtVal*ai1; + sumBI1 += wtVal*bi1; if (MOMENTS) { - MxxAI1 += x*x*ai1; - MyyAI1 += y*y*ai1; - MxxBI2 += x*x*bi2; - MyyBI2 += y*y*bi2; + float winVal = (window) ? window->kernel[y][x] : 1.0; + MxxAI1 += winVal*x*x*ai1; + MyyAI1 += winVal*y*y*ai1; + MxxBI2 += winVal*x*x*bi2; + MyyBI2 += winVal*y*y*bi2; } } @@ -435,8 +442,10 @@ // XXX this makes the Chisq portion independent of the normalization and star flux // but may be mis-scaling between stars of different fluxes +# if (RENORM_BY_FLUX) sumAI1 /= PS_SQR(stamp->normI2); sumBI1 /= PS_SQR(stamp->normI2); sumAI2 /= PS_SQR(stamp->normI2); sumBI2 /= PS_SQR(stamp->normI2); +# endif // Spatial variation @@ -788,5 +797,5 @@ stamp->normSquare2 = normSquare2; - psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "normValue: %f %f %f (%f %f)\n", normI1, normI2, stamp->norm, normSquare1, normSquare2); + // psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "normValue: %f %f %f (%f %f)\n", normI1, normI2, stamp->norm, normSquare1, normSquare2); return true; @@ -800,11 +809,9 @@ pmSubtractionKernels *kernels = job->args->data[1]; // Kernels int index = PS_SCALAR_VALUE(job->args->data[2], S32); // Stamp index - pmSubtractionEquationCalculationMode mode = PS_SCALAR_VALUE(job->args->data[3], S32); // calculation model - - return pmSubtractionCalculateEquationStamp(stamps, kernels, index, mode); -} - -bool pmSubtractionCalculateEquationStamp(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels, - int index, const pmSubtractionEquationCalculationMode mode) + + return pmSubtractionCalculateEquationStamp(stamps, kernels, index); +} + +bool pmSubtractionCalculateEquationStamp(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels, int index) { PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); @@ -833,40 +840,14 @@ psAssert(stamp->status == PM_SUBTRACTION_STAMP_CALCULATE, "We only operate on stamps with this state."); - // Generate convolutions: these are generated once and saved - if (!pmSubtractionConvolveStamp(stamp, kernels, footprint)) { - psError(psErrorCodeLast(), false, "Unable to convolve stamp %d.", index); - return NULL; - } - -#ifdef TESTING - for (int j = 0; j < numKernels; j++) { - if (stamp->convolutions1) { - psString convName = NULL; - psStringAppend(&convName, "conv1_%03d_%03d.fits", index, j); - psFits *fits = psFitsOpen(convName, "w"); - psFree(convName); - psKernel *conv = stamp->convolutions1->data[j]; - psFitsWriteImage(fits, NULL, conv->image, 0, NULL); - psFitsClose(fits); - } - - if (stamp->convolutions2) { - psString convName = NULL; - psStringAppend(&convName, "conv2_%03d_%03d.fits", index, j); - psFits *fits = psFitsOpen(convName, "w"); - psFree(convName); - psKernel *conv = stamp->convolutions2->data[j]; - psFitsWriteImage(fits, NULL, conv->image, 0, NULL); - psFitsClose(fits); - } - } -#endif - - // XXX visualize the set of convolved stamps - - psImage *polyValues = p_pmSubtractionPolynomial(NULL, spatialOrder, - stamp->xNorm, stamp->yNorm); // Polynomial terms - - bool new = stamp->vector ? false : true; // Is this a new run? + psImage *polyValues = p_pmSubtractionPolynomial(NULL, spatialOrder, stamp->xNorm, stamp->yNorm); // Polynomial terms + + // Is this a new run? Have we allocated the correct sized vector/matrix? + bool new = stamp->vector ? false : true; + if (!new && (stamp->vector->n != numParams)) { + psFree (stamp->vector); + psFree (stamp->matrix); + new = true; + } + if (new) { stamp->matrix = psImageAlloc(numParams, numParams, PS_TYPE_F64); @@ -941,6 +922,5 @@ } -bool pmSubtractionCalculateEquation(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels, - const pmSubtractionEquationCalculationMode mode) +bool pmSubtractionCalculateEquation(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels) { PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); @@ -953,4 +933,5 @@ for (int i = 0; i < stamps->num; i++) { pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE) { continue; @@ -969,10 +950,9 @@ psArrayAdd(job->args, 1, (pmSubtractionKernels*)kernels); // Casting away const to put on array PS_ARRAY_ADD_SCALAR(job->args, i, PS_TYPE_S32); - PS_ARRAY_ADD_SCALAR(job->args, mode, PS_TYPE_S32); if (!psThreadJobAddPending(job)) { return false; } } else { - pmSubtractionCalculateEquationStamp(stamps, kernels, i, mode); + pmSubtractionCalculateEquationStamp(stamps, kernels, i); } } @@ -983,11 +963,9 @@ } - pmSubtractionVisualPlotLeastSquares(stamps); pmSubtractionVisualShowKernels((pmSubtractionKernels *)kernels); pmSubtractionVisualShowBasis(stamps); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "Calculate equation: %f sec", - psTimerClear("pmSubtractionCalculateEquation")); - + pmSubtractionVisualPlotLeastSquares(stamps); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Calculate equation: %f sec", psTimerClear("pmSubtractionCalculateEquation")); return true; @@ -998,27 +976,10 @@ bool psFitsWriteImageSimple (char *filename, psImage *image, psMetadata *header); -psImage *p_pmSubSolve_wUt (psVector *w, psImage *U); -psImage *p_pmSubSolve_VwUt (psImage *V, psImage *wUt); - -bool p_pmSubSolve_SetWeights (psVector *wApply, psVector *w, psVector *wMask); - -bool p_pmSubSolve_UtB (psVector **UtB, psImage *U, psVector *B); -bool p_pmSubSolve_wUtB (psVector **wUtB, psVector *w, psVector *UtB); -bool p_pmSubSolve_VwUtB (psVector **VwUtB, psImage *V, psVector *wUtB); - -bool p_pmSubSolve_Ax (psVector **B, psImage *A, psVector *x); -bool p_pmSubSolve_VdV (double *value, psVector *x, psVector *y); -bool p_pmSubSolve_y2 (double *y2, pmSubtractionKernels *kernels, const pmSubtractionStampList *stamps); - -psImage *p_pmSubSolve_Xvar (psImage *V, psVector *w); - -double p_pmSubSolve_ChiSquare (pmSubtractionKernels *kernels, const pmSubtractionStampList *stamps); - -bool pmSubtractionSolveEquation(pmSubtractionKernels *kernels, - const pmSubtractionStampList *stamps, - const pmSubtractionEquationCalculationMode mode) +bool pmSubtractionSolveEquation(pmSubtractionKernels *kernels, const pmSubtractionStampList *stamps) { PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); + + psTimerStart("pmSubtractionSolveEquation"); // Check inputs @@ -1042,4 +1003,7 @@ } + if (stamp->vector->n != numParams) { + fprintf (stderr, "mismatch length\n"); + } PS_ASSERT_VECTOR_NON_NULL(stamp->vector, false); PS_ASSERT_VECTOR_SIZE(stamp->vector, (long)numParams, false); @@ -1080,4 +1044,6 @@ } + pmSubtractionVisualPlotLeastSquaresResid(stamps, sumMatrix, numStamps); + #if 0 psImage *save = psImageCopy(NULL, sumMatrix, PS_TYPE_F32); @@ -1087,4 +1053,15 @@ #endif + // XXX TEST : print the matrix & vector + if (0) { + for (int iy = 0; iy < sumMatrix->numRows; iy++) { + for (int ix = 0; ix < sumMatrix->numCols; ix++) { + fprintf (stderr, "%e ", sumMatrix->data.F64[iy][ix]); + } + fprintf (stderr, " : %e\n", sumVector->data.F64[iy]); + } + } + + psImage *invMatrix = NULL; psVector *solution = NULL; // Solution to equation! solution = psVectorAlloc(numParams, PS_TYPE_F64); @@ -1094,30 +1071,38 @@ // solution = psMatrixSolveSVD(solution, sumMatrix, sumVector, NAN); // SINGLE solution - if (1) { - solution = psMatrixSolveSVD(solution, sumMatrix, sumVector, NAN); - } else { - psVector *PERM = NULL; - psImage *LU = psMatrixLUDecomposition(NULL, &PERM, sumMatrix); - solution = psMatrixLUSolution(solution, LU, sumVector, PERM); - psFree (LU); - psFree (PERM); - } +# if (1) + solution = psMatrixSolveSVD(solution, sumMatrix, sumVector, 1e-10); + invMatrix = psMatrixInvert(NULL, sumMatrix, NULL); +# endif +# if (0) + psMatrixLUSolve(sumMatrixLU, sumVector); + solution = psMemIncrRefCounter(sumVector); + invMatrix = psMemIncrRefCounter(sumMatrix); +# endif +# if (0) + psMatrixGJSolve(sumMatrix, sumVector); + invMatrix = psMemIncrRefCounter(sumMatrix); + solution = psMemIncrRefCounter(sumVector); +# endif # if (0) for (int i = 0; i < solution->n; i++) { - psLogMsg("psModules.imcombine", PS_LOG_DETAIL, "Single solution %d: %lf\n", i, solution->data.F64[i]); + psLogMsg("psModules.imcombine", PS_LOG_DETAIL, "Single solution %d: %lf +/- %lf\n", i, solution->data.F64[i], sqrt(fabs(invMatrix->data.F64[i][i]))); } # endif - if (!kernels->solution1) { - kernels->solution1 = psVectorAlloc(sumVector->n + 2, PS_TYPE_F64); // 1 for norm, 1 for bg - psVectorInit(kernels->solution1, 0.0); - } + // ensure we have a solution vector of the right size + kernels->solution1 = psVectorRecycle(kernels->solution1, sumVector->n + 2, PS_TYPE_F64); // 1 for norm, 1 for bg + kernels->solution1err = psVectorRecycle(kernels->solution1err, sumVector->n + 2, PS_TYPE_F64); // 1 for norm, 1 for bg + psVectorInit(kernels->solution1, 0.0); + psVectorInit(kernels->solution1err, 0.0); int numKernels = kernels->num; int spatialOrder = kernels->spatialOrder; // Order of spatial variation int numPoly = PM_SUBTRACTION_POLYTERMS(spatialOrder); // Number of polynomial terms + for (int i = 0; i < numKernels * numPoly; i++) { kernels->solution1->data.F64[i] = solution->data.F64[i]; + kernels->solution1err->data.F64[i] = sqrt(invMatrix->data.F64[i][i]); } @@ -1131,4 +1116,5 @@ psFree(sumVector); psFree(sumMatrix); + psFree(invMatrix); } else { @@ -1160,4 +1146,6 @@ } + pmSubtractionVisualPlotLeastSquaresResid(stamps, sumMatrix, numStamps); + #if 0 psImage *save = psImageCopy(NULL, sumMatrix, PS_TYPE_F32); @@ -1171,4 +1159,15 @@ } + // XXX TEST : print the matrix & vector + if (0) { + for (int iy = 0; iy < sumMatrix->numRows; iy++) { + for (int ix = 0; ix < sumMatrix->numCols; ix++) { + fprintf (stderr, "%e ", sumMatrix->data.F64[iy][ix]); + } + fprintf (stderr, " : %e\n", sumVector->data.F64[iy]); + } + } + + psImage *invMatrix = NULL; psVector *solution = NULL; // Solution to equation! solution = psVectorAlloc(numParams, PS_TYPE_F64); @@ -1176,20 +1175,36 @@ // DUAL solution - solution = psMatrixSolveSVD(solution, sumMatrix, sumVector, NAN); +# if (1) + solution = psMatrixSolveSVD(solution, sumMatrix, sumVector, 1e-10); + invMatrix = psMatrixInvert(NULL, sumMatrix, NULL); +# endif +# if (0) + psMatrixLUSolve(sumMatrix, sumVector); + solution = psMemIncrRefCounter(sumVector); + invMatrix = psMemIncrRefCounter(sumMatrix); +# endif #if (0) for (int i = 0; i < solution->n; i++) { - fprintf(stderr, "Dual solution %d: %lf\n", i, solution->data.F64[i]); + fprintf(stderr, "Dual solution %d: %lf +/- %lf\n", i, solution->data.F64[i], sqrt(invMatrix->data.F64[i][i])); } #endif - if (!kernels->solution1) { - kernels->solution1 = psVectorAlloc(numSolution1 + 2, PS_TYPE_F64); - psVectorInit (kernels->solution1, 0.0); - } - if (!kernels->solution2) { - kernels->solution2 = psVectorAlloc(numSolution2, PS_TYPE_F64); - psVectorInit (kernels->solution2, 0.0); - } + // XXX TEST: manually set the coeffs to a desired solution + // solution->data.F64[0] = +1.826; + // solution->data.F64[1] = -0.115; + // solution->data.F64[2] = 0.0; + // solution->data.F64[3] = 0.0; + + // ensure we have solution vectors of the right size + kernels->solution1 = psVectorRecycle(kernels->solution1, numSolution1 + 2, PS_TYPE_F64); // 1 for norm, 1 for bg + kernels->solution1err = psVectorRecycle(kernels->solution1err, numSolution1 + 2, PS_TYPE_F64); // 1 for norm, 1 for bg + kernels->solution2 = psVectorRecycle(kernels->solution2, numSolution2, PS_TYPE_F64); // 1 for norm, 1 for bg + kernels->solution2err = psVectorRecycle(kernels->solution2err, numSolution2, PS_TYPE_F64); // 1 for norm, 1 for bg + + psVectorInit(kernels->solution1, 0.0); + psVectorInit(kernels->solution1err, 0.0); + psVectorInit(kernels->solution2, 0.0); + psVectorInit(kernels->solution2err, 0.0); // for DUAL convolution analysis, we apply the normalization to I1 as follows: @@ -1205,4 +1220,8 @@ kernels->solution1->data.F64[i] = solution->data.F64[i] * stamps->normValue; kernels->solution2->data.F64[i] = solution->data.F64[i + numSolution1]; + + kernels->solution1err->data.F64[i] = sqrt(invMatrix->data.F64[i][i]) * stamps->normValue; + int i2 = i + numSolution1; + kernels->solution2err->data.F64[i] = sqrt(invMatrix->data.F64[i2][i2]); } @@ -1213,7 +1232,8 @@ kernels->solution1->data.F64[bgIndex] = 0.0; + psFree(solution); + psFree(sumVector); psFree(sumMatrix); - psFree(sumVector); - psFree(solution); + psFree(invMatrix); } @@ -1224,12 +1244,14 @@ if (psTraceGetLevel("psModules.imcombine") >= 7) { for (int i = 0; i < kernels->solution1->n; i++) { - psTrace("psModules.imcombine", 7, "Solution 1 %d: %f\n", i, kernels->solution1->data.F64[i]); + psTrace("psModules.imcombine", 7, "Solution 1 %d: %f +/- %f\n", i, kernels->solution1->data.F64[i], kernels->solution1err->data.F64[i]); } if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { for (int i = 0; i < kernels->solution2->n; i++) { - psTrace("psModules.imcombine", 7, "Solution 2 %d: %f\n", i, kernels->solution2->data.F64[i]); + psTrace("psModules.imcombine", 7, "Solution 2 %d: %f +/- %f\n", i, kernels->solution2->data.F64[i], kernels->solution2err->data.F64[i]); } } } + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Solve equation: %f sec", psTimerClear("pmSubtractionSolveEquation")); // pmSubtractionVisualPlotLeastSquares((pmSubtractionStampList *) stamps); //casting away const @@ -1283,10 +1305,417 @@ } -psVector *pmSubtractionCalculateDeviations(pmSubtractionStampList *stamps, - pmSubtractionKernels *kernels) +// given the convolved image(s) and the residual image, calculate the second moment(s) and the chisq +bool pmSubtractionChisqStats(psVector *fluxesVector, psVector *chisqDVector, psVector *chisqRVector, psVector *momentVector, psVector *stampMask, psKernel *convolved1, psKernel *convolved2, psKernel *difference, psKernel *residual, psKernel *weight, psKernel *window) { + +# ifndef USE_WEIGHT + psAssert(weight == NULL, "impossible!"); +# endif +# ifndef USE_WINDOW + psAssert(window == NULL, "impossible!"); +# endif + + int npix = 0; + float chisqR = 0; + float chisqD = 0; + + // get the chisq + for (int y = residual->yMin; y <= residual->yMax; y++) { + for (int x = residual->xMin; x <= residual->xMax; x++) { + float valueR = PS_SQR(residual->kernel[y][x]); + if (weight) { + valueR *= weight->kernel[y][x]; + } + // XXX NOTE: do NOT apply the window to the chisq portions of the calculation (that would bias the chisq) + chisqR += valueR; + + float valueD = PS_SQR(difference->kernel[y][x]); + if (weight) { + valueD *= weight->kernel[y][x]; + } + chisqD += valueD; + npix ++; + } + } + psVectorAppend(chisqRVector, chisqR / npix); + psVectorAppend(chisqDVector, chisqD / npix); + + float value1 = 0; + float value2 = 0; + float flux2 = 0; + float fluxX = 0; + float fluxY = 0; + float fluxX2 = 0; + float fluxY2 = 0; + + float fluxC1 = 0; + float fluxC2 = 0; + + float moment = 0; + + // get the moments from convolved1 + if (convolved1) { + for (int y = residual->yMin; y <= residual->yMax; y++) { + for (int x = residual->xMin; x <= residual->xMax; x++) { + value1 = convolved1->kernel[y][x]; + value2 = PS_SQR(value1); + + if (window) { + value1 *= window->kernel[y][x]; + value2 *= window->kernel[y][x]; + } + + fluxC1 += value1; + flux2 += value2; + fluxX += x * value2; + fluxY += y * value2; + // fluxX2 += PS_SQR(x) * value2; + // fluxY2 += PS_SQR(y) * value2; + fluxX2 += PS_SQR(x) * value1; + fluxY2 += PS_SQR(y) * value1; + } + } + // float Mx = fluxX / flux2; + // float My = fluxY / flux2; + // float Mxx = fluxX2 / flux2; + // float Myy = fluxY2 / flux2; + float Mxx = fluxX2 / fluxC1; + float Myy = fluxY2 / fluxC1; + + // fprintf (stderr, "conv1, flux2: %f, Mx: %f, My: %f, Mxx: %f, Myy: %f, chisq: %f, npix: %d\n", flux2, Mx, My, Mxx, Myy, chisq, npix); + moment += Mxx + Myy; + } + + // get the moments from convolved1 + if (convolved2) { + for (int y = residual->yMin; y <= residual->yMax; y++) { + for (int x = residual->xMin; x <= residual->xMax; x++) { + value1 = convolved2->kernel[y][x]; + value2 = PS_SQR(value1); + + // XXX NOTE: do NOT apply the weight to the moments calculation + if (false && weight) { + value2 *= weight->kernel[y][x]; + } + if (window) { + value1 *= window->kernel[y][x]; + value2 *= window->kernel[y][x]; + } + + fluxC2 += value1; + flux2 += value2; + fluxX += x * value2; + fluxY += y * value2; + // fluxX2 += PS_SQR(x) * value2; + // fluxY2 += PS_SQR(y) * value2; + fluxX2 += PS_SQR(x) * value1; + fluxY2 += PS_SQR(y) * value1; + } + } + // float Mx = fluxX / flux2; + // float My = fluxY / flux2; + // float Mxx = fluxX2 / flux2; + // float Myy = fluxY2 / flux2; + float Mxx = fluxX2 / fluxC2; + float Myy = fluxY2 / fluxC2; + + // fprintf (stderr, "conv2, flux2: %f, Mx: %f, My: %f, Mxx: %f, Myy: %f, chisq: %f, npix: %d\n", flux2, Mx, My, Mxx, Myy, chisq, npix); + moment += Mxx + Myy; + } + + float flux = fluxC1 + fluxC2; + + if (convolved1 && convolved2) { + moment *= 0.5; + flux *= 0.5; + } + psVectorAppend(momentVector, moment); + psVectorAppend(fluxesVector, flux); + + // check that the last appended values are ok: + int Nelem = fluxesVector->n - 1; + bool valid = true; + valid &= isfinite(chisqRVector->data.F32[Nelem]); + valid &= isfinite(fluxesVector->data.F32[Nelem]); + valid &= isfinite(momentVector->data.F32[Nelem]); + if (valid) { + psVectorAppend(stampMask, 0); + } else { + psVectorAppend(stampMask, 0x02); + } + return true; +} + +bool pmSubtractionCalculateChisqAndMoments(pmSubtractionQuality **bestMatch, + pmSubtractionStampList *stamps, + pmSubtractionKernels *kernels) { PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, NULL); PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NULL); PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NULL); + + psTimerStart("pmSubtractionCalculateChisqAndMoments"); + + // XXX need to save these somewhere + psVector *fluxes = psVectorAllocEmpty(stamps->num, PS_TYPE_F32); + psVector *chisqD = psVectorAllocEmpty(stamps->num, PS_TYPE_F32); + psVector *chisqR = psVectorAllocEmpty(stamps->num, PS_TYPE_F32); + psVector *moments = psVectorAllocEmpty(stamps->num, PS_TYPE_F32); + psVector *stampMask = psVectorAllocEmpty(stamps->num, PS_TYPE_VECTOR_MASK); + + int footprint = stamps->footprint; // Half-size of stamps + int numKernels = kernels->num; // Number of kernels + + psImage *polyValues = NULL; // Polynomial values + + // storage for the image (convolved2 is not used in SINGLE mode) + psKernel *residual = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image + psKernel *difference = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image + psKernel *convolved1 = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image + psKernel *convolved2 = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image + + int nGood = 0; + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // The stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_USED) { + // mark this stamp as unused (note that we have to append NANs to the other vectors to keep the lengths in sync) + psVectorAppend(moments, NAN); + psVectorAppend(fluxes, NAN); + psVectorAppend(chisqD, NAN); + psVectorAppend(chisqR, NAN); + psVectorAppend(stampMask, 0x01); + continue; + } + nGood ++; + + // Calculate coefficients of the kernel basis functions + polyValues = p_pmSubtractionPolynomial(polyValues, kernels->spatialOrder, stamp->xNorm, stamp->yNorm); + double norm = p_pmSubtractionSolutionNorm(kernels); // Normalisation + double background = p_pmSubtractionSolutionBackground(kernels, polyValues);// Difference in background + + // Calculate residuals + psImageInit(residual->image, 0.0); + psImageInit(difference->image, 0.0); + + psKernel *weight = NULL; + psKernel *window = NULL; + +#ifdef USE_WEIGHT + weight = stamp->weight; +#endif +#ifdef USE_WINDOW + window = stamps->window; +#endif + + if (kernels->mode != PM_SUBTRACTION_MODE_DUAL) { + + // the single-direction psf match code attempts to find the kernel such that: + // source * kernel = target. we need to assign 'source' and 'target' correctly + // depending on which of image1 or image2 we asked to be convolved. + + psKernel *target; // Target postage stamp (convolve source to match the target) + psKernel *source; // Source postage stamp (convolve source to match the target) + psArray *convolutions; // Convolution postage stamps for each kernel basis function + + // init the accumulation image + psImageInit(convolved1->image, 0.0); + + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + target = stamp->image2; + source = stamp->image1; + convolutions = stamp->convolutions1; + break; + case PM_SUBTRACTION_MODE_2: + target = stamp->image1; + source = stamp->image2; + convolutions = stamp->convolutions2; + break; + default: + psAbort("Unsupported subtraction mode: %x", kernels->mode); + } + + // generate the convolved source image (sum over kernels) + for (int j = 0; j < numKernels; j++) { + psKernel *convolution = convolutions->data[j]; // Convolution + double coefficient = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, false); // Coefficient + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + convolved1->kernel[y][x] += convolution->kernel[y][x] * coefficient; + } + } + } + + // Generate the difference, residual, and convolved source images. Note the we + // accumulate the convolution of (A-B), so we need to replace it to generate the + // images of the convolved source image. + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + difference->kernel[y][x] = target->kernel[y][x] - source->kernel[y][x] * norm - background; + residual->kernel[y][x] = difference->kernel[y][x] - convolved1->kernel[y][x]; + convolved1->kernel[y][x] += source->kernel[y][x] * norm; + } + } + + // XXX if we want to have a weight and window, we'll need to pass through to here + pmSubtractionChisqStats(fluxes, chisqD, chisqR, moments, stampMask, convolved1, NULL, difference, residual, weight, window); + + } else { + + // Dual convolution + psArray *convolutions1 = stamp->convolutions1; // Convolutions of the first image + psArray *convolutions2 = stamp->convolutions2; // Convolutions of the second image + psKernel *image1 = stamp->image1; // The first image + psKernel *image2 = stamp->image2; // The second image + + // init the accumulation images + psImageInit(convolved1->image, 0.0); + psImageInit(convolved2->image, 0.0); + + for (int j = 0; j < numKernels; j++) { + psKernel *conv1 = convolutions1->data[j]; // Convolution of first image + psKernel *conv2 = convolutions2->data[j]; // Convolution of second image + double coeff1 = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, false); // Coefficient 1 + double coeff2 = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, true); // Coefficient 2 + + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + // NOTE sign for coeff2 + convolved1->kernel[y][x] += +conv1->kernel[y][x] * coeff1; + convolved2->kernel[y][x] += -conv2->kernel[y][x] * coeff2; + } + } + } + + // Generate the difference, residual, and convolved source images. Note the we + // accumulate the convolutions of (A-B), so we need to replace (A or B) to generate + // the images of the convolved source images. + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + difference->kernel[y][x] = image2->kernel[y][x] - image1->kernel[y][x] * norm - background; + residual->kernel[y][x] = difference->kernel[y][x] + convolved2->kernel[y][x] - convolved1->kernel[y][x]; + convolved1->kernel[y][x] += image1->kernel[y][x] * norm; + convolved2->kernel[y][x] += image2->kernel[y][x]; + } + } + + if (0) { + psFitsWriteImageSimple("conv1.fits", convolved1->image, NULL); + psFitsWriteImageSimple("conv2.fits", convolved2->image, NULL); + psFitsWriteImageSimple("resid.fits", residual->image, NULL); + pmVisualAskUser(NULL); + } + + pmSubtractionChisqStats(fluxes, chisqD, chisqR, moments, stampMask, convolved1, convolved2, difference, residual, weight, window); + } + } + + // find the mean chisq and mean moment + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN); + psVectorStats (stats, chisqD, NULL, stampMask, 0xff); + float chisqDValue = stats->sampleMean; + + psStatsInit(stats); + psVectorStats (stats, chisqR, NULL, stampMask, 0xff); + float chisqRValue = stats->sampleMean; + + psStatsInit(stats); + psVectorStats (stats, moments, NULL, stampMask, 0xff); + float momentValue = stats->sampleMean; + + double sumKernel1 = 0.0, sumKernel2 = 0.0; // Sum of the kernel + + // calculate the variance contribution from this smoothing kernel + psKernel *modelKernel = pmSubtractionKernel(kernels, 0.0, 0.0, false); + for (int y = modelKernel->yMin; y <= modelKernel->yMax; y++) { + for (int x = modelKernel->xMin; x <= modelKernel->xMax; x++) { + if (!isfinite(modelKernel->kernel[y][x])) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Non-finite covariance matrix element in kernel at %d,%d", x, y); + return NULL; + } + sumKernel1 += PS_SQR(modelKernel->kernel[y][x]); + } + } + psFree (modelKernel); + + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + psKernel *modelKernel = pmSubtractionKernel(kernels, 0.0, 0.0, true); + for (int y = modelKernel->yMin; y <= modelKernel->yMax; y++) { + for (int x = modelKernel->xMin; x <= modelKernel->xMax; x++) { + if (!isfinite(modelKernel->kernel[y][x])) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Non-finite covariance matrix element in kernel at %d,%d", x, y); + return NULL; + } + sumKernel2 += PS_SQR(modelKernel->kernel[y][x]); + } + } + psFree (modelKernel); + } else { + sumKernel2 = 1.0; + } + + // if we modify the chisq value by the (sumKernel1 + sumKernel2), we account for the + // smoothing coming from larger kernels adding additional spatial fit terms should be + // penalized by increasing the score somewhat. the 0.01 value is not well-chosen. + float orderFactor = 0.01 * kernels->spatialOrder; + float score = 2.0 * chisqRValue / (sumKernel1 + sumKernel2) + orderFactor; + psLogMsg("psModules.imcombine", PS_LOG_INFO, "chisq: %6.3f, chisqD: %6.3f, moment: %6.3f, sumKernel_1: %6.3f, sumKernel_2, score: %6.3f: %6.3f\n", chisqRValue, chisqDValue, momentValue, sumKernel1, sumKernel2, score); + + // save this result if it is the first or the best (skip if bestMatch is NULL) + if (bestMatch) { + pmSubtractionQuality *match = *bestMatch; + bool keep = false; + if (match == NULL) { + *bestMatch = match = pmSubtractionQualityAlloc(); + keep = true; + } else { + if (score < match->score) { + psFree(match->fluxes); + psFree(match->chisq); + psFree(match->moments); + psFree(match->stampMask); + keep = true; + } + } + if (keep) { + psLogMsg("psModules.imcombine", PS_LOG_INFO, "keeping order: %d, mode: %d, score: %f\n", kernels->spatialOrder, kernels->mode, score); + match->score = score; + match->spatialOrder = kernels->spatialOrder; + match->mode = kernels->mode; + match->nGood = nGood; + match->fluxes = psMemIncrRefCounter(fluxes); + match->chisq = psMemIncrRefCounter(chisqR); + match->moments = psMemIncrRefCounter(moments); + match->stampMask = psMemIncrRefCounter(stampMask); + } + } + + pmSubtractionVisualPlotChisqAndMoments(fluxes, chisqR, moments); + + psFree(stats); + psFree(chisqR); + psFree(chisqD); + psFree(fluxes); + psFree(moments); + psFree(stampMask); + + psFree(residual); + psFree(difference); + psFree(convolved1); + psFree(convolved2); + psFree(polyValues); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Calculate Chisq and Moments: %f sec", psTimerClear("pmSubtractionCalculateChisqAndMoments")); + + return true; +} + +// XXX for now, let's not use this, and let's instead just use values from pmSubtractionCalculateChisqAndMoments +psVector *pmSubtractionCalculateDeviations(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, NULL); + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NULL); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NULL); + + psTimerStart("pmSubtractionCalculateDeviations"); psVector *deviations = psVectorAlloc(stamps->num, PS_TYPE_F32); // Mean deviation for stamps @@ -1298,7 +1727,4 @@ psImage *polyValues = NULL; // Polynomial values psKernel *residual = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image - - // set up holding images for the visualization - pmSubtractionVisualShowFitInit (stamps); psVector *fResSigma = psVectorAllocEmpty(stamps->num, PS_TYPE_F32); @@ -1401,7 +1827,4 @@ } - // XXX visualize the target, source, convolution and residual - pmSubtractionVisualShowFitAddStamp (target, source, residual, background, norm, i); - for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { @@ -1438,7 +1861,4 @@ } - // XXX visualize the target, source, convolution and residual - pmSubtractionVisualShowFitAddStamp (image2, image1, residual, background, norm, i); - for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { @@ -1455,4 +1875,5 @@ } + double flux = 0.0; double deviation = 0.0; // Sum of differences for (int y = - footprint; y <= footprint; y++) { @@ -1460,62 +1881,22 @@ double dev = PS_SQR(residual->kernel[y][x]) * weight->kernel[y][x]; deviation += dev; -#ifdef TESTING - residual->kernel[y][x] = dev; -#endif + flux += stamp->image1->kernel[y][x] + stamp->image2->kernel[y][x]; } } deviations->data.F32[i] = devNorm * deviation; - psTrace("psModules.imcombine", 5, "Deviation for stamp %d (%d,%d): %f\n", - i, (int)(stamp->x - 0.5), (int)(stamp->y - 0.5), deviations->data.F32[i]); + psTrace("psModules.imcombine", 5, "Deviation and Flux for stamp %d (%d,%d): %f %f\n", + i, (int)(stamp->x - 0.5), (int)(stamp->y - 0.5), deviations->data.F32[i], flux); psStringAppend(&log, "Stamp %d (%d,%d): %f\n", i, (int)(stamp->x - 0.5), (int)(stamp->y - 0.5), deviations->data.F32[i]); if (!isfinite(deviations->data.F32[i])) { stamp->status = PM_SUBTRACTION_STAMP_REJECTED; - psTrace("psModules.imcombine", 5, - "Rejecting stamp %d (%d,%d) because of non-finite deviation\n", - i, (int)(stamp->x - 0.5), (int)(stamp->y - 0.5)); + psTrace("psModules.imcombine", 5, "Rejecting stamp %d (%d,%d) because of non-finite deviation\n", i, (int)(stamp->x - 0.5), (int)(stamp->y - 0.5)); continue; } - -#ifdef TESTING - { - psString filename = NULL; - psStringAppend(&filename, "resid_%03d.fits", i); - psFits *fits = psFitsOpen(filename, "w"); - psFree(filename); - psFitsWriteImage(fits, NULL, residual->image, 0, NULL); - psFitsClose(fits); - } - if (stamp->image1) { - psString filename = NULL; - psStringAppend(&filename, "stamp_image1_%03d.fits", i); - psFits *fits = psFitsOpen(filename, "w"); - psFree(filename); - psFitsWriteImage(fits, NULL, stamp->image1->image, 0, NULL); - psFitsClose(fits); - } - if (stamp->image2) { - psString filename = NULL; - psStringAppend(&filename, "stamp_image2_%03d.fits", i); - psFits *fits = psFitsOpen(filename, "w"); - psFree(filename); - psFitsWriteImage(fits, NULL, stamp->image2->image, 0, NULL); - psFitsClose(fits); - } - if (stamp->weight) { - psString filename = NULL; - psStringAppend(&filename, "stamp_weight_%03d.fits", i); - psFits *fits = psFitsOpen(filename, "w"); - psFree(filename); - psFitsWriteImage(fits, NULL, stamp->weight->image, 0, NULL); - psFitsClose(fits); - } -#endif - } psFree(keepStamps); - psLogMsg("psModules.imcombine", PS_LOG_DETAIL, "%s", log); + psLogMsg("psModules.imcombine", PS_LOG_MINUTIA, "%s", log); psFree(log); @@ -1527,7 +1908,4 @@ psLogMsg("psModules.imcombine", PS_LOG_INFO, "normalization: %f, background: %f", norm, background); - pmSubtractionVisualShowFit(norm); - pmSubtractionVisualPlotFit(kernels); - psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); psVectorStats (stats, fResSigma, NULL, NULL, 0); @@ -1560,303 +1938,7 @@ psFree(polyValues); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Calculate Deviations: %f sec", psTimerClear("pmSubtractionCalculateDeviations")); + return deviations; -} - -// we are supplied U, not Ut; w represents a diagonal matrix (also, we apply 1/w instead of w) -psImage *p_pmSubSolve_wUt (psVector *w, psImage *U) { - - psAssert (w->n == U->numCols, "w and U dimensions do not match"); - - // wUt has dimensions transposed relative to Ut. - psImage *wUt = psImageAlloc (U->numRows, U->numCols, PS_TYPE_F64); - psImageInit (wUt, 0.0); - - for (int i = 0; i < wUt->numCols; i++) { - for (int j = 0; j < wUt->numRows; j++) { - if (!isfinite(w->data.F64[j])) continue; - if (w->data.F64[j] == 0.0) continue; - wUt->data.F64[j][i] = U->data.F64[i][j] / w->data.F64[j]; - } - } - return wUt; -} - -// XXX this is just standard matrix multiplication: use psMatrixMultiply? -psImage *p_pmSubSolve_VwUt (psImage *V, psImage *wUt) { - - psAssert (V->numCols == wUt->numRows, "matrix dimensions do not match"); - - psImage *Ainv = psImageAlloc (wUt->numCols, V->numRows, PS_TYPE_F64); - - for (int i = 0; i < Ainv->numCols; i++) { - for (int j = 0; j < Ainv->numRows; j++) { - double sum = 0.0; - for (int k = 0; k < V->numCols; k++) { - sum += V->data.F64[j][k] * wUt->data.F64[k][i]; - } - Ainv->data.F64[j][i] = sum; - } - } - return Ainv; -} - -// we are supplied U, not Ut -bool p_pmSubSolve_UtB (psVector **UtB, psImage *U, psVector *B) { - - psAssert (U->numRows == B->n, "U and B dimensions do not match"); - - UtB[0] = psVectorRecycle (UtB[0], U->numCols, PS_TYPE_F64); - - for (int i = 0; i < U->numCols; i++) { - double sum = 0.0; - for (int j = 0; j < U->numRows; j++) { - sum += B->data.F64[j] * U->data.F64[j][i]; - } - UtB[0]->data.F64[i] = sum; - } - return true; -} - -// w is diagonal -bool p_pmSubSolve_wUtB (psVector **wUtB, psVector *w, psVector *UtB) { - - psAssert (w->n == UtB->n, "w and UtB dimensions do not match"); - - // wUt has dimensions transposed relative to Ut. - wUtB[0] = psVectorRecycle (wUtB[0], w->n, PS_TYPE_F64); - psVectorInit (wUtB[0], 0.0); - - for (int i = 0; i < w->n; i++) { - if (!isfinite(w->data.F64[i])) continue; - if (w->data.F64[i] == 0.0) continue; - wUtB[0]->data.F64[i] = UtB->data.F64[i] / w->data.F64[i]; - } - return true; -} - -// this is basically matrix * vector -bool p_pmSubSolve_VwUtB (psVector **VwUtB, psImage *V, psVector *wUtB) { - - psAssert (V->numCols == wUtB->n, "V and wUtB dimensions do not match"); - - VwUtB[0] = psVectorRecycle (*VwUtB, V->numRows, PS_TYPE_F64); - - for (int j = 0; j < V->numRows; j++) { - double sum = 0.0; - for (int i = 0; i < V->numCols; i++) { - sum += V->data.F64[j][i] * wUtB->data.F64[i]; - } - VwUtB[0]->data.F64[j] = sum; - } - return true; -} - -// this is basically matrix * vector -bool p_pmSubSolve_Ax (psVector **B, psImage *A, psVector *x) { - - psAssert (A->numCols == x->n, "A and x dimensions do not match"); - - B[0] = psVectorRecycle (*B, A->numRows, PS_TYPE_F64); - - for (int j = 0; j < A->numRows; j++) { - double sum = 0.0; - for (int i = 0; i < A->numCols; i++) { - sum += A->data.F64[j][i] * x->data.F64[i]; - } - B[0]->data.F64[j] = sum; - } - return true; -} - -// this is basically Vector * vector -bool p_pmSubSolve_VdV (double *value, psVector *x, psVector *y) { - - psAssert (x->n == y->n, "x and y dimensions do not match"); - - double sum = 0.0; - for (int i = 0; i < x->n; i++) { - sum += x->data.F64[i] * y->data.F64[i]; - } - *value = sum; - return true; -} - -bool p_pmSubSolve_y2 (double *y2, pmSubtractionKernels *kernels, const pmSubtractionStampList *stamps) { - - int footprint = stamps->footprint; // Half-size of stamps - - double sum = 0.0; - for (int i = 0; i < stamps->num; i++) { - - pmSubtractionStamp *stamp = stamps->stamps->data[i]; - if (stamp->status != PM_SUBTRACTION_STAMP_USED) continue; - - psKernel *weight = NULL; - psKernel *window = NULL; - psKernel *input = NULL; - -#ifdef USE_WEIGHT - weight = stamp->weight; -#endif -#ifdef USE_WINDOW - window = stamps->window; -#endif - - switch (kernels->mode) { - // MODE_1 : convolve image 1 to match image 2 (and vice versa) - case PM_SUBTRACTION_MODE_1: - input = stamp->image2; - break; - case PM_SUBTRACTION_MODE_2: - input = stamp->image1; - break; - default: - psAbort ("programming error"); - } - - for (int y = - footprint; y <= footprint; y++) { - for (int x = - footprint; x <= footprint; x++) { - double in = input->kernel[y][x]; - double value = in*in; - if (weight) { - float wtVal = weight->kernel[y][x]; - value *= wtVal; - } - if (window) { - float winVal = window->kernel[y][x]; - value *= winVal; - } - sum += value; - } - } - } - *y2 = sum; - return true; -} - -double p_pmSubSolve_ChiSquare (pmSubtractionKernels *kernels, const pmSubtractionStampList *stamps) { - - int footprint = stamps->footprint; // Half-size of stamps - int numKernels = kernels->num; // Number of kernels - - double sum = 0.0; - - psKernel *residual = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image - psImageInit(residual->image, 0.0); - - psImage *polyValues = NULL; // Polynomial values - - for (int i = 0; i < stamps->num; i++) { - - pmSubtractionStamp *stamp = stamps->stamps->data[i]; - if (stamp->status != PM_SUBTRACTION_STAMP_USED) continue; - - psKernel *weight = NULL; - psKernel *window = NULL; - psKernel *target = NULL; - psKernel *source = NULL; - - psArray *convolutions = NULL; - -#ifdef USE_WEIGHT - weight = stamp->weight; -#endif -#ifdef USE_WINDOW - window = stamps->window; -#endif - - switch (kernels->mode) { - // MODE_1 : convolve image 1 to match image 2 (and vice versa) - case PM_SUBTRACTION_MODE_1: - target = stamp->image2; - source = stamp->image1; - convolutions = stamp->convolutions1; - break; - case PM_SUBTRACTION_MODE_2: - target = stamp->image1; - source = stamp->image2; - convolutions = stamp->convolutions2; - break; - default: - psAbort ("programming error"); - } - - // Calculate coefficients of the kernel basis functions - polyValues = p_pmSubtractionPolynomial(polyValues, kernels->spatialOrder, stamp->xNorm, stamp->yNorm); - double norm = p_pmSubtractionSolutionNorm(kernels); // Normalisation - double background = p_pmSubtractionSolutionBackground(kernels, polyValues);// Difference in background - - psImageInit(residual->image, 0.0); - for (int j = 0; j < numKernels; j++) { - psKernel *convolution = convolutions->data[j]; // Convolution - double coefficient = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, false); // Coefficient - for (int y = - footprint; y <= footprint; y++) { - for (int x = - footprint; x <= footprint; x++) { - residual->kernel[y][x] -= convolution->kernel[y][x] * coefficient; - } - } - } - - for (int y = - footprint; y <= footprint; y++) { - for (int x = - footprint; x <= footprint; x++) { - double resid = target->kernel[y][x] - background - source->kernel[y][x] * norm + residual->kernel[y][x]; - double value = PS_SQR(resid); - if (weight) { - float wtVal = weight->kernel[y][x]; - value *= wtVal; - } - if (window) { - float winVal = window->kernel[y][x]; - value *= winVal; - } - sum += value; - } - } - } - psFree (polyValues); - psFree (residual); - - return sum; -} - -bool p_pmSubSolve_SetWeights (psVector *wApply, psVector *w, psVector *wMask) { - - for (int i = 0; i < w->n; i++) { - wApply->data.F64[i] = wMask->data.U8[i] ? 0.0 : w->data.F64[i]; - } - return true; -} - -// we are supplied V and w; w represents a diagonal matrix (also, we apply 1/w instead of w) -psImage *p_pmSubSolve_Xvar (psImage *V, psVector *w) { - - psAssert (w->n == V->numCols, "w and U dimensions do not match"); - - psImage *Vn = psImageAlloc (V->numCols, V->numRows, PS_TYPE_F64); - psImageInit (Vn, 0.0); - - // generate Vn = V * w^{-1} - for (int j = 0; j < Vn->numRows; j++) { - for (int i = 0; i < Vn->numCols; i++) { - if (!isfinite(w->data.F64[i])) continue; - if (w->data.F64[i] == 0.0) continue; - Vn->data.F64[j][i] = V->data.F64[j][i] / w->data.F64[i]; - } - } - - psImage *Xvar = psImageAlloc (V->numCols, V->numRows, PS_TYPE_F64); - psImageInit (Xvar, 0.0); - - // generate Xvar = Vn * Vn^T - for (int j = 0; j < Vn->numRows; j++) { - for (int i = 0; i < Vn->numCols; i++) { - double sum = 0.0; - for (int k = 0; k < Vn->numCols; k++) { - sum += Vn->data.F64[k][i]*Vn->data.F64[k][j]; - } - Xvar->data.F64[j][i] = sum; - } - } - return Xvar; } @@ -1864,5 +1946,4 @@ // of the elements of an image A(x,y) = A->data.F64[y][x] = A_x,y, a matrix // multiplication is: A_k,j * B_i,k = C_i,j - bool psFitsWriteImageSimple (char *filename, psImage *image, psMetadata *header) { Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.h (revision 30631) @@ -4,12 +4,5 @@ #include "pmSubtractionStamps.h" #include "pmSubtractionKernels.h" - -typedef enum { - PM_SUBTRACTION_EQUATION_NONE = 0x00, - PM_SUBTRACTION_EQUATION_NORM = 0x01, - PM_SUBTRACTION_EQUATION_BG = 0x02, - PM_SUBTRACTION_EQUATION_KERNELS = 0x04, - PM_SUBTRACTION_EQUATION_ALL = 0x07, // value should be NORM | BG | KERNELS -} pmSubtractionEquationCalculationMode; +#include "pmSubtraction.h" /// Execute a thread job to calculate the least-squares equation for a stamp @@ -20,18 +13,15 @@ bool pmSubtractionCalculateEquationStamp(pmSubtractionStampList *stamps, ///< Stamps pmSubtractionKernels *kernels, ///< Kernel parameters - int index, ///< Index of stamp - const pmSubtractionEquationCalculationMode mode + int index ///< Index of stamp ); /// Calculate the least-squares equation to match the image quality bool pmSubtractionCalculateEquation(pmSubtractionStampList *stamps, ///< Stamps - pmSubtractionKernels *kernels, ///< Kernel parameters - const pmSubtractionEquationCalculationMode mode + pmSubtractionKernels *kernels ///< Kernel parameters ); /// Solve the least-squares equation to match the image quality bool pmSubtractionSolveEquation(pmSubtractionKernels *kernels, ///< Kernel parameters - const pmSubtractionStampList *stamps, ///< Stamps - const pmSubtractionEquationCalculationMode mode + const pmSubtractionStampList *stamps ///< Stamps ); @@ -92,3 +82,6 @@ bool pmSubtractionCalculateMomentsKernel(double *Mxx, double *Myy, psKernel *image, int footprint, int window); +bool pmSubtractionChisqStats(psVector *fluxesVector, psVector *chisqDVector, psVector *chisqRVector, psVector *momentVector, psVector *stampMask, psKernel *convolved1, psKernel *convolved2, psKernel *difference, psKernel *residual, psKernel *weight, psKernel *window); + +bool pmSubtractionCalculateChisqAndMoments(pmSubtractionQuality **bestMatch, pmSubtractionStampList *stamps, pmSubtractionKernels *kernels); #endif Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.v0.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.v0.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionEquation.v0.c (revision 30631) @@ -9,4 +9,5 @@ #include "pmErrorCodes.h" #include "pmSubtraction.h" +#include "pmSubtractionTypes.h" #include "pmSubtractionKernels.h" #include "pmSubtractionStamps.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionHermitian.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionHermitian.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionHermitian.c (revision 30631) @@ -8,4 +8,5 @@ #include +#include "pmSubtractionTypes.h" #include "pmSubtractionHermitian.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionIO.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionIO.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionIO.c (revision 30631) @@ -12,4 +12,5 @@ #include "pmConceptsRead.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionKernels.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionKernels.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionKernels.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionKernels.c (revision 30631) @@ -8,4 +8,6 @@ #include +#include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionKernels.h" @@ -20,4 +22,6 @@ { psFree(kernels->description); + psFree(kernels->fwhms); + psFree(kernels->orders); psFree(kernels->u); psFree(kernels->v); @@ -30,4 +34,6 @@ psFree(kernels->solution1); psFree(kernels->solution2); + psFree(kernels->solution1err); + psFree(kernels->solution2err); psFree(kernels->sampleStamps); } @@ -419,18 +425,12 @@ int num = 0; // Number of basis functions - psString params = NULL; // List of parameters for (int i = 0; i < numGaussians; i++) { int gaussOrder = orders->data.S32[i]; // Polynomial order to apply to Gaussian - psStringAppend(¶ms, "(%.1f,%d)", fwhms->data.F32[i], orders->data.S32[i]); num += (gaussOrder + 1) * (gaussOrder + 2) / 2; } - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_ISIS, size, - spatialOrder, penalty, bounds, mode); // Kernels - psStringAppend(&kernels->description, "ISIS(%d,%s,%d,%.2e)", size, params, spatialOrder, penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "ISIS kernel: %s,%d --> %d elements", - params, spatialOrder, num); - psFree(params); + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_ISIS, size, fwhms, orders, spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); # if (!CENTRAL_DELTA && !ZERO_KERNEL_ZERO_FLUX) @@ -503,18 +503,13 @@ int num = 0; // Number of basis functions - psString params = NULL; // List of parameters for (int i = 0; i < numGaussians; i++) { int gaussOrder = orders->data.S32[i]; // Polynomial order to apply to Gaussian - psStringAppend(¶ms, "(%.1f,%d)", fwhms->data.F32[i], orders->data.S32[i]); num += (gaussOrder + 1) * (gaussOrder + 2) / 2; num += (11 - gaussOrder - 1); // include all higher order radial terms } - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_ISIS_RADIAL, size, - spatialOrder, penalty, bounds, mode); // Kernels - psStringAppend(&kernels->description, "ISIS_RADIAL(%d,%s,%d,%.2e)", size, params, spatialOrder, penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "ISIS_RADIAL kernel: %s,%d --> %d elements", params, spatialOrder, num); - psFree(params); + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_ISIS_RADIAL, size, fwhms, orders, spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); // Set the kernel parameters @@ -569,18 +564,12 @@ int num = 0; // Number of basis functions - psString params = NULL; // List of parameters for (int i = 0; i < numGaussians; i++) { int gaussOrder = orders->data.S32[i]; // Polynomial order to apply to Gaussian - psStringAppend(¶ms, "(%.1f,%d)", fwhms->data.F32[i], orders->data.S32[i]); num += (gaussOrder + 1) * (gaussOrder + 2) / 2; } - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_HERM, size, - spatialOrder, penalty, bounds, mode); // Kernels - psStringAppend(&kernels->description, "HERM(%d,%s,%d,%.2e)", size, params, spatialOrder, penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "HERM kernel: %s,%d --> %d elements", - params, spatialOrder, num); - psFree(params); + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_HERM, size, fwhms, orders, spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); // Set the kernel parameters @@ -627,17 +616,12 @@ int num = 0; // Number of basis functions - psString params = NULL; // List of parameters for (int i = 0; i < numGaussians; i++) { int gaussOrder = orders->data.S32[i]; // Polynomial order to apply to Gaussian - psStringAppend(¶ms, "(%.1f,%d)", fwhms->data.F32[i], orders->data.S32[i]); num += PS_SQR(gaussOrder + 1); } - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_DECONV_HERM, size, - spatialOrder, penalty, bounds, mode); // Kernels - psStringAppend(&kernels->description, "DECONV_HERM(%d,%s,%d,%.2e)", size, params, spatialOrder, penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "DECONVOLVED HERM kernel: %s,%d --> %d elements", params, spatialOrder, num); - psFree(params); + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_DECONV_HERM, size, fwhms, orders, spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); // XXXXX hard-wired reference sigma for now of 1.7 pix (== 4.0 pix fwhm == 1.0 arcsec in simtest) @@ -713,5 +697,5 @@ pmSubtractionKernels *pmSubtractionKernelsAlloc(int numBasisFunctions, pmSubtractionKernelsType type, - int size, int spatialOrder, float penalty, psRegion bounds, + int size, psVector *fwhms, psVector *orders, int spatialOrder, float penalty, psRegion bounds, pmSubtractionMode mode) { @@ -722,4 +706,6 @@ kernels->description = NULL; kernels->num = numBasisFunctions; + kernels->fwhms = psMemIncrRefCounter(fwhms); + kernels->orders = psMemIncrRefCounter(orders); kernels->u = psVectorAlloc(numBasisFunctions, PS_TYPE_S32); kernels->v = psVectorAlloc(numBasisFunctions, PS_TYPE_S32); @@ -740,4 +726,6 @@ kernels->size = size; kernels->inner = 0; + kernels->binning = 0; + kernels->ringsOrder = 0; kernels->spatialOrder = spatialOrder; kernels->bgOrder = 0; @@ -745,4 +733,6 @@ kernels->solution1 = NULL; kernels->solution2 = NULL; + kernels->solution1err = NULL; + kernels->solution2err = NULL; kernels->mean = NAN; kernels->rms = NAN; @@ -823,9 +813,7 @@ int num = PS_SQR(2 * size + 1) - 1; // Number of basis functions - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(0, PM_SUBTRACTION_KERNEL_POIS, size, - spatialOrder, penalty, bounds, mode); // Kernels - psStringAppend(&kernels->description, "POIS(%d,%d,%.2e)", size, spatialOrder, penalty); - psLogMsg("psModules.imcombine", PS_LOG_INFO, "POIS kernel: %d,%d --> %d elements", - size, spatialOrder, num); + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(0, PM_SUBTRACTION_KERNEL_POIS, size, NULL, NULL, spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); if (!p_pmSubtractionKernelsAddGrid(kernels, 0, size)) { @@ -871,12 +859,9 @@ psTrace("psModules.imcombine", 3, "Number of basis functions: %d\n", num); - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_SPAM, size, - spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_SPAM, size, NULL, NULL, spatialOrder, penalty, bounds, mode); // Kernels kernels->inner = inner; - psStringAppend(&kernels->description, "SPAM(%d,%d,%d,%d,%.2e)", size, inner, binning, spatialOrder, - penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "SPAM kernel: %d,%d,%d,%d --> %d elements", - size, inner, binning, spatialOrder, num); + kernels->binning = binning; + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); kernels->uStop = psVectorAlloc(num, PS_TYPE_S32); @@ -970,11 +955,8 @@ psTrace("psModules.imcombine", 3, "Number of basis functions: %d\n", num); - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_FRIES, size, - spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_FRIES, size, NULL, NULL, spatialOrder, penalty, bounds, mode); // Kernels kernels->inner = inner; - psStringAppend(&kernels->description, "FRIES(%d,%d,%d,%.2e)", size, inner, spatialOrder, penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "FRIES kernel: %d,%d,%d --> %d elements", - size, inner, spatialOrder, num); + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); kernels->uStop = psVectorAlloc(num, PS_TYPE_S32); @@ -1053,9 +1035,9 @@ PS_ASSERT_INT_LESS_THAN(inner, size, NULL); - pmSubtractionKernels *kernels = p_pmSubtractionKernelsRawISIS(size, spatialOrder, fwhms, orders, - penalty, bounds, mode); // Kernels + pmSubtractionKernels *kernels = p_pmSubtractionKernelsRawISIS(size, spatialOrder, fwhms, orders, penalty, bounds, mode); // Kernels kernels->type = PM_SUBTRACTION_KERNEL_GUNK; - psStringPrepend(&kernels->description, "GUNK="); - psStringAppend(&kernels->description, "+POIS(%d,%d)", inner, spatialOrder); + kernels->inner = inner; + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, (int) kernels->num); int numISIS = kernels->num; // Number of ISIS kernels @@ -1100,12 +1082,9 @@ int num = numRings * numPoly; // Total number of basis functions - pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_RINGS, size, - spatialOrder, penalty, bounds, mode); // Kernels + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_RINGS, size, NULL, NULL, spatialOrder, penalty, bounds, mode); // Kernels kernels->inner = inner; - psStringAppend(&kernels->description, "RINGS(%d,%d,%d,%d,%.2e)", size, inner, ringsOrder, spatialOrder, - penalty); - - psLogMsg("psModules.imcombine", PS_LOG_INFO, "RINGS kernel: %d,%d,%d,%d --> %d elements", - size, inner, ringsOrder, spatialOrder, num); + kernels->ringsOrder = ringsOrder; + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "kernel: %s --> %d elements", kernels->description, num); // Set the Gaussian kernel parameters @@ -1405,4 +1384,63 @@ } +bool pmSubtractionKernelsMakeDescription(pmSubtractionKernels *kernels) { + + // free if it exists + psFree (kernels->description); + + // generate the description parameter string + psString params = NULL; + if (kernels->fwhms) { + for (int i = 0; i < kernels->fwhms->n; i++) { + psStringAppend(¶ms, "(%.1f,%d)", kernels->fwhms->data.F32[i], kernels->orders->data.S32[i]); + } + } + + switch (kernels->type) { + case PM_SUBTRACTION_KERNEL_ISIS: + psStringAppend (&kernels->description, "ISIS(%d,%s,%d,%.2e)", kernels->size, params, kernels->spatialOrder, kernels->penalty); + break; + + case PM_SUBTRACTION_KERNEL_ISIS_RADIAL: + psStringAppend(&kernels->description, "ISIS_RADIAL(%d,%s,%d,%.2e)", kernels->size, params, kernels->spatialOrder, kernels->penalty); + break; + + case PM_SUBTRACTION_KERNEL_HERM: + psStringAppend(&kernels->description, "HERM(%d,%s,%d,%.2e)", kernels->size, params, kernels->spatialOrder, kernels->penalty); + break; + + case PM_SUBTRACTION_KERNEL_DECONV_HERM: + psStringAppend(&kernels->description, "DECONV_HERM(%d,%s,%d,%.2e)", kernels->size, params, kernels->spatialOrder, kernels->penalty); + break; + + case PM_SUBTRACTION_KERNEL_POIS: + psStringAppend(&kernels->description, "POIS(%d,%d,%.2e)", kernels->size, kernels->spatialOrder, kernels->penalty); + break; + + case PM_SUBTRACTION_KERNEL_SPAM: + psStringAppend(&kernels->description, "SPAM(%d,%d,%d,%d,%.2e)", kernels->size, kernels->inner, kernels->binning, kernels->spatialOrder, kernels->penalty); + break; + + case PM_SUBTRACTION_KERNEL_FRIES: + psStringAppend(&kernels->description, "FRIES(%d,%d,%d,%.2e)", kernels->size, kernels->inner, kernels->spatialOrder, kernels->penalty); + break; + + // Grid United with Normal Kernel [description: GUNK=ISIS(...)+POIS(...)] + case PM_SUBTRACTION_KERNEL_GUNK: + psStringAppend(&kernels->description, "GUNK=ISIS(%d,%s,%d,%.2e)", kernels->size, params, kernels->spatialOrder, kernels->penalty); + psStringAppend(&kernels->description, "+POIS(%d,%d)", kernels->inner, kernels->spatialOrder); + break; + + case PM_SUBTRACTION_KERNEL_RINGS: + psStringAppend(&kernels->description, "RINGS(%d,%d,%d,%d,%.2e)", kernels->size, kernels->inner, kernels->ringsOrder, kernels->spatialOrder, kernels->penalty); + break; + + default: + psAbort("unknown kernel"); + } + psFree (params); + return true; +} + pmSubtractionKernels *pmSubtractionKernelsCopy(const pmSubtractionKernels *in) { @@ -1435,4 +1473,6 @@ out->solution1 = in->solution1 ? psVectorCopy(NULL, in->solution1, PS_TYPE_F64) : NULL; out->solution2 = in->solution2 ? psVectorCopy(NULL, in->solution2, PS_TYPE_F64) : NULL; + out->solution1err = in->solution1err ? psVectorCopy(NULL, in->solution1err, PS_TYPE_F64) : NULL; + out->solution2err = in->solution2err ? psVectorCopy(NULL, in->solution2err, PS_TYPE_F64) : NULL; out->sampleStamps = psMemIncrRefCounter(in->sampleStamps); Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionKernels.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionKernels.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionKernels.h (revision 30631) @@ -2,72 +2,6 @@ #define PM_SUBTRACTION_KERNELS_H -#include -#include - -/// Type of subtraction kernel -typedef enum { - PM_SUBTRACTION_KERNEL_NONE, ///< Nothing --- an error - PM_SUBTRACTION_KERNEL_POIS, ///< Pan-STARRS Optimal Image Subtraction --- delta functions - PM_SUBTRACTION_KERNEL_ISIS, ///< Traditional kernel --- gaussians modified by polynomials - PM_SUBTRACTION_KERNEL_ISIS_RADIAL, ///< ISIS + higher-order radial Hermitians - PM_SUBTRACTION_KERNEL_HERM, ///< Hermitian polynomial kernels - PM_SUBTRACTION_KERNEL_DECONV_HERM, ///< Deconvolved Hermitian polynomial kernels - PM_SUBTRACTION_KERNEL_SPAM, ///< Summed Pixels for Advanced Matching --- summed delta functions - PM_SUBTRACTION_KERNEL_FRIES, ///< Fibonacci Radius Increases Excellence of Subtraction - PM_SUBTRACTION_KERNEL_GUNK, ///< Grid United with Normal Kernel --- POIS and ISIS hybrid - PM_SUBTRACTION_KERNEL_RINGS, ///< Rings Instead of the Normal Gaussian Subtraction -} pmSubtractionKernelsType; - -/// Modes --- specifies which image to convolve -typedef enum { - PM_SUBTRACTION_MODE_ERR, // Error in the mode - PM_SUBTRACTION_MODE_1, // Convolve image 1 - PM_SUBTRACTION_MODE_2, // Convolve image 2 - PM_SUBTRACTION_MODE_UNSURE, // Not sure yet which image to convolve so try to satisfy both - PM_SUBTRACTION_MODE_DUAL, // Dual convolution -} pmSubtractionMode; - -/// Kernels specification -typedef struct { - pmSubtractionKernelsType type; ///< Type of kernels --- allowing the use of multiple kernels - psString description; ///< Description of the kernel parameters - int xMin, xMax, yMin, yMax; ///< Bounds of image (for normalisation) - long num; ///< Number of kernel components (not including the spatial ones) - psVector *u, *v; ///< Offset (for POIS) or polynomial order (for ISIS, HERM or DECONV_HERM) - psVector *widths; ///< Gaussian FWHMs (ISIS, HERM or DECONV_HERM) - psVector *uStop, *vStop; ///< Width of kernel element (SPAM,FRIES only) - psArray *preCalc; ///< Array of images containing pre-calculated kernel (for ISIS, HERM or DECONV_HERM) - float penalty; ///< Penalty for wideness - psVector *penalties1; ///< Penalty for each kernel component - psVector *penalties2; ///< Penalty for each kernel component - bool havePenalties; ///< flag to test if we have already calculated the penalties or not. - int size; ///< The half-size of the kernel - int inner; ///< The size of an inner region - int spatialOrder; ///< The spatial order of the kernels - int bgOrder; ///< The order for the background fitting - pmSubtractionMode mode; ///< Mode for subtraction - psVector *solution1, *solution2; ///< Solution for the PSF matching - // Quality information - float mean, rms; ///< Mean and RMS of chi^2 from stamps - int numStamps; ///< Number of good stamps - float fResSigmaMean; ///< mean fractional stdev of residuals - float fResSigmaStdev; ///< stdev of fractional stdev of residuals - float fResOuterMean; ///< mean fractional positive swing in residuals - float fResOuterStdev; ///< stdev of fractional positive swing in residuals - float fResTotalMean; ///< mean fractional negative swing in residuals - float fResTotalStdev; ///< stdev of fractional negative swing in residuals - psArray *sampleStamps; ///< array of brightest set of stamps for output visualizations -} pmSubtractionKernels; - -// pmSubtractionKernels->preCalc is an array of pmSubtractionKernelPreCalc structures -typedef struct { - psVector *uCoords; // used by RINGS - psVector *vCoords; // used by RINGS - psVector *poly; // used by RINGS - - psVector *xKernel; // used by ISIS, HERM, DECONV_HERM - psVector *yKernel; // used by ISIS, HERM, DECONV_HERM - psKernel *kernel; // used by ISIS, HERM, DECONV_HERM -} pmSubtractionKernelPreCalc; +// #include +// #include // Assertion to check pmSubtractionKernels @@ -162,4 +96,6 @@ pmSubtractionKernelsType type, ///< Kernel type int size, ///< Half-size of kernel + psVector *fwhms, ///< requested kernel basis function + psVector *orders, int spatialOrder, ///< Order of spatial variations float penalty, ///< Penalty for wideness @@ -303,4 +239,7 @@ ); +bool pmSubtractionKernelsMakeDescription(pmSubtractionKernels *kernels); + + /// Copy kernels /// Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMask.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMask.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMask.c (revision 30631) @@ -7,4 +7,6 @@ #include "pmErrorCodes.h" +#include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionKernels.h" Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMatch.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMatch.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMatch.c (revision 30631) @@ -11,9 +11,10 @@ #include "pmFPA.h" #include "pmHDUUtils.h" +#include "pmSubtractionTypes.h" +#include "pmSubtraction.h" #include "pmSubtractionParams.h" #include "pmSubtractionKernels.h" #include "pmSubtractionStamps.h" #include "pmSubtractionEquation.h" -#include "pmSubtraction.h" #include "pmSubtractionAnalysis.h" #include "pmSubtractionMask.h" @@ -27,6 +28,4 @@ static bool useFFT = true; // Do convolutions using FFT - -# define SUBMODE PM_SUBTRACTION_EQUATION_ALL //#define TESTING @@ -59,53 +58,4 @@ } - -static bool subtractionGetStamps(pmSubtractionStampList **stamps, // Stamps to read - const pmReadout *ro1, // Readout 1 - const pmReadout *ro2, // Readout 2 - const psImage *subMask, // Mask for subtraction, or NULL - psImage *variance, // Variance map - const psRegion *region, // Region of interest - float thresh1, // Threshold for stamp finding on readout 1 - float thresh2, // Threshold for stamp finding on readout 2 - float stampSpacing, // Spacing between stamps - float normFrac, // Fraction of flux in window for normalisation window - float sysError, // Relative systematic error in images - float skyError, // Relative systematic error in images - int size, // Kernel half-size - int footprint, // Convolution footprint for stamps - pmSubtractionMode mode // Mode for subtraction - ) -{ - PS_ASSERT_PTR_NON_NULL(stamps, false); - PM_ASSERT_READOUT_NON_NULL(ro1, false); - PM_ASSERT_READOUT_NON_NULL(ro2, false); - PS_ASSERT_IMAGE_NON_NULL(subMask, false); - PS_ASSERT_IMAGE_NON_NULL(variance, false); - PS_ASSERT_PTR_NON_NULL(region, false); - - psTrace("psModules.imcombine", 3, "Finding stamps...\n"); - - psImage *image1 = ro1 ? ro1->image : NULL, *image2 = ro2 ? ro2->image : NULL; // Images of interest - - *stamps = pmSubtractionStampsFind(*stamps, image1, image2, subMask, region, thresh1, thresh2, - size, footprint, stampSpacing, normFrac, sysError, skyError, mode); - if (!*stamps) { - psError(psErrorCodeLast(), false, "Unable to find stamps."); - return false; - } - - memCheck(" find stamps"); - - psTrace("psModules.imcombine", 3, "Extracting stamps...\n"); - if (!pmSubtractionStampsExtract(*stamps, ro1->image, ro2->image, variance, size, *region)) { - psError(psErrorCodeLast(), false, "Unable to extract stamps."); - return false; - } - - memCheck(" extract stamps"); - pmSubtractionVisualPlotStamps(*stamps, (pmReadout *) ro1); - return true; -} - // Check input arguments static bool subtractionMatchCheck(pmReadout *conv1, pmReadout *conv2, // Convolved images @@ -123,5 +73,5 @@ float badFrac, // Maximum fraction of bad input pixels to accept pmSubtractionMode subMode // Mode of subtraction - ) + ) { if (subMode != PM_SUBTRACTION_MODE_2) { @@ -473,4 +423,55 @@ } +bool pmSubtractionMatchAttempt(pmSubtractionQuality **bestMatch, pmSubtractionKernels *kernels, pmSubtractionStampList *stamps, pmSubtractionMode mode, int spatialOrder, bool final) { + + pmSubtractionMode nativeMode = kernels->mode; + pmSubtractionMode nativeOrder = kernels->spatialOrder; + + kernels->mode = mode; + kernels->spatialOrder = spatialOrder; + + // we always need to recalculate the matrix equation elements... + pmSubtractionStampsResetStatus(stamps); + + psTrace("psModules.imcombine", 3, "Convolving stamps as needed...\n"); + if (!pmSubtractionConvolveStamps(stamps, kernels)) { + psError(psErrorCodeLast(), false, "Unable to convolve stamps."); + return false; + } + + // step 1: generate the elements of the matrix equation Ax = B + psTrace("psModules.imcombine", 3, "Calculating kernel equations...\n"); + if (!pmSubtractionCalculateEquation(stamps, kernels)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + return false; + } + + // step 2: solve the matrix equation Ax = B + psTrace("psModules.imcombine", 3, "Solving kernel equations...\n"); + if (!pmSubtractionSolveEquation(kernels, stamps)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + return false; + } + memCheck(" solve equation"); + + // calculate the score for this model fit attempt + // XXX store the chisq, flux and moments for stamp rejection + pmSubtractionCalculateChisqAndMoments(bestMatch, stamps, kernels); // Stamp deviations + + // display the input and model stamps + pmSubtractionVisualShowFit(stamps, kernels); + pmSubtractionVisualPlotFit(kernels); + pmSubtractionVisualPlotConvKernels(kernels); + + // reset the kernel if desired (on final pass, do not reset) + if (!final) { + kernels->mode = nativeMode; + kernels->spatialOrder = nativeOrder; + } else { + pmSubtractionKernelsMakeDescription(kernels); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "final kernel: %s", kernels->description); + } + return true; +} bool pmSubtractionMatch(pmReadout *conv1, pmReadout *conv2, const pmReadout *ro1, const pmReadout *ro2, @@ -478,5 +479,5 @@ const psArray *sources, const char *stampsName, pmSubtractionKernelsType type, int size, int spatialOrder, - const psVector *isisWidths, const psVector *isisOrders, + psVector *isisWidths, const psVector *isisOrders, int inner, int ringsOrder, int binning, float penalty, bool optimum, const psVector *optFWHMs, int optOrder, float optThreshold, @@ -559,4 +560,30 @@ psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + pmSubtractionQuality *bestMatch = NULL; + + int N_TEST_MODES; + int N_TEST_ORDER = spatialOrder; + + pmSubtractionMode TestModes[3]; + switch (subMode) { + case PM_SUBTRACTION_MODE_1: + N_TEST_MODES = 1; + TestModes[0] = PM_SUBTRACTION_MODE_1; + break; + case PM_SUBTRACTION_MODE_2: + N_TEST_MODES = 1; + TestModes[0] = PM_SUBTRACTION_MODE_2; + break; + case PM_SUBTRACTION_MODE_DUAL: + N_TEST_MODES = 3; + TestModes[0] = PM_SUBTRACTION_MODE_1; + TestModes[1] = PM_SUBTRACTION_MODE_2; + TestModes[2] = PM_SUBTRACTION_MODE_DUAL; + break; + default: + psError(psErrorCodeLast(), false, "For now, only modes 1, 2, and DUAL are supported."); + goto MATCH_ERROR; + } + memCheck("start"); @@ -628,5 +655,5 @@ regionString = psRegionToString(*region); psLogMsg("psModules.imcombine", PS_LOG_DETAIL, "Iso-kernel region: %s out of %d,%d\n", - regionString, numCols, numRows); + regionString, numCols, numRows); if (stampsName && strlen(stampsName) > 0) { @@ -640,167 +667,182 @@ } - // We get the stamps here; we will also attempt to get stamps at the first iteration, but it - // doesn't matter. - if (!subtractionGetStamps(&stamps, ro1, ro2, subMask, variance, region, stampThresh1, stampThresh2, - stampSpacing, normFrac, sysError, skyError, size, footprint, subMode)) { - goto MATCH_ERROR; - } - - - // generate the window function from the set of stamps - if (!pmSubtractionStampsGetWindow(stamps, size)) { - psError(psErrorCodeLast(), false, "Unable to get stamp window."); - goto MATCH_ERROR; - } - - // Define kernel basis functions - if (optimum && (type == PM_SUBTRACTION_KERNEL_ISIS || type == PM_SUBTRACTION_KERNEL_GUNK)) { - kernels = pmSubtractionKernelsOptimumISIS(type, size, inner, spatialOrder, - optFWHMs, optOrder, stamps, footprint, - optThreshold, penalty, bounds, subMode); - if (!kernels) { - psErrorClear(); - psWarning("Unable to derive optimum ISIS kernel --- switching to default."); - } - } - if (kernels == NULL) { - // Not an ISIS/GUNK kernel, or the optimum kernel search failed - kernels = pmSubtractionKernelsGenerate(type, size, spatialOrder, isisWidths, isisOrders, - inner, binning, ringsOrder, penalty, bounds, subMode); - } - - memCheck("kernels"); - - if (subMode == PM_SUBTRACTION_MODE_UNSURE) { - pmSubtractionMode newMode = pmSubtractionBestMode(&stamps, &kernels, subMask, rej); - switch (newMode) { - case PM_SUBTRACTION_MODE_1: - psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolving image 1 to match image 2."); - break; - case PM_SUBTRACTION_MODE_2: - psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolving image 2 to match image 1."); - break; - default: - psError(psErrorCodeLast(), false, "Unable to determine subtraction order."); - goto MATCH_ERROR; - } - subMode = newMode; - } - - int numRejected = -1; // Number of rejected stamps in each iteration - for (int k = 0; k < iter && numRejected != 0; k++) { - psLogMsg("psModules.imcombine", PS_LOG_INFO, "Iteration %d.", k); - - if (!subtractionGetStamps(&stamps, ro1, ro2, subMask, variance, region, - stampThresh1, stampThresh2, stampSpacing, normFrac, - sysError, skyError, size, footprint, subMode)) { - goto MATCH_ERROR; - } - - // generate the window function from the set of stamps - if (!pmSubtractionStampsGetWindow(stamps, size)) { - psError(psErrorCodeLast(), false, "Unable to get stamps window."); - goto MATCH_ERROR; - } + bool tryAgain = true; + while (tryAgain) { + // We get the stamps here; we will also attempt to get stamps at the first iteration, but it + // doesn't matter. + if (!pmSubtractionStampsSelect(&stamps, ro1, ro2, subMask, variance, region, stampThresh1, stampThresh2, + stampSpacing, normFrac, sysError, skyError, size, footprint, subMode)) { + goto MATCH_ERROR; + } + + // generate the window function from the set of stamps + if (!pmSubtractionStampsGetWindow(&tryAgain, stamps, size)) { + // if we failed, it might be due to the desired normWindow being larger than the current footprint. + // in this case, just adjust the footprint and try again. + if (tryAgain) { + // keep the border constant + int border = footprint - size; + size = PS_MAX(stamps->normWindow1, stamps->normWindow2) + 2; + footprint = size + border; + + // we need to reconstruct everything, so just free the stamps here and retry + psFree(stamps); + } else { + // unrecoverable error + psError(psErrorCodeLast(), false, "Unable to get stamp window."); + goto MATCH_ERROR; + } + } + } + + // check on the kernel scaling -- if the kron-based radial moments are very different, adjust to match them + { + // float fwhm1; + // float fwhm2; + // pmSubtractionGetFWHMs(&fwhm1, &fwhm2); + // psAssert(isfinite(fwhm1), "fwhm 1 not set"); + // psAssert(isfinite(fwhm2), "fwhm 2 not set"); + + // XXX this is BAD: depends on the relationship below: + // stamps->normWindow1 = 2.75*R1; + // stamps->normWindow2 = 2.75*R2; + float radMoment1 = stamps->normWindow1 / 2.75; + float radMoment2 = stamps->normWindow2 / 2.75; + pmSubtractionParamsScale(NULL, NULL, isisWidths, radMoment1, radMoment2); + + // float maxFWHM = PS_MAX(fwhm1, fwhm2); + // float maxRadial = PS_MAX(radMoment1, radMoment2); + + // if (fabs(2.0*(maxFWHM - maxRadial)/(maxFWHM + maxRadial)) > 0.25) { + // if (1) { + // + // float scale = maxRadial / maxFWHM; + // psLogMsg ("psModules.imcombine", PS_LOG_INFO, "Kron and FWHM scales are quite different, re-scale by %f to use Kron", scale); + // + // for (int i = 0; i < isisWidths->n; i++) { + // isisWidths->data.F32[i] *= scale; + // } + // } + } + + // Define kernel basis functions + if (optimum && (type == PM_SUBTRACTION_KERNEL_ISIS || type == PM_SUBTRACTION_KERNEL_GUNK)) { + kernels = pmSubtractionKernelsOptimumISIS(type, size, inner, spatialOrder, + optFWHMs, optOrder, stamps, footprint, + optThreshold, penalty, bounds, subMode); + if (!kernels) { + psErrorClear(); + psWarning("Unable to derive optimum ISIS kernel --- switching to default."); + } + } + if (kernels == NULL) { + // Not an ISIS/GUNK kernel, or the optimum kernel search failed + kernels = pmSubtractionKernelsGenerate(type, size, spatialOrder, isisWidths, isisOrders, + inner, binning, ringsOrder, penalty, bounds, subMode); + } + + memCheck("kernels"); + + if (subMode == PM_SUBTRACTION_MODE_UNSURE) { + pmSubtractionMode newMode = pmSubtractionBestMode(&stamps, &kernels, subMask, rej); + switch (newMode) { + case PM_SUBTRACTION_MODE_1: + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolving image 1 to match image 2."); + break; + case PM_SUBTRACTION_MODE_2: + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolving image 2 to match image 1."); + break; + default: + psError(psErrorCodeLast(), false, "Unable to determine subtraction order."); + goto MATCH_ERROR; + } + subMode = newMode; + } + + int numRejected = -1; // Number of rejected stamps in each iteration + for (int k = 0; (k < iter) && (numRejected != 0); k++) { + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Iteration %d.", k); + + bool tryAgain = true; + while (tryAgain) { + // We get the stamps here; we will also attempt to get stamps at the first iteration, but it + // doesn't matter. + if (!pmSubtractionStampsSelect(&stamps, ro1, ro2, subMask, variance, region, stampThresh1, stampThresh2, + stampSpacing, normFrac, sysError, skyError, size, footprint, subMode)) { + goto MATCH_ERROR; + } + + // generate the window function from the set of stamps + if (!pmSubtractionStampsGetWindow(&tryAgain, stamps, size)) { + // if we failed, it might be due to the desired normWindow being larger than the current footprint. + // in this case, just adjust the footprint and try again. + if (tryAgain) { + footprint = PS_MAX(stamps->normWindow1, stamps->normWindow2) + 2; + + // we need to reconstruct everything, so just free the stamps here and retry + psFree(stamps); + } else { + // unrecoverable error + psError(psErrorCodeLast(), false, "Unable to get stamp window."); + goto MATCH_ERROR; + } + } + } // step 0 : calculate the normalizations, pass along to the next steps via stamps->normValue - psTrace("psModules.imcombine", 3, "Calculating normalization...\n"); - if (!pmSubtractionCalculateNormalization(stamps, kernels->mode)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - goto MATCH_ERROR; - } - - // step 0b : calculate the moments, pass along to the next steps via stamps->normValue - // XXX this step is now skipped -- delete - psTrace("psModules.imcombine", 3, "Calculating normalization...\n"); - if (!pmSubtractionCalculateMoments(kernels, stamps)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - goto MATCH_ERROR; - } - - // step 1: generate the elements of the matrix equation Ax = B - psTrace("psModules.imcombine", 3, "Calculating kernel equations...\n"); - if (!pmSubtractionCalculateEquation(stamps, kernels, SUBMODE)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - goto MATCH_ERROR; - } - - // step 2: solve the matrix equation Ax = B - psTrace("psModules.imcombine", 3, "Solving kernel equations...\n"); - if (!pmSubtractionSolveEquation(kernels, stamps, SUBMODE)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - goto MATCH_ERROR; - } - memCheck(" solve equation"); - - pmSubtractionVisualPlotConvKernels(kernels); - - psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations - if (!deviations) { - psError(psErrorCodeLast(), false, "Unable to calculate deviations."); - goto MATCH_ERROR; - } - memCheck(" calculate deviations"); - - psTrace("psModules.imcombine", 3, "Rejecting stamps...\n"); - numRejected = pmSubtractionRejectStamps(kernels, stamps, deviations, subMask, rej); - if (numRejected < 0) { - psError(psErrorCodeLast(), false, "Unable to reject stamps."); - psFree(deviations); - goto MATCH_ERROR; - } - psFree(deviations); - - memCheck(" reject stamps"); - } - - // if we hit the max number of iterations and we have rejected stamps, re-solve - if (numRejected > 0) { - - // step 1: generate the elements of the matrix equation Ax = B - psTrace("psModules.imcombine", 3, "Calculating equation for normalization...\n"); - if (!pmSubtractionCalculateEquation(stamps, kernels, SUBMODE)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - goto MATCH_ERROR; - } - - // step 2: solve the matrix equation Ax = B - psTrace("psModules.imcombine", 3, "Solving equation for kernels...\n"); - if (!pmSubtractionSolveEquation(kernels, stamps, SUBMODE)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - goto MATCH_ERROR; - } - - pmSubtractionVisualPlotConvKernels(kernels); - - psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations - if (!deviations) { - psError(psErrorCodeLast(), false, "Unable to calculate deviations."); - goto MATCH_ERROR; - } - pmSubtractionRejectStamps(kernels, stamps, deviations, subMask, NAN); - psFree(deviations); - } - psFree(stamps); - stamps = NULL; - - memCheck("solution"); - - if (!pmSubtractionAnalysis(analysis, header, kernels, region, numCols, numRows)) { - psError(psErrorCodeLast(), false, "Unable to generate QA data"); - goto MATCH_ERROR; - } - memCheck("diag outputs"); - - psTrace("psModules.imcombine", 2, "Convolving...\n"); - if (!pmSubtractionConvolve(conv1, conv2, ro1, ro2, subMask, stride, maskBad, maskPoor, poorFrac, - kernelError, covarFrac, region, kernels, true, useFFT)) { - psError(psErrorCodeLast(), false, "Unable to convolve image."); - goto MATCH_ERROR; - } - - psFree(kernels); - kernels = NULL; - } + psTrace("psModules.imcombine", 3, "Calculating normalization...\n"); + if (!pmSubtractionCalculateNormalization(stamps, kernels->mode)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + goto MATCH_ERROR; + } + + // on each iteration, we start from scratch + psFree(bestMatch); + + // choose the spatial order and subtraction direction (1, 2, dual) + // XXX need to make these respect recipe somewhat + for (int order = 0; order <= N_TEST_ORDER; order++) { + for (int j = 0; j < N_TEST_MODES; j++) { + if (!pmSubtractionMatchAttempt(&bestMatch, kernels, stamps, TestModes[j], order, false)) { + goto MATCH_ERROR; + } + } + } + + // reject the deviant stamps based on the stats of the best match + psTrace("psModules.imcombine", 3, "Rejecting stamps...\n"); + numRejected = pmSubtractionRejectStamps(kernels, stamps, bestMatch, subMask, rej); + if (numRejected < 0) { + psError(psErrorCodeLast(), false, "Unable to reject stamps."); + goto MATCH_ERROR; + } + memCheck(" reject stamps"); + } + + // apply the best fit so we are ready to roll + psLogMsg("psModules.imcombine", PS_LOG_INFO, "applying order: %d, mode: %d\n", bestMatch->spatialOrder, bestMatch->mode); + if (!pmSubtractionMatchAttempt(NULL, kernels, stamps, bestMatch->mode, bestMatch->spatialOrder, true)) { + goto MATCH_ERROR; + } + psFree(stamps); + psFree(bestMatch); + memCheck("solution"); + + if (!pmSubtractionAnalysis(analysis, header, kernels, region, numCols, numRows)) { + psError(psErrorCodeLast(), false, "Unable to generate QA data"); + goto MATCH_ERROR; + } + memCheck("diag outputs"); + + psTrace("psModules.imcombine", 2, "Convolving...\n"); + if (!pmSubtractionConvolve(conv1, conv2, ro1, ro2, subMask, stride, maskBad, maskPoor, poorFrac, + kernelError, covarFrac, region, kernels, true, useFFT)) { + psError(psErrorCodeLast(), false, "Unable to convolve image."); + goto MATCH_ERROR; + } + + psFree(kernels); + kernels = NULL; + } } psFree(rng); @@ -816,10 +858,10 @@ if (conv1 && !pmSubtractionBorder(conv1->image, conv1->variance, conv1->mask, size, maskBad)) { - psError(psErrorCodeLast(), false, "Unable to set border of convolved image."); - goto MATCH_ERROR; + psError(psErrorCodeLast(), false, "Unable to set border of convolved image."); + goto MATCH_ERROR; } if (conv2 && !pmSubtractionBorder(conv2->image, conv2->variance, conv2->mask, size, maskBad)) { - psError(psErrorCodeLast(), false, "Unable to set border of convolved image."); - goto MATCH_ERROR; + psError(psErrorCodeLast(), false, "Unable to set border of convolved image."); + goto MATCH_ERROR; } @@ -832,15 +874,15 @@ #ifdef TESTING { - if (subMode == PM_SUBTRACTION_MODE_1 || subMode == PM_SUBTRACTION_MODE_DUAL) { - psFits *fits = psFitsOpen("convolved1.fits", "w"); - psFitsWriteImage(fits, NULL, conv1->image, 0, NULL); - psFitsClose(fits); - } - - if (subMode == PM_SUBTRACTION_MODE_2 || subMode == PM_SUBTRACTION_MODE_DUAL) { - psFits *fits = psFitsOpen("convolved2.fits", "w"); - psFitsWriteImage(fits, NULL, conv2->image, 0, NULL); - psFitsClose(fits); - } + if (subMode == PM_SUBTRACTION_MODE_1 || subMode == PM_SUBTRACTION_MODE_DUAL) { + psFits *fits = psFitsOpen("convolved1.fits", "w"); + psFitsWriteImage(fits, NULL, conv1->image, 0, NULL); + psFitsClose(fits); + } + + if (subMode == PM_SUBTRACTION_MODE_2 || subMode == PM_SUBTRACTION_MODE_DUAL) { + psFits *fits = psFitsOpen("convolved2.fits", "w"); + psFitsWriteImage(fits, NULL, conv2->image, 0, NULL); + psFitsClose(fits); + } } #endif @@ -858,4 +900,5 @@ psFree(variance); psFree(rng); + psFree(bestMatch); return false; } @@ -866,8 +909,8 @@ // increment). static int subtractionOrderWidth(const psKernel *kernel, // Image - float bg, // Background in image - int size, // Maximum size - const psArray *models, // Buffer of models - const psVector *modelSums // Buffer of model sums + float bg, // Background in image + int size, // Maximum size + const psArray *models, // Buffer of models + const psVector *modelSums // Buffer of model sums ) { @@ -882,20 +925,20 @@ psVector *chi2 = psVectorAlloc(size, PS_TYPE_F32); // chi^2 as a function of radius for (int sigma = 0; sigma < size; sigma++) { - double sumFG = 0.0; // Sum for calculating the normalisation of the Gaussian - psKernel *model = models->data[sigma]; // Model of interest - for (int y = yMin; y <= yMax; y++) { - for (int x = xMin; x <= xMax; x++) { - sumFG += model->kernel[y][x] * (kernel->kernel[y][x] - bg); - } - } - float norm = sumFG * modelSums->data.F64[sigma]; // Normalisation for Gaussian - double sumDev2 = 0.0; // Sum of square deviations - for (int y = yMin; y <= yMax; y++) { - for (int x = xMin; x <= xMax; x++) { - float dev = kernel->kernel[y][x] - bg - norm * model->kernel[y][x]; // Deviation - sumDev2 += PS_SQR(dev); - } - } - chi2->data.F32[sigma] = sumDev2; + double sumFG = 0.0; // Sum for calculating the normalisation of the Gaussian + psKernel *model = models->data[sigma]; // Model of interest + for (int y = yMin; y <= yMax; y++) { + for (int x = xMin; x <= xMax; x++) { + sumFG += model->kernel[y][x] * (kernel->kernel[y][x] - bg); + } + } + float norm = sumFG * modelSums->data.F64[sigma]; // Normalisation for Gaussian + double sumDev2 = 0.0; // Sum of square deviations + for (int y = yMin; y <= yMax; y++) { + for (int x = xMin; x <= xMax; x++) { + float dev = kernel->kernel[y][x] - bg - norm * model->kernel[y][x]; // Deviation + sumDev2 += PS_SQR(dev); + } + } + chi2->data.F32[sigma] = sumDev2; } @@ -904,8 +947,8 @@ float bestChi2 = INFINITY; // Best chi^2 for (int i = 0; i < size; i++) { - if (chi2->data.F32[i] < bestChi2) { - bestChi2 = chi2->data.F32[i]; - bestIndex = i; - } + if (chi2->data.F32[i] < bestChi2) { + bestChi2 = chi2->data.F32[i]; + bestIndex = i; + } } psFree(chi2); @@ -916,6 +959,6 @@ bool pmSubtractionOrderStamp(psVector *ratios, psVector *mask, const pmSubtractionStampList *stamps, - const psArray *models, const psVector *modelSums, - int index, float bg1, float bg2) + const psArray *models, const psVector *modelSums, + int index, float bg1, float bg2) { PS_ASSERT_VECTOR_NON_NULL(ratios, false); @@ -931,5 +974,5 @@ pmSubtractionStamp *stamp = stamps->stamps->data[index]; // Stamp of interest psAssert(stamp->status == PM_SUBTRACTION_STAMP_CALCULATE || stamp->status == PM_SUBTRACTION_STAMP_USED, - "We checked this earlier."); + "We checked this earlier."); // Widths of stars @@ -938,11 +981,11 @@ if (width1 == 0 || width2 == 0) { - ratios->data.F32[index] = NAN; - mask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0xff; + ratios->data.F32[index] = NAN; + mask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0xff; } else { - ratios->data.F32[index] = (float)width1 / (float)width2; - mask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0; - psTrace("psModules.imcombine", 3, "Stamp %d (%.1f,%.1f) widths: %d, %d --> %f\n", - index, stamp->x, stamp->y, width1, width2, ratios->data.F32[index]); + ratios->data.F32[index] = (float)width1 / (float)width2; + mask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0; + psTrace("psModules.imcombine", 3, "Stamp %d (%.1f,%.1f) widths: %d, %d --> %f\n", + index, stamp->x, stamp->y, width1, width2, ratios->data.F32[index]); } @@ -978,60 +1021,60 @@ psVector *modelSums = psVectorAlloc(size, PS_TYPE_F64); // Gaussian model sums for (int sigma = 0; sigma < size; sigma++) { - psKernel *model = psKernelAlloc(-size, size, -size, size); // Gaussian model - float invSigma2 = 1.0 / (float)PS_SQR(1 + sigma); // Inverse sigma squared - double sumGG = 0.0; // Sum of square of Gaussian - for (int y = -size; y <= size; y++) { - int y2 = PS_SQR(y); // y squared - for (int x = -size; x <= size; x++) { - float rad2 = PS_SQR(x) + y2; // Radius squared - float value = expf(-rad2 * invSigma2); // Model value - model->kernel[y][x] = value; - sumGG += PS_SQR(value); - } - } - models->data[sigma] = model; - modelSums->data.F64[sigma] = 1.0 / sumGG; + psKernel *model = psKernelAlloc(-size, size, -size, size); // Gaussian model + float invSigma2 = 1.0 / (float)PS_SQR(1 + sigma); // Inverse sigma squared + double sumGG = 0.0; // Sum of square of Gaussian + for (int y = -size; y <= size; y++) { + int y2 = PS_SQR(y); // y squared + for (int x = -size; x <= size; x++) { + float rad2 = PS_SQR(x) + y2; // Radius squared + float value = expf(-rad2 * invSigma2); // Model value + model->kernel[y][x] = value; + sumGG += PS_SQR(value); + } + } + models->data[sigma] = model; + modelSums->data.F64[sigma] = 1.0 / sumGG; } // Fit models to stamps for (int i = 0; i < stamps->num; i++) { - pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest - if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE && stamp->status != PM_SUBTRACTION_STAMP_USED) { - mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xff; - continue; - } - - if (pmSubtractionThreaded()) { - psThreadJob *job = psThreadJobAlloc("PSMODULES_SUBTRACTION_ORDER"); - psArrayAdd(job->args, 1, ratios); - psArrayAdd(job->args, 1, mask); - psArrayAdd(job->args, 1, stamps); - psArrayAdd(job->args, 1, models); - psArrayAdd(job->args, 1, modelSums); - PS_ARRAY_ADD_SCALAR(job->args, i, PS_TYPE_S32); - PS_ARRAY_ADD_SCALAR(job->args, bg1, PS_TYPE_F32); - PS_ARRAY_ADD_SCALAR(job->args, bg2, PS_TYPE_F32); - if (!psThreadJobAddPending(job)) { - return false; - } - } else { - if (!pmSubtractionOrderStamp(ratios, mask, stamps, models, modelSums, i, bg1, bg2)) { - psError(psErrorCodeLast(), false, "Unable to measure PSF width for stamp %d", i); - psFree(models); - psFree(modelSums); - psFree(ratios); - psFree(mask); - return false; - } - } + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE && stamp->status != PM_SUBTRACTION_STAMP_USED) { + mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xff; + continue; + } + + if (pmSubtractionThreaded()) { + psThreadJob *job = psThreadJobAlloc("PSMODULES_SUBTRACTION_ORDER"); + psArrayAdd(job->args, 1, ratios); + psArrayAdd(job->args, 1, mask); + psArrayAdd(job->args, 1, stamps); + psArrayAdd(job->args, 1, models); + psArrayAdd(job->args, 1, modelSums); + PS_ARRAY_ADD_SCALAR(job->args, i, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, bg1, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, bg2, PS_TYPE_F32); + if (!psThreadJobAddPending(job)) { + return false; + } + } else { + if (!pmSubtractionOrderStamp(ratios, mask, stamps, models, modelSums, i, bg1, bg2)) { + psError(psErrorCodeLast(), false, "Unable to measure PSF width for stamp %d", i); + psFree(models); + psFree(modelSums); + psFree(ratios); + psFree(mask); + return false; + } + } } if (!psThreadPoolWait(true)) { - psError(psErrorCodeLast(), false, "Error waiting for threads."); - psFree(models); - psFree(modelSums); - psFree(ratios); - psFree(mask); - return false; + psError(psErrorCodeLast(), false, "Error waiting for threads."); + psFree(models); + psFree(modelSums); + psFree(ratios); + psFree(mask); + return false; } @@ -1041,9 +1084,9 @@ psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); if (!psVectorStats(stats, ratios, NULL, mask, 0xff)) { - psError(psErrorCodeLast(), false, "Unable to calculate statistics for moments ratio."); - psFree(mask); - psFree(ratios); - psFree(stats); - return PM_SUBTRACTION_MODE_ERR; + psError(psErrorCodeLast(), false, "Unable to calculate statistics for moments ratio."); + psFree(mask); + psFree(ratios); + psFree(stats); + return PM_SUBTRACTION_MODE_ERR; } psFree(ratios); @@ -1052,6 +1095,6 @@ // XXX raise an error here or not? if (isnan(stats->robustMedian)) { - psFree(stats); - return PM_SUBTRACTION_MODE_ERR; + psFree(stats); + return PM_SUBTRACTION_MODE_ERR; } @@ -1066,23 +1109,31 @@ // Test a subtraction mode by performing a single iteration static bool subtractionModeTest(pmSubtractionStampList *stamps, // Stamps to use to find best mode - pmSubtractionKernels *kernels, // Kernel description - const char *description, // Description for trace - psImage *subMask, // Subtraction mask - float rej // Rejection threshold - ) + pmSubtractionKernels *kernels, // Kernel description + const char *description, // Description for trace + psImage *subMask, // Subtraction mask + float rej // Rejection threshold + ) { assert(stamps); assert(kernels); + psAbort("this function is not working"); +# if (0) + psTrace("psModules.imcombine", 3, "Convolving stamps as needed...\n"); + if (!pmSubtractionConvolveStamps(stamps, kernels)) { + psError(psErrorCodeLast(), false, "Unable to convolve stamps."); + return false; + } + psTrace("psModules.imcombine", 3, "Calculating %s normalization equation...\n", description); - if (!pmSubtractionCalculateEquation(stamps, kernels, SUBMODE)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - return false; + if (!pmSubtractionCalculateEquation(stamps, kernels)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + return false; } psTrace("psModules.imcombine", 3, "Solving %s normalization equation...\n", description); - if (!pmSubtractionSolveEquation(kernels, stamps, SUBMODE)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - return false; + if (!pmSubtractionSolveEquation(kernels, stamps)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + return false; } @@ -1090,47 +1141,48 @@ psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations if (!deviations) { - psError(psErrorCodeLast(), false, "Unable to calculate deviations."); - return false; - } - + psError(psErrorCodeLast(), false, "Unable to calculate deviations."); + return false; + } + + // XXX this needs to be made consistent with the modified 'reject stamps' function psTrace("psModules.imcombine", 3, "Rejecting %s stamps...\n", description); long numRejected = pmSubtractionRejectStamps(kernels, stamps, deviations, subMask, rej); if (numRejected < 0) { - psError(psErrorCodeLast(), false, "Unable to reject stamps."); - psFree(deviations); - return false; + psError(psErrorCodeLast(), false, "Unable to reject stamps."); + psFree(deviations); + return false; } psFree(deviations); if (numRejected > 0) { - // Allow re-fit with reduced stamps set - psTrace("psModules.imcombine", 3, "Calculating %s normalization equation...\n", description); - if (!pmSubtractionCalculateEquation(stamps, kernels, PM_SUBTRACTION_EQUATION_ALL)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - return false; - } - - psTrace("psModules.imcombine", 3, "Resolving %s equation...\n", description); - if (!pmSubtractionSolveEquation(kernels, stamps, PM_SUBTRACTION_EQUATION_ALL)) { - psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); - return false; - } - psTrace("psModules.imcombine", 3, "Recalculate %s deviations...\n", description); - - psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations - if (!deviations) { - psError(psErrorCodeLast(), false, "Unable to calculate deviations."); - return false; - } - psTrace("psModules.imcombine", 3, "Measuring %s quality...\n", description); - long numRejected = pmSubtractionRejectStamps(kernels, stamps, deviations, subMask, NAN); - if (numRejected < 0) { - psError(psErrorCodeLast(), false, "Unable to reject stamps."); - psFree(deviations); - return false; - } - psFree(deviations); - } - + // Allow re-fit with reduced stamps set + psTrace("psModules.imcombine", 3, "Calculating %s normalization equation...\n", description); + if (!pmSubtractionCalculateEquation(stamps, kernels)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + return false; + } + + psTrace("psModules.imcombine", 3, "Resolving %s equation...\n", description); + if (!pmSubtractionSolveEquation(kernels, stamps)) { + psError(psErrorCodeLast(), false, "Unable to calculate least-squares equation."); + return false; + } + psTrace("psModules.imcombine", 3, "Recalculate %s deviations...\n", description); + + psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations + if (!deviations) { + psError(psErrorCodeLast(), false, "Unable to calculate deviations."); + return false; + } + psTrace("psModules.imcombine", 3, "Measuring %s quality...\n", description); + long numRejected = pmSubtractionRejectStamps(kernels, stamps, deviations, subMask, NAN); + if (numRejected < 0) { + psError(psErrorCodeLast(), false, "Unable to reject stamps."); + psFree(deviations); + return false; + } + psFree(deviations); + } +# endif return true; } @@ -1138,5 +1190,5 @@ pmSubtractionMode pmSubtractionBestMode(pmSubtractionStampList **stamps, pmSubtractionKernels **kernels, - const psImage *subMask, float rej) + const psImage *subMask, float rej) { PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(*stamps, PM_SUBTRACTION_MODE_ERR); @@ -1150,9 +1202,9 @@ if (!subtractionModeTest(stamps1, kernels1, "convolve 1", subMask1, rej)) { - psError(psErrorCodeLast(), false, "Unable to test subtraction with convolution of image 1"); - psFree(stamps1); - psFree(kernels1); - psFree(subMask1); - return PM_SUBTRACTION_MODE_ERR; + psError(psErrorCodeLast(), false, "Unable to test subtraction with convolution of image 1"); + psFree(stamps1); + psFree(kernels1); + psFree(subMask1); + return PM_SUBTRACTION_MODE_ERR; } psFree(subMask1); @@ -1165,11 +1217,11 @@ if (!subtractionModeTest(stamps2, kernels2, "convolve 2", subMask2, rej)) { - psError(psErrorCodeLast(), false, "Unable to test subtraction with convolution of image 2"); - psFree(stamps2); - psFree(kernels2); - psFree(subMask2); - psFree(stamps1); - psFree(kernels1); - return PM_SUBTRACTION_MODE_ERR; + psError(psErrorCodeLast(), false, "Unable to test subtraction with convolution of image 2"); + psFree(stamps2); + psFree(kernels2); + psFree(subMask2); + psFree(stamps1); + psFree(kernels1); + return PM_SUBTRACTION_MODE_ERR; } psFree(subMask2); @@ -1179,14 +1231,14 @@ pmSubtractionKernels *bestKernels = NULL; // Best choice for kernels psLogMsg("psModules.imcombine", PS_LOG_INFO, - "Image 1: %f +/- %f from %d stamps\nImage 2: %f +/- %f from %d stamps\n", - kernels1->mean, kernels1->rms, kernels1->numStamps, - kernels2->mean, kernels2->rms, kernels2->numStamps); + "Image 1: %f +/- %f from %d stamps\nImage 2: %f +/- %f from %d stamps\n", + kernels1->mean, kernels1->rms, kernels1->numStamps, + kernels2->mean, kernels2->rms, kernels2->numStamps); if (kernels1->mean < kernels2->mean) { - bestStamps = stamps1; - bestKernels = kernels1; + bestStamps = stamps1; + bestKernels = kernels1; } else { - bestStamps = stamps2; - bestKernels = kernels2; + bestStamps = stamps2; + bestKernels = kernels2; } @@ -1204,42 +1256,62 @@ } - -bool pmSubtractionParamsScale(int *kernelSize, int *stampSize, psVector *widths, - float scaleRef, float scaleMin, float scaleMax) +static float scaleRefOption = NAN; +static float scaleMinOption = NAN; +static float scaleMaxOption = NAN; +static bool scaleOption = false; + +bool pmSubtractionParamScaleOptions(bool scale, float scaleRef, float scaleMin, float scaleMax) { + + if (scale) { + PS_ASSERT_FLOAT_LARGER_THAN(scaleRef, 0.0, false); + PS_ASSERT_FLOAT_LARGER_THAN(scaleMin, 0.0, false); + PS_ASSERT_FLOAT_LARGER_THAN(scaleMax, 0.0, false); + PS_ASSERT_FLOAT_LARGER_THAN(scaleMax, scaleMin, false); + } + + scaleRefOption = scaleRef; + scaleMinOption = scaleMin; + scaleMaxOption = scaleMax; + scaleOption = scale; + + return true; +} + +bool pmSubtractionParamsScale(int *kernelSize, int *stampSize, psVector *widths, float fwhm1, float fwhm2) { - PS_ASSERT_PTR_NON_NULL(kernelSize, false); - PS_ASSERT_PTR_NON_NULL(stampSize, false); + // PS_ASSERT_PTR_NON_NULL(kernelSize, false); + // PS_ASSERT_PTR_NON_NULL(stampSize, false); PS_ASSERT_VECTOR_NON_NULL(widths, false); PS_ASSERT_VECTOR_TYPE(widths, PS_TYPE_F32, false); - PS_ASSERT_FLOAT_LARGER_THAN(scaleRef, 0.0, false); - PS_ASSERT_FLOAT_LARGER_THAN(scaleMin, 0.0, false); - PS_ASSERT_FLOAT_LARGER_THAN(scaleMax, 0.0, false); - PS_ASSERT_FLOAT_LARGER_THAN(scaleMax, scaleMin, false); - - float fwhm1; - float fwhm2; - - pmSubtractionGetFWHMs(&fwhm1, &fwhm2); - psAssert(isfinite(fwhm1), "fwhm 1 not set"); - psAssert(isfinite(fwhm2), "fwhm 2 not set"); + + if (!scaleOption) return true; + + // pmSubtractionGetFWHMs(&fwhm1, &fwhm2); + // psAssert(isfinite(fwhm1), "fwhm 1 not set"); + // psAssert(isfinite(fwhm2), "fwhm 2 not set"); // float diff = sqrtf(PS_SQR(PS_MAX(fwhm1, fwhm2)) - PS_SQR(PS_MIN(fwhm1, fwhm2))); // Difference - float scale = PS_MAX(fwhm1, fwhm2) / scaleRef; // Scaling factor - - if (isfinite(scaleMin) && scale < scaleMin) { - scale = scaleMin; - } - if (isfinite(scaleMax) && scale > scaleMax) { - scale = scaleMax; + float scale = PS_MAX(fwhm1, fwhm2) / scaleRefOption; // Scaling factor + + if (isfinite(scaleMinOption) && scale < scaleMinOption) { + scale = scaleMinOption; + } + if (isfinite(scaleMaxOption) && scale > scaleMaxOption) { + scale = scaleMaxOption; } for (int i = 0; i < widths->n; i++) { - widths->data.F32[i] *= scale; - } - *kernelSize = *kernelSize * scale + 0.5; - *stampSize = *stampSize * scale + 0.5; - - psLogMsg("psModules.imcombine", PS_LOG_INFO, - "Scaling kernel parameters by %f: %d %d", scale, *kernelSize, *stampSize); + widths->data.F32[i] *= scale; + } + if (kernelSize) { + *kernelSize = *kernelSize * scale + 0.5; + } + if (stampSize) { + *stampSize = *stampSize * scale + 0.5; + } + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Scaling kernel parameters by %f", scale); + if (kernelSize) psLogMsg("psModules.imcombine", PS_LOG_INFO, " modified kernel size %d", *kernelSize); + if (stampSize) psLogMsg("psModules.imcombine", PS_LOG_INFO, " modified stamp size %d", *stampSize); return true; Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMatch.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMatch.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionMatch.h (revision 30631) @@ -8,4 +8,5 @@ #include #include +#include /// Match two images @@ -26,5 +27,5 @@ int size, ///< Kernel half-size int order, ///< Spatial polynomial order - const psVector *widths, ///< ISIS Gaussian widths + psVector *widths, ///< ISIS Gaussian widths const psVector *orders, ///< ISIS Polynomial orders int inner, ///< Inner radius for various kernel types @@ -100,11 +101,24 @@ /// Scale subtraction parameters according to the FWHMs of the inputs -bool pmSubtractionParamsScale( - int *kernelSize, ///< Half-size of the kernel - int *stampSize, ///< Half-size of the stamp (footprint) - psVector *widths, ///< ISIS widths - float scaleRef, ///< Reference width for scaling - float scaleMin, ///< Minimum scaling ratio, or NAN - float scaleMax ///< Maximum scaling ratio, or NAN +// bool pmSubtractionParamsScale( +// int *kernelSize, ///< Half-size of the kernel +// int *stampSize, ///< Half-size of the stamp (footprint) +// psVector *widths, ///< ISIS widths +// float scaleRef, ///< Reference width for scaling +// float scaleMin, ///< Minimum scaling ratio, or NAN +// float scaleMax ///< Maximum scaling ratio, or NAN +// ); + +bool pmSubtractionParamsScale(int *kernelSize, int *stampSize, psVector *widths, float fwhm1, float fwhm2); + +bool pmSubtractionParamScaleOptions(bool scale, float scaleRef, float scaleMin, float scaleMax); + +bool pmSubtractionMatchAttempt( + pmSubtractionQuality **bestMatch, + pmSubtractionKernels *kernels, + pmSubtractionStampList *stamps, + pmSubtractionMode mode, + int spatialOrder, + bool final ); Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionParams.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionParams.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionParams.c (revision 30631) @@ -9,6 +9,8 @@ #include "pmErrorCodes.h" +#include "pmFPA.h" +#include "pmSubtractionTypes.h" +#include "pmSubtraction.h" #include "pmSubtractionStamps.h" -#include "pmSubtraction.h" #include "pmSubtractionParams.h" @@ -16,4 +18,5 @@ #if 0 +// XXX this was moved to pmSubtraction.c in r15443 -- delete // Convolve the reference stamp by the kernel static psKernel *convolveStamp(const pmSubtractionStamp *stamp, // Stamp to be convolved Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionStamps.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionStamps.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionStamps.c (revision 30631) @@ -27,6 +27,8 @@ #include "pmSource.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionStamps.h" +#include "pmSubtractionVisual.h" #define STAMP_LIST_BUFFER 20 // Number of stamps to add to list at a time @@ -122,4 +124,5 @@ if ((image1 && image1->data.F32[y][x] < thresh1) || (image2 && image2->data.F32[y][x] < thresh2)) { + // fprintf (stderr, "%f,%f : thresh\n", xRaw, yRaw); continue; } @@ -366,10 +369,63 @@ } - -pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, const psImage *image1, - const psImage *image2, const psImage *subMask, - const psRegion *region, float thresh1, float thresh2, - int size, int footprint, float spacing, float normFrac, - float sysErr, float skyErr, pmSubtractionMode mode) +bool pmSubtractionStampsSelect(pmSubtractionStampList **stamps, // Stamps to read + const pmReadout *ro1, // Readout 1 + const pmReadout *ro2, // Readout 2 + const psImage *subMask, // Mask for subtraction, or NULL + psImage *variance, // Variance map + const psRegion *region, // Region of interest + float thresh1, // Threshold for stamp finding on readout 1 + float thresh2, // Threshold for stamp finding on readout 2 + float stampSpacing, // Spacing between stamps + float normFrac, // Fraction of flux in window for normalisation window + float sysError, // Relative systematic error in images + float skyError, // Relative systematic error in images + int size, // Kernel half-size + int footprint, // Convolution footprint for stamps + pmSubtractionMode mode // Mode for subtraction + ) +{ + PS_ASSERT_PTR_NON_NULL(stamps, false); + PM_ASSERT_READOUT_NON_NULL(ro1, false); + PM_ASSERT_READOUT_NON_NULL(ro2, false); + PS_ASSERT_IMAGE_NON_NULL(subMask, false); + PS_ASSERT_IMAGE_NON_NULL(variance, false); + PS_ASSERT_PTR_NON_NULL(region, false); + + psTrace("psModules.imcombine", 3, "Finding stamps...\n"); + + psImage *image1 = ro1 ? ro1->image : NULL, *image2 = ro2 ? ro2->image : NULL; // Images of interest + + *stamps = pmSubtractionStampsFind(*stamps, image1, image2, subMask, region, thresh1, thresh2, + size, footprint, stampSpacing, normFrac, sysError, skyError, mode); + if (!*stamps) { + psError(psErrorCodeLast(), false, "Unable to find stamps."); + return false; + } + + psTrace("psModules.imcombine", 3, "Extracting stamps...\n"); + if (!pmSubtractionStampsExtract(*stamps, ro1->image, ro2->image, variance, size, *region)) { + psError(psErrorCodeLast(), false, "Unable to extract stamps."); + return false; + } + + pmSubtractionVisualPlotStamps(*stamps, (pmReadout *) ro1); + return true; +} + +pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, + const psImage *image1, + const psImage *image2, + const psImage *subMask, + const psRegion *region, + float thresh1, + float thresh2, + int size, + int footprint, + float spacing, + float normFrac, + float sysErr, + float skyErr, + pmSubtractionMode mode) { if (!image1 && !image2) { @@ -429,4 +485,19 @@ stamps = pmSubtractionStampListAlloc(numCols, numRows, region, footprint, spacing, normFrac, sysErr, skyErr); + } + + // XXX TEST : dump all stars in the stamps here + if (0) { + FILE *f = fopen ("stamp.dat", "w"); + for (int i = 0; i < stamps->num; i++) { + psVector *xList = stamps->x->data[i]; + psVector *yList = stamps->y->data[i]; // Coordinate lists + psVector *fluxList = stamps->flux->data[i]; // List of stamp fluxes + + for (int j = 0; j < xList->n; j++) { + fprintf (f, "%d %d %f %f %f\n", i, j, xList->data.F32[j], yList->data.F32[j], fluxList->data.F32[j]); + } + } + fclose (f); } @@ -636,4 +707,6 @@ } + int nTotal = 0; + // Sort the list by flux, with the brightest last for (int i = 0; i < numStamps; i++) { @@ -662,17 +735,35 @@ stamps->y->data[i] = ySorted; stamps->flux->data[i] = fluxSorted; - } - + nTotal += num; + } + // fprintf (stderr, "nTotal %d\n", nTotal); + return stamps; } - -bool pmSubtractionStampsGetWindow(pmSubtractionStampList *stamps, int kernelSize) +// we are essentially using aperture photometry to determine the photometric match between the +// images. we need to choose an appropriate-sized aperture for this analysis. If it is too +// large, the measurement will be noisy (and possibly biased) due to the sky noise. If it is +// too small, or inconsistent, the measurement will be biased. We use Kron-mag like aperture +// scaled by the first radial moment. +bool pmSubtractionStampsGetWindow(bool *tryAgain, pmSubtractionStampList *stamps, int kernelSize) { PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); PS_ASSERT_INT_NONNEGATIVE(kernelSize, false); + // if we succeed, or fail with an unrecoverable error, do not try again + if (tryAgain) { + *tryAgain = false; + } + int size = stamps->footprint; // Size of postage stamps + // window for moments calculations downstream + psFree (stamps->window); + stamps->window = psKernelAlloc(-size, size, -size, size); + psImageInit(stamps->window->image, 0.0); + + // window1 and window2 are mean stamp images used here to measure the + // first radial moment, and thus the normalization window psFree (stamps->window1); stamps->window1 = psKernelAlloc(-size, size, -size, size); @@ -683,22 +774,16 @@ psImageInit(stamps->window2->image, 0.0); - // Generate a weighting window based on the fwhms (20% larger than the largest) - { - float fwhm1, fwhm2; - - // XXX this is annoyingly hack-ish - pmSubtractionGetFWHMs(&fwhm1, &fwhm2); - - float sigma = 1.5 * PS_MAX(fwhm1, fwhm2) / 2.35; - - psFree (stamps->window); - stamps->window = psKernelAlloc(-size, size, -size, size); - psImageInit(stamps->window->image, 0.0); - - for (int y = -size; y <= size; y++) { - for (int x = -size; x <= size; x++) { - stamps->window->kernel[y][x] = exp(-0.5*(x*x + y*y)/(sigma*sigma)); - } - } + // Generate an initial weighting window based on the fwhms (50% larger than the largest) + float fwhm1, fwhm2; + + // XXX this is annoyingly hack-ish + pmSubtractionGetFWHMs(&fwhm1, &fwhm2); + + float sigma = 1.5 * PS_MAX(fwhm1, fwhm2) / 2.35; + + for (int y = -size; y <= size; y++) { + for (int x = -size; x <= size; x++) { + stamps->window->kernel[y][x] = exp(-0.5*(x*x + y*y)/(sigma*sigma)); + } } @@ -790,6 +875,5 @@ psTrace("psModules.imcombine", 3, "Window total (2): %f, threshold: %f\n", sum2, (1.0 - stamps->normFrac) * sum2); -# if (1) - // this block attempts to calculate the radius based on the first radial moment + // attempt to calculate the normalization window based on the first radial moment double Sr1 = 0.0; double Sr2 = 0.0; @@ -809,71 +893,56 @@ float R2 = Sr2 / Sf2; - stamps->normWindow1 = 2.0*R1; - stamps->normWindow2 = 2.0*R2; - psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Kron Radii: %f for 1, %f for 2\n", stamps->normWindow1, stamps->normWindow2); - -# else - // XXX : this block attempts to calculate the radius by looking at the curve of growth (or something vaguely equivalent). - // It did not do very well (though a true curve-of-growth analysis might be better...) - bool done1 = false; - bool done2 = false; - double prior1 = 0.0; - double prior2 = 0.0; - double delta1o = 1.0; - double delta2o = 1.0; - for (int radius = 1; radius <= size && !(done1 && done2); radius++) { - double within1 = 0.0; - double within2 = 0.0; - for (int y = -radius; y <= radius; y++) { - for (int x = -radius; x <= radius; x++) { - if (PS_SQR(x) + PS_SQR(y) <= PS_SQR(radius)) { - within1 += stamps->window1->kernel[y][x]; - } - if (PS_SQR(x) + PS_SQR(y) <= PS_SQR(radius)) { - within2 += stamps->window2->kernel[y][x]; - } - } - } - double delta1 = (within1 - prior1) / within1; - if (!done1 && (fabs(delta1) < stamps->normFrac)) { - // interpolate to the radius at which delta2 is normFrac: - stamps->normWindow1 = radius - (stamps->normFrac - delta1) / (delta1o - delta1); - psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "choosing %f (%d) for 1 (%f : %f)\n", stamps->normWindow1, radius, within1, delta1); - done1 = true; - } - double delta2 = (within2 - prior2) / within2; - if (!done2 && (fabs(delta2) < stamps->normFrac)) { - // interpolate to the radius at which delta2 is normFrac: - stamps->normWindow2 = radius - (stamps->normFrac - delta2) / (delta2o - delta2); - psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "choosing %f (%d) for 2 (%f : %f)\n", stamps->normWindow2, radius, within2, delta2); - done2 = true; - } - psTrace("psModules.imcombine", 5, "Radius %d: %f (%f) and %f (%f)\n", radius, within1, delta1, within2, delta2); - - prior1 = within1; - prior2 = within2; - delta1o = delta1; - delta2o = delta2; - - // if (!done1 && (within1 > (1.0 - stamps->normFrac) * sum1)) { - // stamps->normWindow1 = radius; - // done1 = true; - // } - // if (!done2 && (within2 > (1.0 - stamps->normFrac) * sum2)) { - // stamps->normWindow2 = radius; - // done2 = true; - // } - - } -# endif - - psTrace("psModules.imcombine", 3, "Normalisation window radii set to %f and %f\n", stamps->normWindow1, stamps->normWindow2); - if (stamps->normWindow1 == 0 || stamps->normWindow1 >= size) { + // Compare the Kron Radii (R1 & R2) to above to the FWHMs : if they are too discrepant, we will need to rescale + psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Kron Radii vs FWHMs 1: fwhm: %f, kron %f\n", fwhm1, R1); + psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Kron Radii vs FWHMs 2: fwhm: %f, kron %f\n", fwhm2, R2); + + // XXX CAREFUL : in pmSubtractionMatch.c:703, we rely on this factor of 2.75.. + stamps->normWindow1 = 2.75*R1; + stamps->normWindow2 = 2.75*R2; + psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Windows from Kron Radii: %f for 1, %f for 2\n", stamps->normWindow1, stamps->normWindow2); + + // if the calculated normWindows are too large, we will fall off the stamps. In this case, we need to try again. + if ((stamps->normWindow1 > size) || (stamps->normWindow2 > size)) { + if (tryAgain) { + *tryAgain = true; + } + psFree (stats); + psFree (flux1); + psFree (flux2); + psFree (norm1); + psFree (norm2); + return false; + } + + // this is an unrecoverable error : something really bogus in the data + if (stamps->normWindow1 == 0) { psError(PM_ERR_STAMPS, true, "Unable to determine normalisation window size (1)."); + psFree (stats); + psFree (flux1); + psFree (flux2); + psFree (norm1); + psFree (norm2); return false; } - if (stamps->normWindow2 == 0 || stamps->normWindow2 >= size) { + if (stamps->normWindow2 == 0) { psError(PM_ERR_STAMPS, true, "Unable to determine normalisation window size (2)."); + psFree (stats); + psFree (flux1); + psFree (flux2); + psFree (norm1); + psFree (norm2); return false; + } + + // Generate a weighting window based on the kron radii + float radius = 2.0 * PS_MAX(R1, R2); + psImageInit(stamps->window->image, 0.0); + + // we use a top-hat window for the moments analysis + for (int y = -size; y <= size; y++) { + for (int x = -size; x <= size; x++) { + if (hypot(x,y) > radius) continue; + stamps->window->kernel[y][x] = 1.0; + } } @@ -890,16 +959,4 @@ } } - -#if 0 - { - psFits *fits = NULL; - fits = psFitsOpen ("window1.norm.fits", "w"); - psFitsWriteImage (fits, NULL, stamps->window1->image, 0, NULL); - psFitsClose (fits); - fits = psFitsOpen ("window2.norm.fits", "w"); - psFitsWriteImage (fits, NULL, stamps->window2->image, 0, NULL); - psFitsClose (fits); - } -#endif psFree (stats); @@ -1128,4 +1185,9 @@ continue; } + + // XXX this is somewhat arbitrary... + if (source->errMag > 0.05) continue; + if (fabs(source->psfMag - source->apMag) > 0.5) continue; + if (source->modelPSF) { x->data.F32[numOut] = source->modelPSF->params->data.F32[PM_PAR_XPOS]; @@ -1135,5 +1197,4 @@ y->data.F32[numOut] = source->peak->yf; } - // fprintf (stderr, "stamp: %5.1f %5.1f\n", x->data.F32[numOut], y->data.F32[numOut]); numOut++; } Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionStamps.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionStamps.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionStamps.h (revision 30631) @@ -1,38 +1,4 @@ #ifndef PM_SUBTRACTION_STAMPS_H #define PM_SUBTRACTION_STAMPS_H - -#include - -#include "pmSubtractionKernels.h" - -/// Status of stamp -typedef enum { - PM_SUBTRACTION_STAMP_INIT, ///< Initial state - PM_SUBTRACTION_STAMP_FOUND, ///< Found a suitable source for this stamp - PM_SUBTRACTION_STAMP_CALCULATE, ///< Calculate matrix and vector values for this stamp - PM_SUBTRACTION_STAMP_USED, ///< Use this stamp - PM_SUBTRACTION_STAMP_REJECTED, ///< This stamp has been rejected - PM_SUBTRACTION_STAMP_NONE ///< No stamp in this region -} pmSubtractionStampStatus; - -/// A list of stamps -typedef struct { - long num; ///< Number of stamps - psArray *stamps; ///< The stamps - psArray *regions; ///< Regions for each stamp - psArray *x, *y; ///< Coordinates for possible stamps (or NULL) - psArray *flux; ///< Fluxes for possible stamps (or NULL) - int footprint; ///< Half-size of stamps - float normFrac; ///< Fraction of flux in window for normalisation window - float normValue; ///< calculated normalization - float normValue2; ///< calculated normalization - psKernel *window1; ///< window function generated from ensemble of stamps (input 1) - psKernel *window2; ///< window function generated from ensemble of stamps (input 2) - psKernel *window; ///< weighting window function (sigma = 1.1 * MAX(fwhm)) - float normWindow1; ///< Size of window for measuring normalisation - float normWindow2; ///< Size of window for measuring normalisation - float sysErr; ///< Systematic error - float skyErr; ///< increase effective readnoise -} pmSubtractionStampList; /// Allocate a list of stamps @@ -74,35 +40,25 @@ ); - -/// A stamp for image subtraction -typedef struct { - float x, y; ///< Position - float flux; ///< Flux - float xNorm, yNorm; ///< Normalised position - psKernel *image1; ///< Reference image postage stamp - psKernel *image2; ///< Input image postage stamp - psKernel *weight; ///< Weight image (1/variance) postage stamp, or NULL - psArray *convolutions1; ///< Convolutions of image 1 for each kernel component, or NULL - psArray *convolutions2; ///< Convolutions of image 2 for each kernel component, or NULL - psImage *matrix; ///< Least-squares matrix, or NULL - psVector *vector; ///< Least-squares vector, or NULL - double norm; ///< Normalisation difference - double normI1; ///< Sum(flux) for image 1 - double normI2; ///< Sum(flux) for image 2 - double normSquare1; ///< Sum(flux^2) for image 1 (used for penalty) - double normSquare2; ///< Sum(flux^2) for image 2 (used for penalty) - pmSubtractionStampStatus status; ///< Status of stamp - psVector *MxxI1; ///< second moments of convolution images - psVector *MyyI1; ///< second moments of convolution images - psVector *MxxI2; ///< second moments of convolution images - psVector *MyyI2; ///< second moments of convolution images - double MxxI1raw; - double MyyI1raw; - double MxxI2raw; - double MyyI2raw; -} pmSubtractionStamp; - /// Allocate a stamp pmSubtractionStamp *pmSubtractionStampAlloc(void); + +// find and extract the stamps +bool pmSubtractionStampsSelect(pmSubtractionStampList **stamps, // Stamps to read + const pmReadout *ro1, // Readout 1 + const pmReadout *ro2, // Readout 2 + const psImage *subMask, // Mask for subtraction, or NULL + psImage *variance, // Variance map + const psRegion *region, // Region of interest + float thresh1, // Threshold for stamp finding on readout 1 + float thresh2, // Threshold for stamp finding on readout 2 + float stampSpacing, // Spacing between stamps + float normFrac, // Fraction of flux in window for normalisation window + float sysError, // Relative systematic error in images + float skyError, // Relative systematic error in images + int size, // Kernel half-size + int footprint, // Convolution footprint for stamps + pmSubtractionMode mode // Mode for subtraction + ); + /// Find stamps on an image @@ -172,4 +128,5 @@ /// Calculate the window and normalisation window from the stamps bool pmSubtractionStampsGetWindow( + bool *tryAgain, ///< re-try with new stamp size? pmSubtractionStampList *stamps, ///< List of stamps int kernelSize ///< Half-size of kernel Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionThreads.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionThreads.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionThreads.c (revision 30631) @@ -6,4 +6,5 @@ #include +#include "pmSubtractionTypes.h" #include "pmSubtractionMatch.h" #include "pmSubtractionEquation.h" @@ -33,6 +34,13 @@ { - psThreadTask *task = psThreadTaskAlloc("PSMODULES_SUBTRACTION_CALCULATE_EQUATION", 4); + psThreadTask *task = psThreadTaskAlloc("PSMODULES_SUBTRACTION_CALCULATE_EQUATION", 3); task->function = &pmSubtractionCalculateEquationThread; + psThreadTaskAdd(task); + psFree(task); + } + + { + psThreadTask *task = psThreadTaskAlloc("PSMODULES_SUBTRACTION_CONVOLVE_STAMP", 3); + task->function = &pmSubtractionConvolveStampThread; psThreadTaskAdd(task); psFree(task); Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionTypes.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionTypes.h (revision 30631) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionTypes.h (revision 30631) @@ -0,0 +1,172 @@ +/* @file pmSubtraction.h + * + * PSF-matched image subtraction, based on the Alard & Lupton (1998) and Alard (2000) methods. + * + * @author Paul Price, IfA + * @author GLG, MHPCC + * + * @version $Revision: 1.36 $ $Name: not supported by cvs2svn $ + * @date $Date: 2009-02-06 02:31:25 $ + * Copyright 2004-207 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_SUBTRACTION_TYPES_H +#define PM_SUBTRACTION_TYPES_H + +/// @addtogroup imcombine Image Combinations +/// @{ + +/// Mask values for the subtraction mask +typedef enum { + PM_SUBTRACTION_MASK_CLEAR = 0x00, // No masking + PM_SUBTRACTION_MASK_BAD_1 = 0x01, // Image 1 is bad + PM_SUBTRACTION_MASK_BAD_2 = 0x02, // Image 2 is bad + PM_SUBTRACTION_MASK_CONVOLVE_1 = 0x04, // If image 1 is convolved, would be poor or bad + PM_SUBTRACTION_MASK_CONVOLVE_2 = 0x08, // If image 2 is convolved, would be poor or bad + PM_SUBTRACTION_MASK_CONVOLVE_BAD_1 = 0x10, // If image 1 is convolved, would be bad + PM_SUBTRACTION_MASK_CONVOLVE_BAD_2 = 0x20, // If image 2 is convolved, would be bad + PM_SUBTRACTION_MASK_BORDER = 0x40, // Image border + PM_SUBTRACTION_MASK_REJ = 0x80, // Previously tried as a stamp, and rejected +} pmSubtractionMasks; + +/// Type of subtraction kernel +typedef enum { + PM_SUBTRACTION_KERNEL_NONE, ///< Nothing --- an error + PM_SUBTRACTION_KERNEL_POIS, ///< Pan-STARRS Optimal Image Subtraction --- delta functions + PM_SUBTRACTION_KERNEL_ISIS, ///< Traditional kernel --- gaussians modified by polynomials + PM_SUBTRACTION_KERNEL_ISIS_RADIAL, ///< ISIS + higher-order radial Hermitians + PM_SUBTRACTION_KERNEL_HERM, ///< Hermitian polynomial kernels + PM_SUBTRACTION_KERNEL_DECONV_HERM, ///< Deconvolved Hermitian polynomial kernels + PM_SUBTRACTION_KERNEL_SPAM, ///< Summed Pixels for Advanced Matching --- summed delta functions + PM_SUBTRACTION_KERNEL_FRIES, ///< Fibonacci Radius Increases Excellence of Subtraction + PM_SUBTRACTION_KERNEL_GUNK, ///< Grid United with Normal Kernel --- POIS and ISIS hybrid + PM_SUBTRACTION_KERNEL_RINGS, ///< Rings Instead of the Normal Gaussian Subtraction +} pmSubtractionKernelsType; + +/// Modes --- specifies which image to convolve +typedef enum { + PM_SUBTRACTION_MODE_ERR, // Error in the mode + PM_SUBTRACTION_MODE_1, // Convolve image 1 + PM_SUBTRACTION_MODE_2, // Convolve image 2 + PM_SUBTRACTION_MODE_UNSURE, // Not sure yet which image to convolve so try to satisfy both + PM_SUBTRACTION_MODE_DUAL, // Dual convolution +} pmSubtractionMode; + +/// Status of stamp +typedef enum { + PM_SUBTRACTION_STAMP_INIT, ///< Initial state + PM_SUBTRACTION_STAMP_FOUND, ///< Found a suitable source for this stamp + PM_SUBTRACTION_STAMP_CALCULATE, ///< Calculate matrix and vector values for this stamp + PM_SUBTRACTION_STAMP_USED, ///< Use this stamp + PM_SUBTRACTION_STAMP_REJECTED, ///< This stamp has been rejected + PM_SUBTRACTION_STAMP_NONE ///< No stamp in this region +} pmSubtractionStampStatus; + +typedef struct { + double score; + pmSubtractionMode mode; + int spatialOrder; + int nGood; + psVector *fluxes; + psVector *chisq; + psVector *moments; + psVector *stampMask; +} pmSubtractionQuality; + +/// Kernels specification +typedef struct { + pmSubtractionKernelsType type; ///< Type of kernels --- allowing the use of multiple kernels + psString description; ///< Description of the kernel parameters + int xMin, xMax, yMin, yMax; ///< Bounds of image (for normalisation) + long num; ///< Number of kernel components (not including the spatial ones) + psVector *fwhms; ///< requested fwhms of the kernel Gaussians (ISIS, HERM or DECONV_HERM) + psVector *orders; ///< polynomial orders for each Gaussian (ISIS, HERM or DECONV_HERM) + psVector *u, *v; ///< Offset (for POIS) or polynomial order (for ISIS, HERM or DECONV_HERM) + psVector *widths; ///< measured Gaussian FWHMs of Gauss*poly (ISIS, HERM or DECONV_HERM) + psVector *uStop, *vStop; ///< Width of kernel element (SPAM,FRIES only) + psArray *preCalc; ///< Array of images containing pre-calculated kernel (for ISIS, HERM or DECONV_HERM) + float penalty; ///< Penalty for wideness + psVector *penalties1; ///< Penalty for each kernel component + psVector *penalties2; ///< Penalty for each kernel component + bool havePenalties; ///< flag to test if we have already calculated the penalties or not. + int size; ///< The half-size of the kernel + int inner; ///< The size of an inner region + int binning; ///< Binning used for the SPAM kernels + int ringsOrder; ///< + int spatialOrder; ///< The spatial order of the kernels + int bgOrder; ///< The order for the background fitting + pmSubtractionMode mode; ///< Mode for subtraction + psVector *solution1, *solution2; ///< Solution for the PSF matching + psVector *solution1err, *solution2err; ///< error in solution for the PSF matching + // Quality information + float mean, rms; ///< Mean and RMS of chi^2 from stamps + int numStamps; ///< Number of good stamps + float fResSigmaMean; ///< mean fractional stdev of residuals + float fResSigmaStdev; ///< stdev of fractional stdev of residuals + float fResOuterMean; ///< mean fractional positive swing in residuals + float fResOuterStdev; ///< stdev of fractional positive swing in residuals + float fResTotalMean; ///< mean fractional negative swing in residuals + float fResTotalStdev; ///< stdev of fractional negative swing in residuals + psArray *sampleStamps; ///< array of brightest set of stamps for output visualizations +} pmSubtractionKernels; + +// pmSubtractionKernels->preCalc is an array of pmSubtractionKernelPreCalc structures +typedef struct { + psVector *uCoords; // used by RINGS + psVector *vCoords; // used by RINGS + psVector *poly; // used by RINGS + + psVector *xKernel; // used by ISIS, HERM, DECONV_HERM + psVector *yKernel; // used by ISIS, HERM, DECONV_HERM + psKernel *kernel; // used by ISIS, HERM, DECONV_HERM +} pmSubtractionKernelPreCalc; + +/// A list of stamps +typedef struct { + long num; ///< Number of stamps + psArray *stamps; ///< The stamps + psArray *regions; ///< Regions for each stamp + psArray *x, *y; ///< Coordinates for possible stamps (or NULL) + psArray *flux; ///< Fluxes for possible stamps (or NULL) + int footprint; ///< Half-size of stamps + float normFrac; ///< Fraction of flux in window for normalisation window + float normValue; ///< calculated normalization + float normValue2; ///< calculated normalization + psKernel *window1; ///< window function generated from ensemble of stamps (input 1) + psKernel *window2; ///< window function generated from ensemble of stamps (input 2) + psKernel *window; ///< weighting window function (sigma = 1.1 * MAX(fwhm)) + float normWindow1; ///< Size of window for measuring normalisation + float normWindow2; ///< Size of window for measuring normalisation + float sysErr; ///< Systematic error + float skyErr; ///< increase effective readnoise +} pmSubtractionStampList; + +/// A stamp for image subtraction +typedef struct { + float x, y; ///< Position + float flux; ///< Flux + float xNorm, yNorm; ///< Normalised position + psKernel *image1; ///< Reference image postage stamp + psKernel *image2; ///< Input image postage stamp + psKernel *weight; ///< Weight image (1/variance) postage stamp, or NULL + psArray *convolutions1; ///< Convolutions of image 1 for each kernel component, or NULL + psArray *convolutions2; ///< Convolutions of image 2 for each kernel component, or NULL + psImage *matrix; ///< Least-squares matrix, or NULL + psVector *vector; ///< Least-squares vector, or NULL + double norm; ///< Normalisation difference + double normI1; ///< Sum(flux) for image 1 + double normI2; ///< Sum(flux) for image 2 + double normSquare1; ///< Sum(flux^2) for image 1 (used for penalty) + double normSquare2; ///< Sum(flux^2) for image 2 (used for penalty) + pmSubtractionStampStatus status; ///< Status of stamp + psVector *MxxI1; ///< second moments of convolution images + psVector *MyyI1; ///< second moments of convolution images + psVector *MxxI2; ///< second moments of convolution images + psVector *MyyI2; ///< second moments of convolution images + double MxxI1raw; + double MyyI1raw; + double MxxI2raw; + double MyyI2raw; +} pmSubtractionStamp; + +#endif Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionVisual.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionVisual.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionVisual.c (revision 30631) @@ -16,8 +16,11 @@ #include "pmKapaPlots.h" +#include "pmFPA.h" +#include "pmSubtractionTypes.h" #include "pmSubtraction.h" #include "pmSubtractionStamps.h" #include "pmSubtractionEquation.h" #include "pmSubtractionKernels.h" +#include "pmSubtractionVisual.h" #include "pmVisual.h" @@ -210,4 +213,18 @@ psImageOverlaySection(canvas, im, x0, y0, "="); stampNum++; + + // renormalize the section + float maxValue = 0; + for (int iy = 0; iy < im->numRows; iy++) { + for (int ix = 0; ix < im->numCols; ix++) { + maxValue = PS_MAX(maxValue, im->data.F64[iy][ix]); + } + } + if (maxValue == 0.0) continue; + for (int iy = 0; iy < im->numRows; iy++) { + for (int ix = 0; ix < im->numCols; ix++) { + canvas->data.F64[y0 + iy][x0 + ix] /= maxValue; + } + } } @@ -231,4 +248,109 @@ } +/** Plot the least-squares matrix of each stamp */ +bool pmSubtractionVisualPlotLeastSquaresResid (const pmSubtractionStampList *stamps, psImage *matrixIn, int nUsed) { + + if (!pmVisualTestLevel("ppsub.chisq", 1)) return true; + + if (!plotLeastSquares) return true; + + if (!pmVisualInitWindow (&kapa1, "PPSub:Images")) { + return false; + } + + psImage *matrixNorm = psImageCopy(NULL, matrixIn, PS_TYPE_F64); + { + // renormalize the matrix + float maxValue = 0; + for (int iy = 0; iy < matrixNorm->numRows; iy++) { + for (int ix = 0; ix < matrixNorm->numCols; ix++) { + maxValue = PS_MAX(maxValue, matrixNorm->data.F64[iy][ix]); + } + } + for (int iy = 0; iy < matrixNorm->numRows; iy++) { + for (int ix = 0; ix < matrixNorm->numCols; ix++) { + matrixNorm->data.F64[iy][ix] /= maxValue; + } + } + } + + // Find the stamp size + int imageMax = -1; + int numStamps = 0; + psElemType type = PS_TYPE_F64; + + for(int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; + if (stamp == NULL) continue; + + psImage *im = stamp->matrix; + if (im == NULL) continue; + + imageMax = PS_MAX(imageMax, im->numCols); + imageMax = PS_MAX(imageMax, im->numRows); + numStamps++; + type = im->type.type; + } + if (imageMax == -1) return false; + + int border = 15; + imageMax += border; + int tileRowCount = (int) ceil(sqrt(numStamps)); + int canvasX = tileRowCount * (imageMax); + int canvasY = tileRowCount * (imageMax); + psImage *canvas = psImageAlloc (canvasX, canvasY, type); + psImageInit (canvas, NAN); + + // overlay the images + int stampNum = 0; + int stampListNum = 0; + while (stampNum < numStamps) { + int x0 = (imageMax) * (stampNum % tileRowCount); + int y0 = (imageMax) * (stampNum / tileRowCount); + + pmSubtractionStamp *stamp = stamps->stamps->data[stampListNum++]; + if (stamp == NULL) continue; + + psImage *im = stamp->matrix; + if (im == NULL) continue; + + stampNum++; + + if (stamp->status != PM_SUBTRACTION_STAMP_USED) continue; + + // renormalize the section + float maxValue = 0; + for (int iy = 0; iy < im->numRows; iy++) { + for (int ix = 0; ix < im->numCols; ix++) { + maxValue = PS_MAX(maxValue, im->data.F64[iy][ix]); + } + } + if (maxValue == 0.0) continue; + for (int iy = 0; iy < im->numRows; iy++) { + for (int ix = 0; ix < im->numCols; ix++) { + canvas->data.F64[y0 + iy][x0 + ix] = (im->data.F64[iy][ix] / maxValue - matrixNorm->data.F64[iy][ix]) / matrixNorm->data.F64[iy][ix]; + } + } + } + + psImage *canvas32 = pmVisualImageToFloat(canvas); + pmVisualRangeImage(kapa2, canvas32, "Least_Squares", 0, -100.0, 100.0); + + if (0) { + static int count = 0; + char filename[64]; + sprintf (filename, "chisq.%02d.fits", count); + count ++; + psFits *fits = psFitsOpen (filename, "w"); + psFitsWriteImage (fits, NULL, canvas32, 0, NULL); + psFitsClose (fits); + } + + pmVisualAskUser(&plotLeastSquares); + psFree(canvas); + psFree(canvas32); + return true; +} + bool pmSubtractionVisualShowSubtraction(psImage *image, psImage *ref, psImage *sub) { @@ -304,5 +426,6 @@ } - // XXX clear the overlay(s) (red at least!) + // clear the overlay (red at least!) + KiiEraseOverlay (kapa2, "red"); // get the kernel sizes @@ -337,4 +460,5 @@ int nKernels = 0; + // paste in the kernel images, scaled by sum2 if (maxStamp->convolutions1) { // output image is a grid of NXsub by NYsub sub-images @@ -359,14 +483,16 @@ int yPix = ySub * (2*footprint + 1 + 3) + footprint; - double sum = 0.0; double sum2 = 0.0; for (int y = -footprint; y <= footprint; y++) { for (int x = -footprint; x <= footprint; x++) { - output->data.F32[y + yPix][x + xPix] = kernel->kernel[y][x]; - sum += kernel->kernel[y][x]; sum2 += PS_SQR(kernel->kernel[y][x]); } } - // fprintf (stderr, "kernel %d, sum %f, sum2: %e\n", i, sum, sum2); + float scale = sqrt(sum2) / PS_SQR(2*footprint + 1); + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + output->data.F32[y + yPix][x + xPix] = kernel->kernel[y][x] / scale; + } + } } pmVisualScaleImage(kapa2, output, "Image", 0, true); @@ -401,14 +527,16 @@ int yPix = ySub * (2*footprint + 1 + 3) + footprint; - double sum = 0.0; double sum2 = 0.0; for (int y = -footprint; y <= footprint; y++) { for (int x = -footprint; x <= footprint; x++) { - output->data.F32[y + yPix][x + xPix] = kernel->kernel[y][x]; - sum += kernel->kernel[y][x]; sum2 += PS_SQR(kernel->kernel[y][x]); } } - // fprintf (stderr, "kernel %d, sum %f, sum2: %e\n", i, sum, sum2); + float scale = sqrt(sum2) / PS_SQR(2*footprint + 1); + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + output->data.F32[y + yPix][x + xPix] = kernel->kernel[y][x] / scale; + } + } } pmVisualScaleImage(kapa2, output, "Image", 1, true); @@ -468,122 +596,4 @@ KiiLoadOverlay (kapa2, overlay, Noverlay, "red"); FREE (overlay); - return true; -} - -static int footprint = 0; -static int NX = 0; -static int NY = 0; -static psImage *sourceImage = NULL; -static psImage *targetImage = NULL; -static psImage *residualImage = NULL; -static psImage *fresidualImage = NULL; -static psImage *differenceImage = NULL; -static psImage *convolutionImage = NULL; - -bool pmSubtractionVisualShowFitInit(pmSubtractionStampList *stamps) { - - if (!pmVisualTestLevel("ppsub.fit", 1)) return true; - - // generate 4 storage images large enough to hold the N stamps: - - footprint = stamps->footprint; - - float NXf = sqrt(stamps->num); - NX = (int) NXf == NXf ? NXf : NXf + 1.0; - - float NYf = stamps->num / NX; - NY = (int) NYf == NY ? NYf : NYf + 1.0; - - int NXpix = (2*footprint + 1) * NX; - NXpix += (NX > 1) ? 3 * NX : 0; - - int NYpix = (2*footprint + 1) * NY; - NYpix += (NY > 1) ? 3 * NY : 0; - - sourceImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); - targetImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); - residualImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); - fresidualImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); - differenceImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); - convolutionImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); - - psImageInit (sourceImage, 0.0); - psImageInit (targetImage, 0.0); - psImageInit (residualImage, 0.0); - psImageInit (fresidualImage, 0.0); - psImageInit (differenceImage, 0.0); - psImageInit (convolutionImage, 0.0); - - return true; -} - -bool pmSubtractionVisualShowFitAddStamp(psKernel *target, psKernel *source, psKernel *convolution, double background, double norm, int index) { - - if (!pmVisualTestLevel("ppsub.stamp", 1)) return true; - - double sum; - - int NXoff = index % NX; - int NYoff = index / NX; - - int NXpix = NXoff * (2*footprint + 1 + 3) + footprint; - int NYpix = NYoff * (2*footprint + 1 + 3) + footprint; - - // insert the (target) kernel into the (target) image: - sum = 0.0; - for (int y = -footprint; y <= footprint; y++) { - for (int x = -footprint; x <= footprint; x++) { - targetImage->data.F32[y + NYpix][x + NXpix] = target->kernel[y][x]; - sum += targetImage->data.F32[y + NYpix][x + NXpix]; - } - } - targetImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; - - // insert the (source) kernel into the (source) image: - sum = 0.0; - for (int y = -footprint; y <= footprint; y++) { - for (int x = -footprint; x <= footprint; x++) { - sourceImage->data.F32[y + NYpix][x + NXpix] = source->kernel[y][x]; - sum += sourceImage->data.F32[y + NYpix][x + NXpix]; - } - } - sourceImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; - - // insert the (convolution) kernel into the (convolution) image: - sum = 0.0; - for (int y = -footprint; y <= footprint; y++) { - for (int x = -footprint; x <= footprint; x++) { - convolutionImage->data.F32[y + NYpix][x + NXpix] = convolution->kernel[y][x]; - sum += convolutionImage->data.F32[y + NYpix][x + NXpix]; - } - } - convolutionImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; - - // insert the (difference) kernel into the (difference) image: - sum = 0.0; - for (int y = -footprint; y <= footprint; y++) { - for (int x = -footprint; x <= footprint; x++) { - differenceImage->data.F32[y + NYpix][x + NXpix] = target->kernel[y][x] - background - source->kernel[y][x] * norm; - sum += differenceImage->data.F32[y + NYpix][x + NXpix]; - } - } - differenceImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; - - // insert the (residual) kernel into the (residual) image: - sum = 0.0; - for (int y = -footprint; y <= footprint; y++) { - for (int x = -footprint; x <= footprint; x++) { - residualImage->data.F32[y + NYpix][x + NXpix] = target->kernel[y][x] - background - source->kernel[y][x] * norm - convolution->kernel[y][x]; - sum += residualImage->data.F32[y + NYpix][x + NXpix]; - } - } - residualImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; - - // insert the (fresidual) kernel into the (fresidual) image: - for (int y = -footprint; y <= footprint; y++) { - for (int x = -footprint; x <= footprint; x++) { - fresidualImage->data.F32[y + NYpix][x + NXpix] = residualImage->data.F32[y + NYpix][x + NXpix] / sqrt(PS_MAX(target->kernel[y][x], 100.0)); - } - } return true; } @@ -611,15 +621,15 @@ } -bool pmSubtractionVisualShowFit(double norm) { - - // for testing, dump the residual image and exit - if (0) { - psMetadata *header = psMetadataAlloc(); - psMetadataAddF32 (header, PS_LIST_TAIL, "NORM", 0, "Normalization", norm); - psFits *fits = psFitsOpen("resid.fits", "w"); - psFitsWriteImage(fits, header, residualImage, 0, NULL); - psFitsClose(fits); - // exit (0); - } +static int footprint = 0; +static int NX = 0; +static int NY = 0; +static psImage *sourceImage = NULL; +static psImage *targetImage = NULL; +static psImage *residualImage = NULL; +static psImage *fresidualImage = NULL; +static psImage *differenceImage = NULL; +static psImage *convolutionImage = NULL; + +bool pmSubtractionVisualShowFit(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels) { if (!pmVisualTestLevel("ppsub.fit", 1)) return true; @@ -627,4 +637,187 @@ if (!pmVisualInitWindow(&kapa1, "ppSub:Images")) return false; if (!pmVisualInitWindow(&kapa2, "ppSub:Misc")) return false; + + // set up holding images for the visualization + pmSubtractionVisualShowFitInit (stamps); + + int numKernels = kernels->num; // Number of kernels + + psImage *polyValues = NULL; // Polynomial values + psKernel *residual = psKernelAlloc(-stamps->footprint, stamps->footprint, -stamps->footprint, stamps->footprint); // Residual image + + double norm = p_pmSubtractionSolutionNorm(kernels); // Normalisation + + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // The stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_USED) { continue; } + + // Calculate coefficients of the kernel basis functions + polyValues = p_pmSubtractionPolynomial(polyValues, kernels->spatialOrder, stamp->xNorm, stamp->yNorm); + double background = p_pmSubtractionSolutionBackground(kernels, polyValues); // Difference in background + + psImageInit(residual->image, 0.0); + + if (kernels->mode != PM_SUBTRACTION_MODE_DUAL) { + psKernel *target; // Target postage stamp + psKernel *source; // Source postage stamp + psArray *convolutions; // Convolution postage stamps for each kernel basis function + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + target = stamp->image2; + source = stamp->image1; + convolutions = stamp->convolutions1; + break; + case PM_SUBTRACTION_MODE_2: + target = stamp->image1; + source = stamp->image2; + convolutions = stamp->convolutions2; + break; + default: + psAbort("Unsupported subtraction mode: %x", kernels->mode); + } + + for (int j = 0; j < numKernels; j++) { + psKernel *convolution = convolutions->data[j]; // Convolution + double coefficient = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, false); // Coefficient + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + residual->kernel[y][x] += convolution->kernel[y][x] * coefficient; + } + } + } + // visualize the target, source, convolution and residual + pmSubtractionVisualShowFitAddStamp (target, source, residual, background, norm, i); + } else { + // Dual convolution + psArray *convolutions1 = stamp->convolutions1; // Convolutions of the first image + psArray *convolutions2 = stamp->convolutions2; // Convolutions of the second image + psKernel *image1 = stamp->image1; // The first image + psKernel *image2 = stamp->image2; // The second image + + for (int j = 0; j < numKernels; j++) { + psKernel *conv1 = convolutions1->data[j]; // Convolution of first image + psKernel *conv2 = convolutions2->data[j]; // Convolution of second image + double coeff1 = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, false); // Coefficient 1 + double coeff2 = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, true); // Coefficient 2 + + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + residual->kernel[y][x] += conv2->kernel[y][x] * coeff2 + conv1->kernel[y][x] * coeff1; + } + } + } + // visualize the target, source, convolution and residual + pmSubtractionVisualShowFitAddStamp (image2, image1, residual, background, norm, i); + } + } + pmSubtractionVisualShowFitImage(norm); + + return true; +} + +// generate 4 storage images large enough to hold the stamps: +bool pmSubtractionVisualShowFitInit(pmSubtractionStampList *stamps) { + + footprint = stamps->footprint; + + float NXf = sqrt(stamps->num); + NX = (int) NXf == NXf ? NXf : NXf + 1.0; + + float NYf = stamps->num / NX; + NY = (int) NYf == NY ? NYf : NYf + 1.0; + + int NXpix = (2*footprint + 1) * NX; + NXpix += (NX > 1) ? 3 * NX : 0; + + int NYpix = (2*footprint + 1) * NY; + NYpix += (NY > 1) ? 3 * NY : 0; + + sourceImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); + targetImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); + residualImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); + fresidualImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); + differenceImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); + convolutionImage = psImageAlloc (NXpix, NYpix, PS_TYPE_F32); + + psImageInit (sourceImage, 0.0); + psImageInit (targetImage, 0.0); + psImageInit (residualImage, 0.0); + psImageInit (fresidualImage, 0.0); + psImageInit (differenceImage, 0.0); + psImageInit (convolutionImage, 0.0); + + return true; +} + +bool pmSubtractionVisualShowFitAddStamp(psKernel *target, psKernel *source, psKernel *convolution, double background, double norm, int index) { + + double sum; + + int NXoff = index % NX; + int NYoff = index / NX; + + int NXpix = NXoff * (2*footprint + 1 + 3) + footprint; + int NYpix = NYoff * (2*footprint + 1 + 3) + footprint; + + // insert the (target) kernel into the (target) image: + sum = 0.0; + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + targetImage->data.F32[y + NYpix][x + NXpix] = target->kernel[y][x]; + sum += targetImage->data.F32[y + NYpix][x + NXpix]; + } + } + targetImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; + + // insert the (source) kernel into the (source) image: + sum = 0.0; + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + sourceImage->data.F32[y + NYpix][x + NXpix] = source->kernel[y][x]; + sum += sourceImage->data.F32[y + NYpix][x + NXpix]; + } + } + sourceImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; + + // insert the (convolution) kernel into the (convolution) image: + sum = 0.0; + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + convolutionImage->data.F32[y + NYpix][x + NXpix] = convolution->kernel[y][x]; + sum += convolutionImage->data.F32[y + NYpix][x + NXpix]; + } + } + convolutionImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; + + // insert the (difference) kernel into the (difference) image: + sum = 0.0; + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + differenceImage->data.F32[y + NYpix][x + NXpix] = target->kernel[y][x] - background - source->kernel[y][x] * norm; + sum += differenceImage->data.F32[y + NYpix][x + NXpix]; + } + } + differenceImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; + + // insert the (residual) kernel into the (residual) image: + sum = 0.0; + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + residualImage->data.F32[y + NYpix][x + NXpix] = target->kernel[y][x] - background - source->kernel[y][x] * norm - convolution->kernel[y][x]; + sum += residualImage->data.F32[y + NYpix][x + NXpix]; + } + } + residualImage->data.F32[footprint + 1 + NYpix][NXpix] = sum; + + // insert the (fresidual) kernel into the (fresidual) image: + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + fresidualImage->data.F32[y + NYpix][x + NXpix] = residualImage->data.F32[y + NYpix][x + NXpix] / sqrt(PS_MAX(target->kernel[y][x], 100.0)); + } + } + return true; +} + +bool pmSubtractionVisualShowFitImage(double norm) { KiiEraseOverlay (kapa1, "red"); @@ -673,4 +866,5 @@ psVector *x = psVectorAllocEmpty (kernels->num, PS_TYPE_F32); psVector *y = psVectorAllocEmpty (kernels->num, PS_TYPE_F32); + psVector *dy = psVectorAllocEmpty (kernels->num, PS_TYPE_F32); graphdata.xmin = -1.0; @@ -685,8 +879,9 @@ x->data.F32[i] = i; y->data.F32[i] = p_pmSubtractionSolutionCoeff(kernels, polyValues, i, false); + dy->data.F32[i] = kernels->solution1err->data.F64[i]; graphdata.ymin = PS_MIN(graphdata.ymin, y->data.F32[i]); graphdata.ymax = PS_MAX(graphdata.ymax, y->data.F32[i]); } - x->n = y->n = kernels->num; + x->n = y->n = dy->n = kernels->num; float range; @@ -709,12 +904,128 @@ graphdata.size = 0.5; graphdata.style = 2; + graphdata.etype |= 0x01; KapaPrepPlot (kapa3, x->n, &graphdata); KapaPlotVector (kapa3, x->n, x->data.F32, "x"); KapaPlotVector (kapa3, x->n, y->data.F32, "y"); + KapaPlotVector (kapa3, x->n, dy->data.F32, "dym"); + KapaPlotVector (kapa3, x->n, dy->data.F32, "dyp"); psFree (x); psFree (y); + psFree (dy); psFree (polyValues); + + pmVisualAskUser(NULL); + return true; +} + +// plot log(flux) vs log(chisq), log(flux) vs log(moments), log(chisq) vs log(moments) +bool pmSubtractionVisualPlotChisqAndMoments(psVector *fluxes, psVector *chisq, psVector *moments) { + + Graphdata graphdata; + KapaSection section; + + if (!pmVisualTestLevel("ppsub.fit", 1)) return true; + + if (!pmVisualInitWindow(&kapa3, "ppSub:plots")) return false; + + KapaClearSections (kapa3); + KapaInitGraph (&graphdata); + KiiResize(kapa3, 1500, 500); + + psVector *lchi = psVectorAlloc (fluxes->n, PS_TYPE_F32); + psVector *lflx = psVectorAlloc (fluxes->n, PS_TYPE_F32); + psVector *lMxx = psVectorAlloc (fluxes->n, PS_TYPE_F32); + + // construct the plot vectors + for (int i = 0; i < fluxes->n; i++) { + lchi->data.F32[i] = log10(chisq->data.F32[i]); + lflx->data.F32[i] = log10(fluxes->data.F32[i]); + lMxx->data.F32[i] = log10(moments->data.F32[i]); + } + + section.bg = KapaColorByName ("none"); // XXX probably should be 'none' + + // section 1: lflux vs lchi + section.dx = 0.33; + section.dy = 1.00; + section.x = 0.00; + section.y = 0.00; + section.name = psStringCopy ("flux.v.chi"); + KapaSetSection (kapa3, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + pmVisualScaleGraphdata(&graphdata, lflx, lchi, false); + KapaSetLimits (kapa3, &graphdata); + + KapaSetFont (kapa3, "helvetica", 14); + KapaBox (kapa3, &graphdata); + KapaSendLabel (kapa3, "log(flux)", KAPA_LABEL_XM); + KapaSendLabel (kapa3, "log(chisq)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + + KapaPrepPlot (kapa3, lflx->n, &graphdata); + KapaPlotVector (kapa3, lflx->n, lflx->data.F32, "x"); + KapaPlotVector (kapa3, lflx->n, lchi->data.F32, "y"); + + // section 2: lflux vs lMxx + section.dx = 0.33; + section.dy = 1.00; + section.x = 0.33; + section.y = 0.00; + section.name = psStringCopy ("flux.v.mom"); + KapaSetSection (kapa3, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + pmVisualScaleGraphdata(&graphdata, lflx, lMxx, false); + KapaSetLimits (kapa3, &graphdata); + + KapaSetFont (kapa3, "helvetica", 14); + KapaBox (kapa3, &graphdata); + KapaSendLabel (kapa3, "log(flux)", KAPA_LABEL_XM); + KapaSendLabel (kapa3, "log(moments)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + + KapaPrepPlot (kapa3, lflx->n, &graphdata); + KapaPlotVector (kapa3, lflx->n, lflx->data.F32, "x"); + KapaPlotVector (kapa3, lflx->n, lMxx->data.F32, "y"); + + // section 1: lflux vs lchi + section.dx = 0.33; + section.dy = 1.00; + section.x = 0.66; + section.y = 0.00; + section.name = psStringCopy ("chi.v.mom"); + KapaSetSection (kapa3, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + pmVisualScaleGraphdata(&graphdata, lchi, lMxx, false); + KapaSetLimits (kapa3, &graphdata); + + KapaSetFont (kapa3, "helvetica", 14); + KapaBox (kapa3, &graphdata); + KapaSendLabel (kapa3, "log(chisq)", KAPA_LABEL_XM); + KapaSendLabel (kapa3, "log(moments)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + + KapaPrepPlot (kapa3, lflx->n, &graphdata); + KapaPlotVector (kapa3, lflx->n, lchi->data.F32, "x"); + KapaPlotVector (kapa3, lflx->n, lMxx->data.F32, "y"); pmVisualAskUser(NULL); Index: /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionVisual.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionVisual.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/imcombine/pmSubtractionVisual.h (revision 30631) @@ -6,11 +6,16 @@ bool pmSubtractionVisualPlotStamps(pmSubtractionStampList *stamps, pmReadout *ro); bool pmSubtractionVisualPlotLeastSquares(pmSubtractionStampList *stamps); +bool pmSubtractionVisualPlotLeastSquaresResid (const pmSubtractionStampList *stamps, psImage *matrixIn, int nUsed); bool pmSubtractionVisualShowSubtraction(psImage *image, psImage *ref, psImage *sub); + +bool pmSubtractionVisualShowFit(pmSubtractionStampList *stamps, pmSubtractionKernels *kernels); bool pmSubtractionVisualShowFitInit(pmSubtractionStampList *stamps); bool pmSubtractionVisualShowFitAddStamp(psKernel *target, psKernel *source, psKernel *convolution, double background, double norm, int index); -bool pmSubtractionVisualShowFit(double norm); +bool pmSubtractionVisualShowFitImage(double norm); + bool pmSubtractionVisualPlotFit(const pmSubtractionKernels *kernels); bool pmSubtractionVisualShowKernels(pmSubtractionKernels *kernels); bool pmSubtractionVisualShowBasis(pmSubtractionStampList *stamps); +bool pmSubtractionVisualPlotChisqAndMoments(psVector *fluxes, psVector *chisq, psVector *moments); #endif Index: /branches/czw_branch/20101203/psModules/src/objects/pmFootprint.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmFootprint.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmFootprint.c (revision 30631) @@ -255,3 +255,14 @@ } +// create a new footprint with the same span set as the input footprint +pmFootprint *pmFootprintCopyData(pmFootprint *inFoot, psImage *image) { + + pmFootprint *outFoot = pmFootprintAlloc(inFoot->nspans, image); + for (int i = 0; i < inFoot->nspans; i++) { + pmSpan *span = inFoot->spans->data[i]; + pmFootprintAddSpan(outFoot, span->y, span->x0, span->x1); + } + return outFoot; +} + /************************************************************************************************************/ Index: /branches/czw_branch/20101203/psModules/src/objects/pmFootprint.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmFootprint.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmFootprint.h (revision 30631) @@ -27,4 +27,6 @@ bool pmFootprintAllocEmptySpans (pmFootprint *footprint, int nSpans); + +pmFootprint *pmFootprintCopyData(pmFootprint *inFoot, psImage *image); pmFootprint *pmFootprintNormalize(pmFootprint *fp); Index: /branches/czw_branch/20101203/psModules/src/objects/pmFootprintAssignPeaks.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmFootprintAssignPeaks.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmFootprintAssignPeaks.c (revision 30631) @@ -24,5 +24,5 @@ /* * Given a psArray of pmFootprints and another of pmPeaks, assign the peaks to the - * footprints in which that fall; if they _don't_ fall in a footprint, add a suitable + * footprints in which they fall; if they _don't_ fall in a footprint, add a suitable * one to the list. */ @@ -83,8 +83,18 @@ // XXX are we allowed to have peak-less footprints?? - if (!fp->peaks->n) continue; + if (fp->peaks->n == 0) continue; + if (fp->peaks->n == 1) continue; fp->peaks = psArraySort(fp->peaks, pmPeakSortBySN); + // XXX check for an assert on duplicates (I don't think they can happen, but + // let's double check for now) + + for (int j = 1; j < fp->peaks->n; j++) { + psAssert (fp->peaks->data[j] != fp->peaks->data[j-1], "duplicate peak!"); + } + continue; + + // XXX WHY am I culling duplicates? how can there be duplicates? // XXX EAM : the algorithm below should be much faster than using psArrayRemove if // the number of peaks in the footprint is large, or if there are no duplicates. @@ -93,5 +103,5 @@ // track the number of good peaks in the footprint - int nGood = 1; + int lastGood = 0; // check for duplicates @@ -100,12 +110,13 @@ // (if sorted list has A, B, A, B ...) for (int j = 1; j < fp->peaks->n; j++) { - if (fp->peaks->data[j] == fp->peaks->data[nGood]) { + if (fp->peaks->data[j] == fp->peaks->data[lastGood]) { // everything on the array has its own mem reference; free and drop this one psFree (fp->peaks->data[j]); fp->peaks->data[j] = NULL; } else { - nGood ++; + lastGood ++; } } + int nGood = lastGood + 1; // no deleted peaks, go to next footprint Index: /branches/czw_branch/20101203/psModules/src/objects/pmFootprintCullPeaks.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmFootprintCullPeaks.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmFootprintCullPeaks.c (revision 30631) @@ -28,5 +28,5 @@ * Examine the peaks in a pmFootprint, and throw away the ones that are not sufficiently * isolated. More precisely, for each peak find the highest coll that you'd have to traverse - * to reach a still higher peak --- and if that coll's more than nsigma DN below your + * to reach a still higher peak --- and if that coll's more (less?) than nsigma DN below your * starting point, discard the peak. */ @@ -99,5 +99,5 @@ float threshold = flux - nsigma_delta*stdev_pad; - if (isnan(threshold) || threshold < min_threshold) { + if (isnan(threshold)) { // min_threshold is assumed to be below the detection threshold, // so all the peaks are pmFootprint, and this isn't the brightest @@ -109,4 +109,8 @@ threshold = subImg->data.F32[y][x] - 10*FLT_EPSILON; } + + if (threshold < min_threshold) { + threshold = min_threshold; + } // init peakFootprint here? Index: /branches/czw_branch/20101203/psModules/src/objects/pmModel.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmModel.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmModel.c (revision 30631) @@ -70,5 +70,6 @@ tmp->fitRadius = 0; tmp->flags = PM_MODEL_STATUS_NONE; - tmp->residuals = NULL; // XXX should the model own this memory? + tmp->residuals = NULL; // do not free: the model does not own this memory + tmp->isPCM = false; psS32 Nparams = pmModelClassParameterCount(type); @@ -108,6 +109,7 @@ pmModel *pmModelCopy (pmModel *model) { - PS_ASSERT_PTR_NON_NULL(model, NULL); - + if (model == NULL) { + return NULL; + } pmModel *new = pmModelAlloc (model->type); Index: /branches/czw_branch/20101203/psModules/src/objects/pmModel.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmModel.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmModel.h (revision 30631) @@ -43,4 +43,5 @@ float fitRadius; ///< fit radius actually used pmResiduals *residuals; ///< normalized PSF residuals + bool isPCM; ///< is this model fitted with PSF-convolution? // functions for this model which depend on the model class Index: /branches/czw_branch/20101203/psModules/src/objects/pmPCMdata.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmPCMdata.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmPCMdata.h (revision 30631) @@ -83,4 +83,6 @@ pmPCMdata *pcm); +psKernel *pmPCMkernelFromPSF (pmSource *source, int nPix); + bool pmSourceModelGuessPCM (pmPCMdata *pcm, pmSource *source, psImageMaskType maskVal, psImageMaskType markVal); Index: /branches/czw_branch/20101203/psModules/src/objects/pmPSFtry.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmPSFtry.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmPSFtry.c (revision 30631) @@ -84,5 +84,13 @@ for (int i = 0; i < sources->n; i++) { - test->sources->data[i] = pmSourceCopy (sources->data[i]); + pmSource *sourceOld = sources->data[i]; + pmSource *sourceNew = pmSourceCopy (sourceOld); + + // save a reference so we can get back to the original + // this is specifically used in psphotChooosePSF to unflag the candidate PSF sources + // which were not actually used to generate a PSF model + sourceNew->parent = sourceOld; + + test->sources->data[i] = sourceNew; } Index: /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryFitEXT.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryFitEXT.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryFitEXT.c (revision 30631) @@ -65,7 +65,9 @@ if (!source->moments) { psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_EXT_FAIL; + psTrace ("psModules.objects", 4, "masking %d (%d,%d) : no moments\n", i, source->peak->x, source->peak->y); continue; } if (!source->moments->nPixels) { + psTrace ("psModules.objects", 4, "masking %d (%d,%d) : no pixels\n", i, source->peak->x, source->peak->y); psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_EXT_FAIL; continue; @@ -74,6 +76,6 @@ source->modelEXT = pmSourceModelGuess (source, options->type); if (source->modelEXT == NULL) { + psTrace ("psModules.objects", 4, "masking %d (%d,%d) : failed to generate model guess\n", i, source->peak->x, source->peak->y); psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_EXT_FAIL; - psTrace ("psModules.objects", 4, "masking %d (%d,%d) : failed to generate model guess\n", i, source->peak->x, source->peak->y); continue; } @@ -91,6 +93,6 @@ // exclude the poor fits if (!status) { + psTrace ("psModules.objects", 4, "masking %d (%d,%d) : status is poor\n", i, source->peak->x, source->peak->y); psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_EXT_FAIL; - psTrace ("psModules.objects", 4, "masking %d (%d,%d) : status is poor\n", i, source->peak->x, source->peak->y); continue; } Index: /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryFitPSF.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryFitPSF.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryFitPSF.c (revision 30631) @@ -81,4 +81,5 @@ if (source->modelPSF == NULL) { psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_BAD_MODEL; + psTrace ("psModules.objects", 4, "dropping %d (%d,%d) : bad PSF fit\n", i, source->peak->x, source->peak->y); continue; } Index: /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryModel.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryModel.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmPSFtryModel.c (revision 30631) @@ -52,4 +52,23 @@ // mask values indicate the reason the source was rejected: +// XXX some test code (delete eventually) +// bool countMaskedSources(pmPSFtry *try, char *msg) { +// +// int N1 = 0; +// int N2 = 0; +// for (int i = 0; i < try->sources->n; i++) { +// pmSource *source = try->sources->data[i]; +// // fprintf (stderr, "%llx : %d\n", (long long int) source, (source->mode & PM_SOURCE_MODE_PSFSTAR)); +// if (source->mode & PM_SOURCE_MODE_PSFSTAR) { +// N1 ++; +// } +// if (try->mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { +// N2 ++; +// } +// } +// fprintf (stderr, "%s : masked: %d or %d of %ld\n", msg, N1, N2, try->sources->n); +// return true; +// } + // generate a pmPSFtry with a copy of the test PSF sources pmPSFtry *pmPSFtryModel (const psArray *sources, const char *modelName, pmPSFOptions *options, psImageMaskType maskVal, psImageMaskType markVal) @@ -203,4 +222,5 @@ chisq->data.F32[i] = 0.0; mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xff; + psTrace ("psModules.objects", 4, "dropping %d (%d,%d) : no PSF model\n", i, source->peak->x, source->peak->y); } else { flux->data.F32[i] = source->modelPSF->params->data.F32[PM_PAR_I0]; Index: /branches/czw_branch/20101203/psModules/src/objects/pmSource.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSource.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSource.c (revision 30631) @@ -61,5 +61,5 @@ psFree(tmp->moments); psFree(tmp->diffStats); - psFree(tmp->radial); + psFree(tmp->radialAper); psFree(tmp->blends); psTrace("psModules.objects", 10, "---- end ----\n"); @@ -126,5 +126,6 @@ source->extpars = NULL; source->diffStats = NULL; - source->radial = NULL; + source->radialAper = NULL; + source->parent = NULL; source->region = psRegionSet(NAN, NAN, NAN, NAN); @@ -159,5 +160,5 @@ affecting the original. This Copy can be used to allow multiple fit attempts on the same object. The pixels, variance, and mask arrays all point to the same original subarrays. The -peak and moments point at the original values. +peak and moments point at the original values. The models, blends, and XXX are NOT copied *****************************************************************************/ pmSource *pmSourceCopy(pmSource *in) @@ -167,7 +168,4 @@ } pmSource *source = pmSourceAlloc (); - - // keep the original ID so we can find map back to the original - P_PM_SOURCE_SET_ID(source, in->id); // peak has the same values as the original @@ -203,50 +201,4 @@ } -/****************************************************************************** -pmSourceCopyData(): this creates a new, duplicate source with the same parameters as the -original (but is actually a new source at the same location) -*****************************************************************************/ -pmSource *pmSourceCopyData(pmSource *in) -{ - if (in == NULL) { - return(NULL); - } - // this copy is used to allow multiple fit attempts on the - // same object. the pixels, variance, and mask arrays all point to - // the same original subarrays. the peak and moments point at - // the original values. - pmSource *source = pmSourceAlloc (); - - // this is actually the same peak as the original, is this the correct way to handle this? - if (in->peak != NULL) { - source->peak = pmPeakAlloc (in->peak->x, in->peak->y, in->peak->value, in->peak->type); - source->peak->xf = in->peak->xf; - source->peak->yf = in->peak->yf; - source->peak->flux = in->peak->flux; - source->peak->SN = in->peak->SN; - } - - // copy the values in the moments structure - if (in->moments != NULL) { - source->moments = pmMomentsAlloc(); - *source->moments = *in->moments; - } - - // These images are all views to the parent. - // We want a new view, but pointing at the same pixels. - source->pixels = psImageCopyView(NULL, in->pixels); - source->variance = psImageCopyView(NULL, in->variance); - source->maskView = in->maskView ? psImageCopyView(NULL, in->maskView) : NULL; - - // the maskObj is a unique mask array; create a new mask image - source->maskObj = in->maskObj ? psImageCopy (NULL, in->maskObj, PS_TYPE_IMAGE_MASK) : NULL; - - source->type = in->type; - source->mode = in->mode; - source->imageID = in->imageID; - - return(source); -} - // x,y are defined in the parent image coords of readout->image bool pmSourceDefinePixels(pmSource *mySource, @@ -300,4 +252,5 @@ newRegion = psRegionForImage (readout->image, newRegion); + // re-define if required by region or absence of pixels extend = false; extend |= (int)(newRegion.x0) < (int)(mySource->region.x0); @@ -335,4 +288,37 @@ } return extend; +} + +bool pmSourceRedefinePixelsByRegion(pmSource *mySource, + const pmReadout *readout, + psRegion newRegion) +{ + PS_ASSERT_PTR_NON_NULL(mySource, false); + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(readout->image, false); + + // re-create the subimage + psFree (mySource->pixels); + psFree (mySource->variance); + psFree (mySource->maskView); + + mySource->pixels = psImageSubset(readout->image, newRegion); + mySource->variance = psImageSubset(readout->variance, newRegion); + mySource->maskView = psImageSubset(readout->mask, newRegion); + mySource->region = newRegion; + + // re-copy the main mask pixels. NOTE: the user will need to reset the object mask + // pixels (eg, with psImageKeepCircle) + mySource->maskObj = psImageCopy (mySource->maskObj, mySource->maskView, PS_TYPE_IMAGE_MASK); + + // drop the old modelFlux pixels and force the user to re-create + psFree (mySource->modelFlux); + mySource->modelFlux = NULL; + + // drop the old psfImage pixels and force the user to re-create + psFree (mySource->psfImage); + mySource->psfImage = NULL; + + return true; } @@ -675,5 +661,5 @@ if ((source->moments->SN > PSF_SN_LIM) && (radius < PSF_CLUMP_NSIGMA)) { source->type = PM_SOURCE_TYPE_STAR; - source->mode |= PM_SOURCE_MODE_PSFSTAR; + source->tmpFlags |= PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; Npsf ++; continue; @@ -934,6 +920,5 @@ } -// construct a realization of the source model -// XXX this function should optionally save an existing psf image from modelFlux +// construct a realization of the PSF model at the location of this source bool pmSourceCachePSF (pmSource *source, psImageMaskType maskVal) { PS_ASSERT_PTR_NON_NULL(source, false); @@ -1020,12 +1005,10 @@ } - if (!addNoise) { - if (add) { - status = pmModelAddWithOffset (target, source->maskObj, model, PM_MODEL_OP_FULL, maskVal, dx, dy); - source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; - } else { - status = pmModelSubWithOffset (target, source->maskObj, model, PM_MODEL_OP_FULL, maskVal, dx, dy); - source->tmpFlags |= PM_SOURCE_TMPF_SUBTRACTED; - } + if (add) { + status = pmModelAddWithOffset (target, source->maskObj, model, PM_MODEL_OP_FULL, maskVal, dx, dy); + source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + } else { + status = pmModelSubWithOffset (target, source->maskObj, model, PM_MODEL_OP_FULL, maskVal, dx, dy); + source->tmpFlags |= PM_SOURCE_TMPF_SUBTRACTED; } Index: /branches/czw_branch/20101203/psModules/src/objects/pmSource.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSource.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSource.h (revision 30631) @@ -35,4 +35,5 @@ PM_SOURCE_TMPF_SIZE_CR_CANDIDATE = 0x0008, PM_SOURCE_TMPF_MOMENTS_MEASURED = 0x0010, + PM_SOURCE_TMPF_CANDIDATE_PSFSTAR = 0x0020, } pmSourceTmpF; @@ -42,4 +43,16 @@ * simplest measurement of a source is the location and flux of the peak pixel * associated with the source: + * + * a pmSource is the information about a (possible) blob of flux in a specific image. A source + * may represent an insignificant or undetected source. There may be multiple representations + * of an image (eg, alternate smoothed copies); sources on alternate images may have a pointer + * to the version on the primary image (source->parent). A set of sources on different, but + * related images (eg, multiple exposures or different filters) which (may) represent the same + * astronomical object are grouped together with the pmPhotObj type (set pmPhotObj.h). + * + * A single source may be fitted with multiple models (not at the same time!). The PSF model + * fit is a fit of the position (optionally) and the flux to the PSF model at the location of + * the source. Alternate model fits are extended source models. The best model fit is used to + * subtract the object from the image. * * XXX do I have to re-organize this (again!) to allow an arbitrary set of extended model fits?? @@ -54,9 +67,9 @@ pmPeak *peak; ///< Description of peak pixel. psImage *pixels; ///< Rectangular region including object pixels. - psImage *variance; ///< Image variance. + psImage *variance; ///< Image variance. psImage *maskObj; ///< unique mask for this object which marks included pixels associated with objects. psImage *maskView; ///< view into global image mask for this object region psImage *modelFlux; ///< cached copy of the best model for this source - psImage *psfImage; ///< cached copy of the psf model for this source + psImage *psfImage; ///< cached copy of the psf model for this source pmMoments *moments; ///< Basic moments measured for the object. pmModel *modelPSF; ///< PSF Model fit (parameters and type) @@ -89,5 +102,6 @@ pmSourceExtendedPars *extpars; ///< extended source parameters pmSourceDiffStats *diffStats; ///< extra parameters for difference detections - pmSourceRadialApertures *radial; ///< radial flux in circular apertures + psArray *radialAper; ///< radial flux in circular apertures + pmSource *parent; ///< reference to the master source from which this is derived int imageID; }; @@ -162,4 +176,10 @@ ); +bool pmSourceRedefinePixelsByRegion ( + pmSource *mySource, ///< source to be re-defined + const pmReadout *readout, ///< base the source on this readout + psRegion newRegion ///< region for source pixel definition +); + /** pmSourcePSFClump() * Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceExtendedPars.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceExtendedPars.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceExtendedPars.c (revision 30631) @@ -17,21 +17,5 @@ #endif -// #include -// #include -// #include #include -// #include "pmHDU.h" -// #include "pmFPA.h" -// #include "pmFPAMaskWeight.h" -// #include "pmSpan.h" -// #include "pmFootprint.h" -// #include "pmPeaks.h" -// #include "pmMoments.h" -// #include "pmResiduals.h" -// #include "pmGrowthCurve.h" -// #include "pmTrend2D.h" -// #include "pmPSF.h" -// #include "pmModel.h" -// #include "pmSource.h" #include "pmSourceExtendedPars.h" Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceFitPCM.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceFitPCM.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceFitPCM.c (revision 30631) @@ -89,4 +89,7 @@ if (!fitStatus) pcm->modelConv->flags |= PM_MODEL_STATUS_NONCONVERGE; + // once we have fitted a model, we need to record that this model is a PCM model: + pcm->modelConv->isPCM = true; + // models can go insane: reject these bool onPic = true; Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO.c (revision 30631) @@ -59,12 +59,13 @@ // lookup the EXTNAME values used for table data and image header segments -static bool sourceExtensions(psString *headname, // Extension name for header - psString *dataname, // Extension name for data - psString *deteffname, // Extension name for detection efficiency - psString *xsrcname, // Extension name for extended sources - psString *xfitname, // Extension name for extended fits - const pmFPAfile *file, // File of interest - const pmFPAview *view // View to level of interest - ) +bool pmSourceIOextnames(psString *headname, // Extension name for image header + psString *dataname, // Extension name for PSF table data + psString *deteffname, // Extension name for detection efficiency + psString *xsrcname, // Extension name for extended non-parametric measurements + psString *xfitname, // Extension name for extended fitted measurements + psString *xradname, // Extension name for radial apertures + const pmFPAfile *file, // File of interest + const pmFPAview *view // View to level of interest + ) { bool status; // Status of MD lookup @@ -87,5 +88,5 @@ } - // EXTNAME for table data + // EXTNAME for PSF table data if (dataname) { const char *rule = psMetadataLookupStr(&status, menu, "CMF.DATA"); @@ -107,5 +108,5 @@ } - // EXTNAME for extended source data table + // EXTNAME for extended source non-parametric measurements if (xsrcname) { const char *rule = psMetadataLookupStr(&status, menu, "CMF.XSRC"); @@ -117,6 +118,6 @@ } + // EXTNAME for extended source fitted measurements if (xfitname) { - // EXTNAME for extended source data table const char *rule = psMetadataLookupStr(&status, menu, "CMF.XFIT"); if (!rule) { @@ -125,4 +126,14 @@ } *xfitname = pmFPAfileNameFromRule (rule, file, view); + } + + // EXTNAME for radial apertures + if (xradname) { + const char *rule = psMetadataLookupStr(&status, menu, "CMF.XRAD"); + if (!rule) { + psError(PS_ERR_UNKNOWN, true, "missing entry for CMF.XRAD in EXTNAME.RULES in camera.config"); + return false; + } + *xradname = pmFPAfileNameFromRule (rule, file, view); } @@ -344,4 +355,7 @@ status &= pmSourcesWrite_##TYPE##_XFIT (file->fits, readout, sources, file->header, xfitname); \ } \ + if (xradname) { \ + status &= pmSourcesWrite_##TYPE##_XRAD (file->fits, readout, sources, file->header, xradname, recipe); \ + } \ } @@ -449,4 +463,5 @@ bool XSRC_OUTPUT = psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_ANALYSIS"); bool XFIT_OUTPUT = psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_FITS"); + bool XRAD_OUTPUT = psMetadataLookupBool(&status, recipe, "RADIAL_APERTURES"); // define the EXTNAME values for the different data segments: @@ -456,8 +471,10 @@ psString xsrcname = NULL; psString xfitname = NULL; - if (!sourceExtensions(&headname, &dataname, &deteffname, - XSRC_OUTPUT ? &xsrcname : NULL, - XFIT_OUTPUT ? &xfitname : NULL, - file, view)) { + psString xradname = NULL; + if (!pmSourceIOextnames(&headname, &dataname, &deteffname, + XSRC_OUTPUT ? &xsrcname : NULL, + XFIT_OUTPUT ? &xfitname : NULL, + XRAD_OUTPUT ? &xradname : NULL, + file, view)) { return false; } @@ -539,5 +556,7 @@ psMetadataAddStr (outhead, PS_LIST_TAIL, "XFITNAME", PS_META_REPLACE, "name of XFIT table extension", xfitname); } - + if (xradname) { + psMetadataAddStr (outhead, PS_LIST_TAIL, "XRADNAME", PS_META_REPLACE, "name of XRAD table extension", xradname); + } // these are case-sensitive since the EXTYPE is case-sensitive @@ -563,5 +582,4 @@ } - // write out the detection efficiency TABLE segments if (deteffname) { @@ -583,4 +601,5 @@ psFree (xsrcname); psFree (xfitname); + psFree (xradname); psFree (deteffname); @@ -593,4 +612,5 @@ psFree (xsrcname); psFree (xfitname); + psFree (xradname); psFree (deteffname); return false; @@ -939,5 +959,5 @@ psString dataname = NULL; psString deteffname = NULL; - if (!sourceExtensions(&headname, &dataname, &deteffname, NULL, NULL, file, view)) { + if (!pmSourceIOextnames(&headname, &dataname, &deteffname, NULL, NULL, NULL, file, view)) { return false; } Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO.h (revision 30631) @@ -30,40 +30,50 @@ bool pmSourcesWrite_SMPDATA_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_SMPDATA_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_SMPDATA_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_DEV_0(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_DEV_0_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_DEV_0_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_PS1_DEV_0_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_DEV_1(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_DEV_1_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_DEV_1_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_PS1_DEV_1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_CAL_0(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_CAL_0_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_PS1_CAL_0_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_PS1_CAL_0_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V1(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V1_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V1_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_V1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V2(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V2_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V2_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_V2_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V3(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V3_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_V3_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_V3_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_SV1(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_SV1_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_SV1_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_SV1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_DV1(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_DV1_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_DV1_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_DV1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_DV2(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_DV2_XSRC(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); bool pmSourcesWrite_CMF_PS1_DV2_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_DV2_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); psArray *pmSourcesReadCMP (char *filename, psMetadata *header); Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_DV1.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_DV1.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_DV1.c (revision 30631) @@ -676,2 +676,7 @@ return true; } + +bool pmSourcesWrite_CMF_PS1_DV1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c (revision 30631) @@ -737,2 +737,7 @@ return true; } + +bool pmSourcesWrite_CMF_PS1_DV2_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c (revision 30631) @@ -20,4 +20,5 @@ #include "pmConfig.h" +#include "pmErrorCodes.h" #include "pmDetrendDB.h" @@ -71,5 +72,4 @@ psF32 errMag, chisq, apRadius; psS32 nPix, nDOF; - char keyword1[80], keyword2[80]; pmChip *chip = readout->parent->parent; @@ -103,4 +103,5 @@ table = psArrayAllocEmpty (sources->n); +# if (0) // we use this just to define the output vectors (which must be present for all objects) bool status = false; @@ -118,5 +119,5 @@ psMetadataAddF32 (imageHeader, PS_LIST_TAIL, keyword2, PS_META_REPLACE, "min radius for SB profile", radMax->data.F32[i]); } - +# endif // we write out PSF-fits for all sources, regardless of quality. the source flags tell us the state @@ -258,4 +259,6 @@ psMetadataAdd (row, PS_LIST_TAIL, "FLAGS2", PS_DATA_U32, "psphot analysis flags", source->mode2); +# if (0) + // XXX if we have raw radial apertures, write them out here psVector *radFlux = psVectorAlloc(radMax->n, PS_TYPE_F32); psVector *radFluxErr = psVectorAlloc(radMax->n, PS_TYPE_F32); @@ -284,4 +287,5 @@ psFree (radFluxErr); psFree (radFill); +# endif // XXX not sure how to get this : need to load Nimages with weight? @@ -781,2 +785,137 @@ return true; } + +// **** write out the radial flux values for the sources for a given matched-PSF image +// **** how do we distinguish the matched-PSF images from the non-matched version +bool pmSourcesWrite_CMF_PS1_SV1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + + psArray *table; + psMetadata *row; + psF32 xPos, yPos; + char keyword1[80], keyword2[80]; + + // create a header to hold the output data + psMetadata *outhead = psMetadataAlloc (); + + // write the links to the image header + psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "radial flux table extension", extname); + + // we use this just to define the output vectors (which must be present for all objects) + bool status = false; + psVector *radMin = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.LOWER"); + psVector *radMax = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + psAssert (radMax, "this must have been defined and tested earlier!"); + psAssert (radMax->n, "this must have been defined and tested earlier!"); + psAssert (radMin->n == radMax->n, "inconsistent annular bins"); + + // write the radial profile apertures to header + for (int i = 0; i < radMax->n; i++) { + sprintf (keyword1, "RMIN_%02d", i); + sprintf (keyword2, "RMAX_%02d", i); + psMetadataAddF32 (imageHeader, PS_LIST_TAIL, keyword1, PS_META_REPLACE, "min radius for SB profile", radMin->data.F32[i]); + psMetadataAddF32 (imageHeader, PS_LIST_TAIL, keyword2, PS_META_REPLACE, "min radius for SB profile", radMax->data.F32[i]); + } + + psVector *fwhmValues = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + if (!fwhmValues) { + psError (PM_ERR_CONFIG, true, "convolved or measured FWHM is not defined for this readout"); + return false; + } + + // let's write these out in S/N order + sources = psArraySort (sources, pmSourceSortBySN); + + table = psArrayAllocEmpty (sources->n); + + // we write out all sources, regardless of quality. the source flags tell us the state + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // skip sources without radial aper measurements (or insufficient) + if (source->radialAper == NULL) continue; + psAssert (source->radialAper->n == fwhmValues->n, "inconsistent radial aperture set"); + + for (int entry = 0; entry < fwhmValues->n; entry++) { + + // choose the convolved EXT model, if available, otherwise the simple one + pmSourceRadialApertures *radialAper = source->radialAper->data[entry]; + assert (radialAper); + + bool useMoments = true; + useMoments = (useMoments && source->moments); // can't if there are no moments + useMoments = (useMoments && source->moments->nPixels); // can't if the moments were not measured + useMoments = (useMoments && !(source->mode && PM_SOURCE_MODE_MOMENTS_FAILURE)); // can't if the moments failed... + + if (useMoments) { + xPos = source->moments->Mx; + yPos = source->moments->My; + } else { + xPos = source->peak->xf; + yPos = source->peak->yf; + } + + row = psMetadataAlloc (); + + // XXX we are not writing out the mode (flags) or the type (psf, ext, etc) + psMetadataAddU32 (row, PS_LIST_TAIL, "IPP_IDET", 0, "IPP detection identifier index", source->seq); + psMetadataAddF32 (row, PS_LIST_TAIL, "X_APER", 0, "Center of aperture measurements", xPos); + psMetadataAddF32 (row, PS_LIST_TAIL, "Y_APER", 0, "Center of aperture measurements", yPos); + psMetadataAddF32 (row, PS_LIST_TAIL, "PSF_FWHM", 0, "FWHM of matched PSF", fwhmValues->data.F32[entry]); + + // XXX if we have raw radial apertures, write them out here + psVector *radFlux = psVectorAlloc(radMax->n, PS_TYPE_F32); + psVector *radFluxErr = psVectorAlloc(radMax->n, PS_TYPE_F32); + psVector *radFill = psVectorAlloc(radMax->n, PS_TYPE_F32); + psVectorInit (radFlux, NAN); + psVectorInit (radFluxErr, NAN); + psVectorInit (radFill, NAN); + if (!radialAper->flux) goto write_annuli; + if (!radialAper->fill) goto write_annuli; + psAssert (radialAper->flux->n <= radFlux->n, "inconsistent vector lengths"); + psAssert (radialAper->fill->n <= radFlux->n, "inconsistent vector lengths"); + + // copy the data from fluxVal (which is not guaranteed to be the full length) to radFlux + for (int j = 0; j < radialAper->flux->n; j++) { + radFlux->data.F32[j] = radialAper->flux->data.F32[j]; + radFluxErr->data.F32[j] = radialAper->fluxErr->data.F32[j]; + radFill->data.F32[j] = radialAper->fill->data.F32[j]; + } + + write_annuli: + psMetadataAdd (row, PS_LIST_TAIL, "APER_FLUX", PS_DATA_VECTOR, "flux within annuli", radFlux); + psMetadataAdd (row, PS_LIST_TAIL, "APER_FLUX_ERR", PS_DATA_VECTOR, "flux error in annuli", radFluxErr); + psMetadataAdd (row, PS_LIST_TAIL, "APER_FILL", PS_DATA_VECTOR, "fill factor of annuli", radFill); + psFree (radFlux); + psFree (radFluxErr); + psFree (radFill); + + psArrayAdd (table, 100, row); + psFree (row); + } + } + + if (table->n == 0) { + if (!psFitsWriteBlank (fits, outhead, extname)) { + psError(psErrorCodeLast(), false, "Unable to write empty sources file."); + psFree(outhead); + psFree(table); + return false; + } + psFree (outhead); + psFree (table); + return true; + } + + psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname); + if (!psFitsWriteTable (fits, outhead, table, extname)) { + psError(psErrorCodeLast(), false, "writing ext data %s\n", extname); + psFree (outhead); + psFree(table); + return false; + } + psFree (outhead); + psFree (table); + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V1.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V1.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V1.c (revision 30631) @@ -721,2 +721,7 @@ return false; } + +bool pmSourcesWrite_CMF_PS1_V1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V2.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V2.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V2.c (revision 30631) @@ -724,2 +724,7 @@ return true; } + +bool pmSourcesWrite_CMF_PS1_V2_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c (revision 30631) @@ -762,2 +762,7 @@ return true; } + +bool pmSourcesWrite_CMF_PS1_V3_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_CAL_0.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_CAL_0.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_CAL_0.c (revision 30631) @@ -702,2 +702,7 @@ return true; } + +bool pmSourcesWrite_PS1_CAL_0_XRAD (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_DEV_0.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_DEV_0.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_DEV_0.c (revision 30631) @@ -248,2 +248,7 @@ return true; } + +bool pmSourcesWrite_PS1_DEV_0_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_DEV_1.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_DEV_1.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_PS1_DEV_1.c (revision 30631) @@ -588,2 +588,7 @@ return true; } + +bool pmSourcesWrite_PS1_DEV_1_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_SMPDATA.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_SMPDATA.c (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/objects/pmSourceIO_SMPDATA.c (revision 30631) @@ -222,2 +222,7 @@ return true; } + +bool pmSourcesWrite_SMPDATA_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe) +{ + return true; +} Index: /branches/czw_branch/20101203/psModules/src/psmodules.h =================================================================== --- /branches/czw_branch/20101203/psModules/src/psmodules.h (revision 30630) +++ /branches/czw_branch/20101203/psModules/src/psmodules.h (revision 30631) @@ -98,4 +98,5 @@ // the following headers are from psModule:imcombine #include +#include #include #include Index: /branches/czw_branch/20101203/psconfig/psbuild =================================================================== --- /branches/czw_branch/20101203/psconfig/psbuild (revision 30630) +++ /branches/czw_branch/20101203/psconfig/psbuild (revision 30631) @@ -13,4 +13,5 @@ $operations = 0; $start = ""; +$start_after = ""; $stop = ""; $verbose = 0; @@ -85,4 +86,5 @@ } if ($ARGV[0] eq "-only") { + if ($start || $stop || $start_after) { die "do not mix -only (-start, -stop) or (-after, -stop)\n"; } $start = $ARGV[1]; $stop = $ARGV[1]; @@ -90,8 +92,16 @@ } if ($ARGV[0] eq "-start") { + if ($start || $start_after) { die "do not mix -only (-start, -stop) or (-after, -stop)\n"; } $start = $ARGV[1]; shift; shift; next; } + if ($ARGV[0] eq "-after") { + if ($start || $start_after) { die "do not mix -only (-start, -stop) or (-after, -stop)\n"; } + $start_after = $ARGV[1]; + $start = "INVALID_VALUE"; # need to have a value to skip earlier entries + shift; shift; next; + } if ($ARGV[0] eq "-stop") { + if ($stop) { die "do not mix -only (-start, -stop) or (-after, -stop)\n"; } $stop = $ARGV[1]; shift; shift; next; @@ -258,6 +268,7 @@ $stop_now = 0; for ($i = 0; !$stop_now && ($i < @module); $i++) { - if (($stop ne "") && ($stop eq $module[$i])) { $stop_now = 1; } - if (($start ne "") && ($start ne $module[$i])) { next; } + if ($stop && ($stop eq $module[$i])) { $stop_now = 1; } + if ($start_after && ($start_after eq $module[$i])) { $start = $module[$i+1]; next; } + if ($start && ($start ne $module[$i])) { next; } $start = ""; @@ -348,4 +359,7 @@ chdir $homedir; } + if ($start) { if ($start_after) { $start = $start_after; } &failure($start, "unknown -start or -after module $start\n"); } + if ($stop && !$stop_now) { &failure($start, "unknown -stop module $stop\n"); } + print "\033]0; ** psbuild: finished ** \007"; exit 0; Index: /branches/czw_branch/20101203/psphot/doc/notes.20101222.txt =================================================================== --- /branches/czw_branch/20101203/psphot/doc/notes.20101222.txt (revision 30631) +++ /branches/czw_branch/20101203/psphot/doc/notes.20101222.txt (revision 30631) @@ -0,0 +1,18 @@ + +20101222 : EAM + + continuing work on psphot stack. Some notes: + + * visualization now has duplicate image display calls + * visual needs to do a better jobs scaling the smoothed, subtracted images. + * the fit radius for the convolved images should be a bit bigger than the unconvolved one + * *** fitRadius vs apRadius *** + I have been using 'markVal' to define the valid pixels for different operations + I think this has been inconsistent. + + ** Have pmSourceMagnitudes apply the defined apRadius on the fly + when measuring aperture photometry + + ** Set the fitRadius and keep it until redefined? + + ** RadialApertures does not use the marked region? Index: /branches/czw_branch/20101203/psphot/doc/stack.txt =================================================================== --- /branches/czw_branch/20101203/psphot/doc/stack.txt (revision 30630) +++ /branches/czw_branch/20101203/psphot/doc/stack.txt (revision 30631) @@ -1,2 +1,92 @@ + +20101221 + + psphotStackReadout is now correctly subtracting the PSF models from + the images as it measures the radial aperture fluxes. + + Some issues: + + * the source properties get buggered up by the radial aperture flux + analysis (we need to re-determine the psf, re-create the source + pixels, and re-fit the models (linearly) to subtract them + correctly). + + * the standard analysis sequence is not doing a second pass + + * the choice of the best model is ambiguous + + * the radial aperture (and source addition / subtraction) is only + using the PSF model + + * make sure psphotEfficiency actually subtracts the inserted fake + sources + + * XXX how does the PSF-convolved model work with changing psf models? + +20101207 + + header fields for PSPS: + + * stack needs to count # of inputs + * propagate the input header to the output image (DONE) + * save PSF parameters in header + * stack type is only known to the launcher + + * what are we doing for the stack calibrations?? + +skycellID : code +surveyID : code +filterID : code +stackMetaID : stack_id +photoCalID : photcode -> number +magSat : FSATUR saturation magnitude level ** not correctly set +completMag : FLIMIT 95% completion level in mag ** not correctly set +stackTypeID : STK_TYPE stack type identifier ** deep stack, nightly stack, best IQ stack? +refImageID : identifier of image used as reference for analysis +subtrImageID : N/A (stack) identifier of image subtracted to generate difference image +analVer : analysis version index ** index for tess_id + skycell_id + filter? +nP2Images : NINPUTS number of P2 images contributing to this cell ** missing from input stack +astroScat : astrometric scatter for chip ** measure scatter on stack creation? +photoScat : photometric scatter for chip ** internal scatter? +nAstroRef : number of astrometric reference sources ** (connected to above) +nPhoRef : number of photometric reference sources ** same +psfFwhm : PSF full width at half maximum ** 0.5(FWHM_MAJ + FWHM_MIN) +psfmodelID : * PSFMODEL PSF model identifier ** save the PSF model name in the header +psfWidMajor : * FWHM_MAJ PSF parameters +psfWidMinor : * FWHM_MIN PSF parameters +psfTheta : * ANGLE PSF parameters +psfExtra1 : * PSF_EXT1 PSF parameters ** (at field center?) +psfExtra2 : * PSF_EXT2 PSF parameters ** (not set for all models) +photoZero : local derived photometric zero point +photoColor : local derived photometric color term +ctype1 : * CTYPE1 name of astrometric projection in RA ** propagate from input stacks +ctype2 : * CTYPE2 name of astrometric projection in DEC +crval1 : * CRVAL1 RA corresponding to reference pixel +crval2 : * CRVAL2 DEC corresponding to reference pixel +crpix1 : * CRPIX1 reference pixel value for RA +crpix2 : * CRPIX2 reference pixel value for DEC +cdelt1 : * CDELT1 scale factor for RA +cdelt2 : * CDELT2 scale factor for DEC +pc001001 : * PC001001 elements of rotation/Dcale matrix +pc001002 : * PC001002 elements of rotation/Dcale matrix +pc002001 : * PC002001 elements of rotation/Dcale matrix +pc002002 : * PC002002 elements of rotation/Dcale matrix +polyOrder : * NPLYTERM polynomial order of astrometry fit ** default to 1 +pca1x3y0 : * PCA1X3Y0 polynomial coefficients for the astrometric fit +pca1x2y1 : * PCA1X2Y1 polynomial coefficients for the astrometric fit +pca1x1y2 : * PCA1X1Y2 polynomial coefficients for the astrometric fit +pca1x0y3 : * PCA1X0Y3 polynomial coefficients for the astrometric fit +pca1x2y0 : * PCA1X2Y0 polynomial coefficients for the astrometric fit +pca1x1y1 : * PCA1X1Y1 polynomial coefficients for the astrometric fit +pca1x0y2 : * PCA1X0Y2 polynomial coefficients for the astrometric fit +pca2x3y0 : * PCA2X3Y0 polynomial coefficients for the astrometric fit +pca2x2y1 : * PCA2X2Y1 polynomial coefficients for the astrometric fit +pca2x1y2 : * PCA2X1Y2 polynomial coefficients for the astrometric fit +pca2x0y3 : * PCA2X0Y3 polynomial coefficients for the astrometric fit +pca2x2y0 : * PCA2X2Y0 polynomial coefficients for the astrometric fit +pca2x1y1 : * PCA2X1Y1 polynomial coefficients for the astrometric fit +pca2x0y2 : * PCA2X0Y2 polynomial coefficients for the astrometric fit +calibModNum : calibration modification number +dataRelease : Data release 20100506: Index: /branches/czw_branch/20101203/psphot/src/Makefile.am =================================================================== --- /branches/czw_branch/20101203/psphot/src/Makefile.am (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/Makefile.am (revision 30631) @@ -98,4 +98,5 @@ psphotStackMatchPSFsUtils.c \ psphotStackMatchPSFsPrepare.c \ + psphotStackMatchPSFsNext.c \ psphotStackOptions.c \ psphotStackObjects.c \ Index: /branches/czw_branch/20101203/psphot/src/psphot.h =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphot.h (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphot.h (revision 30631) @@ -75,6 +75,6 @@ bool psphotImageQualityReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); -bool psphotChoosePSF (pmConfig *config, const pmFPAview *view, const char *filerule); -bool psphotChoosePSFReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotChoosePSF (pmConfig *config, const pmFPAview *view, const char *filerule, bool newSources); +bool psphotChoosePSFReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool newSources); bool psphotGuessModels (pmConfig *config, const pmFPAview *view, const char *filerule); @@ -260,5 +260,6 @@ bool psphotVisualShowSourceSize (pmReadout *readout, psArray *sources); bool psphotVisualPlotSourceSizeAlt (psMetadata *recipe, psMetadata *analysis, psArray *sources); -bool psphotVisualShowResidualImage (pmReadout *readout); +bool psphotVisualShowResidualImage (pmReadout *readout, bool reshow); +bool psphotVisualShowObjectRegions (pmReadout *readout, psMetadata *recipe, psArray *sources); bool psphotVisualPlotApResid (psArray *sources, float mean, float error, bool useApMag); bool psphotVisualPlotChisq (psArray *sources); @@ -312,4 +313,5 @@ int psphotFileruleCount(const pmConfig *config, const char *filerule); +bool psphotFileruleCountSet(const pmConfig *config, const char *filerule, int num); bool psphotAddKnownSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources); @@ -403,4 +405,6 @@ bool psphotStackMatchPSFsReadout (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index); bool psphotStackMatchPSFsPrepare (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index); +bool psphotStackMatchPSFsNext (bool *smoothAgain, pmConfig *config, const pmFPAview *view, const char *filerule, int lastSize); +bool psphotStackMatchPSFsNextReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, psVector *fwhmValues, int lastSize); // psphotStackMatchPSFsUtils @@ -426,8 +430,8 @@ bool psphotRadialApertures (pmConfig *config, const pmFPAview *view, const char *filerule); bool psphotRadialAperturesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); -bool psphotRadialApertureSource (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal, const psVector *radMax); +bool psphotRadialApertureSource (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal, const psVector *radMax, int entry); bool psphotExtendedSourceAnalysisByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule); -bool psphotRadialAperturesByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule); +bool psphotRadialAperturesByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule, int nMatchedPSF); bool psphotStackObjectsUnifyPosition (psArray *objects); @@ -440,3 +444,13 @@ bool psphotCleanInputs (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotResetModels (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotResetModelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotRedefinePixels (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotRedefinePixelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotSourceChildren (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); +bool psphotSourceChildrenReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); +psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objectsSrc); + #endif Index: /branches/czw_branch/20101203/psphot/src/psphotBlendFit.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotBlendFit.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotBlendFit.c (revision 30631) @@ -187,5 +187,6 @@ psLogMsg ("psphot.psphotBlendFit", PS_LOG_INFO, "fit models: %f sec for %d objects (%d psf, %d ext, %d failed, %ld skipped)\n", psTimerMark ("psphot.fit.nonlinear"), Nfit, Npsf, Next, Nfail, sources->n - Nfit); - psphotVisualShowResidualImage (readout); + psphotVisualShowResidualImage (readout, false); + psphotVisualShowObjectRegions (readout, recipe, sources); psphotVisualShowFlags (sources); Index: /branches/czw_branch/20101203/psphot/src/psphotCheckStarDistribution.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotCheckStarDistribution.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotCheckStarDistribution.c (revision 30631) @@ -34,5 +34,5 @@ pmSource *source = stars->data[i]; if (source->peak == NULL) continue; - if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + if (!(source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR)) continue; int binX = psImageBinningGetRuffX (binning, source->peak->xf); @@ -85,5 +85,5 @@ if (source->peak == NULL) continue; if (source->moments == NULL) continue; - if (source->mode & PM_SOURCE_MODE_PSFSTAR) continue; + if (source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR) continue; if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; if (source->type != PM_SOURCE_TYPE_STAR) continue; @@ -97,5 +97,5 @@ if (y > Ye) continue; - source->mode |= PM_SOURCE_MODE_PSFSTAR; + source->tmpFlags |= PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; psArrayAdd (stars, 200, source); Index: /branches/czw_branch/20101203/psphot/src/psphotChoosePSF.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotChoosePSF.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotChoosePSF.c (revision 30631) @@ -2,5 +2,5 @@ // generate a PSF model for inputs without PSF models already loaded -bool psphotChoosePSF (pmConfig *config, const pmFPAview *view, const char *filerule) +bool psphotChoosePSF (pmConfig *config, const pmFPAview *view, const char *filerule, bool newSources) { bool status = true; @@ -19,5 +19,5 @@ for (int i = 0; i < num; i++) { if (i == chisqNum) continue; // skip chisq image - if (!psphotChoosePSFReadout (config, view, filerule, i, recipe)) { + if (!psphotChoosePSFReadout (config, view, filerule, i, recipe, newSources)) { psError (PSPHOT_ERR_CONFIG, false, "failed to choose a psf model for %s entry %d", filerule, i); return false; @@ -28,5 +28,5 @@ // try PSF models and select best option -bool psphotChoosePSFReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { +bool psphotChoosePSFReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool newSources) { bool status; @@ -50,5 +50,5 @@ psAssert (detections, "missing detections?"); - psArray *sources = detections->newSources; + psArray *sources = newSources ? detections->newSources : detections->allSources; psAssert (sources, "missing sources?"); @@ -166,10 +166,8 @@ for (int i = 0; i < sources->n; i++) { pmSource *source = sources->data[i]; - if (source->mode & PM_SOURCE_MODE_PSFSTAR) { - // keep NSTARS PSF stars, unmark the rest + if (source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR) { + // keep NSTARS PSF stars if (stars->n < NSTARS) { psArrayAdd (stars, 200, source); - } else { - source->mode &= ~PM_SOURCE_MODE_PSFSTAR; } } @@ -214,9 +212,5 @@ psFree(options); - // unset the PSFSTAR flags (none are used): - for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - source->mode &= ~PM_SOURCE_MODE_PSFSTAR; - } + // no sources are used as PSF stars // XXX set sxx, etc from FWHM in recipe @@ -292,9 +286,5 @@ psLogMsg ("psphot.pspsf", PS_LOG_INFO, "Using guess PSF model"); - // unset the PSFSTAR flags (none are used): - for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - source->mode &= ~PM_SOURCE_MODE_PSFSTAR; - } + // no sources are used as PSF stars // XXX set sxx, etc from FWHM in recipe @@ -322,45 +312,21 @@ pmPSFtry *try = models->data[bestN]; - // unset the PSFSTAR flag for stars not used for PSF model - // XXX a more efficient way of achieving this would be to record a pair of arrays - // of the source index and the source id for the psf stars. but that would require we do - // not re-sort the source list in the meanwhile - int nDrop = 0; + // set the PSFSTAR flag for stars used for the PSF model + int nKeep = 0; for (int i = 0; i < try->sources->n; i++) { pmSource *source = try->sources->data[i]; - if (try->mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { - // need to find this source in the original list (these are copies, not pointers) - for (int j = 0; j < sources->n; j++) { - pmSource *realSource = sources->data[j]; - if (realSource->id != source->id) continue; - realSource->mode &= ~PM_SOURCE_MODE_PSFSTAR; - source->mode &= ~PM_SOURCE_MODE_PSFSTAR; - nDrop ++; - break; - } - } - } - // fprintf (stderr, "drop %d stars as PSF stars\n", nDrop); - - // XXX is this working? - // int N1 = 0; - // for (int i = 0; i < try->sources->n; i++) { - // pmSource *source = try->sources->data[i]; - // fprintf (stderr, "%llx : %d\n", (long long int) source, (source->mode & PM_SOURCE_MODE_PSFSTAR)); - // if (source->mode & PM_SOURCE_MODE_PSFSTAR) { - // N1 ++; - // } - // } - // int N2 = 0; - // for (int i = 0; i < sources->n; i++) { - // pmSource *source = sources->data[i]; - // fprintf (stderr, "%llx : %d\n", (long long int) source, (source->mode & PM_SOURCE_MODE_PSFSTAR)); - // if (source->mode & PM_SOURCE_MODE_PSFSTAR) { - // N2 ++; - // } - // } - // fprintf (stderr, "N1: %d, N2: %d\n", N1, N2); + if (try->mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue; + source->mode |= PM_SOURCE_MODE_PSFSTAR; + + // this source was used: find the parent and set its PSFSTAR flag + pmSource *realSource = source->parent; + psAssert (realSource, "pmPSFtryAlloc should have set the parent pointers"); + realSource->mode |= PM_SOURCE_MODE_PSFSTAR; + nKeep ++; + } + psLogMsg ("psphot.pspsf", PS_LOG_DETAIL, "used %d of %ld candidate PSF objects\n", nKeep, try->sources->n); // build a PSF residual image + // we need to use the 'try' set because they have the associated models defined if (!psphotMakeResiduals (try->sources, recipe, try->psf, maskVal)) { psError(PSPHOT_ERR_PSF, false, "Unable to construct residual table for PSF"); @@ -386,5 +352,5 @@ } - // XXX test dump of psf star data and psf-subtracted image + // test dump of psf star data and psf-subtracted image if (psTraceGetLevel("psphot.psfstars") > 5) { psphotDumpPSFStars (readout, try, options->fitRadius, maskVal, markVal); @@ -413,7 +379,6 @@ return false; } - psFree (psf); // XXX double-check - - // move into psphotChoosePSF + psFree (psf); + psphotVisualShowPSFModel (readout, psf); @@ -436,4 +401,6 @@ psVector *fwhmMajor = psVectorAllocEmpty (100, PS_DATA_F32); psVector *fwhmMinor = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *psfExtra1 = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *psfExtra2 = psVectorAllocEmpty (100, PS_DATA_F32); for (float ix = -0.4; ix <= +0.4; ix += 0.1) { @@ -459,13 +426,21 @@ shape.sxy = modelPSF->params->data.F32[PM_PAR_SXY]; axes = psEllipseShapeToAxes (shape, 20.0); - psFree (modelPSF); float FWHM_MINOR = FWHM_MAJOR * (axes.minor / axes.major); if (!isfinite(FWHM_MAJOR) || !isfinite(FWHM_MINOR)) { fprintf (stderr, "!"); + psFree (modelPSF); continue; } psVectorAppend (fwhmMajor, FWHM_MAJOR); psVectorAppend (fwhmMinor, FWHM_MINOR); + + if (modelPSF->params->n >= 7) { + psVectorAppend (psfExtra1, modelPSF->params->data.F32[7]); + } + if (modelPSF->params->n >= 8) { + psVectorAppend (psfExtra2, modelPSF->params->data.F32[8]); + } + psFree (modelPSF); } } @@ -474,5 +449,5 @@ if (!psVectorStats (stats, fwhmMajor, NULL, NULL, 0)) { psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MAJOR"); - return false; + goto escape; } @@ -486,5 +461,5 @@ if (!psVectorStats (stats, fwhmMinor, NULL, NULL, 0)) { psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MINOR"); - return false; + goto escape; } psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MIN", PS_META_REPLACE, "PSF FWHM Minor axis (mean)", stats->sampleMean); @@ -494,19 +469,64 @@ float fwhmMin = stats->sampleMean; // FWHM on minor axis + if (readout->parent) { + + // we now have 2 possible measurements of the seeing : the PSF model based version and + // the Moments based value we need to define a definitive "CHIP.SEEING" value. This + // value is used by the PSF-matching programs (ppSub and ppStack). The moments-based + // value is probably more representative of the value needed by those tools (it is also + // more stable?) + bool status = false; + float fwhmMajorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1"); + float fwhmMinorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1"); + pmChip *chip = readout->parent->parent; // Parent chip psAssert(chip, "Cell should be attached to a chip."); psMetadataItem *item = psMetadataLookup(chip->concepts, "CHIP.SEEING"); // Item with chip - item->data.F32 = 0.5 * (fwhmMaj + fwhmMin); + item->data.F32 = 0.5 * (fwhmMajorMoments + fwhmMinorMoments); + + psLogMsg ("psphot", PS_LOG_DETAIL, "fwhm (psf): %f,%f (moments): %f,%f", fwhmMaj, fwhmMin, fwhmMajorMoments, fwhmMinorMoments); + } + + if (psfExtra1->n) { + if (!psVectorStats (stats, psfExtra1, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 1"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT1", PS_META_REPLACE, "PSF extra param 1", stats->sampleMean); + } + + if (psfExtra2->n) { + if (!psVectorStats (stats, psfExtra2, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 2"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT2", PS_META_REPLACE, "PSF extra param 2", stats->sampleMean); } psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE", PS_META_REPLACE, "PSF angle", axes.theta); psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", psf->nPSFstars); - psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "Valid PSF Model?", true); + + char *psfModelName = pmModelClassGetName(psf->type); + psMetadataAddStr(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "PSF Model Name", psfModelName); + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSF_OK", PS_META_REPLACE, "Valid PSF Model?", true); + + int nParams = pmModelClassParameterCount(psf->type); + psMetadataAddS32(readout->analysis, PS_LIST_TAIL, "PSF_NPAR", PS_META_REPLACE, "Number of PSF parameters", nParams); psFree (fwhmMajor); psFree (fwhmMinor); + psFree (psfExtra1); + psFree (psfExtra2); psFree (stats); return true; + +escape: + psFree (fwhmMajor); + psFree (fwhmMinor); + psFree (psfExtra1); + psFree (psfExtra2); + psFree (stats); + return false; } @@ -568,7 +588,8 @@ psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_LQ", PS_META_REPLACE, "PSF FWHM Minor axis (lower quartile)", 0); psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_UQ", PS_META_REPLACE, "PSF FWHM Minor axis (upper quartile)", 0); - psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE", PS_META_REPLACE, "PSF angle", FWHM_T); - psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", 0); - psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "Valid PSF Model?", false); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE", PS_META_REPLACE, "PSF angle", FWHM_T); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", 0); + psMetadataAddStr(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "PSF Model Name", "NONE"); + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSF_OK", PS_META_REPLACE, "Valid PSF Model?", false); return true; Index: /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceAnalysis.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceAnalysis.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceAnalysis.c (revision 30631) @@ -173,5 +173,5 @@ psLogMsg ("psphot", PS_LOG_INFO, " %d kron\n", Nkron); - psphotVisualShowResidualImage (readout); + psphotVisualShowResidualImage (readout, false); if (doPetrosian) { Index: /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceAnalysisByObject.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceAnalysisByObject.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceAnalysisByObject.c (revision 30631) @@ -53,4 +53,7 @@ pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); psAssert (readout, "missing readout?"); + + psLogMsg("psphot", PS_LOG_INFO, "petrosians for image %d", i); + psphotVisualShowImage(readout); readouts->data[i] = psMemIncrRefCounter(readout); Index: /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceFits.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceFits.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotExtendedSourceFits.c (revision 30631) @@ -18,6 +18,11 @@ int num = psphotFileruleCount(config, filerule); + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + // loop over the available readouts for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image if (!psphotExtendedSourceFitsReadout (config, view, filerule, i, recipe)) { psError (PSPHOT_ERR_CONFIG, false, "failed on to fit extended sources for %s entry %d", filerule, i); @@ -37,4 +42,5 @@ int Nplain = 0; int NplainPass = 0; + int Nfaint = 0; psTimerStart ("psphot.extended"); @@ -46,4 +52,7 @@ pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); psAssert (readout, "missing readout?"); + + psLogMsg("psphot", PS_LOG_INFO, "extended source fits for image %d", index); + psphotVisualShowImage(readout); pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); @@ -167,4 +176,5 @@ PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nplain PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for NplainPass + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfain if (false && !psThreadJobAddPending(job)) { @@ -189,4 +199,6 @@ scalar = job->args->data[11]; NplainPass += scalar->data.S32; + scalar = job->args->data[12]; + Nfaint += scalar->data.S32; psFree(job); } @@ -217,4 +229,6 @@ scalar = job->args->data[11]; NplainPass += scalar->data.S32; + scalar = job->args->data[12]; + Nfaint += scalar->data.S32; } psFree(job); @@ -227,4 +241,5 @@ psLogMsg ("psphot", PS_LOG_INFO, " %d convolved models (%d passed)\n", Nconvolve, NconvolvePass); psLogMsg ("psphot", PS_LOG_INFO, " %d plain models (%d passed)\n", Nplain, NplainPass); + psLogMsg ("psphot", PS_LOG_INFO, " %d too faint to fit\n", Nfaint); return true; } @@ -238,4 +253,5 @@ int NconvolvePass = 0; int Nplain = 0; + int Nfaint = 0; int NplainPass = 0; bool savePics = false; @@ -326,9 +342,9 @@ // array to store the pointers to the model flux images while the models are being fitted - psArray *modelFluxes = psArrayAllocEmpty (4); + psArray *modelFluxes = psArrayAllocEmpty (models->list->n); // allocate the array to store the model fits if (source->modelFits == NULL) { - source->modelFits = psArrayAllocEmpty (4); + source->modelFits = psArrayAllocEmpty (models->list->n); } @@ -350,5 +366,8 @@ // limit selection to some SN limit assert (source->peak); // how can a source not have a peak? - if (source->peak->SN < SNlim) continue; + if (source->peak->SN < SNlim) { + Nfaint ++; + continue; + } // check on the model type @@ -497,4 +516,7 @@ scalar->data.S32 = NplainPass; + scalar = job->args->data[12]; + scalar->data.S32 = Nfaint; + return true; } Index: /branches/czw_branch/20101203/psphot/src/psphotFindDetections.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotFindDetections.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotFindDetections.c (revision 30631) @@ -86,9 +86,9 @@ psImage *significance = psphotSignificanceImage (readout, recipe, maskVal); - // display the significance image - psphotVisualShowSignificance (significance, -1.0, PS_SQR(3.0*NSIGMA_PEAK)); - // display the log significance image psphotVisualShowLogSignificance (significance, 0.0, 4.5); + + // display the significance image + psphotVisualShowSignificance (significance, 0.98*threshold, 1.02*threshold); // detect the peaks in the significance image @@ -108,4 +108,6 @@ } + // XXX do a second (or third?) pass with rebinning (to detected more extended sources) + psFree (significance); Index: /branches/czw_branch/20101203/psphot/src/psphotFindFootprints.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotFindFootprints.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotFindFootprints.c (revision 30631) @@ -21,6 +21,7 @@ psArray *footprints = pmFootprintsFind (significance, threshold, npixMin); - pmFootprintsAssignPeaks(footprints, detections->peaks); - // XXX handle the error conditions here + if (pmFootprintsAssignPeaks(footprints, detections->peaks) != PS_ERR_NONE) { + psAbort ("inconsistent peaks and footprints"); + } // footprints now owns the peaks; after culling (below), we will rebuild the peaks array Index: /branches/czw_branch/20101203/psphot/src/psphotFitSourcesLinear.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotFitSourcesLinear.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotFitSourcesLinear.c (revision 30631) @@ -46,4 +46,8 @@ return false; } + + psphotVisualShowResidualImage (readout, (num > 0)); + psphotVisualShowPeaks (detections); + psphotVisualShowObjectRegions (readout, recipe, sources); } return true; @@ -151,4 +155,23 @@ if (x > AnalysisRegion.x1) continue; if (y > AnalysisRegion.y1) continue; + + // check the integral of the model : is it large enough? + float modelSum = 0.0; + for (int iy = 0; iy < source->modelFlux->numRows; iy++) { + for (int ix = 0; ix < source->modelFlux->numCols; ix++) { + modelSum += source->modelFlux->data.F32[iy][ix]; + } + } + if (modelSum < 0.5) continue; // skip sources with no model constraint (somewhat arbitrary limit) + // if (modelSum < 0.01) continue; // skip sources with no model constraint (somewhat arbitrary limit) + if (modelSum < 0.8) { + fprintf (stderr, "low-sig model @ %f, %f (%f sum, %f peak)\n", + source->peak->xf, source->peak->yf, modelSum, source->peak->flux); + } + + pmModel *model = pmSourceGetModel (NULL, source); + + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, model->fitRadius, "OR", markVal); source->mode |= PM_SOURCE_MODE_LINEAR_FIT; @@ -270,5 +293,4 @@ model->params->data.F32[PM_PAR_I0] = norm->data.F32[i]; model->dparams->data.F32[PM_PAR_I0] = errors->data.F32[i]; - // XXX is the value of 'errors' modified by the sky fit? // subtract object @@ -297,5 +319,4 @@ psLogMsg ("psphot.ensemble", PS_LOG_INFO, "measure ensemble of PSFs: %f sec\n", psTimerMark ("psphot.linear")); - psphotVisualShowResidualImage (readout); psphotVisualPlotChisq (sources); // psphotVisualShowFlags (sources); Index: /branches/czw_branch/20101203/psphot/src/psphotFitSourcesLinearStack.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotFitSourcesLinearStack.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotFitSourcesLinearStack.c (revision 30631) @@ -176,5 +176,4 @@ psLogMsg ("psphot.ensemble", PS_LOG_INFO, "measure ensemble of PSFs: %f sec\n", psTimerMark ("psphot.linear")); - return true; } Index: /branches/czw_branch/20101203/psphot/src/psphotImageQuality.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotImageQuality.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotImageQuality.c (revision 30631) @@ -79,5 +79,5 @@ // select by PSFSTAR mode? // ?? - if (source->type != PM_SOURCE_TYPE_STAR || !(source->mode & PM_SOURCE_MODE_PSFSTAR)) { + if (source->type != PM_SOURCE_TYPE_STAR || !(source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR)) { psTrace("psphot", 10, "Ignoring source for image quality because not a good star"); continue; @@ -133,4 +133,12 @@ if (num == 0) { psLogMsg ("psphot", PS_LOG_INFO, "no valid sources for image quality, skipping"); + psFree(FWHM_MAJOR); + psFree(FWHM_MINOR); + psFree(M2); + psFree(M2c); + psFree(M2s); + psFree(M3); + psFree(M4); + return true; } Index: /branches/czw_branch/20101203/psphot/src/psphotMakePSFReadout.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotMakePSFReadout.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotMakePSFReadout.c (revision 30631) @@ -60,5 +60,5 @@ // Use bright stellar objects to measure PSF. If we do not have enough stars to generate // the PSF, build one from the SEEING guess and model class - if (!psphotChoosePSF (config, view, filerule)) { + if (!psphotChoosePSF (config, view, filerule, true)) { psLogMsg ("psphot", 3, "failure to construct a psf model"); return psphotReadoutCleanup (config, view, filerule); Index: /branches/czw_branch/20101203/psphot/src/psphotMakeResiduals.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotMakeResiduals.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotMakeResiduals.c (revision 30631) @@ -136,4 +136,7 @@ psFree (interp); } + xSize = PS_MIN(xSize, 2*radiusMax+3); + ySize = PS_MIN(ySize, 2*radiusMax+3); + pmResiduals *resid = pmResidualsAlloc (xSize, ySize, xBin, yBin); psImageInit (resid->mask, 0); @@ -316,5 +319,5 @@ psFree (B); - psLogMsg ("psphot.pspsf", PS_LOG_MINUTIA, "generate residuals for %ld objects: %f sec\n", input->n, psTimerMark ("psphot.residuals")); + psLogMsg ("psphot.pspsf", PS_LOG_DETAIL, "generate residuals for %ld objects: %f sec\n", input->n, psTimerMark ("psphot.residuals")); psFree (xC); Index: /branches/czw_branch/20101203/psphot/src/psphotMergeSources.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotMergeSources.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotMergeSources.c (revision 30631) @@ -513,5 +513,236 @@ } - return true; -} - + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + + + return true; +} + +// create source children from ruleSrc for ruleOut +bool psphotSourceChildren (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc) +{ + bool status = true; + + int num = psphotFileruleCount(config, ruleSrc); + + // skip the chisq image because it is a duplicate of the detection version + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotSourceChildrenReadout (config, view, ruleOut, ruleSrc, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to copy sources from %s to %s entry %d", ruleSrc, ruleOut, i); + return false; + } + } + return true; +} + +// create source children from ruleSrc for ruleOut for this entry. XXX currently, this is only +// used by psphotStackReadout (sources go on allSources so that psphotChoosePSF can be called +// repeatedly) +bool psphotSourceChildrenReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, ruleSrc, index); // File of interest + psAssert (fileSrc, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmDetections *detectionsSrc = psMetadataLookupPtr (&status, readoutSrc->analysis, "PSPHOT.DETECTIONS"); + psAssert (detectionsSrc, "missing detections?"); + + psArray *sourcesSrc = detectionsSrc->allSources; + psAssert (sourcesSrc, "missing sources?"); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, ruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + // generate a new detection structure for the output filerule + pmDetections *detectionsOut = psMetadataLookupPtr (&status, readoutOut->analysis, "PSPHOT.DETECTIONS"); + if (!detectionsOut) { + detectionsOut = pmDetectionsAlloc(); + detectionsOut->allSources = psArrayAllocEmpty (100); + // save detections on the readout->analysis + if (!psMetadataAddPtr (readoutOut->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detectionsOut)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + psFree(detectionsOut); // a copy remains on the analysis metadata + } + + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + + // copy the sources from sourceSrcs to the new detection structure + for (int i = 0; i < sourcesSrc->n; i++) { + pmSource *sourceSrc = sourcesSrc->data[i]; + + pmSource *sourceOut = pmSourceCopy(sourceSrc); + sourceOut->parent = sourceSrc; + + // keep the original source flags + sourceOut->type = sourceSrc->type; + sourceOut->mode = sourceSrc->mode; + sourceOut->mode2 = sourceSrc->mode2; + sourceOut->tmpFlags = sourceSrc->tmpFlags; + + // does this copy all model data? (NO) + sourceOut->modelPSF = pmModelCopy(sourceSrc->modelPSF); + sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); + + if (sourceSrc->modelFits) { + sourceOut->modelFits = psArrayAlloc(sourceSrc->modelFits->n); + for (int j = 0; j < sourceSrc->modelFits->n; j++) { + sourceOut->modelFits->data[j] = pmModelCopy(sourceSrc->modelFits->data[j]); + } + } + + // drop the references to the original image pixels: + pmSourceFreePixels (sourceOut); + + // allocate image, weight, mask for the new image for each peak + pmSourceRedefinePixels (sourceOut, readoutOut, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); + + // child sources have not been subtracted in this image, but this flag may be raised if + // they were subtracted in the parent's image + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + + psArrayAdd (detectionsOut->allSources, 100, sourceOut); + psFree (sourceOut); + } + psLogMsg ("psphot", 3, "%ld known sources supplied", detectionsOut->allSources->n); + + + return true; +} + +// create source children associated with 'filerule' from the objectsSrc. XXX currently, this +// is only used by psphotStackReadout (sources go on allSources so that psphotChoosePSF can be +// called repeatedly) +psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objectsSrc) { + + bool status; + + int nImages = psphotFileruleCount(config, filerule); + + // generate look-up arrays for detections and readouts + psArray *detArrays = psArrayAlloc(nImages); + psArray *readouts = psArrayAlloc(nImages); + + for (int i = 0; i < nImages; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + detections = pmDetectionsAlloc(); + detections->allSources = psArrayAllocEmpty (100); + detections->peaks = psArrayAllocEmpty (100); + // save detections on the readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return NULL; + } + psFree(detections); // a copy remains on the analysis metadata + psAssert (detections, "missing detections?"); + } + detArrays->data[i] = psMemIncrRefCounter(detections); + readouts->data[i] = psMemIncrRefCounter(readout); + } + + psArray *objectsOut = psArrayAlloc(objectsSrc->n); + + // copy all sources for each object + for (int k = 0; k < objectsSrc->n; k++) { + + pmPhotObj *objectSrc = objectsSrc->data[k]; + if (!objectSrc) continue; + if (!objectSrc->sources) continue; + + pmPhotObj *objectOut = pmPhotObjAlloc(); + objectsOut->data[k] = objectOut; + + objectOut->SN = objectSrc->SN; + objectOut->x = objectSrc->x; + objectOut->y = objectSrc->y; + + objectOut->sources = psArrayAlloc(objectSrc->sources->n); + + // copy the sources from sourceSrcs to the new detection structure + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + for (int i = 0; i < objectSrc->sources->n; i++) { + + pmSource *sourceSrc = objectSrc->sources->data[i]; + + pmSource *sourceOut = pmSourceCopy(sourceSrc); + objectOut->sources->data[i] = sourceOut; + + sourceOut->parent = sourceSrc; + + // keep the original source flags + sourceOut->type = sourceSrc->type; + sourceOut->mode = sourceSrc->mode; + sourceOut->mode2 = sourceSrc->mode2; + sourceOut->tmpFlags = sourceSrc->tmpFlags; + + // does this copy all model data? (NO) + sourceOut->modelPSF = pmModelCopy(sourceSrc->modelPSF); + sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); + + if (sourceSrc->modelFits) { + sourceOut->modelFits = psArrayAlloc(sourceSrc->modelFits->n); + for (int j = 0; j < sourceSrc->modelFits->n; j++) { + sourceOut->modelFits->data[j] = pmModelCopy(sourceSrc->modelFits->data[j]); + } + } + + // drop the references to the original image pixels: + pmSourceFreePixels (sourceOut); + + // set the output readotu + int index = sourceOut->imageID; + if (index >= readouts->n) continue; // skip the sources generated by the chisq image + pmReadout *readout = readouts->data[index]; + + // allocate image, weight, mask for the new image for each peak + pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); + + // child sources have not been subtracted in this image, but this flag may be raised if + // they were subtracted in the parent's image + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + + // set the output detections: + pmDetections *detectionsOut = detArrays->data[index]; + psArrayAdd (detectionsOut->allSources, 100, sourceOut); + psArrayAdd (detectionsOut->peaks, 100, sourceOut->peak); + } + } + + for (int i = 0; i < nImages; i++) { + pmDetections *detections = detArrays->data[i]; + psLogMsg ("psphot", 3, "%ld source children for image %d", detections->allSources->n, i); + } + + psFree (detArrays); + psFree (readouts); + + return objectsOut; +} + Index: /branches/czw_branch/20101203/psphot/src/psphotOutput.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotOutput.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotOutput.c (revision 30631) @@ -15,4 +15,16 @@ psFree (name); return num; +} + +// convert filerule to filerule.NUM and look up in the config->arguments metadata +bool psphotFileruleCountSet(const pmConfig *config, const char *filerule, int num) { + + psString name = NULL; + psStringAppend(&name, "%s.NUM", filerule); + + bool status = psMetadataAddS32(config->arguments, PS_LIST_TAIL, name, PS_META_REPLACE, "", num); + psFree (name); + + return status; } @@ -261,4 +273,7 @@ psMetadataItemSupplement (&status, header, analysis, "ANGLE"); + psMetadataItemSupplement (&status, header, analysis, "PSFMODEL"); + psMetadataItemSupplement (&status, header, analysis, "PSF_OK"); + // Image Quality measurements psMetadataItemSupplement (&status, header, analysis, "IQ_NSTAR"); Index: /branches/czw_branch/20101203/psphot/src/psphotPetrosianAnalysis.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotPetrosianAnalysis.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotPetrosianAnalysis.c (revision 30631) @@ -64,5 +64,5 @@ } - psphotVisualShowResidualImage (readout); + psphotVisualShowResidualImage (readout, false); return true; } Index: /branches/czw_branch/20101203/psphot/src/psphotPetrosianProfile.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotPetrosianProfile.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotPetrosianProfile.c (revision 30631) @@ -68,5 +68,4 @@ // XXX this will only work in the psphot context, not the psphotPetrosianStudy... // XXX add the petrosian to the pmSource structure... - // psphotVisualShowResidualImage (readout); psphotVisualShowPetrosian (source, petrosian); Index: /branches/czw_branch/20101203/psphot/src/psphotRadialApertures.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotRadialApertures.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotRadialApertures.c (revision 30631) @@ -102,4 +102,9 @@ if (source->peak->x > AnalysisRegion.x1) continue; if (source->peak->y > AnalysisRegion.y1) continue; + + // allocate pmSourceExtendedParameters, if not already defined + if (!source->radialAper) { + source->radialAper = psArrayAlloc(1); + } // replace object in image @@ -116,5 +121,5 @@ pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 1.5*radius); - if (!psphotRadialApertureSource (source, recipe, skynoise, maskVal, radMax)) { + if (!psphotRadialApertureSource (source, recipe, skynoise, maskVal, radMax, 0)) { psTrace ("psphot", 5, "failed to extract radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); } else { @@ -130,12 +135,122 @@ } -bool psphotRadialApertureSource (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal, const psVector *radMax) { - - // allocate pmSourceExtendedParameters, if not already defined - if (!source->radial) { - source->radial = pmSourceRadialAperturesAlloc (); +bool psphotRadialApertureSource (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal, const psVector *aperRadii, int entry) { + + // if we are a child source, save the results to the parent source radial aperture array + psArray *radialAperSet = source->radialAper; + if (source->parent) { + radialAperSet = source->parent->radialAper; + } + psAssert(radialAperSet, "this should be defined before calling"); + psAssert(radialAperSet->data[entry] == NULL, "why is this already defined?"); + + pmSourceRadialApertures *radialAper = pmSourceRadialAperturesAlloc (); + radialAperSet->data[entry] = radialAper; + + // storage for the derived pixel values + psVector *pixRadius2 = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *pixFlux = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *pixVar = psVectorAllocEmpty(100, PS_TYPE_F32); + + // outer-most radius for initial truncation + float Rmax = aperRadii->data.F32[aperRadii->n - 1]; + float Rmax2 = PS_SQR(Rmax); + + // store the R^2 values for the apertures + psVector *aperRadii2 = psVectorAlloc(aperRadii->n, PS_TYPE_F32); + for (int i = 0; i < aperRadii->n; i++) { + aperRadii2->data.F32[i] = PS_SQR(aperRadii->data.F32[i]); + } + + // center of the apertures + float xCM = source->moments->Mx - 0.5 - source->pixels->col0; // coord of peak in subimage + float yCM = source->moments->My - 0.5 - source->pixels->row0; // coord of peak in subimage + + // one pass through the pixels to select the valid pixels and calculate R^2 + for (int iy = 0; iy < source->pixels->numRows; iy++) { + + float yDiff = iy - yCM; + if (fabs(yDiff) > Rmax) continue; + + float *vPix = source->pixels->data.F32[iy]; + float *vWgt = source->variance->data.F32[iy]; + psImageMaskType *vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy]; + + for (int ix = 0; ix < source->pixels->numCols; ix++, vPix++, vWgt++) { + + if (vMsk) { + if (*vMsk & maskVal) { + vMsk++; + continue; + } + vMsk++; + } + if (isnan(*vPix)) continue; + + float xDiff = ix - xCM; + if (fabs(xDiff) > Rmax) continue; + + // radius is just a function of (xDiff, yDiff) + float r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > Rmax2) continue; + + psVectorAppend(pixRadius2, r2); + psVectorAppend(pixFlux, *vPix); + psVectorAppend(pixVar, *vWgt); + } + } + + psVector *flux = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fluxErr = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fill = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + + psVectorInit (flux, 0.0); + psVectorInit (fluxErr, 0.0); + psVectorInit (fill, 0.0); + + float *rPix2 = pixRadius2->data.F32; + for (int i = 0; i < pixRadius2->n; i++, rPix2++) { + + float *aRad2 = aperRadii2->data.F32; + for (int j = 0; (*rPix2 < *aRad2) && (j < aperRadii2->n); j++, aRad2++) { + flux->data.F32[j] += pixFlux->data.F32[i]; + fluxErr->data.F32[j] += pixVar->data.F32[i]; + fill->data.F32[j] += 1.0; + } + } + + for (int i = 0; i < flux->n; i++) { + // calculate the total flux for bin 'nOut' + float Area = M_PI*aperRadii2->data.F32[i]; + fluxErr->data.F32[i] = sqrt(fluxErr->data.F32[i]); + fill->data.F32[i] /= Area; + psTrace ("psphot", 5, "radial bins: %3d %5.1f : %8.1f +/- %7.2f : %4.2f %6.1f\n", + i, aperRadii->data.F32[i], flux->data.F32[i], fluxErr->data.F32[i], fill->data.F32[i], Area); } - psVector *radius = psVectorAllocEmpty(100, PS_TYPE_F32); + radialAper->flux = flux; + radialAper->fluxErr = fluxErr; + radialAper->fill = fill; + + psFree (aperRadii2); + psFree (pixRadius2); + psFree (pixFlux); + psFree (pixVar); + + return true; +} + +static int nCalls = 0; +static int nPass = 0; +static int nPix = 0; + +bool psphotRadialApertureSource_With_Sort (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal, const psVector *radMax, int entry) { + + psAssert(source->radialAper->data[entry] == NULL, "why is this already defined?"); + + pmSourceRadialApertures *radialAper = pmSourceRadialAperturesAlloc (); + source->radialAper->data[entry] = radialAper; + + psVector *pixRadius = psVectorAllocEmpty(100, PS_TYPE_F32); psVector *pixFlux = psVectorAllocEmpty(100, PS_TYPE_F32); psVector *pixVar = psVectorAllocEmpty(100, PS_TYPE_F32); @@ -144,5 +259,5 @@ for (int ix = 0; ix < source->pixels->numCols; ix++) { - // 0.5 PIX: get radius as a function of pixel coord + // 0.5 PIX: get pixRadius as a function of pixel coord float x = ix + 0.5 - source->peak->xf + source->pixels->col0; float y = iy + 0.5 - source->peak->yf + source->pixels->row0; @@ -150,10 +265,12 @@ float r = hypot(x, y); - psVectorAppend(radius, r); + psVectorAppend(pixRadius, r); psVectorAppend(pixFlux, source->pixels->data.F32[iy][ix]); psVectorAppend(pixVar, source->variance->data.F32[iy][ix]); - } - } - psphotRadialAperturesSortFlux(radius, pixFlux, pixVar); + nPix ++; + // if (nPix % 10000 == 0) {fprintf (stderr, "?");} + } + } + psphotRadialAperturesSortFlux(pixRadius, pixFlux, pixVar); psVector *flux = psVectorAllocEmpty(radMax->n, PS_TYPE_F32); // surface brightness of radial bin @@ -174,6 +291,6 @@ // XXX assume (or enforce) that the bins are contiguous and non-overlapping (Rmax[i] = Rmin[i+1]) - for (int i = 0; !done && (i < radius->n); i++) { - if (radius->data.F32[i] > Rmax) { + for (int i = 0; !done && (i < pixRadius->n); i++) { + if (pixRadius->data.F32[i] > Rmax) { // calculate the total flux for bin 'nOut' float Area = M_PI*PS_SQR(Rmax); @@ -185,4 +302,7 @@ nOut, radMax->data.F32[nOut], flux->data.F32[nOut], fluxErr->data.F32[nOut], fill->data.F32[nOut], Area); + nPass ++; + // if (nPass % 1000 == 0) {fprintf (stderr, "!");} + nOut ++; if (nOut >= radMax->n) break; @@ -195,16 +315,14 @@ flux->n = fluxErr->n = fill->n = nOut; - psFree(source->radial->flux); - psFree(source->radial->fluxErr); - psFree(source->radial->fill); - - source->radial->flux = flux; - source->radial->fluxErr = fluxErr; - source->radial->fill = fill; - - psFree (radius); + radialAper->flux = flux; + radialAper->fluxErr = fluxErr; + radialAper->fill = fill; + + psFree (pixRadius); psFree (pixFlux); psFree (pixVar); + nCalls ++; + // if (nCalls % 100 == 0) {fprintf (stderr, "*");} return true; } Index: /branches/czw_branch/20101203/psphot/src/psphotRadialAperturesByObject.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotRadialAperturesByObject.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotRadialAperturesByObject.c (revision 30631) @@ -3,5 +3,14 @@ // aperture-like measurements for extended sources // flux in simple, circular apertures -bool psphotRadialAperturesByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule) { + +// **** it looks like this function will re-point the source pixels at the specified FILERULE +// **** I need to distinguish PSF-matched images from raw +// **** save (somewhere) the PSF-matched PSF values + +// this function measures the radial aperture fluxes for the set of readouts. this function +// may be called multiple times (presumably with different matched PSF sizes). we must have +// already added an entry to the readout->analysis identifying the FWHM of this version. + +bool psphotRadialAperturesByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule, int nMatchedPSF) { bool status; @@ -28,4 +37,5 @@ psAssert (radMax, "annular bins (RADIAL.ANNULAR.BINS.UPPER) are not defined in the recipe"); psAssert (radMax->n, "no valid annular bins (RADIAL.ANNULAR.BINS.UPPER) are define"); + float outerRadius = radMax->data.F32[radMax->n - 1]; // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) @@ -39,4 +49,17 @@ float SN_LIM = psMetadataLookupF32 (&status, recipe, "RADIAL_APERTURES_SN_LIM"); + // how many target PSFs do we want? + int nPSFsizes = 0; + { + psMetadataLookupF32 (&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); + if (status) { + nPSFsizes = 1; + } else { + psVector *fwhmValues = psMetadataLookupVector(&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); // Magnitude offsets + psAssert (status, "missing psphot recipe value PSPHOT.STACK.TARGET.PSF.FWHM"); + nPSFsizes = fwhmValues->n; + } + } + // source analysis is done in S/N order (brightest first) objects = psArraySort (objects, pmPhotObjSortBySN); @@ -52,4 +75,15 @@ pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); psAssert (readout, "missing readout?"); + + psVector *fwhmValues = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + if (!fwhmValues) { + psError (PSPHOT_ERR_CONFIG, true, "convolved or measured FWHM is not defined for this readout"); + return false; + } + if (fwhmValues->n != nMatchedPSF + 1) { + psError (PSPHOT_ERR_CONFIG, true, "convolved or measured FWHM sequence is inconsistent this readout"); + return false; + } + psLogMsg ("psphot", PS_LOG_DETAIL, "PSF FWHM of %s : %f pixels\n", file->name, fwhmValues->data.F32[nMatchedPSF]); readouts->data[i] = psMemIncrRefCounter(readout); @@ -77,21 +111,47 @@ if (source->peak->SN < SN_LIM) continue; + int index = source->imageID; + if (index >= readouts->n) continue; // skip the sources generated by the chisq image + pmReadout *readout = readouts->data[index]; + + // psLogMsg("psphot", PS_LOG_INFO, "radial apertures for %d", index); + // psphotVisualShowImage(readout); + + // allocate pmSourceExtendedParameters, if not already defined + if (source->parent) { + if (!source->parent->radialAper) { + source->parent->radialAper = psArrayAlloc(nPSFsizes); + } + } else { + if (!source->radialAper) { + source->radialAper = psArrayAlloc(nPSFsizes); + } + } + // replace object in image if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); } + + // we need to change the view for the radial aperture analysis, but we want to recover exactly + // the original view; the following elements get destroyed by pmSourceRedefinePixels so save them: + psImage *oldMaskObj = psMemIncrRefCounter(source->maskObj); + psImage *oldModelFlux = psMemIncrRefCounter(source->modelFlux); + psImage *oldPSFimage = psMemIncrRefCounter(source->psfImage); + psRegion oldRegion = source->region; + Nradial ++; - int index = source->imageID; - pmReadout *readout = readouts->data[index]; + // psLogMsg("psphot", PS_LOG_INFO, "radial apertures for %d", index); + // psphotVisualShowImage(readout); // force source image to be a bit larger... - float radius = source->peak->xf - source->pixels->col0; - radius = PS_MAX (radius, source->peak->yf - source->pixels->row0); - radius = PS_MAX (radius, source->pixels->numRows - source->peak->yf + source->pixels->row0); - radius = PS_MAX (radius, source->pixels->numCols - source->peak->xf + source->pixels->col0); - pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 1.5*radius); + // float radius = source->peak->xf - source->pixels->col0; + // radius = PS_MAX (radius, source->peak->yf - source->pixels->row0); + // radius = PS_MAX (radius, source->pixels->numRows - source->peak->yf + source->pixels->row0); + // radius = PS_MAX (radius, source->pixels->numCols - source->peak->xf + source->pixels->col0); + pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, outerRadius + 2); - if (!psphotRadialApertureSource (source, recipe, skynoise, maskVal, radMax)) { + if (!psphotRadialApertureSource (source, recipe, skynoise, maskVal, radMax, nMatchedPSF)) { psTrace ("psphot", 5, "failed to extract radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); } else { @@ -99,6 +159,14 @@ } + pmSourceRedefinePixelsByRegion (source, readout, oldRegion); + psFree(source->maskObj); source->maskObj = oldMaskObj; + psFree(source->modelFlux); source->modelFlux = oldModelFlux; + psFree(source->psfImage); source->psfImage = oldPSFimage; + // re-subtract the object, leave local sky pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + // psLogMsg("psphot", PS_LOG_INFO, "radial apertures for %d", index); + // psphotVisualShowImage(readout); } } Index: /branches/czw_branch/20101203/psphot/src/psphotReadout.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotReadout.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotReadout.c (revision 30631) @@ -112,5 +112,5 @@ // use bright stellar objects to measure PSF if we were supplied a PSF for any input file, // this step is skipped - if (!psphotChoosePSF (config, view, filerule)) { // pass 1 + if (!psphotChoosePSF (config, view, filerule, true)) { // pass 1 psLogMsg ("psphot", 3, "failure to construct a psf model"); return psphotReadoutCleanup (config, view, filerule); Index: /branches/czw_branch/20101203/psphot/src/psphotReadoutFindPSF.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotReadoutFindPSF.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotReadoutFindPSF.c (revision 30631) @@ -49,5 +49,5 @@ } - if (!psphotChoosePSF(config, view, filerule)) { + if (!psphotChoosePSF(config, view, filerule, true)) { psError(PSPHOT_ERR_PSF, false, "Failed to construct a psf model"); return psphotReadoutCleanup (config, view, filerule); Index: /branches/czw_branch/20101203/psphot/src/psphotReadoutKnownSources.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotReadoutKnownSources.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotReadoutKnownSources.c (revision 30631) @@ -43,5 +43,5 @@ } - if (!psphotChoosePSF (config, view, filerule)) { + if (!psphotChoosePSF (config, view, filerule, true)) { psError(PSPHOT_ERR_PSF, false, "Failed to construct a psf model"); return psphotReadoutCleanup (config, view, filerule); Index: /branches/czw_branch/20101203/psphot/src/psphotReplaceUnfit.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotReplaceUnfit.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotReplaceUnfit.c (revision 30631) @@ -75,4 +75,6 @@ pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); } + + psphotVisualShowImage(readout); psLogMsg ("psphot.replace", PS_LOG_INFO, "replaced models for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.replace")); return true; @@ -101,2 +103,182 @@ return true; } + +// modify the sources to point at the corresponding pixels for the given filerule +bool psphotRedefinePixels (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotRedefinePixelsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to replace all sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotRedefinePixelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + pmSource *source; + + psTimerStart ("psphot.replace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // XXX the sources have already been copied (merge into here?) + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // sources have not yet been subtracted in this image (but this flag may be raised) + source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + + float Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + float Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + float radius = source->modelPSF->fitRadius; + + // force a redefine to this image + pmSourceFreePixels(source); + pmSourceRedefinePixels (source, readout, Xo, Yo, radius); + } + return true; +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotResetModels (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotResetModelsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to replace all sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotResetModelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + pmSource *source; + + psTimerStart ("psphot.replace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // XXX the sources have already been copied (merge into here?) + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + // what fraction of the PSF is used? (radius in pixels : 2 -> 5x5 box) + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // *** we need to cache the 'best' model, and we have 3 cases: + // 1) model is the psf model --> generate from the new psf + // 2) model is an unconvolved extended model --> just cache the copy (not perfect) + // 3) model is a convolved extended model --> re-generate + + // use the 'best' model to cache the model (PSF or EXT : EXT may point at one of modelFits + bool isPSF = false; + pmModel *model = pmSourceGetModel(&isPSF, source); + float radius = model->fitRadius; // save for future use below + + // regenerate the PSF if the model is a PSF, or if we need the PSF for a PCM + if (isPSF || model->isPCM) { + // the guess central intensity comes from the peak: + float Io = source->peak->flux; + float Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + float Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + + // generate a model for this object with Io = 1.0 + pmModel *modelPSF = pmModelFromPSFforXY(psf, Xo, Yo, Io); + if (modelPSF == NULL) { + psWarning ("Failed to determine PSF model for source at (%f,%f), skipping", Xo, Yo); + continue; + } + + // set the source PSF model + psFree (source->modelPSF); + source->modelPSF = modelPSF; + source->modelPSF->fitRadius = radius; + } + + if (model->isPCM) { + psAssert(false, "this section is not complete"); + + pmSourceCachePSF (source, maskVal); + + psKernel *psfKernel = pmPCMkernelFromPSF(source, psfSize); + if (!psfKernel) { + psWarning ("no psf kernel"); + } + + // generate an image of the right size + psImage *rawModelFlux = psImageCopy (NULL, source->pixels, PS_TYPE_F32); + psImageInit (rawModelFlux, 0.0); + + // insert the model image normalized to 1.0 + pmModelAdd (rawModelFlux, NULL, model, PM_MODEL_OP_FULL | PM_MODEL_OP_NORM, maskVal); + + psImageConvolveFFT (source->modelFlux, rawModelFlux, NULL, 0, psfKernel); + + psFree (psfKernel); + psFree (rawModelFlux); + } + + pmSourceCacheModel (source, maskVal); // ALLOC x14 (!) + } + + psLogMsg ("psphot.replace", PS_LOG_INFO, "subtracted models for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.replace")); + return true; +} + Index: /branches/czw_branch/20101203/psphot/src/psphotSetThreads.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotSetThreads.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotSetThreads.c (revision 30631) @@ -35,5 +35,5 @@ psFree(task); - task = psThreadTaskAlloc("PSPHOT_EXTENDED_FIT", 12); + task = psThreadTaskAlloc("PSPHOT_EXTENDED_FIT", 13); task->function = &psphotExtendedSourceFits_Threaded; psThreadTaskAdd(task); Index: /branches/czw_branch/20101203/psphot/src/psphotSourceFits.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotSourceFits.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotSourceFits.c (revision 30631) @@ -365,5 +365,5 @@ // copy most data from the primary source (modelEXT, blends stay NULL) - pmSource *newSrc = pmSourceCopyData (source); + pmSource *newSrc = pmSourceCopy (source); newSrc->modelPSF = psMemIncrRefCounter (DBL->data[1]); @@ -385,5 +385,5 @@ psLogMsg ("psphot", 1, "PAR %d : %f +/- %f\n", i, source->modelPSF->params->data.F32[i], source->modelPSF->dparams->data.F32[i]); } - psphotVisualShowResidualImage (readout); + psphotVisualShowResidualImage (readout, false); } # endif Index: /branches/czw_branch/20101203/psphot/src/psphotSourceMatch.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotSourceMatch.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotSourceMatch.c (revision 30631) @@ -187,4 +187,8 @@ pmPhotObj *obj = objects->data[i]; + // we will find the input source with the max number of spans and reproduce that footprint + int nSpansMax = 0; + int iSpansMax = -1; + // mark the images for which sources have been found psVectorInit (found, 0); @@ -196,5 +200,22 @@ psAssert (index < found->n, "invalid index"); + if (src->peak && src->peak->footprint && src->peak->footprint->nspans > nSpansMax) { + nSpansMax = src->peak->footprint->nspans; + iSpansMax = j; + } + found->data.U8[index] = 1; + } + + // we make a copy of the largest footprint; this will be used for all new sources associated with this object + pmFootprint *footprint = NULL; + if (iSpansMax != -1) { // copy the footprint info + pmSource *src = obj->sources->data[iSpansMax]; + psAssert(src->peak, "source does not exist?"); + psAssert(src->peak->footprint, "footprint does not exist"); + psAssert(src->peak->footprint->nspans == nSpansMax, "wrong footprint?"); + + // we only care about the spans, do not worry about the image of this footprint + footprint = pmFootprintCopyData(src->peak->footprint, NULL); } @@ -219,6 +240,12 @@ peak->dy = NAN; - // XXX assign to a footprint? - + // assign to a footprint on this readout->image + peak->footprint = pmFootprintCopyData(footprint, readout->image); + + // the peak does not claim ownership of the footprint (it does not free it). save a copy of this + // footprint on detections->footprints so we can free it later + psArrayAdd(detections->footprints, 100, peak->footprint); + psFree (peak->footprint); + // create a new source pmSource *source = pmSourceAlloc(); @@ -227,5 +254,5 @@ // add the peak - source->peak = psMemIncrRefCounter(peak); + source->peak = peak; // allocate space for moments @@ -239,6 +266,6 @@ psArrayAdd (detections->newSources, 100, source); psFree (source); - psFree (peak); - } + } + psFree (footprint); } Index: /branches/czw_branch/20101203/psphot/src/psphotSourceSize.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotSourceSize.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotSourceSize.c (revision 30631) @@ -499,11 +499,10 @@ // clear the mask bit and set the circular mask pixels - psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); - psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); - + // psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + // psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); pmSourceMagnitudes (source, psf, photMode, maskVal, markVal); // clear the mask bit - psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + // psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); // re-subtract the object, leave local sky Index: /branches/czw_branch/20101203/psphot/src/psphotStackArguments.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackArguments.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackArguments.c (revision 30631) @@ -41,5 +41,5 @@ if ((N = psArgumentGet (argc, argv, "-break"))) { if (argc <= N+1) { - psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + psErrorStackPrint(stderr, "Expected to see an argument for -break"); exit(PS_EXIT_CONFIG_ERROR); } Index: /branches/czw_branch/20101203/psphot/src/psphotStackChisqImage.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackChisqImage.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackChisqImage.c (revision 30631) @@ -6,5 +6,5 @@ // XXX supply filename or keep PSPHOT.INPUT fixed? -bool psphotStackChisqImage (pmConfig *config, const pmFPAview *view, const char *ruleDet, const char *ruleCnv) +bool psphotStackChisqImage (pmConfig *config, const pmFPAview *view, const char *ruleDet, const char *ruleSrc) { psTimerStart ("psphot.chisq.image"); @@ -27,6 +27,7 @@ psMetadataAddS32(config->arguments, PS_LIST_TAIL, "PSPHOT.CHISQ.NUM", PS_META_REPLACE, "", num); + + // we need to increment the counter for ruleDet and ruleSrc: num++; - psMetadataAddS32(config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "", num); // save the resulting image in the 'detection' set @@ -35,9 +36,13 @@ return false; } + psphotFileruleCountSet(config, ruleDet, num); - // save the resulting image in the 'convolved' set - if (!psMetadataAddPtr(config->files, PS_LIST_TAIL, ruleCnv, PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "", chisqFile)) { - psError(PM_ERR_CONFIG, false, "could not add chisqFPA to config files"); - return false; + // also save the resulting image in the 'source' set (analysis set) + if (strcmp(ruleDet, ruleSrc)) { + if (!psMetadataAddPtr(config->files, PS_LIST_TAIL, ruleSrc, PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "", chisqFile)) { + psError(PM_ERR_CONFIG, false, "could not add chisqFPA to config files"); + return false; + } + psphotFileruleCountSet(config, ruleSrc, num); } @@ -119,10 +124,10 @@ psAssert (status, "programming error: must define PSPHOT.CHISQ.NUM"); - int inputNum = psphotFileruleCount(config, "PSPHOT.INPUT"); + int inputNum = psphotFileruleCount(config, filerule); pmFPAfileRemoveSingle (config->files, filerule, chisqNum); inputNum --; - psMetadataAddS32(config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "", inputNum); + psphotFileruleCountSet(config, filerule, inputNum); return true; Index: /branches/czw_branch/20101203/psphot/src/psphotStackImageLoop.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackImageLoop.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackImageLoop.c (revision 30631) @@ -101,5 +101,5 @@ */ -# define UPDATE_HEADER 0 +# define UPDATE_HEADER 1 bool GetAstrometryFPA (pmConfig *config, pmFPAview *view) { @@ -174,12 +174,13 @@ pmChip *outChip = pmFPAviewThisChip(view, output->fpa); ///< Chip in the output -# if (UPDATE_HEADER) pmHDU *outHDU = pmFPAviewThisHDU (view, output->fpa); if (!outHDU) { - pmFPAAddSourceFromView(output->fpa, "name", view, output->format); + pmFPAAddSourceFromView(output->fpa, view, output->format); outHDU = pmFPAviewThisHDU (view, output->fpa); psAssert (outHDU, "failed to make HDU"); } -# endif + if (!outHDU->header) { + outHDU->header = psMetadataAlloc(); + } if (bilevelAstrometry) { @@ -188,10 +189,8 @@ continue; } -# if (UPDATE_HEADER) if (!pmAstromWriteBilevelChip(outHDU->header, outChip, WCS_NONLIN_TOL)) { psWarning("Unable to generate WCS header."); continue; } -# endif } else { // we use a default FPA pixel scale of 1.0 @@ -200,10 +199,8 @@ continue; } -# if (UPDATE_HEADER) - if (UPDATE_HEADER && !pmAstromWriteWCS(outHDU->header, output->fpa, outChip, WCS_NONLIN_TOL)) { + if (!pmAstromWriteWCS(outHDU->header, output->fpa, outChip, WCS_NONLIN_TOL)) { psWarning("Unable to generate WCS header."); continue; } -# endif } } @@ -225,16 +222,17 @@ psAssert (output, "missing file?"); -# if (UPDATE_HEADER) pmHDU *PHU = pmFPAviewThisPHU(view, output->fpa); if (!PHU) { - pmFPAAddSourceFromView(output->fpa, "name", view, output->format); + pmFPAAddSourceFromView(output->fpa, view, output->format); PHU = pmFPAviewThisPHU (view, output->fpa); psAssert (PHU, "failed to make PHU"); } + if (!PHU->header) { + PHU->header = psMetadataAlloc(); + } if (!pmAstromWriteBilevelMosaic(PHU->header, output->fpa, WCS_NONLIN_TOL)) { psWarning("Unable to generate WCS header."); } -# endif } Index: /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFs.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFs.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFs.c (revision 30631) @@ -65,4 +65,13 @@ bool psphotStackMatchPSFsReadout (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index) { + psImageMaskType maskValue; + psImageMaskType markValue; + + // get the PSPHOT.MASK value from the config + if (!pmConfigMaskSetBits (&maskValue, &markValue, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + pmFPAfile *fileSrc = psphotStackGetConvolveSource(config, options, index); if (!fileSrc) { @@ -84,7 +93,6 @@ // set NAN pixels to 'SAT' - // XXX replace this is pmReadoutMaskInvalid? - psImageMaskType maskVal = pmConfigMaskGet("SAT", config); - if (!pmReadoutMaskNonfinite(readoutSrc, maskVal)) { + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); + if (!pmReadoutMaskInvalid(readoutSrc, maskValue, maskSat)) { psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels in readout."); return false; @@ -95,27 +103,18 @@ matchKernel(config, readoutOut, readoutSrc, options, index); saveMatchData(readoutOut, options, index); - // renormKernel(readoutCnv, options, index); - } else { - // only match the flux (NO! not for multi-filter, at least!) - // XXX do not generate readoutCnv in this case? - // float norm = powf(10.0, -0.4 * options->norm->data.F32[index]); // Normalisation - // psBinaryOp(readoutRaw->image, readoutRaw->image, "*", psScalarAlloc(norm, PS_TYPE_F32)); - // psBinaryOp(readoutRaw->variance, readoutRaw->variance, "*", psScalarAlloc(PS_SQR(norm), PS_TYPE_F32)); } - rescaleData(readoutOut, config, options, index); - // dumpImage(readoutOut, readoutSrc, index, "convolved"); + // save the output fwhm values in the readout->analysis. we may have / will have multiple output PSF sizes, + // so we save this in a vector. if the vector is not yet defined, create it + bool mdok = false; + psVector *fwhmValues = psMetadataLookupVector(&mdok, readoutOut->analysis, "STACK.PSF.FWHM.VALUES"); + if (!fwhmValues) { + fwhmValues = psVectorAllocEmpty(10, PS_TYPE_F32); + psMetadataAddVector(readoutOut->analysis, PS_LIST_TAIL, "STACK.PSF.FWHM.VALUES", PS_META_REPLACE, "PSF sizes", fwhmValues); + psFree(fwhmValues); // drops the extra copy + } + psVectorAppend(fwhmValues, options->targetSeeing); return true; } - - -# if (0) -// Read previously produced kernel -if (psMetadataLookupBool(NULL, config->arguments, "PPSTACK.DEBUG.STACK")) { - loadKernel(config, readoutCnv, options, index); -} else { - matchKernel(config, readoutCnv, readoutRaw, options, index); -} -# endif Index: /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFsNext.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFsNext.c (revision 30631) +++ /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFsNext.c (revision 30631) @@ -0,0 +1,151 @@ +# include "psphotInternal.h" + +// smooth the input image to match the next target PSF +// this function assumes the image has already been smoothed to match the first value (array element 0), +// and that the smoothing can use a 1D Gaussian kernel of width sqrt(TARGET^2 - CURRENT^2) +// each subsequent call +bool psphotStackMatchPSFsNext(bool *smoothAgain, pmConfig *config, const pmFPAview *view, const char *filerule, int lastSize) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + psVector *fwhmValues = psMetadataLookupVector(&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); // Magnitude offsets + if (!status) { + // must not be a vector, only one value requested + *smoothAgain = false; + return true; + } + + if (lastSize + 1 >= fwhmValues->n) { + // all done with target FWHM values + *smoothAgain = false; + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // smooth the image and variance map + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading in psImageConvolve + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotStackMatchPSFsNextReadout (config, view, filerule, i, recipe, fwhmValues, lastSize)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to smooth image %s (%d) to target PSF %f", filerule, i, fwhmValues->data.F32[lastSize+1]); + psImageConvolveSetThreads(oldThreads); + return false; + } + } + + psImageConvolveSetThreads(oldThreads); + return true; +} + +bool psphotStackMatchPSFsNextReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, psVector *fwhmValues, int lastSize) { + + bool status = false; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + psLogMsg("psphot", PS_LOG_INFO, "smooth %d to next psf", index); + psphotVisualShowImage(readout); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + float minGauss = psMetadataLookupF32(NULL, recipe, "PEAKS_MIN_GAUSS"); // Minimum valid fraction of kernel + if (!isfinite(minGauss)) { + psWarning("PEAKS_MIN_GAUSS is not set in recipe; using default value"); + minGauss = 0.5; + } + + float targetFWHM = fwhmValues->data.F32[lastSize + 1]; + float currentFWHM = fwhmValues->data.F32[lastSize]; + + if (targetFWHM <= currentFWHM) { + psError (PSPHOT_ERR_CONFIG, true, "target FWHM cannot be smaller than current FWHM"); + return false; + } + + float smoothFWHM = sqrt(PS_SQR(targetFWHM) - PS_SQR(currentFWHM)); + float SIGMA_SMTH = smoothFWHM / (2.0*sqrt(2.0*log(2.0))); + float NSIGMA_SMTH = 3.0; + + // record the actual smoothing sigma + // psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SIGMA_SMOOTH", PS_META_REPLACE, "Smoothing sigma for detections", SIGMA_SMTH); + + // smooth the image, applying the mask as we go + psImageSmoothMask_Threaded(readout->image, readout->image, readout->mask, maskVal, SIGMA_SMTH, NSIGMA_SMTH, minGauss); + psLogMsg("psphot", PS_LOG_MINUTIA, "smooth image: %f sec\n", psTimerMark("psphot.smooth")); + + // Smooth the variance, applying the mask as we go. The variance is smoothed by the PSF^2, + // renomalized to maintain the input level of the variance. We achieve this by smoothing + // with a Gaussian with sigma = SIGMA_SMTH/sqrt(2) with unity normalization. Note that + // this process yields a smoothed image with correlated errors. The pixel-to-pixel + // variations in smooth_im will be decreased by a factor of 4*pi*SIGMA_SMTH^2, but for + // measurements based on apertures comparable to or larger than the smoothing kernel, the + // effective per-pixel variance is maintained. + psImageSmoothMask_Threaded(readout->variance, readout->variance, readout->mask, maskVal, SIGMA_SMTH * M_SQRT1_2, NSIGMA_SMTH, minGauss); + psLogMsg("psphot", PS_LOG_MINUTIA, "smooth variance: %f sec\n", psTimerMark("psphot.smooth")); + + psLogMsg("psphot", PS_LOG_INFO, "smoothed"); + psphotVisualShowImage(readout); + + // optionally save example images under trace + if (psTraceGetLevel("psphot") > 5) { + static int pass = 0; + char name[64]; + sprintf (name, "stksm.v%d.fits", pass); + psphotSaveImage(NULL, readout->image, name); + sprintf (name, "stkwt.v%d.fits", pass); + psphotSaveImage(NULL, readout->variance, name); + pass ++; + } + + // XXX need to apply this to the radial apertures somehow. + // Calculate correction factor for the covariance produced by the (potentially multiple) smoothing + // psKernel *kernel = psImageSmoothKernel(SIGMA_SMTH, NSIGMA_SMTH); // Kernel used for smoothing + // double sum2 = 0.0; // Sum of kernel squared + // for (int y = kernel->yMin; y <= kernel->yMax; y++) { + // for (int x = kernel->xMin; x <= kernel->xMax; x++) { + // sum2 += PS_SQR(kernel->kernel[y][x]); + // } + // } + // float factor = 1.0 / (sum2 * psImageCovarianceCalculateFactor(kernel, readout->covariance)); + // psFree(kernel); + + // record the effective area and significance scaling factor + float effArea = 8.0 * M_PI * PS_SQR(SIGMA_SMTH); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "EFFECTIVE_AREA", PS_META_REPLACE, "Effective Area", effArea); + // psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SIGNIFICANCE_SCALE_FACTOR", PS_META_REPLACE, "Signicance scale factor", factor); + + // save the output fwhm values in the readout->analysis. we may have / will have multiple output PSF sizes, + // so we save this in a vector. if the vector is not yet defined, create it + + psVector *fwhmValuesOut = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + psAssert(fwhmValuesOut, "should already exist.."); + psVectorAppend(fwhmValuesOut, targetFWHM); + + // do not generate a PSF if we already were supplied one + pmPSF *psfOld = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + if (psfOld) { + // save PSF on readout->analysis + char psfEntry[64]; + snprintf (psfEntry, 64, "PSPHOT.PSF.V%d", lastSize); + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, psfEntry, PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot psf model", psfOld)) { + psError (PSPHOT_ERR_UNKNOWN, false, "problem saving sources on readout"); + return false; + } + psMetadataRemoveKey(readout->analysis, "PSPHOT.PSF"); + } + + return true; +} Index: /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFsUtils.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFsUtils.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackMatchPSFsUtils.c (revision 30631) @@ -365,8 +365,10 @@ pmSubtractionSetFWHMs(options->inputSeeing->data.F32[index], options->targetSeeing); - if (scale && !pmSubtractionParamsScale(&size, &footprint, widthsCopy, scaleRef, scaleMin, scaleMax)) { - psError(psErrorCodeLast(), false, "Unable to scale kernel parameters"); - goto escape; - } + pmSubtractionParamScaleOptions(scale, scaleRef, scaleMin, scaleMax); + + // if (scale && !pmSubtractionParamsScale(&size, &footprint, widthsCopy, scaleRef, scaleMin, scaleMax)) { + // psError(psErrorCodeLast(), false, "Unable to scale kernel parameters"); + // goto escape; + // } if (!pmSubtractionMatch(NULL, readoutOut, fake, readoutSrc, footprint, stride, regionSize, spacing, threshold, stampSources, stampsName, type, size, order, widthsCopy, orders, inner, ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter, rej, normFrac, sysError, skyErr, kernelError, covarFrac, maskVal, maskBad, maskPoor, poorFrac, badFrac, PM_SUBTRACTION_MODE_2)) { @@ -383,4 +385,10 @@ goto escape; } + + // save the PSF on the new readout->analysis: + // if (!psMetadataAddPtr (readoutOut->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot psf model", options->psf)) { + // psError (PSPHOT_ERR_UNKNOWN, false, "problem saving sources on readout"); + // return false; + // } // dumpImage(readoutOut, readoutSrc, index, "conv"); Index: /branches/czw_branch/20101203/psphot/src/psphotStackPSF.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackPSF.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackPSF.c (revision 30631) @@ -12,12 +12,13 @@ bool autoPSF = psMetadataLookupBool (&mdok, psphotRecipe, "PSPHOT.STACK.TARGET.PSF.AUTO"); + // Get the recipe values + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, "PPSTACK"); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + char *psfModel = psMetadataLookupStr(NULL, recipe, "PSF.MODEL"); // Model for PSF + if (autoPSF) { - // Get the recipe values - psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, "PPSTACK"); // ppStack recipe - psAssert(recipe, "We've thrown an error on this before."); - int psfInstances = psMetadataLookupS32(NULL, recipe, "PSF.INSTANCES"); // Number of instances for PSF float psfRadius = psMetadataLookupF32(NULL, recipe, "PSF.RADIUS"); // Radius for PSF - const char *psfModel = psMetadataLookupStr(NULL, recipe, "PSF.MODEL"); // Model for PSF int psfOrder = psMetadataLookupS32(NULL, recipe, "PSF.ORDER"); // Spatial order for PSF @@ -49,10 +50,14 @@ float targetFWHM = psMetadataLookupF32 (&mdok, psphotRecipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); - psAssert (isfinite(targetFWHM), "missing psphot recipe value PSPHOT.STACK.TARGET.PSF.FWHM"); + if (!mdok) { + psVector *fwhmValues = psMetadataLookupVector(&mdok, psphotRecipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); // Magnitude offsets + psAssert (mdok, "missing psphot recipe value PSPHOT.STACK.TARGET.PSF.FWHM"); + targetFWHM = fwhmValues->data.F32[0]; + } float Sxx = sqrt(2.0)*targetFWHM / 2.35; // XXX probably should make the model type (and par 7) optional from recipe - psf = pmPSFBuildSimple("PS_MODEL_PS1_V1", Sxx, Sxx, 0.0, 1.0); + psf = pmPSFBuildSimple(psfModel, Sxx, Sxx, 0.0, 1.0); if (!psf) { psError(PSPHOT_ERR_PSF, false, "Unable to build dummy PSF."); Index: /branches/czw_branch/20101203/psphot/src/psphotStackParseCamera.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackParseCamera.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackParseCamera.c (revision 30631) @@ -15,4 +15,6 @@ } + int nRaw = 0; + int nCnv = 0; int nInputs = inputs->list->n; for (int i = 0; i < nInputs; i++) { @@ -57,4 +59,5 @@ } } + nRaw ++; } @@ -88,4 +91,5 @@ } } + nCnv ++; } @@ -95,6 +99,13 @@ } + // XXX what if they do not match in length + if (nCnv && nRaw) { + if (nCnv != nRaw) { + psError (PSPHOT_ERR_CONFIG, true, "if both RAW and CNV images are supplied, the number must match"); + return false; + } + } + psString sources = psMetadataLookupStr(&status, input, "SOURCES"); // Name of mask - // pmFPAfile *srcInputFile = rawInputFile ? rawInputFile : cnvInputFile; if (sources && strlen(sources) > 0) { if (!defineFile(config, NULL, "PSPHOT.STACK.SOURCES", sources, PM_FPA_FILE_CMF)) { @@ -107,4 +118,5 @@ // XXX output of these files should be optional { + // pmFPAfile *srcInputFile = rawInputFile ? rawInputFile : cnvInputFile; pmFPAfile *outputImage = pmFPAfileDefineOutput(config, NULL, "PSPHOT.STACK.OUTPUT.IMAGE"); if (!outputImage) { @@ -150,4 +162,7 @@ } psMetadataRemoveKey(config->arguments, "FILENAMES"); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.INPUT.RAW.NUM", PS_META_REPLACE, "number of inputs", nRaw); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.INPUT.CNV.NUM", PS_META_REPLACE, "number of inputs", nCnv); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.OUTPUT.IMAGE.NUM", PS_META_REPLACE, "number of inputs", nInputs); psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", nInputs); Index: /branches/czw_branch/20101203/psphot/src/psphotStackReadout.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotStackReadout.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotStackReadout.c (revision 30631) @@ -1,6 +1,44 @@ # include "psphotInternal.h" +// we have 3 possible real filesets: # define STACK_RAW "PSPHOT.STACK.INPUT.RAW" -# define STACK_OUT "PSPHOT.STACK.OUTPUT.IMAGE" +# define STACK_CNV "PSPHOT.STACK.INPUT.CNV" +# define STACK_OUT "PSPHOT.STACK.OUTPUT.IMAGE" /* the psf-matched image */ + +// we have 3 files on which we operate: +// DET (detection image) : nominally RAW (optionally CNV?) +// SRC (source analysis image) : nominally CNV (optionally RAW) +// OUT (psf-matched images) : always OUT + +bool psphotStackVisualFilerule(pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status = false; + + int num = psphotFileruleCount(config, filerule); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + psphotVisualShowResidualImage (readout, true); + psphotVisualShowObjectRegions (readout, recipe, sources); + } + return true; +} bool psphotStackReadout (pmConfig *config, const pmFPAview *view) { @@ -20,8 +58,13 @@ PS_ASSERT_PTR_NON_NULL (breakPt, false); + // XXX or do I set OUT to be a pmFPAfile pointing at the input of interest? + bool useRaw = psMetadataLookupBool (NULL, recipe, "PSPHOT.STACK.USE.RAW"); + char *STACK_SRC = useRaw ? STACK_RAW : STACK_CNV; + char *STACK_DET = STACK_RAW; // XXX optionally allow this to be CNV? + // we have 3 relevant files: RAW, CNV, OUT // set the photcode for each image - if (!psphotAddPhotcode (config, view, STACK_OUT)) { + if (!psphotAddPhotcode (config, view, STACK_SRC)) { psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); return false; @@ -30,9 +73,9 @@ // Generate the mask and weight images // XXX this should be done before we perform the convolutions - if (!psphotSetMaskAndVariance (config, view, STACK_RAW)) { - return psphotReadoutCleanup (config, view, STACK_OUT); + if (!psphotSetMaskAndVariance (config, view, STACK_DET)) { + return psphotReadoutCleanup (config, view, STACK_SRC); } if (!strcasecmp (breakPt, "NOTHING")) { - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } @@ -40,57 +83,63 @@ // XXX I think this is not defined correctly for an array of images. // XXX probably need to subtract the model (same model?) for both RAW and OUT - if (!psphotModelBackground (config, view, STACK_RAW)) { - return psphotReadoutCleanup (config, view, STACK_OUT); - } - if (!psphotSubtractBackground (config, view, STACK_RAW)) { - return psphotReadoutCleanup (config, view, STACK_OUT); + if (!psphotModelBackground (config, view, STACK_DET)) { + return psphotReadoutCleanup (config, view, STACK_SRC); + } + if (!psphotSubtractBackground (config, view, STACK_DET)) { + return psphotReadoutCleanup (config, view, STACK_SRC); } if (!strcasecmp (breakPt, "BACKMDL")) { - return psphotReadoutCleanup (config, view, STACK_OUT); - } - - // load the psf model, if suppled. FWHM_X,FWHM_Y,etc are determined and saved on - // readout->analysis XXX this function currently only works with a single PSPHOT.INPUT - if (!psphotLoadPSF (config, view, STACK_RAW)) { - psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); - return psphotReadoutCleanup (config, view, STACK_OUT); - } - - if (!psphotStackChisqImage(config, view, STACK_RAW, STACK_OUT)) { + return psphotReadoutCleanup (config, view, STACK_SRC); + } + + if (!psphotStackChisqImage(config, view, STACK_DET, STACK_SRC)) { psError (PSPHOT_ERR_UNKNOWN, false, "failure to generate chisq image"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } if (!strcasecmp (breakPt, "CHISQ")) { - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } // find the detections (by peak and/or footprint) in the image. // This finds the detections on Chisq image as well as the individuals - if (!psphotFindDetections (config, view, STACK_RAW, true)) { // pass 1 + if (!psphotFindDetections (config, view, STACK_DET, true)) { // pass 1 // this only happens if we had an error in psphotFindDetections psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); - return psphotReadoutCleanup (config, view, STACK_OUT); - } - - // copy the detections from RAW to OUT - if (!psphotCopySources (config, view, STACK_OUT, STACK_RAW)) { - psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); + } + + // copy the detections from DET to SRC + if (strcmp(STACK_SRC, STACK_DET)) { + if (!psphotCopySources (config, view, STACK_SRC, STACK_DET)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, STACK_SRC); + } } // construct sources and measure basic stats (saved on detections->newSources) - // only run this on detections from the input images, not chisq image - if (!psphotSourceStats (config, view, STACK_OUT, true)) { // pass 1 + if (!psphotSourceStats (config, view, STACK_SRC, true)) { // pass 1 psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); + } + + if (!strcasecmp (breakPt, "TEST1")) { + return psphotReadoutCleanup (config, view, STACK_SRC); } // generate the objects (object unify the sources from the different images) - psArray *objects = psphotMatchSources (config, view, STACK_OUT); + // XXX this could just match the detections for the chisq image, and not bother measuring the + // source stats in that case... + psArray *objects = psphotMatchSources (config, view, STACK_SRC); + + if (!strcasecmp (breakPt, "TEST2")) { + psFree(objects); + return psphotReadoutCleanup (config, view, STACK_SRC); + } // construct sources for the newly-generated sources (from other images) - if (!psphotSourceStats (config, view, STACK_OUT, false)) { // pass 1 + if (!psphotSourceStats (config, view, STACK_SRC, false)) { // pass 1 + psFree(objects); psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } @@ -98,5 +147,5 @@ // if (!psphotDeblendSatstars (config, view)) { // psError (PSPHOT_ERR_UNKNOWN, false, "failed on satstar deblend analysis"); - // return psphotReadoutCleanup (config, view, STACK_OUT); + // return psphotReadoutCleanup (config, view, STACK_SRC); // } @@ -104,66 +153,102 @@ // if (!psphotBasicDeblend (config, view)) { // psError (PSPHOT_ERR_UNKNOWN, false, "failed on deblend analysis"); - // return psphotReadoutCleanup (config, view, STACK_OUT); + // return psphotReadoutCleanup (config, view, STACK_SRC); // } // classify sources based on moments, brightness // only run this on detections from the input images, not chisq image - if (!psphotRoughClass (config, view, STACK_OUT)) { + if (!psphotRoughClass (config, view, STACK_SRC)) { + psFree(objects); psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough classifications"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } // if we were not supplied a PSF model, determine the IQ stats here (detections->newSources) // only run this on detections from the input images, not chisq image - if (!psphotImageQuality (config, view, STACK_OUT)) { // pass 1 + if (!psphotImageQuality (config, view, STACK_SRC)) { // pass 1 + psFree(objects); psError (PSPHOT_ERR_UNKNOWN, false, "failed to measure image quality"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } if (!strcasecmp (breakPt, "MOMENTS")) { - return psphotReadoutCleanup (config, view, STACK_OUT); + psFree(objects); + return psphotReadoutCleanup (config, view, STACK_SRC); } // use bright stellar objects to measure PSF if we were supplied a PSF for any input file, // this step is skipped - if (!psphotChoosePSF (config, view, STACK_OUT)) { // pass 1 + if (!psphotChoosePSF (config, view, STACK_SRC, true)) { // pass 1 + psFree(objects); psLogMsg ("psphot", 3, "failure to construct a psf model"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } if (!strcasecmp (breakPt, "PSFMODEL")) { - return psphotReadoutCleanup (config, view, STACK_OUT); + psFree(objects); + return psphotReadoutCleanup (config, view, STACK_SRC); } // construct an initial model for each object, set the radius to fitRadius, set circular fit mask - psphotGuessModels (config, view, STACK_OUT); + psphotGuessModels (config, view, STACK_SRC); // merge the newly selected sources into the existing list // NOTE: merge OLD and NEW - psphotMergeSources (config, view, STACK_OUT); + psphotMergeSources (config, view, STACK_SRC); // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) psphotFitSourcesLinearStack (config, objects, FALSE); + psphotStackVisualFilerule(config, view, STACK_SRC); // identify CRs and extended sources - psphotSourceSize (config, view, STACK_OUT, TRUE); + psphotSourceSize (config, view, STACK_SRC, TRUE); + + // XXX do we want to do a preliminary (unconvolved) model fit here, and then + // do a second detection pass? (like standard psphot) // measure aperture photometry corrections - if (!psphotApResid (config, view, STACK_OUT)) { + if (!psphotApResid (config, view, STACK_SRC)) { psFree (objects); psLogMsg ("psphot", 3, "failed on psphotApResid"); - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } psphotStackObjectsUnifyPosition (objects); - // measure circular, radial apertures (objects sorted by S/N) - psphotRadialAperturesByObject (config, objects, view, STACK_OUT); - // measure elliptical apertures, petrosians (objects sorted by S/N) - psphotExtendedSourceAnalysisByObject (config, objects, view, STACK_OUT); // pass 1 (detections->allSources) + psphotExtendedSourceAnalysisByObject (config, objects, view, STACK_SRC); // pass 1 (detections->allSources) // measure non-linear extended source models (exponential, deVaucouleur, Sersic) (sources sorted by S/N) - psphotExtendedSourceFits (config, view, STACK_OUT); // pass 1 (detections->allSources) + psphotExtendedSourceFits (config, view, STACK_SRC); // pass 1 (detections->allSources) // calculate source magnitudes - psphotMagnitudes(config, view, STACK_OUT); + psphotMagnitudes(config, view, STACK_SRC); + + // create source children for the OUT filerule (for radial aperture photometry) + psArray *objectsRadial = psphotSourceChildrenByObject (config, view, STACK_OUT, objects); + if (!objectsRadial) { + psFree(objects); + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, STACK_SRC); + } + + bool smoothAgain = true; + for (int nMatchedPSF = 0; smoothAgain; nMatchedPSF++) { + + // re-measure the PSF for the smoothed image (using entries in 'allSources') + psphotChoosePSF (config, view, STACK_OUT, false); + + // this is necessary to update the models based on the new PSF + psphotResetModels (config, view, STACK_OUT); + + // this is necessary to get the right normalization for the new models + psphotFitSourcesLinear (config, view, STACK_OUT, false); + + // measure circular, radial apertures (objects sorted by S/N) + psphotRadialAperturesByObject (config, objectsRadial, view, STACK_OUT, nMatchedPSF); + + // replace the flux in the image so it is returned to its original state + psphotReplaceAllSources (config, view, STACK_OUT); + + // smooth to the next FWHM, or set 'smoothAgain' to false if no more + psphotStackMatchPSFsNext(&smoothAgain, config, view, STACK_OUT, nMatchedPSF); + } if (0 && !psphotEfficiency(config, view, STACK_OUT)) { @@ -176,16 +261,116 @@ // replace background in residual image - psphotSkyReplace (config, view, STACK_RAW); + psphotSkyReplace (config, view, STACK_DET); // drop the references to the image pixels held by each source - psphotSourceFreePixels (config, view, STACK_OUT); - - // remove chisq image from config->file:PSPHOT.INPUT (why?) - psphotStackRemoveChisqFromInputs(config, STACK_RAW); + // psphotSourceFreePixels (config, view, STACK_OUT); + psphotSourceFreePixels (config, view, STACK_SRC); + + // remove chisq image from config->file:PSPHOT.INPUT + psphotStackRemoveChisqFromInputs(config, STACK_DET); + if (strcmp(STACK_SRC, STACK_DET)) { + psphotStackRemoveChisqFromInputs(config, STACK_SRC); + } psFree (objects); + psFree (objectsRadial); // create the exported-metadata and free local data - return psphotReadoutCleanup (config, view, STACK_OUT); + return psphotReadoutCleanup (config, view, STACK_SRC); } +/* here is the process: + + * we have three(*) images: + * RAW : unconvolved image stack + * CNV : input convolved image + + * OUT : psf-matched output image (there may be more than one of + * these. we will generate the first matched image by selecting the + * target PSF and doing a full psf-maching process (as used by ppStack + * and ppSub). But, additional target output files should use a + * simple gaussian convolution kernel determind from therms of the + * current and the target). + + * the output should be / could be one of the matched images, but not + * all. should we ensure the first gets written out, and ot save the + * others (or only optionally). + + * by default, we probably only sve the cmf ffile outputs. + + * load the RAW image (unconvolved stacks) + * add photcode to the output headers / readout->analysis + * generate mask and variance image (this is probably never needed in + practice: we always load an input mask & var. + * generate & subtract a model background for ?? (RAW? CNV? OUT? all?) + * load a PSF (probably not yet working) + + * generate the CHISQ image from the RAW input images (why save on OUT?) + + * find detections on RAW + + * copy detections to OUT + + * generate source stats (moments) for OUT + + * match sources across inputs (on OUT?) + + * generate source stats for the new constructions + + * rough class (star, galaxy, cosmic, etc) + + * Image quality + + * generate PSF + + * guess models + + * merge sources (new -> old) + + * linear fit to the psf + + * find ApResid + + * assign common positions + + * radial apertures (** this should be on the PSF-matched images + + * extended analysis (elliptical profile & petrosian) + + * extended fits (sersic, etc) + + * psphot magnitudes + + + ****** + + the above is all wrong: first, we should be doing the full + morphology analysis (ExtendedAnalysis & ExtendedFits) on the CNV or + RAW image (as desired optionally), etc. + + In the discussion below, 'BST' (best) means optionally RAW or CNV + + * detection : RAW & CHISQ (of RAW) + * moments : used by psf analysis & classification (BST) + * rough class : uses moments, not pixels + * image quality : uses moments as well + * generate PSF : (BST) + * guess models (BST) + * linear fit (BST) + * find ApResid (BST) -- uses sources not pixels + * extended analysis (BST) + * extended fits (BST) + * detection efficiency (BST) + + * somehow need to copy the sources so they point at the pixels on the + * OUT image + + * foreach target PSF + * radial aperture + * convolve to next target PSF + + * somehow need to organize the output file to have the values from + * the different PSFs in separate tables (with header info to + * specify the size of that PSF) + + */ Index: /branches/czw_branch/20101203/psphot/src/psphotVisual.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotVisual.c (revision 30630) +++ /branches/czw_branch/20101203/psphot/src/psphotVisual.c (revision 30631) @@ -93,11 +93,4 @@ strcpy (coords.ctype, "RA---TAN"); - psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); - psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); - if (!psImageBackground(stats, NULL, inImage, NULL, 0, rng)) { - fprintf (stderr, "failed to get background values\n"); - return false; - } - image.Nx = inImage->numCols; image.Ny = inImage->numRows; @@ -125,7 +118,101 @@ free (image.data2d); + return true; +} + +bool psphotVisualShowObjectRegions (pmReadout *readout, psMetadata *recipe, psArray *sources) { + + KiiImage image; + KapaImageData data; + Coords coords; + + bool status = false; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + maskVal |= markVal; + + if (!pmVisualTestLevel("psphot.image.objects", 2)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + strcpy (coords.ctype, "RA---TAN"); + + psImage *inImage = readout->image; + psImage *inMask = readout->mask; + image.Nx = inImage->numCols; + image.Ny = inImage->numRows; + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + if (!psImageBackground(stats, NULL, inImage, inMask, 0xffff, rng)) { + fprintf (stderr, "failed to get background values\n"); + return false; + } + + ALLOCATE (image.data2d, float *, image.Ny); + for (int iy = 0; iy < image.Ny; iy++) { + ALLOCATE (image.data2d[iy], float, image.Nx); + for (int ix = 0; ix < image.Nx; ix++) { + image.data2d[iy][ix] = 0; + } + } + + // loop over sources and set unmasked pixels to 0 + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (source == NULL) continue; + + psImage *mask = source->maskObj; + if (mask == NULL) continue; + + for (int iy = 0; iy < mask->numRows; iy++) { + int jy = iy + mask->row0; + if (jy < 0) continue; + if (jy >= inImage->numRows) continue; + for (int ix = 0; ix < mask->numCols; ix++) { + int jx = ix + mask->col0; + if (jx < 0) continue; + if (jx >= inImage->numCols) continue; + + if (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal) continue; + image.data2d[jy][jx] = 1; + } + } + } + + for (int iy = 0; iy < image.Ny; iy++) { + for (int ix = 0; ix < image.Nx; ix++) { + image.data2d[iy][ix] = (image.data2d[iy][ix] == 0.0) ? NAN : inImage->data.F32[iy][ix]; + } + } + + strcpy (data.name, "maskObj"); + strcpy (data.file, "maskObj"); + // data.zero = 0.0; + // data.range = 1.0; + data.zero = stats->robustMedian - stats->robustStdev; + data.range = 5*stats->robustStdev; + data.logflux = 0; + + KiiSetChannel (kapa, 2); + KiiNewPicture2D (kapa, &image, &data, &coords); + + for (int iy = 0; iy < image.Ny; iy++) { + free (image.data2d[iy]); + } + free (image.data2d); + psFree (stats); psFree (rng); - + + pmVisualAskUser(NULL); return true; } @@ -2401,5 +2488,6 @@ } -bool psphotVisualShowResidualImage (pmReadout *readout) { +// option to redo variance since in some cases we may have displayed a different image in the meanwhile +bool psphotVisualShowResidualImage (pmReadout *readout, bool reshow) { if (!pmVisualTestLevel("psphot.image.resid", 2)) return true; @@ -2408,5 +2496,9 @@ if (myKapa == -1) return false; - psphotVisualScaleImage (myKapa, readout->image, readout->mask, "resid", 1); + if (reshow) { + psphotVisualShowMask (myKapa, readout->mask, "mask", 2); + psphotVisualScaleImage (myKapa, readout->variance, readout->mask, "variance", 1); + } + psphotVisualScaleImage (myKapa, readout->image, readout->mask, "resid", 0); pmVisualAskUser(NULL); Index: /branches/czw_branch/20101203/psphot/test/tap_psphot_deteff.pro =================================================================== --- /branches/czw_branch/20101203/psphot/test/tap_psphot_deteff.pro (revision 30631) +++ /branches/czw_branch/20101203/psphot/test/tap_psphot_deteff.pro (revision 30631) @@ -0,0 +1,396 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# detection efficiency: +# * does it depend on the stellar density? +# * does it scale correctly with the depth? + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 +$BaseOptions = $BaseOptions -nx 2500 -ny 2500 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 +$RefOptions = $RefOptions -Db PSF.CONVOLVE T +# add the density in mkref +# $RefOptions = $RefOptions -Df STARS.DENSITY 10.0 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE T +# we add the catdir in mkexp so we can change the refbase +# $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR catdir.$refbase + +# add galaxies for this test or not? +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + mkref + + local i Npass imname stname + + $Npass = $1 + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf N "%02d" $i + onestack $N + end +end + +macro summary.stats + if ($0 != 2) + echo "USAGE: summary.stats (Npass)" + break + end + + local i Npass base stack imname stname + + $Npass = $1 + + exec /bin/rm -f psf.image.summary.stats + exec /bin/rm -f psf.stack.inputs.summary.stats + exec /bin/rm -f psf.stack.conv.summary.stats + exec /bin/rm -f psf.stack.unconv.summary.stats + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf base "$imname.%02d/$imname.stats" $i + sprintf stack "$stname.%02d/$stname" $i + echo $i $base $stack + exec grep PSF_INST_MAG $base >> psf.image.summary.stats + exec grep PSF_INST_MAG $stack.conv.stats >> psf.stack.inputs.summary.stats + exec grep PSF_INST_MAG $stack.pht.stats >> psf.stack.conv.summary.stats + exec grep PSF_INST_MAG $stack.unconv.pht.stats >> psf.stack.unconv.summary.stats + end +end + +macro onestack + if ($0 != 2) + echo "USAGE: onestack (N)" + break + end + + mkstack image.$1/image stack.$1/stack + ckstack image.$1/image stack.$1/stack + rmstack image.$1/image stack.$1/stack +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + if ($0 != 3) + echo "mkref (refbase) (density)" + break + end + + local refbase + $refbase = $1 + + exec rm -rf $refbase.catdir + exec rm -f $refbase.fits + + $RefOptions = $RefOptions -Df STARS.DENSITY $2 + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + exec ppSim $RefOptions $RefConfig $refbase + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR $refbase.catdir -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes $refbase.cmf + exec relphot -averages -D CATDIR $refbase.catdir -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 4) + echo "USAGE: mkexp basename fwhm refbase" + break + end + + local fwhm basename refbase + $basename = $1 + $fwhm = $2 + $refbase = $3 + + $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename +end + +macro completeness + if ($0 != 5) + echo "USAGE: completeness (raw) (out) (output) (radius)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + line -c red -20 {log(1/25.0)} to 20 {log(1/25.0)} + line -c blue -20 {log(1/5.0)} to 20 {log(1/5.0)} + + png -name $3 +end + +macro completestack + if ($0 != 6) + echo "USAGE: completeness (raw) (out) (output) (radius) (zeropt)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set Mraw = PSF_INST_MAG_raw + $5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + histogram Mraw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (Mraw > dMag[$i]) && (Mraw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot Mraw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot Mraw ldmag + label +y log(S/N) + + line -c red -20 {log(1/25.0)} to 20 {log(1/25.0)} + line -c blue -20 {log(1/5.0)} to 20 {log(1/5.0)} + + png -name $3 +end + +macro falsepositive + if ($0 != 5) + echo "USAGE: falsepositive (raw) (out) (output) (radius)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_out nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + set fAll = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_out > dMag[$i]) && (PSF_INST_MAG_out <= dMag[$i+1]) + subset all = index2 if (inrange) + subset got = index2 if (inrange) && (index2 == -1) + if (all[] == 0) + fMag[$i] = 0 + else + # fraction of detections that are false + fMag[$i] = got[] / all[] + end + # fraction of detections from this bin that are fake + fAll[$i] = got[] / PSF_INST_MAG_out[] + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + # lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + lim dMag fAll; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fAll + label -x mag_inst -y false_frac + + section overlay1 0 0 1 1 + lim dMag fMag; plot -x 1 dMag fMag -c blue + + set found = (index2 >= 0) + plot PSF_INST_MAG_out found -c red + + set ldmag = log(PSF_INST_MAG_SIG_out) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_out ldmag + label +y log(S/N) + + line -c red -20 {log(1/25.0)} to 20 {log(1/25.0)} + line -c blue -20 {log(1/5.0)} to 20 {log(1/5.0)} + + png -name $3 +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +# for completeness, we only need a few of the fields +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG +# PSF_INST_FLUX +# PSF_INST_FLUX_SIG +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/czw_branch/20101203/psphot/test/tap_psphot_diff.pro =================================================================== --- /branches/czw_branch/20101203/psphot/test/tap_psphot_diff.pro (revision 30631) +++ /branches/czw_branch/20101203/psphot/test/tap_psphot_diff.pro (revision 30631) @@ -0,0 +1,576 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE F + +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.00 + 1.25 +end + +list fwhmMax + 1.25 + 1.00 +end + +list aratio + 1.0 + 1.0 +end + +list thetas + 0.0 + 0.0 +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkdiff + if ($0 != 5) + echo "USAGE: mkdiff (rawbase) (diffbase) (init) (order)" + break + end + + # XXX check for local reference db and run mkref if not found + file catdir.ref found + if (not($found)) + echo "make reference catalog" + mkref + end + + local i base diff + + $base = $1 + $diff = $2 + + dirname $base -var dir + mkdir $dir + + dirname $diff -var dir + mkdir $dir + + $command = ppSub + + for i 0 $fwhm:n + if ($3 == 1) + mkexp $base.$i $fwhm:$i $aratio:$i $thetas:$i $fwhmMax:$i + end + + if ($i == 0) + $command = $command -inimage $base.$i.wrp.fits + $command = $command -inmask $base.$i.wrp.mask.fits + $command = $command -invariance $base.$i.wrp.wt.fits + $command = $command -insources $base.$i.wrp.cmf + else + $command = $command -refimage $base.$i.wrp.fits + $command = $command -refmask $base.$i.wrp.mask.fits + $command = $command -refvariance $base.$i.wrp.wt.fits + $command = $command -refsources $base.$i.wrp.cmf + end + end + +# $command = $command -recipe PSPHOT DIFF +# $command = $command -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE +# $command = $command -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF +# $command = $command -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF -D PSPHOT:OUTPUT.FORMAT PS1_DV2 + $command = $command -Db DUAL F -Db ADD.NOT.SUBTRACT T + $command = $command -Db INVERSE T -convolve 1 -Di SPATIAL.ORDER $4 + $command = $command -save-inconv -save-refconv + $command = $command -threads 4 + $command = $command -image_id 1 -source_id 1 + + echo $command $diff + # exec $command $diff +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 6) + echo "USAGE: mkexp basename fwhm aratio theta fwhmMax" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + $aratio = $3 + $theta = $4 + $fwhmMax = $5 + + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits -D PSPHOT:OUTPUT.FORMAT PS1_V2 +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (Next)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + + # M_raw is the predicted mag based on the raw input mags (need to add the zero point) + + set M_raw = PSF_INST_MAG_raw - 2.5*log($4) + set dM_raw = PSF_INST_MAG_SIG_raw / sqrt($4) + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out dM_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end + +macro ckvar + if ($0 != 2) + echo "ckvar (buffer)" + break + end + + cursor + stats $1 {$X1-10} {$Y1-10} 21 21 + $K2sum0 = $TOTAL + stats $1 {$X2-10} {$Y2-10} 21 21 + $K2sum1 = $TOTAL + echo {$K2sum0 * 193 + $K2sum1 * 193} +end + +macro ckresid + + rd res resid.fits + rd con1 conv1.fits + rd con2 conv2.fits + dev -n im; tvch 1; tv res -50 100; tvch 2; tv con1 -20 100; tvch 3; tv con2 -20 100 +end + +macro ckchisq + data chisq2.dat + read flux2 3 Mxx 9 Myy 11 chisq 13 npix 15 + set chisqv = chisq / npix + set flux = sqrt(flux2) + + set lflux = log(flux) + set lchi = log(chisqv) + set lMxx = log(Mxx) + + set dev = sqrt(chisq) + set ndev = sqrt(chisq)/npix + set lndev = log(ndev) + + lim -n 0 lchi lMxx; clear; box; plot lchi lMxx + lim -n 1 lflux lMxx; clear; box; plot lflux lMxx + lim -n 2 lflux lchi; clear; box; plot lflux lchi + lim -n 3 lflux lndev; clear; box; plot lflux lndev + + # set flux2 = flux + # set flux = sqrt(flux2) + # set dev = sqrt(chisq) + # set ndev = sqrt(chisq)/npix + # set lndev = log(ndev) + # set lflux = log(flux) + # lim -n 0 lflux lndev; clear; box; plot lflux lndev + # lim -n 1 lndev lMxx; clear; box; plot lndev lMxx + +end Index: /branches/czw_branch/20101203/psphot/test/tap_psphot_diffsuite.pro =================================================================== --- /branches/czw_branch/20101203/psphot/test/tap_psphot_diffsuite.pro (revision 30631) +++ /branches/czw_branch/20101203/psphot/test/tap_psphot_diffsuite.pro (revision 30631) @@ -0,0 +1,629 @@ +#!/usr/bin/env mana +# -*-sh-*- + +if (not($?TESTING)) set TESTING = 0 + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE F + +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list params + 1.00 1.00 1.0 0.0 + 1.10 1.10 1.0 0.0 + 1.25 1.25 1.0 0.0 + 1.40 1.40 1.0 0.0 + 1.00 1.00 1.3 30.0 + 1.10 1.10 1.3 30.0 + 1.25 1.25 1.3 30.0 + 1.40 1.40 1.3 30.0 + 1.00 1.00 1.3 -30.0 + 1.10 1.10 1.3 -30.0 + 1.25 1.25 1.3 -30.0 + 1.40 1.40 1.3 -30.0 + 1.00 1.25 1.0 0.0 + 1.10 1.25 1.0 0.0 + 1.25 1.25 1.0 0.0 + 1.40 1.25 1.0 0.0 + 1.00 1.25 1.3 30.0 + 1.10 1.25 1.3 30.0 + 1.25 1.25 1.3 30.0 + 1.40 1.25 1.3 30.0 + 1.00 1.25 1.3 -30.0 + 1.10 1.25 1.3 -30.0 + 1.25 1.25 1.3 -30.0 + 1.40 1.25 1.3 -30.0 +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a set of input exposures using the range of parameters given in the lists +macro mkdiffsuite + if ($0 != 3) + echo "USAGE: mkdiffsuite (rawbase) (diffbase)" + break + end + + local i1 i2 rawbase diffbase + $rawbase = $1 + $diffbase = $2 + + for i1 0 $params:n + for i2 {$i1 + 1} $params:n + mkdiff $rawbase $diffbase.$i1.$i2 $i1 $i2 + end + end +end + +# create diff for a single pair of input +macro mkdiff + if ($0 != 5) + echo "USAGE: mkdiff (rawbase) (diffbase) (in1) (in2)" + break + end + + # XXX check for local reference db and run mkref if not found + + local in1 in2 diff + + $base = $1 + + $diff = $2 + dirname $diff -var dir + mkdir $dir + + $in1 = $3 + $in2 = $4 + + $command = ppSub + + $command = $command -inimage $base.$in1.wrp.fits + $command = $command -inmask $base.$in1.wrp.mask.fits + $command = $command -invariance $base.$in1.wrp.wt.fits + $command = $command -insources $base.$in1.wrp.cmf + + $command = $command -refimage $base.$in2.wrp.fits + $command = $command -refmask $base.$in2.wrp.mask.fits + $command = $command -refvariance $base.$in2.wrp.wt.fits + $command = $command -refsources $base.$in2.wrp.cmf + +# no photometry for now +# $command = $command -recipe PSPHOT DIFF +# $command = $command -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE +# $command = $command -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF +# $command = $command -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF -D PSPHOT:OUTPUT.FORMAT PS1_DV2 + + $command = $command -Db DUAL T -Db INVERSE T -convolve 1 -Df PENALTY 1000000 -Di SPATIAL.ORDER 1 + # $command = $command -Db DUAL T -Db INVERSE T -convolve 1 -Df PENALTY 3000000 -Di SPATIAL.ORDER 1 + # $command = $command -Db DUAL T -Db INVERSE T -convolve 1 -Df PENALTY 3000000000 -Di SPATIAL.ORDER 1 + $command = $command -Di KERNEL.SIZE 5 -Di STAMP.FOOTPRINT 5 + + $command = $command -save-inconv -save-refconv + $command = $command -threads 4 + $command = $command -image_id 1 -source_id 1 + + echo $command $diff + if ($TESTING) + exec $command $diff -visual ppsub 10 + else + exec $command $diff >& $diff.log + end +end + +# create a set of input exposures using the range of parameters given in the lists +macro mkexpsuite + if ($0 != 2) + echo "USAGE: mkexpsuite (rawbase)" + break + end + + local i base + + $base = $1 + + dirname $base -var dir + mkdir $dir + + for i 0 $params:n + list words -split $params:$i + mkexp $base.$i $words:0 $words:1 $words:2 $words:3 + end +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 6) + echo "USAGE: mkexp basename fwhmMin fwhmMax aratio theta" + break + end + + local fwhm basename + $basename = $1 + $fwhmMin = $2 + $fwhmMax = $3 + $aratio = $4 + $theta = $5 + + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhmMin -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhmMin -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits -D PSPHOT:OUTPUT.FORMAT PS1_V2 +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro ckvar + if ($0 != 2) + echo "ckvar (buffer)" + break + end + + cursor + stats $1 {$X1-10} {$Y1-10} 21 21 + $K2sum0 = $TOTAL + stats $1 {$X2-10} {$Y2-10} 21 21 + $K2sum1 = $TOTAL + echo {$K2sum0 * 193 + $K2sum1 * 193} +end + +macro ckresid + + rd res resid.fits + rd con1 conv1.fits + rd con2 conv2.fits + dev -n im; tvch 1; tv res -50 100; tvch 2; tv con1 -20 100; tvch 3; tv con2 -20 100 +end + +macro ckchisq + data chisq2.dat + read flux2 3 Mxx 9 Myy 11 chisq 13 npix 15 + set chisqv = chisq / npix + set flux = sqrt(flux2) + + set lflux = log(flux) + set lchi = log(chisqv) + set lMxx = log(Mxx) + + set dev = sqrt(chisq) + set ndev = sqrt(chisq)/npix + set lndev = log(ndev) + + lim -n 0 lchi lMxx; clear; box; plot lchi lMxx + lim -n 1 lflux lMxx; clear; box; plot lflux lMxx + lim -n 2 lflux lchi; clear; box; plot lflux lchi + lim -n 3 lflux lndev; clear; box; plot lflux lndev + + # set flux2 = flux + # set flux = sqrt(flux2) + # set dev = sqrt(chisq) + # set ndev = sqrt(chisq)/npix + # set lndev = log(ndev) + # set lflux = log(flux) + # lim -n 0 lflux lndev; clear; box; plot lflux lndev + # lim -n 1 lndev lMxx; clear; box; plot lndev lMxx + +end + +macro ckscore + if ($0 != 3) + echo "USAGE: ckscore (diffbase) (output)" + break + end + + local i j diffbase + $diffbase = $1 + + output $2 + for i 0 $params:n + for j {$i+1} $params:n + $line = `grep applying $diffbase.$i.$j.log` + list word -split $line + $order = $word:2 + $mode = $word:4 + $val = `grep chisq: $diffbase.$i.$j.log | head -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 2 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 3 | tail -n 1` + sprintf chisq3 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + echo "$order $mode $chisq1 $chisq2 $chisq3 $params:$i $params:$j" + end + end + output stdout +end + +macro ckslope + if ($0 != 3) + echo "USAGE: ckslope (diffbase) (output)" + break + end + + local i j diffbase + $diffbase = $1 + + output $2 + for i 0 $params:n + for j {$i+1} $params:n + $line = `grep applying $diffbase.$i.$j.log` + list word -split $line + $order = $word:2 + $mode = $word:4 + if ($mode == 1) + $val = `grep chisq: $diffbase.$i.$j.log | head -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 4 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + end + if ($mode == 2) + $val = `grep chisq: $diffbase.$i.$j.log | head -n 2 | tail -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 5 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + end + if ($mode == 4) + $val = `grep chisq: $diffbase.$i.$j.log | head -n 3 | tail -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 6 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + end + echo "$order $mode $chisq1 $chisq2 $params:$i $params:$j" + end + end + output stdout +end + +macro plotscore + if ($0 != 2) + echo "USAGE: plotscore (input)" + break + end + + data $1 + read x1 4 f1 10 x2 14 f2 20 xd 24 fd1 30 fd2 32 + set X1 = 2 * x1 / (1 + f1) + set X2 = 2 * x2 / (1 + f2) + set Xd = 2 * xd / (fd1 + fd2) + + subset X1_1 = X1 if (X1 < X2) && (X1 < Xd) + subset X2_1 = X2 if (X1 < X2) && (X1 < Xd) + subset Xd_1 = Xd if (X1 < X2) && (X1 < Xd) + + subset X1_2 = X1 if (X1 > X2) && (X2 < Xd) + subset X2_2 = X2 if (X1 > X2) && (X2 < Xd) + subset Xd_2 = Xd if (X1 > X2) && (X2 < Xd) + + subset X1_d = X1 if (Xd < X2) && (X1 > Xd) + subset X2_d = X2 if (Xd < X2) && (X1 > Xd) + subset Xd_d = Xd if (Xd < X2) && (X1 > Xd) + + lim -n 0 0.3 1.1 0.3 1.1; clear; box + line 0 0 to 2 2 + + plot X1_1 X2_1 -pt 0 -c blue -sz 0.5 + plot X1_2 X2_2 -pt 2 -c red + plot X1_d X2_d -pt 7 -c black + + lim -n 1 0.3 1.1 0.3 1.1; clear; box + line 0 0 to 2 2 + + plot X1_1 Xd_1 -pt 0 -c blue -sz 0.5 + plot X1_2 Xd_2 -pt 2 -c red + plot X1_d Xd_d -pt 7 -c black + + lim -n 2 0.3 1.1 0.3 1.1; clear; box + line 0 0 to 2 2 + + plot X2_1 Xd_1 -pt 0 -c blue -sz 0.5 + plot X2_2 Xd_2 -pt 2 -c red + plot X2_d Xd_d -pt 7 -c black + +end + +if ($SCRIPT) + # mkexpsuite image/image + mkdiffsuite image/image diff.wt4/diff + exit 0 +end + Index: /branches/czw_branch/20101203/psphot/test/tap_psphot_stack.pro =================================================================== --- /branches/czw_branch/20101203/psphot/test/tap_psphot_stack.pro (revision 30631) +++ /branches/czw_branch/20101203/psphot/test/tap_psphot_stack.pro (revision 30631) @@ -0,0 +1,768 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE T +$RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 +$RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 +$RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 +$RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 +$RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + mkref + + local i Npass imname stname + + $Npass = $1 + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf N "%02d" $i + onestack $N + end +end + +macro summary.stats + if ($0 != 2) + echo "USAGE: summary.stats (Npass)" + break + end + + local i Npass base stack imname stname + + $Npass = $1 + + exec /bin/rm -f psf.image.summary.stats + exec /bin/rm -f psf.stack.inputs.summary.stats + exec /bin/rm -f psf.stack.conv.summary.stats + exec /bin/rm -f psf.stack.unconv.summary.stats + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf base "$imname.%02d/$imname.stats" $i + sprintf stack "$stname.%02d/$stname" $i + echo $i $base $stack + exec grep PSF_INST_MAG $base >> psf.image.summary.stats + exec grep PSF_INST_MAG $stack.conv.stats >> psf.stack.inputs.summary.stats + exec grep PSF_INST_MAG $stack.pht.stats >> psf.stack.conv.summary.stats + exec grep PSF_INST_MAG $stack.unconv.pht.stats >> psf.stack.unconv.summary.stats + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +macro onestack + if ($0 != 2) + echo "USAGE: onestack (N)" + break + end + + mkstack image.$1/image stack.$1/stack + ckstack image.$1/image stack.$1/stack + rmstack image.$1/image stack.$1/stack +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i base stack + + $base = $1 + $stack = $2 + + dirname $base -var dir + mkdir $dir + + dirname $stack -var dir + mkdir $dir + + exec echo "INPUT MULTI" > $stack.mdc + exec echo "" >> $stack.mdc + + for i 0 $fwhm:n + mkexp $base.$i $fwhm:$i + + exec echo "INPUT METADATA" >> $stack.mdc + exec echo "IMAGE STR $base.$i.wrp.fits" >> $stack.mdc + exec echo "MASK STR $base.$i.wrp.mask.fits" >> $stack.mdc + exec echo "VARIANCE STR $base.$i.wrp.wt.fits" >> $stack.mdc + exec echo "PSF STR $base.$i.wrp.psf" >> $stack.mdc + exec echo "SOURCES STR $base.$i.wrp.cmf" >> $stack.mdc + exec echo "END" >> $stack.mdc + exec echo "" >> $stack.mdc + end + + exec ppStack -threads 4 -input $stack.mdc $stack -Db TEMP.DELETE F + + basename $stack -var stackbase + for i 0 $fwhm:n + exec /bin/cp -f /tmp/$stackbase.$i.conv.im.fits $stack.$i.conv.im.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.var.fits $stack.$i.conv.var.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.mk.fits $stack.$i.conv.mk.fits + end + + # XXX note that the output convolved variance is missing the covariance + for i 0 $fwhm:n + exec psphot -file $stack.$i.conv.im.fits -mask $stack.$i.conv.mk.fits $stack.$i.conv + end + + # basic photometry for the stack and unconvolved stack + exec psphot -file $stack.fits -mask $stack.mask.fits -variance $stack.weight.fits $stack.pht + exec psphot -file $stack.unconv.fits -mask $stack.unconv.mask.fits -variance $stack.unconv.wt.fits $stack.unconv.pht +end + +# run this on the files created with mkstack +macro ckstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i ZPT base stack + + $base = $1 + $stack = $2 + + # generate plots and stats for the per-image chip analysis + # and the psf-matched convolved images + $ZPT = -2.5*log($fwhm:n) + for i 0 $fwhm:n + ckchip $base.$i.in.cmf $base.$i.cmf $base.$i 0.0 + ckwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.$i.conv $ZPT + + stchip $base.$i.in.cmf $base.$i.cmf $base.stats 0.0 + stwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.conv.stats $ZPT + + completeness $base.$i.in.cmf $base.$i.cmf $base.$i.complete.png + end + + ckwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht $ZPT + ckwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht $ZPT + + stwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht.stats $ZPT + stwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht.stats $ZPT +end + +macro rmstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local base stack + $base = $1 + $stack = $2 + + for i 0 $fwhm:n + exec rm -f $base.$i.fits + exec rm -f $base.$i.ch.fits + exec rm -f $base.$i.ch.mk.fits + exec rm -f $base.$i.ch.wt.fits + + exec rm -f $base.$i.wrp.fits + exec rm -f $base.$i.wrp.mask.fits + exec rm -f $base.$i.wrp.wt.fits + + exec rm -f $stack.$i.conv.im.fits + exec rm -f $stack.$i.conv.mk.fits + exec rm -f $stack.$i.conv.wt.fits + end + + exec rm -f $stack.fits + exec rm -f $stack.mask.fits + exec rm -f $stack.weight.fits + + exec rm -f $stack.exp.fits + exec rm -f $stack.num.fits + exec rm -f $stack.expwt.fits + + exec rm -f $stack.unconv.fits + exec rm -f $stack.unconv.mask.fits + exec rm -f $stack.unconv.wt.fits + + exec rm -f $stack.unconv.exp.fits + exec rm -f $stack.unconv.num.fits + exec rm -f $stack.unconv.expwt.fits +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 3) + echo "USAGE: mkexp basename fwhm" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + + echo psphot -file $basename.wrp.fits -mask $basename.wrp.mask.fits -variance $basename.wrp.wt.fits $basename.wrp.pht + exec psphot -file $basename.wrp.fits -mask $basename.wrp.mask.fits -variance $basename.wrp.wt.fits $basename.wrp.pht +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stchip + if ($0 != 5) + echo "USAGE: stchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +macro stwarp + if ($0 != 5) + echo "USAGE: stwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +macro completeness + if ($0 != 4) + echo "USAGE: completeness (raw) (out) (output)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out 1.5 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + png -name $3 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y '$word:0' -x '$word:3' +end + +# this list is used to compare a pair of vectors with an error it is +# assumed that the vector sets have been loaded and matched with +# match2d to generate index vectors 'index1' and index2'. the macro +# show.dpair generates a plot of the range vector vs (v1 - v2) / dv1. +# The indices for v1, dv1, and v2 are index1,1, and 2 respectively. The +# index for the range vector is defined by the integer following that +# vector. The y-limits of the plot are given by the last two numbers +# (use 'def') for the full default range of the delta vector +list dpairs + # v1 dv v2 range + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl (revision 30630) +++ /branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl (revision 30631) @@ -153,4 +153,5 @@ # assume the fault is transient. + # fault the dependent to give the fault a chance to correct itself my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR); } @@ -410,5 +411,5 @@ if ($warp->{data_state} ne 'full') { - my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic); + my $warp_status = check_states_warp($warp, $rlabel, $need_magic); if ($warp_status eq $PSTAMP_GONE) { faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); @@ -426,5 +427,5 @@ if ($warp1->{data_state} ne 'full') { $warps_ready = 0; - $warp_status = check_states_warp($warp1, 0, $rlabel, $need_magic); + $warp_status = check_states_warp($warp1, $rlabel, $need_magic); if ($warp_status) { if ($warp_status eq $PSTAMP_GONE) { @@ -440,5 +441,5 @@ if ($warp2->{data_state} ne 'full') { $warps_ready = 0; - $warp_status = check_states_warp($warp2, 0, $rlabel, $need_magic); + $warp_status = check_states_warp($warp2, $rlabel, $need_magic); if ($warp_status eq $PSTAMP_GONE) { faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); @@ -493,5 +494,5 @@ if ($warp->{data_state} ne 'full') { - my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic); + my $warp_status = check_states_warp($warp, $rlabel, $need_magic); if ($warp_status eq $PSTAMP_GONE) { faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); Index: /branches/czw_branch/20101203/tools/roc.exposure =================================================================== --- /branches/czw_branch/20101203/tools/roc.exposure (revision 30631) +++ /branches/czw_branch/20101203/tools/roc.exposure (revision 30631) @@ -0,0 +1,315 @@ +#!/usr/bin/env perl + +# USAGE +if (@ARGV != 2) { die "USAGE: roc.exposure (subdir) (expname)\n" ; } + +$dbname = "gpc1"; +$subdir = $ARGV[0]; +$expname = $ARGV[1]; +$savelists = 0; +$loadlists = 0; + +@rochost = ("ipp005", "ipp006", "ipp007", "ipp040", "ipp045"); + +if ($loadlists) { + &load_from_file(); +} else { + &load_from_nebulous(); +} + +# we have 60 unique chips; attempt to make 6 groups of 10 chips +$Ntotal = @chiplist; +$Nalloc = 0; +for (my $j = 0; $j < 6; $j++) { + $Npass = mkgroup($j, 10); + $Nalloc += $Npass; +} + +print "$expname : $Nalloc of $Ntotal (first pass)\n"; + +# re-attempt to add any missed chips to the groups +if ($Nalloc < @chiplist) { + for (my $i = 0; $i < @chiplist; $i++) { + my $thischip = $chiplist[$i]; + if ($chipused[$thischip]) { next; } + $status = 0; + for (my $j = 0; ($status == 0) && ($j < 6); $$j ++) { + $status = &ckchip ($thischip, $j); + } + } + + # re-check the totals + $Nalloc = 0; + for (my $j = 0; $j < 6; $j++) { + $Nalloc += $Nchip[$j]; + } + print "$expname : $Nalloc of $Ntotal (second pass)\n"; + + if ($Nalloc < @chiplist) { + die "unable to assign all files for $expname, skipping\n"; + } +} + +for (my $j = 0; $j < 6; $j++) { + &mkroc($j); +} + +for (my $i = 0; $i < @cullURIs; $i++) { + $command = "neb-cull $cullURIs[$i] --volume $cullHosts[$i]"; + my $status = vsystem ($command); + if ($status) { die "failed to neb-cull with $command\n"; } +} + +vsystem ("regtool -dbname $dbname -updatebyquery -set_state roc -exp_name $expname"); + +exit 0; + +# query database for imfiles, query nebulous for instances, generate: +# %chiphost - hash array of all instance hosts (key = chipID.seq) +# %chipfile - hash array of all instance files (key = chipID.seq) +# @chiplist - array of all unique chips +# %chipused - hash array noting if chip has already been used (key = chipID) +sub load_from_nebulous { + + %chipused = (); + @chiplist = (); + %chiphost = (); + %chipfile = (); + + @reglist = `regtool -dbname $dbname -processedimfile -exp_name $expname | grep uri`; + if ($?) { die "failed to get uri list from regtool\n"; } + + foreach $line (@reglist) { + + ($tmp1, $tmp2, $uri) = split (" ", $line); + + # get the chip id + ($chipID) = $uri =~ m|ota(\d\d).fits|; + if (!$chipID) { die "failed to get chip ID for $uri\n"; } + + # printf "uri: $uri, chipID: %s\n", $chipID; + + push @chiplist, $chipID; + $chipused{$chipID} = 0; + $urilist{$chipID} = $uri; + + @imfiles = `neb-locate --all --path $uri`; + + for ($seq = 0; $seq < @imfiles; $seq ++) { + $imfile = $imfiles[$seq]; + chomp $imfile; + + ($hostID) = $imfile =~ m|/data/(ipp\d\d\d).0/nebulous/|; + + # printf "chipID: $chipID, hostID: $hostID\n"; + + # push @hostIDs, $hostID; + # push @chipIDs, $chipID; + # push @fileIDs, $imfile; + + $thischip = "$chipID.$seq"; + $chiphost{$thischip} = $hostID; + $chipfile{$thischip} = $imfile; + } + } + + # (optionally) save the results + if ($savelists) { + open (FILE, ">chiplist.dat"); + for (my $i = 0; $i < @chiplist; $i++) { + $chipID = $chiplist[$i]; + print FILE "$urilist{$chipID} $chiplist[$i]\n"; + } + close (FILE); + + @keys = keys %chiphost; + open (FILE, ">chiphost.dat"); + foreach $key (@keys) { + printf FILE "$key %s %s\n", $chiphost{$key}, $chipfile{$key}; + } + close (FILE); + } +} + +# load data from a file (for testing) +# %chiphost - hash array of all instance hosts (key = chipID.seq) +# %chipfile - hash array of all instance files (key = chipID.seq) +# @chiplist - array of all unique chips +# %chipused - hash array noting if chip has already been used (key = chipID) +sub load_from_file { + + %chipused = (); + @chiplist = (); + %chiphost = (); + %chipfile = (); + + # (optionally) save the results + open (FILE, "chiplist.dat"); + @lines = ; + close (FILE); + foreach $line (@lines) { + ($uri, $chipID) = split (" ", $line); + + push @chiplist, $chipID; + $urilist{$chipID} = $uri; + $chipused{$chipID} = 0; + } + + + open (FILE, "chiphost.dat"); + @lines = ; + close (FILE); + + foreach $line (@lines) { + ($key, $hostID, $file) = split (" ", $line); + $chiphost{$key} = $hostID; + $chipfile{$key} = $file; + $chipkeep{$key} = 0; + } + + foreach $chipID (@chiplist) { + # print "chipID: $chipID\n"; + } + + @keys = keys %chiphost; + foreach $key (@keys) { + # printf "$key %s %s\n", $chiphost{$key}, $chipfile{$key}; + } +} + +# select 10 chipIDs that (a) have not already been used and (b) are on a host that is not +# yet used for this group +sub mkgroup { + + my $group = $_[0]; + my $Nmax = $_[1]; + my $N = 0; + my($i, $status); + + $Nchip[$group] = 0; + + for ($i = 0; ($N < $Nmax) && ($i < @chiplist); $i++) { + my $thischip = $chiplist[$i]; + if ($chipused[$thischip]) { next; } + $status = &ckchip($thischip, $group); + if ($status) { + $chipused[$thischip] = 1; + $N ++; + } + } + return $N; +} + +sub ckchip { + + my($chipID) = $_[0]; + my($group) = $_[1]; + + $seq = 0; + while (1) { + + # does this chip entry exist? + my $thischip = "$chipID.$seq"; + my $thisfile = $chipfile{$thischip}; + # print "thischip $thischip, thisfile: $thisfile\n"; + if ("$thisfile" eq "") { + return 0; + } + + $thishost = $chiphost{$thischip}; + # print "thishost $thishost\n"; + if ($host[$group]{$thishost} == 0) { + # use this host: + $host[$group]{$thishost} = 1; + $N = $Nchip[$group]; + + # note the entries we are keeping + $chipkeep{$thischip} = 1; + + $file[$group][$N] = $thisfile; + $chip[$group][$N] = $chipID; + $Nchip[$group] ++; + + # print "$group, $chip[$group][$N], $file[$group][$N]\n"; + + &mkcull($chipID, $seq); + + return 1; + } + $seq ++; + } +} + +sub mkroc { + + my $group = $_[0]; + + $inlist = ""; + + $N = $Nchip[$group]; + + # choose an acceptable outhost + $skiphost = 1; + $usehost = ""; + for (my $j = 0; $skiphost && ($j < @rochost); $j++) { + if ($host[$group]{$rochost[$j]} == 0) { + $skiphost = 0; + $usehost = $rochost[$j]; + } + } + if ($skiphost) { + die "no valid ROC host for group $group\n"; + } + + # generate the system command to create the roc file for this group: + for (my $i = 0; $i < $N; $i ++) { + $inlist = "$inlist $file[$group][$i]"; + } + + $outdir = "/data/$usehost.0/roc/$subdir"; + if (! -d $outdir) { + mkdir $outdir; + } + $command = "roc -create $outdir/$expname.$group.roc $inlist"; + + my $status = vsystem ($command); + if ($status) { die "failed to create roc file with $command\n"; } + + open (ROCFILE, ">>rocfile.$usehost.txt"); + print ROCFILE "$command\n"; + close (ROCFILE); +} + +sub mkcull { + + my($chipID) = $_[0]; + my($seqKeep) = $_[1]; + + # find the instances which need to be culled: + my $seq = 0; + while (1) { + if ($seq == $seqKeep) { $seq ++; next; } + + my $thischip = "$chipID.$seq"; + my $thisfile = $chipfile{$thischip}; + + if ("$thisfile" eq "") { + return 1; + } + + my $thishost = $chiphost{$thischip}; + my $thisuri = $urilist{$chipID}; + + push @cullURIs, $thisuri; + push @cullHosts, $thishost; + $seq ++; + } + return 1; +} + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + return $status; + # return 0; +}