Changeset 33640

Timestamp:

Mar 30, 2012, 2:54:17 PM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20111122/psphot

Files:

• . (modified) (1 prop)
• src (modified) (1 prop)
• src/psphotApResid.c (modified) (1 diff)
• src/psphotBlendFit.c (modified) (1 diff)
• src/psphotExtendedSourceAnalysis.c (modified) (1 diff)
• src/psphotExtendedSourceFits.c (modified) (1 diff)
• src/psphotFindFootprints.c (modified) (1 diff)
• src/psphotFitSourcesLinear.c (modified) (1 diff)
• src/psphotForcedReadout.c (modified) (1 diff)
• src/psphotGuessModels.c (modified) (1 diff)
• src/psphotKronIterate.c (modified) (1 diff)
• src/psphotMagnitudes.c (modified) (2 diffs)
• src/psphotMergeSources.c (modified) (2 diffs)
• src/psphotRadialApertures.c (modified) (1 diff)
• src/psphotRadialProfileWings.c (modified) (2 diffs)
• src/psphotReadout.c (modified) (2 diffs)
• src/psphotSersicModelClass.c (modified) (1 diff)
• src/psphotSourceMatch.c (modified) (8 diffs)
• src/psphotSourceStats.c (modified) (2 diffs)
• src/psphotStackImageLoop.c (modified) (1 prop)
• src/psphotStackMatchPSFs.c (modified) (1 diff)
• src/psphotStackMatchPSFsPrepare.c (modified) (1 diff)
• src/psphotTest.c (modified) (1 diff)

branches/eam_branches/ipp-20111122/psphot/src/psphotApResid.c

@@ -154 +154 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+            psFree(cellGroups);
             return false;
         }

branches/eam_branches/ipp-20111122/psphot/src/psphotBlendFit.c

@@ -151 +151 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             psFree (fitOptions);

branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceAnalysis.c

@@ -134 +134 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             psFree(AnalysisRegion);

branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceFits.c

@@ -218 +218 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             psFree(AnalysisRegion);

branches/eam_branches/ipp-20111122/psphot/src/psphotFindFootprints.c

@@ -35 +35 @@
         psArray *tmp = pmFootprintArrayGrow(footprints, growRadius);
         psImageConvolveSetThreads(oldThreads);
-        psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp->n, psTimerMark ("psphot.footprints"));
-        psFree(footprints);
-        footprints = tmp;
+        psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp ? tmp->n : 0, psTimerMark ("psphot.footprints"));
+        if (tmp) {
+            psFree(footprints);
+            footprints = tmp;
+        } else {
+            psLogMsg ("psphot", PS_LOG_WARN, "pmFootprintArray grow returned NULL\n");
+        }
+            
     }
 

branches/eam_branches/ipp-20111122/psphot/src/psphotFitSourcesLinear.c

@@ -280 +280 @@
     psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built fitSources: %f sec (%ld objects)\n", psTimerMark ("psphot.linear"), sources->n);
 
-    // fprintf (stderr, "****** Nsat : %d ********\n", Nsat);
-
     if (fitSources->n == 0) {
         psFree(fitSources);

branches/eam_branches/ipp-20111122/psphot/src/psphotForcedReadout.c

@@ -55 +55 @@
     psphotLoadExtSources (config, view, filerule);
 
+    // merge the newly selected sources into the existing list
+    // NOTE: merge OLD and NEW
+    psphotMergeSources (config, view, filerule);
+
     // Construct an initial model for each object, set the radius to fitRadius, set circular
     // fit mask.  NOTE: only applied to sources without guess models
     psphotGuessModels (config, view, filerule);
-
-    // merge the newly selected sources into the existing list
-    // NOTE: merge OLD and NEW
-    psphotMergeSources (config, view, filerule);
 
     // linear PSF fit to source peaks, subtract the models from the image (in PSF mask)

branches/eam_branches/ipp-20111122/psphot/src/psphotGuessModels.c

@@ -112 +112 @@
         // wait here for the threaded jobs to finish
         // fprintf (stderr, "wait for threads (%d, %d)\n", jx, jy);
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
+            // harvest our jobs
+            psFree(cellGroups);
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;

branches/eam_branches/ipp-20111122/psphot/src/psphotKronIterate.c

@@ -163 +163 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;

branches/eam_branches/ipp-20111122/psphot/src/psphotMagnitudes.c

@@ -129 +129 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;
@@ -309 +309 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;

branches/eam_branches/ipp-20111122/psphot/src/psphotMergeSources.c

@@ -677 +677 @@
       if (sourceOut->modelPSF) {
         pmSourceRedefinePixels (sourceOut, readoutOut, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius);
-      }
+      } else {
+        // if we have no pixels we can't use it to determine the psf so make sure this bit is off
+        sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_CANDIDATE_PSFSTAR;
+       }
 
       // child sources have not been subtracted in this image, but this flag may be raised if
@@ -900 +903 @@
             // allocate image, weight, mask for the new image for each peak
             if (sourceOut->modelPSF) {
-              pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius);
-            }
+                pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, 
+                                                                        sourceOut->modelPSF->fitRadius);
+            } else {
+                // if we have no pixels we can't use it to determine the psf so make sure this bit is off
+                sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_CANDIDATE_PSFSTAR;
+            }
 
             // child sources have not been subtracted in this image, but this flag may be raised if

branches/eam_branches/ipp-20111122/psphot/src/psphotRadialApertures.c

@@ -176 +176 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             psFree(AnalysisRegion);

branches/eam_branches/ipp-20111122/psphot/src/psphotRadialProfileWings.c

@@ -147 +147 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;
@@ -307 +307 @@
             // linearly interpolate to the radius at which we hit the sky, using the last flux and the limiting slope
             if (isfinite(lastFlux)) {
-                limitRadius = lastRadius + lastFlux / 3.0;
+                float interpolatedRadius = lastRadius + lastFlux / 3.0;
+                if (interpolatedRadius < MAX_RADIUS) {
+                    limitRadius = interpolatedRadius;
+                } else {
+                    // XXX should we keep going in this case?
+                    limitRadius = meanRadius;
+                }
             } else {
                 limitRadius = meanRadius;

branches/eam_branches/ipp-20111122/psphot/src/psphotReadout.c

@@ -133 +133 @@
     // Construct an initial model for each object, set the radius to fitRadius, set circular
     // fit mask.  NOTE: only applied to sources without guess models
-    psphotGuessModels (config, view, filerule); // pass 1
+    // pass 1
+    if (!psphotGuessModels (config, view, filerule)) {
+        psLogMsg ("psphot", 3, "failure to Guess Model - pass 1");
+        return psphotReadoutCleanup (config, view, filerule);
+    }
 
     // linear PSF fit to source peaks, subtract the models from the image (in PSF mask)
@@ -155 +159 @@
     // non-linear PSF and EXT fit to brighter sources
     // replace model flux, adjust mask as needed, fit, subtract the models (full stamp)
+    // XXX: can leave faulted job in done queue
     psphotBlendFit (config, view, filerule); // pass 1 (detections->allSources)
 

branches/eam_branches/ipp-20111122/psphot/src/psphotSersicModelClass.c

@@ -546 +546 @@
     // set the normalization by linear fit between model and data
     psphotSersicModelNorm (pcm, source);
+    if (!isfinite(PAR[PM_PAR_I0])) {
+        fprintf(stderr, "psphotSersicModelClassGuessPCM: psphotSerisicModelNorm set PM_PAR_I0 to NAN\n");
+        return false;
+    }
 
     if (TIMING) { t5 = psTimerMark ("SersicGuess"); }

branches/eam_branches/ipp-20111122/psphot/src/psphotSourceMatch.c

@@ -3 +3 @@
 bool psphotMatchSourcesAddMissing (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects);
 bool psphotMatchSourcesSetIDs (psArray *objects);
+
+static psArray     *psphotMatchFootprintCacheAlloc (int nImages);
+static pmFootprint *psphotMatchLookupFootprint (psArray *cache, int footprintID, int imageID);
+static pmFootprint *psphotMatchCopyFootprint (psArray *cache, pmFootprint *footprint, int imageID, psImage *image, psArray *footprints);
  
 psArray *psphotMatchSources (pmConfig *config, const pmFPAview *view, const char *filerule) 
@@ -193 +197 @@
     // vector to track if source for an image is found
     psVector *found = psVectorAlloc(nImages, PS_TYPE_U8);
+    psArray *footprintCache = psphotMatchFootprintCacheAlloc(nImages);
 
     for (int i = 0; i < objects->n; i++) { 
@@ -219 +224 @@
 
         // we make a copy of the largest footprint; this will be used for all new sources associated with this object
-        pmFootprint *footprint = NULL;
+        pmFootprint *largestFootprint = NULL;
         if (iSpansMax != -1) { // copy the footprint info
             pmSource *src = obj->sources->data[iSpansMax]; 
@@ -226 +231 @@
             psAssert(src->peak->footprint->nspans == nSpansMax, "wrong footprint?");
             
-            // we only care about the spans, do not worry about the image of this footprint
-            footprint = pmFootprintCopyData(src->peak->footprint, NULL);
+            largestFootprint = src->peak->footprint;
         }
 
@@ -248 +252 @@
             
             // assign to a footprint on this readout->image
-            if (footprint) {
-                peak->footprint = pmFootprintCopyData(footprint, readout->image);
-
-                // the peak does not claim ownership of the footprint (it does not free it). save a copy of this 
-                // footprint on detections->footprints so we can free it later
-                psArrayAdd(detections->footprints, 100, peak->footprint); 
-                psFree (peak->footprint);
+            if (largestFootprint) {
+                // we save the copies that we make of the of the footprints in a hash so that we can reuse them
+                // for all sources that share the fooprint. Without this we had serious memory explosion in
+                // dense fields with lots of footprints (spans are small but when you have enough of them ...)
+                peak->footprint = psphotMatchLookupFootprint(footprintCache, largestFootprint->id, index);
+                if (!peak->footprint) {
+                    // the peak does not claim ownership of the footprint (it does not free it). 
+                    // psphotMatchCopyFootprint saves a copy of this 
+                    // footprint on detections->footprints so we can free it later
+                    peak->footprint = psphotMatchCopyFootprint(footprintCache, largestFootprint,
+                                            index, readout->image, detections->footprints);
+                }
             }
             
@@ -272 +281 @@
             pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER);
 
+#if (0)
+            fprintf(stderr, "Add mising source for obj: %5d %5d image: %d flux: %f size: %4d %4d\n",
+                                                      i, obj->id, index, peakFlux, source->pixels->numRows, source->pixels->numCols);
+#endif
+
             peak->assigned = true;
             pmPhotObjAddSource(obj, source);
@@ -277 +291 @@
             psFree (source);
         }
-        psFree (footprint);
-    }
+    }
+    psFree(footprintCache);
 
     // how many sources do we have now?
@@ -309 +323 @@
     return true;
 }
+
+// Cache of footprints created for unmatched sources.
+static psArray * psphotMatchFootprintCacheAlloc (int nImages) {
+    psArray *cache = psArrayAlloc(nImages);
+    for (int i = 0; i < nImages; i++) {
+        psHash *hash = psHashAlloc(5000);
+        cache->data[i] = hash;
+    }
+    return cache;
+}
+
+// Find copy of footprint with given ID made for a given image
+static pmFootprint *psphotMatchLookupFootprint (psArray *cache, int footprintID, int imageID) {
+    psHash *hash = (psHash *) cache->data[imageID];
+
+    psAssert(hash != NULL, "missing hash for image %d", imageID);
+
+    // footprintID is the id of the original footprint.
+    char key[32];
+    sprintf(key, "%d", footprintID); 
+
+    // The footprints in our hashes are the copies of that footprint for the respective images
+    pmFootprint *copy = psHashLookup(hash, key);
+
+    return copy;
+}
+
+// Create a copy of a given footprint for a given image
+static pmFootprint *psphotMatchCopyFootprint (psArray *cache, pmFootprint *footprint, int imageID, psImage *image, psArray *footprints) {
+
+    psAssert((imageID >= 0 && imageID < cache->n), "invalid imageID %d", imageID);
+
+    psHash *hash = (psHash *) cache->data[imageID];
+
+    psAssert(hash != NULL, "missing hash for image %d", imageID);
+
+    char key[32];
+    sprintf(key, "%d", footprint->id); 
+
+    pmFootprint *copy = pmFootprintCopyData (footprint, image);
+
+    // save a copy in this image's hash
+    psHashAdd(hash, key, copy);
+
+    // save a copy of this footprint on the passed in footprints Array so we can free it later
+    psArrayAdd(footprints, 100, copy);
+
+    // drop our ref
+    psFree(copy);
+
+    return copy;
+}

branches/eam_branches/ipp-20111122/psphot/src/psphotSourceStats.c

@@ -216 +216 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS");
             psFree(detections->newSources);
@@ -347 +347 @@
 
         // wait for the threads to finish and manage results
-        if (!psThreadPoolWait (false)) {
+        if (!psThreadPoolWait (false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS");
             return NULL;

branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFs.c

@@ -101 +101 @@
     // Image Matching (PSFs or just flux)
     if (options->convolve) {
-        matchKernel(config, readoutOut, readoutSrc, options, index);
+        if (!matchKernel(config, readoutOut, readoutSrc, options, index)){
+            psError(psErrorCodeLast(), false, "Unable to match image PSF in readout.");
+            return false;
+        }
         saveMatchData(readoutOut, options, index);
     }

branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFsPrepare.c

@@ -32 +32 @@
         pmDetections *detections = psMetadataLookupPtr(NULL, readout->analysis, "PSPHOT.DETECTIONS"); // Sources
         if (!detections || !detections->allSources) {
-            psWarning("No detections found for image %d --- rejecting.", index);
-            options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0x01;
-            return true;  // XXX not an error: we continue processing other images
+            // An input must have sources. If we got here without any the input file was likely corrupt
+            psError(PSPHOT_ERR_IO, true, "Input %d has no sources.\n\tCheck all instances of: %s", index, inputSrc->origname);
+            return false;
         }
         psAssert (detections->allSources, "missing sources?");

branches/eam_branches/ipp-20111122/psphot/src/psphotTest.c

@@ -85 +85 @@
 
     // wait for the threads to finish and manage results
-    if (!psThreadPoolWait (true)) {
+    if (!psThreadPoolWait (true, true)) {
         fprintf (stderr, "failure to run FillImage (2)");
         exit (1);