Index: branches/eam_branches/ipp-20111122/psphot/src/psphotApResid.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotApResid.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotApResid.c (revision 33640) @@ -154,6 +154,7 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(cellGroups); return false; } Index: branches/eam_branches/ipp-20111122/psphot/src/psphotBlendFit.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotBlendFit.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotBlendFit.c (revision 33640) @@ -151,5 +151,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); psFree (fitOptions); Index: branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceAnalysis.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceAnalysis.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceAnalysis.c (revision 33640) @@ -134,5 +134,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); psFree(AnalysisRegion); Index: branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceFits.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceFits.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotExtendedSourceFits.c (revision 33640) @@ -218,5 +218,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); psFree(AnalysisRegion); Index: branches/eam_branches/ipp-20111122/psphot/src/psphotFindFootprints.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotFindFootprints.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotFindFootprints.c (revision 33640) @@ -35,7 +35,12 @@ psArray *tmp = pmFootprintArrayGrow(footprints, growRadius); psImageConvolveSetThreads(oldThreads); - psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp->n, psTimerMark ("psphot.footprints")); - psFree(footprints); - footprints = tmp; + psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp ? tmp->n : 0, psTimerMark ("psphot.footprints")); + if (tmp) { + psFree(footprints); + footprints = tmp; + } else { + psLogMsg ("psphot", PS_LOG_WARN, "pmFootprintArray grow returned NULL\n"); + } + } Index: branches/eam_branches/ipp-20111122/psphot/src/psphotFitSourcesLinear.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotFitSourcesLinear.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotFitSourcesLinear.c (revision 33640) @@ -280,6 +280,4 @@ psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built fitSources: %f sec (%ld objects)\n", psTimerMark ("psphot.linear"), sources->n); - // fprintf (stderr, "****** Nsat : %d ********\n", Nsat); - if (fitSources->n == 0) { psFree(fitSources); Index: branches/eam_branches/ipp-20111122/psphot/src/psphotForcedReadout.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotForcedReadout.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotForcedReadout.c (revision 33640) @@ -55,11 +55,11 @@ psphotLoadExtSources (config, view, filerule); + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + // Construct an initial model for each object, set the radius to fitRadius, set circular // fit mask. NOTE: only applied to sources without guess models psphotGuessModels (config, view, filerule); - - // merge the newly selected sources into the existing list - // NOTE: merge OLD and NEW - psphotMergeSources (config, view, filerule); // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) Index: branches/eam_branches/ipp-20111122/psphot/src/psphotGuessModels.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotGuessModels.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotGuessModels.c (revision 33640) @@ -112,5 +112,7 @@ // wait here for the threaded jobs to finish // fprintf (stderr, "wait for threads (%d, %d)\n", jx, jy); - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { + // harvest our jobs + psFree(cellGroups); psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); return false; Index: branches/eam_branches/ipp-20111122/psphot/src/psphotKronIterate.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotKronIterate.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotKronIterate.c (revision 33640) @@ -163,5 +163,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); return false; Index: branches/eam_branches/ipp-20111122/psphot/src/psphotMagnitudes.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotMagnitudes.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotMagnitudes.c (revision 33640) @@ -129,5 +129,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); return false; @@ -309,5 +309,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); return false; Index: branches/eam_branches/ipp-20111122/psphot/src/psphotMergeSources.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotMergeSources.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotMergeSources.c (revision 33640) @@ -677,5 +677,8 @@ if (sourceOut->modelPSF) { pmSourceRedefinePixels (sourceOut, readoutOut, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); - } + } else { + // if we have no pixels we can't use it to determine the psf so make sure this bit is off + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; + } // child sources have not been subtracted in this image, but this flag may be raised if @@ -900,6 +903,10 @@ // allocate image, weight, mask for the new image for each peak if (sourceOut->modelPSF) { - pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); - } + pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, + sourceOut->modelPSF->fitRadius); + } else { + // if we have no pixels we can't use it to determine the psf so make sure this bit is off + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; + } // child sources have not been subtracted in this image, but this flag may be raised if Index: branches/eam_branches/ipp-20111122/psphot/src/psphotRadialApertures.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotRadialApertures.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotRadialApertures.c (revision 33640) @@ -176,5 +176,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); psFree(AnalysisRegion); Index: branches/eam_branches/ipp-20111122/psphot/src/psphotRadialProfileWings.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotRadialProfileWings.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotRadialProfileWings.c (revision 33640) @@ -147,5 +147,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); return false; @@ -307,5 +307,11 @@ // linearly interpolate to the radius at which we hit the sky, using the last flux and the limiting slope if (isfinite(lastFlux)) { - limitRadius = lastRadius + lastFlux / 3.0; + float interpolatedRadius = lastRadius + lastFlux / 3.0; + if (interpolatedRadius < MAX_RADIUS) { + limitRadius = interpolatedRadius; + } else { + // XXX should we keep going in this case? + limitRadius = meanRadius; + } } else { limitRadius = meanRadius; Index: branches/eam_branches/ipp-20111122/psphot/src/psphotReadout.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotReadout.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotReadout.c (revision 33640) @@ -133,5 +133,9 @@ // Construct an initial model for each object, set the radius to fitRadius, set circular // fit mask. NOTE: only applied to sources without guess models - psphotGuessModels (config, view, filerule); // pass 1 + // pass 1 + if (!psphotGuessModels (config, view, filerule)) { + psLogMsg ("psphot", 3, "failure to Guess Model - pass 1"); + return psphotReadoutCleanup (config, view, filerule); + } // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) @@ -155,4 +159,5 @@ // non-linear PSF and EXT fit to brighter sources // replace model flux, adjust mask as needed, fit, subtract the models (full stamp) + // XXX: can leave faulted job in done queue psphotBlendFit (config, view, filerule); // pass 1 (detections->allSources) Index: branches/eam_branches/ipp-20111122/psphot/src/psphotSersicModelClass.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotSersicModelClass.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotSersicModelClass.c (revision 33640) @@ -546,4 +546,8 @@ // set the normalization by linear fit between model and data psphotSersicModelNorm (pcm, source); + if (!isfinite(PAR[PM_PAR_I0])) { + fprintf(stderr, "psphotSersicModelClassGuessPCM: psphotSerisicModelNorm set PM_PAR_I0 to NAN\n"); + return false; + } if (TIMING) { t5 = psTimerMark ("SersicGuess"); } Index: branches/eam_branches/ipp-20111122/psphot/src/psphotSourceMatch.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotSourceMatch.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotSourceMatch.c (revision 33640) @@ -3,4 +3,8 @@ bool psphotMatchSourcesAddMissing (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects); bool psphotMatchSourcesSetIDs (psArray *objects); + +static psArray *psphotMatchFootprintCacheAlloc (int nImages); +static pmFootprint *psphotMatchLookupFootprint (psArray *cache, int footprintID, int imageID); +static pmFootprint *psphotMatchCopyFootprint (psArray *cache, pmFootprint *footprint, int imageID, psImage *image, psArray *footprints); psArray *psphotMatchSources (pmConfig *config, const pmFPAview *view, const char *filerule) @@ -193,4 +197,5 @@ // vector to track if source for an image is found psVector *found = psVectorAlloc(nImages, PS_TYPE_U8); + psArray *footprintCache = psphotMatchFootprintCacheAlloc(nImages); for (int i = 0; i < objects->n; i++) { @@ -219,5 +224,5 @@ // we make a copy of the largest footprint; this will be used for all new sources associated with this object - pmFootprint *footprint = NULL; + pmFootprint *largestFootprint = NULL; if (iSpansMax != -1) { // copy the footprint info pmSource *src = obj->sources->data[iSpansMax]; @@ -226,6 +231,5 @@ psAssert(src->peak->footprint->nspans == nSpansMax, "wrong footprint?"); - // we only care about the spans, do not worry about the image of this footprint - footprint = pmFootprintCopyData(src->peak->footprint, NULL); + largestFootprint = src->peak->footprint; } @@ -248,11 +252,16 @@ // assign to a footprint on this readout->image - if (footprint) { - peak->footprint = pmFootprintCopyData(footprint, readout->image); - - // the peak does not claim ownership of the footprint (it does not free it). save a copy of this - // footprint on detections->footprints so we can free it later - psArrayAdd(detections->footprints, 100, peak->footprint); - psFree (peak->footprint); + if (largestFootprint) { + // we save the copies that we make of the of the footprints in a hash so that we can reuse them + // for all sources that share the fooprint. Without this we had serious memory explosion in + // dense fields with lots of footprints (spans are small but when you have enough of them ...) + peak->footprint = psphotMatchLookupFootprint(footprintCache, largestFootprint->id, index); + if (!peak->footprint) { + // the peak does not claim ownership of the footprint (it does not free it). + // psphotMatchCopyFootprint saves a copy of this + // footprint on detections->footprints so we can free it later + peak->footprint = psphotMatchCopyFootprint(footprintCache, largestFootprint, + index, readout->image, detections->footprints); + } } @@ -272,4 +281,9 @@ pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); +#if (0) + fprintf(stderr, "Add mising source for obj: %5d %5d image: %d flux: %f size: %4d %4d\n", + i, obj->id, index, peakFlux, source->pixels->numRows, source->pixels->numCols); +#endif + peak->assigned = true; pmPhotObjAddSource(obj, source); @@ -277,6 +291,6 @@ psFree (source); } - psFree (footprint); - } + } + psFree(footprintCache); // how many sources do we have now? @@ -309,2 +323,54 @@ return true; } + +// Cache of footprints created for unmatched sources. +static psArray * psphotMatchFootprintCacheAlloc (int nImages) { + psArray *cache = psArrayAlloc(nImages); + for (int i = 0; i < nImages; i++) { + psHash *hash = psHashAlloc(5000); + cache->data[i] = hash; + } + return cache; +} + +// Find copy of footprint with given ID made for a given image +static pmFootprint *psphotMatchLookupFootprint (psArray *cache, int footprintID, int imageID) { + psHash *hash = (psHash *) cache->data[imageID]; + + psAssert(hash != NULL, "missing hash for image %d", imageID); + + // footprintID is the id of the original footprint. + char key[32]; + sprintf(key, "%d", footprintID); + + // The footprints in our hashes are the copies of that footprint for the respective images + pmFootprint *copy = psHashLookup(hash, key); + + return copy; +} + +// Create a copy of a given footprint for a given image +static pmFootprint *psphotMatchCopyFootprint (psArray *cache, pmFootprint *footprint, int imageID, psImage *image, psArray *footprints) { + + psAssert((imageID >= 0 && imageID < cache->n), "invalid imageID %d", imageID); + + psHash *hash = (psHash *) cache->data[imageID]; + + psAssert(hash != NULL, "missing hash for image %d", imageID); + + char key[32]; + sprintf(key, "%d", footprint->id); + + pmFootprint *copy = pmFootprintCopyData (footprint, image); + + // save a copy in this image's hash + psHashAdd(hash, key, copy); + + // save a copy of this footprint on the passed in footprints Array so we can free it later + psArrayAdd(footprints, 100, copy); + + // drop our ref + psFree(copy); + + return copy; +} Index: branches/eam_branches/ipp-20111122/psphot/src/psphotSourceStats.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotSourceStats.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotSourceStats.c (revision 33640) @@ -216,5 +216,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS"); psFree(detections->newSources); @@ -347,5 +347,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (false)) { + if (!psThreadPoolWait (false, true)) { psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS"); return NULL; Index: branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFs.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFs.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFs.c (revision 33640) @@ -101,5 +101,8 @@ // Image Matching (PSFs or just flux) if (options->convolve) { - matchKernel(config, readoutOut, readoutSrc, options, index); + if (!matchKernel(config, readoutOut, readoutSrc, options, index)){ + psError(psErrorCodeLast(), false, "Unable to match image PSF in readout."); + return false; + } saveMatchData(readoutOut, options, index); } Index: branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFsPrepare.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFsPrepare.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotStackMatchPSFsPrepare.c (revision 33640) @@ -32,7 +32,7 @@ pmDetections *detections = psMetadataLookupPtr(NULL, readout->analysis, "PSPHOT.DETECTIONS"); // Sources if (!detections || !detections->allSources) { - psWarning("No detections found for image %d --- rejecting.", index); - options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0x01; - return true; // XXX not an error: we continue processing other images + // An input must have sources. If we got here without any the input file was likely corrupt + psError(PSPHOT_ERR_IO, true, "Input %d has no sources.\n\tCheck all instances of: %s", index, inputSrc->origname); + return false; } psAssert (detections->allSources, "missing sources?"); Index: branches/eam_branches/ipp-20111122/psphot/src/psphotTest.c =================================================================== --- branches/eam_branches/ipp-20111122/psphot/src/psphotTest.c (revision 33612) +++ branches/eam_branches/ipp-20111122/psphot/src/psphotTest.c (revision 33640) @@ -85,5 +85,5 @@ // wait for the threads to finish and manage results - if (!psThreadPoolWait (true)) { + if (!psThreadPoolWait (true, true)) { fprintf (stderr, "failure to run FillImage (2)"); exit (1);