Index: trunk/ippScripts/scripts/dist_defineruns.pl =================================================================== --- trunk/ippScripts/scripts/dist_defineruns.pl (revision 36632) +++ trunk/ippScripts/scripts/dist_defineruns.pl (revision 36633) @@ -88,5 +88,5 @@ push @stages, $stage; } else { - @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle); + @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle ff); } Index: trunk/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- trunk/ippScripts/scripts/dist_make_fileset.pl (revision 36632) +++ trunk/ippScripts/scripts/dist_make_fileset.pl (revision 36633) @@ -223,5 +223,5 @@ my $dbname = shift; - if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal') { + if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal' or $stage eq "ff") { return ""; } Index: trunk/ippScripts/scripts/psphot_fullforce_summary.pl =================================================================== --- trunk/ippScripts/scripts/psphot_fullforce_summary.pl (revision 36632) +++ trunk/ippScripts/scripts/psphot_fullforce_summary.pl (revision 36633) @@ -79,4 +79,6 @@ my ($listFile, $listName) = tempfile("/tmp/fullforce.summary.list.XXXX", UNLINK => !$save_temps ); +my $cff_file; + { my $mdcParser = PS::IPP::Metadata::Config->new; @@ -107,7 +109,14 @@ &my_die("Couldn't find input cmf: $cmf", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cmf); print $listFile "SOURCES STR $cmf\n"; + if (!$cff_file) { + $cff_file = $ipprc->filename('PSPHOT.OUTPUT.CFF', $result->{sources_path_base}); + &my_die("Couldn't find input cff: $cff_file", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cff_file); + } } close $listFile; } + +&my_die("No CFF found in results: $cff_file", $ff_id, $PS_EXIT_PROG_ERROR) unless $cff_file; + # Recipes to use based on reduction class @@ -141,4 +150,5 @@ my $command = "$psphotFullForceSummary $outroot"; $command .= " -input $listName"; + $command .= " -cff $cff_file"; $command .= " -threads $threads" if defined $threads; if ($dump_config) { Index: trunk/ippTasks/dist.pro =================================================================== --- trunk/ippTasks/dist.pro (revision 36632) +++ trunk/ippTasks/dist.pro (revision 36633) @@ -32,4 +32,5 @@ list DIST_STAGE -add "sky" list DIST_STAGE -add "skycal" +list DIST_STAGE -add "ff" $currentStage = 0 Index: trunk/ippTools/share/fftool_definebyquery_select_warps.sql =================================================================== --- trunk/ippTools/share/fftool_definebyquery_select_warps.sql (revision 36632) +++ trunk/ippTools/share/fftool_definebyquery_select_warps.sql (revision 36633) @@ -5,4 +5,5 @@ JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) + JOIN skycell ON skycell.tess_id = warpSkyfile.tess_id AND skycell.skycell_id = warpSkyfile.skycell_id WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s' AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0 Index: trunk/ippTools/share/fftool_result.sql =================================================================== --- trunk/ippTools/share/fftool_result.sql (revision 36632) +++ trunk/ippTools/share/fftool_result.sql (revision 36633) @@ -6,4 +6,5 @@ fullForceRun.skycal_id, fullForceRun.workdir, + fullForceRun.sources_path_base, CONCAT_WS('.', warp_id, skycell_id) AS dist_component, stackRun.tess_id, Index: trunk/ippTools/share/fftool_summary.sql =================================================================== --- trunk/ippTools/share/fftool_summary.sql (revision 36632) +++ trunk/ippTools/share/fftool_summary.sql (revision 36633) @@ -6,4 +6,5 @@ fullForceRun.skycal_id, fullForceRun.workdir, + fullForceRun.sources_path_base, 'summary' AS dist_component, stackRun.tess_id, Index: trunk/ippTools/share/fftool_toadvance.sql =================================================================== --- trunk/ippTools/share/fftool_toadvance.sql (revision 36632) +++ trunk/ippTools/share/fftool_toadvance.sql (revision 36633) @@ -15,5 +15,6 @@ JOIN rawExp USING(exp_id) LEFT JOIN fullForceResult USING(ff_id, warp_id) -WHERE fullForceRun.state = 'new' + LEFT JOIN fullForceSummary USING(ff_id) +WHERE fullForceRun.state = 'new' AND fullForceSummary.ff_id IS NULL -- WHERE hook %s GROUP BY ff_id Index: trunk/ippTools/src/fftool.c =================================================================== --- trunk/ippTools/src/fftool.c (revision 36632) +++ trunk/ippTools/src/fftool.c (revision 36633) @@ -139,7 +139,7 @@ psMetadata *warpWhereMD = psMetadataAlloc(); - pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "fullForceRun.label", "LIKE"); - pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","fullForceRun.data_group", "LIKE"); - PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "fullForceRun.warp_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); if (!psListLength(warpWhereMD->list)) { psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required"); @@ -148,20 +148,7 @@ return false; } - if (!psListLength(warpWhereMD->list)) { - psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); - psFree(skycalWhereMD); - psFree(warpWhereMD); - return false; - } - - PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "fullForceRun.tess_id", "=="); - pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "fullForceRun.filter", "LIKE"); - if (!pxskycellAddWhere(config, warpWhereMD)) { - psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); - psFree(skycalWhereMD); - psFree(warpWhereMD); - return false; - } - + PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "rawExp.filter", "LIKE"); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -186,5 +173,5 @@ psString joinHook = NULL; if (!rerun) { - psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = fullForceRun.skycal_id"); + psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = skycalRun.skycal_id"); psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label); psStringAppend(&select, "\nAND ff_id IS NULL"); @@ -502,5 +489,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); @@ -510,5 +497,5 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); - PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); pxskycellAddWhere(config, where); Index: trunk/ippTools/src/fftoolConfig.c =================================================================== --- trunk/ippTools/src/fftoolConfig.c (revision 36632) +++ trunk/ippTools/src/fftoolConfig.c (revision 36633) @@ -58,4 +58,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0); pxskycellAddArguments(definebyqueryArgs); Index: trunk/ippTools/src/staticskytool.c =================================================================== --- trunk/ippTools/src/staticskytool.c (revision 36632) +++ trunk/ippTools/src/staticskytool.c (revision 36633) @@ -1045,8 +1045,8 @@ PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); pxAddLabelSearchArgs(config, whereMD, "-select_label", "staticskyRun.label", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE"); - pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); Index: trunk/ippconfig/recipes/psphot.config =================================================================== --- trunk/ippconfig/recipes/psphot.config (revision 36632) +++ trunk/ippconfig/recipes/psphot.config (revision 36633) @@ -195,4 +195,6 @@ EXT.NSIGMA.LIMIT.USE BOOL TRUE + +EXT_MODEL_TYPE_FOR_CFF STR BEST # Limits on extended source fits by galactic coordinates @@ -440,9 +442,9 @@ GALAXY_SHAPES_STYLE STR R_MAJ_R_MIN -GALAXY_SHAPES_FR_MAJOR_MIN F32 0.5 -GALAXY_SHAPES_FR_MAJOR_MAX F32 2.00 +GALAXY_SHAPES_FR_MAJOR_MIN F32 0.8 +GALAXY_SHAPES_FR_MAJOR_MAX F32 1.60 GALAXY_SHAPES_FR_MAJOR_DEL F32 0.05 -GALAXY_SHAPES_FR_MINOR_MIN F32 0.5 -GALAXY_SHAPES_FR_MINOR_MAX F32 2.00 +GALAXY_SHAPES_FR_MINOR_MIN F32 0.8 +GALAXY_SHAPES_FR_MINOR_MAX F32 1.60 GALAXY_SHAPES_FR_MINOR_DEL F32 0.05 @@ -489,4 +491,5 @@ PEAKS_NMAX_TOTAL S32 0 # set this to limit the allowed number of peaks - Yields fault instead of avoid memory explosion + EXT_MODEL_TYPE_FOR_CFF STR PS_MODEL_DEV SAVE.RESID BOOL TRUE Index: trunk/psModules/src/objects/pmSourceIO_CFF.c =================================================================== --- trunk/psModules/src/objects/pmSourceIO_CFF.c (revision 36632) +++ trunk/psModules/src/objects/pmSourceIO_CFF.c (revision 36633) @@ -126,5 +126,7 @@ pmModel *model = pmModelAlloc (modelType); source->modelPSF = model; - source->type = PM_SOURCE_TYPE_STAR; // XXX this should be added to the flags +// RoughClass wants source type to be unknown +// source->type = PM_SOURCE_TYPE_STAR; // XXX this should be added to the flags + source->type = PM_SOURCE_TYPE_UNKNOWN; // XXX we can set this in general, but for a specific image, we need to weed out SATSTARS and @@ -178,6 +180,8 @@ source->moments->My = Y; source->moments->Mrf = kronRadius * 0.4; // kronRadius is 2.5 * first radial moment - source->tmpFlags |= PM_SOURCE_TMPF_MOMENTS_MEASURED; - // XXX does the above cause a problem with pmSourceMoments (do we run it? yes we do) + + // Don't mark the moments as measured because that causes many fields to be left blank. + // The moments code knows not to change the position or the Mrf for external sources + // source->tmpFlags |= PM_SOURCE_TMPF_MOMENTS_MEASURED; if (isfinite(petRadius)) { @@ -246,4 +250,12 @@ pmModelType sersicModelType = pmModelClassGetType("PS_MODEL_SERSIC"); + + psString modelToChoose = psMetadataLookupStr(&mdok, recipe, "EXT_MODEL_TYPE_FOR_CFF"); + pmModelType selectedModelType = -1; + if (mdok && modelToChoose != NULL) { + if (strcmp(modelToChoose, "BEST")) { + selectedModelType = pmModelClassGetType(modelToChoose); + } + } for (int i = 0; i < sources->n; i++) { @@ -273,12 +285,9 @@ theta = 0; } else { - // Choose the extended model to use for this source. - // For now we use the one set as the modelEXT for this source which had the best fit. - // XXX: make this controllable by recipe - // use best (as implemented here) - // choose specific type + // Find the model with the selected type. If selected type is -1 choose the one selected ad + // modelEXT which was the best int iModel = -1; if (source->modelEXT) { - pmModelType ext_model_type = source->modelEXT->type; + pmModelType ext_model_type = selectedModelType != -1 ? selectedModelType : source->modelEXT->type; for (int j=0; jmodelFits->n; j++) { pmModel *aModel = source->modelFits->data[j]; Index: trunk/psphot/src/psphotFullForce.SourceStats.c =================================================================== --- trunk/psphot/src/psphotFullForce.SourceStats.c (revision 36632) +++ trunk/psphot/src/psphotFullForce.SourceStats.c (revision 36633) @@ -85,5 +85,6 @@ psAssert (markVal, "missing MARK.PSPHOT"); - psArray *sources = detections->allSources; + // psArray *sources = detections->allSources; + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; // generate the array of sources, define the associated pixel Index: trunk/psphot/src/psphotFullForceReadout.c =================================================================== --- trunk/psphot/src/psphotFullForceReadout.c (revision 36632) +++ trunk/psphot/src/psphotFullForceReadout.c (revision 36633) @@ -52,7 +52,10 @@ psphotLoadExtSources (config, view, filerule); +#ifdef notmoved + // XXX: moved down below // merge the newly selected sources into the existing list (detections->allSources) // NOTE: merge OLD and NEW psphotMergeSources (config, view, filerule); +#endif // construct sources and measure moments and other basic stats (saved on detections->allSources) @@ -63,4 +66,16 @@ return psphotReadoutCleanup (config, view, filerule); } + + // classify sources based on moments, brightness. if a PSF model has been loaded, the PSF + // clump defined for it is used not measured (detections->newSources) + if (!psphotRoughClass (config, view, filerule)) { // pass 1 + psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough classifications"); + return psphotReadoutCleanup (config, view, filerule); + } + + // merge the newly selected sources into the existing list (detections->allSources) + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + // generate a psf model for any readouts which need one Index: trunk/psphot/src/psphotFullForceSummary.c =================================================================== --- trunk/psphot/src/psphotFullForceSummary.c (revision 36632) +++ trunk/psphot/src/psphotFullForceSummary.c (revision 36633) @@ -78,4 +78,18 @@ usage(); } + if ((N = psArgumentGet(argc, argv, "-cff"))) { + if (argc <= N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + usage(); + } + psArgumentRemove(N, &argc, argv); + + // We read CFF file as table not as sources. All we want are the nominal parameters input to psphotFullForce + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "CFF_FILE", 0, "CFF file with nominal parameters", argv[N]); + psArgumentRemove(N, &argc, argv); + } else { + psErrorStackPrint(stderr, "-cff must be supplied."); + usage(); + } if (argc < 2) { @@ -148,4 +162,45 @@ } output->save = true; + + // We read cff file as a table not as sources + psString cffName = psMetadataLookupStr(NULL, config->arguments, "CFF_FILE"); + if (!cffName) { + psError(PSPHOT_ERR_CONFIG, true, "CFF_FILE is missing from arguments"); + return false; + } + psString resolvedName = pmConfigConvertFilename(cffName, config, false, false); + if (!resolvedName) { + psError(PSPHOT_ERR_CONFIG, false, "failed to resolve CFF_FILE %s", cffName); + return false; + } + psFits *fits = psFitsOpen(resolvedName, "r"); + if (!fits) { + psError(PSPHOT_ERR_CONFIG, false, "failed to open fits CFF_FILE %s", resolvedName); + return false; + } + psArray *inTable = psFitsReadTable(fits); + if (!inTable) { + psError(PSPHOT_ERR_CONFIG, false, "failed to read cff fits table from CFF_FILE %s", resolvedName); + return false; + } + psFitsClose(fits); + + // Convert to an array indexed by ID + psArray *sortedTable = psArrayAlloc(4*inTable->n); + for (int i=0; in; i++) { + psMetadata *row = inTable->data[i]; + psS32 ID = psMetadataLookupS32(&status, row, "ID"); + if (ID >= sortedTable->n) { + sortedTable = psArrayRealloc(sortedTable, 2*ID); + } + if (sortedTable->data[ID]) { + psError(PSPHOT_ERR_CONFIG, true, "Duplicate row with ID %d", ID); + return false; + } + sortedTable->data[ID] = psMemIncrRefCounter(row); + } + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "CFF_TABLE", PS_META_REPLACE, "cff table", sortedTable); + psFree(inTable); + psFree(sortedTable); return true; Index: trunk/psphot/src/psphotFullForceSummaryReadout.c =================================================================== --- trunk/psphot/src/psphotFullForceSummaryReadout.c (revision 36632) +++ trunk/psphot/src/psphotFullForceSummaryReadout.c (revision 36633) @@ -14,4 +14,5 @@ psArray *zeroPt; psArray *exptime; + psArray *cffTable; } galaxyShapeOptions; @@ -69,4 +70,9 @@ if (!setOptions(&options, readout, recipe, true)) { psError (PS_ERR_UNKNOWN, false, "problem determining galaxy shape options."); + return false; + } + options.cffTable = psMetadataLookupPtr(NULL, config->arguments, "CFF_TABLE"); + if (!options.cffTable) { + psError (PS_ERR_UNKNOWN, true, "Cannot find cff table in arguments."); return false; } @@ -309,6 +315,30 @@ // copy the best fit params to the model psEllipseAxes axes = pmPSF_ModelToAxes(outSrc->modelEXT->params->data.F32, outSrc->modelEXT->type); - axes.major *= options->rMajor->data.F32[min_j]; - axes.minor *= options->rMinor->data.F32[min_j]; + if (outSrc->seq >= options->cffTable->n) { + psError(PS_ERR_PROGRAMMING, true, "object sequence number %d is larger than cff table length %ld", + outSrc->seq, options->cffTable->n); + return NULL; + } + psMetadata *row = options->cffTable->data[outSrc->seq]; + if (!row) { + psError (PS_ERR_PROGRAMMING, true, "NO cff data for object %d", outSrc->seq); + psFree(outSrc); + return NULL; + } + bool mdok; + psF32 rMajor = psMetadataLookupF32(&mdok, row, "R_MAJOR"); + if (!mdok) { + psError (PS_ERR_PROGRAMMING, true, "can't find R_MAJOR for object %d", outSrc->seq); + psFree(outSrc); + return NULL; + } + psF32 rMinor = psMetadataLookupF32(&mdok, row, "R_MINOR"); + if (!mdok) { + psError (PS_ERR_PROGRAMMING, true, "can't find R_MINOR for object %d", outSrc->seq); + psFree(outSrc); + return NULL; + } + axes.major = rMajor * options->rMajor->data.F32[min_j]; + axes.minor = rMinor * options->rMinor->data.F32[min_j]; pmPSF_AxesToModel (outSrc->modelEXT->params->data.F32, axes, outSrc->modelEXT->type); outSrc->modelEXT->chisq = minChisq; Index: trunk/psphot/src/psphotGalaxyShape.c =================================================================== --- trunk/psphot/src/psphotGalaxyShape.c (revision 36632) +++ trunk/psphot/src/psphotGalaxyShape.c (revision 36633) @@ -205,4 +205,5 @@ if (!(source->tmpFlags & PM_SOURCE_TMPF_MOMENTS_MEASURED)) continue; if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + if (!source->modelPSF) continue; // psphotSetRadiusMomentsExact sets the radius based on Mrf @@ -274,13 +275,19 @@ int i = source->galaxyFits->chisq->n - 1; - float thisChisq = source->galaxyFits->chisq->data.F32[i]; - if (isfinite(thisChisq) && (!isfinite(chisqBest) || thisChisq < chisqBest)) { - chisqBest = thisChisq; - fRmajorBest = fRmajor; - fRminorBest = fRminor; + float flux = source->galaxyFits->Flux->data.F32[i]; + if (isfinite(flux)) { + float thisChisq = source->galaxyFits->chisq->data.F32[i]; + if (isfinite(thisChisq) && isfinite(flux) && (!isfinite(chisqBest) || thisChisq < chisqBest)) { + chisqBest = thisChisq; + fRmajorBest = fRmajor; + fRminorBest = fRminor; + } } + // reset I0 to avoid potential problems on the next iteration + PAR[PM_PAR_I0] = 1.0; } } +#define SAVE_BEST_MODEL #ifdef SAVE_BEST_MODEL // Save model with smallest chisq @@ -301,15 +308,16 @@ psphotGalaxyShapeSource (pcm, source, maskVal, psfSize, false); - // Replace modelEXT with this model - // XXX: only do this if the model is good - psFree (source->modelEXT); - - source->modelEXT = psMemIncrRefCounter (pcm->modelConv); - source->type = PM_SOURCE_TYPE_EXTENDED; - source->mode |= PM_SOURCE_MODE_EXTMODEL; - source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; - - // cache the model flux - pmPCMCacheModel (source, maskVal, psfSize, fitOptions->nsigma); + // Replace modelEXT with this model, if the model is good + if (isfinite(PAR[PM_PAR_I0])) { + psFree (source->modelEXT); + + source->modelEXT = psMemIncrRefCounter (pcm->modelConv); + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXTMODEL; + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + + // cache the model flux + pmPCMCacheModel (source, maskVal, psfSize, fitOptions->nsigma); + } }