Index: branches/eam_branches/ipp-20150616/ippScripts/scripts/staticsky.pl =================================================================== --- branches/eam_branches/ipp-20150616/ippScripts/scripts/staticsky.pl (revision 38551) +++ branches/eam_branches/ipp-20150616/ippScripts/scripts/staticsky.pl (revision 38552) @@ -43,4 +43,5 @@ print "USAGE: staticsky.pl --sky_id (sky_id) --outroot (root) --camera (camera) [options]\n"; print " --sky_id (sky_id) : run identifier\n"; + print " --run-state (state) : run state (new or update)\n"; print " --camera (camera) : Camera name\n"; print " --dbname (dbname) : Database name\n"; @@ -62,6 +63,9 @@ my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $require_sources, $verbose, $no_update, $no_op, $redirect, $save_temps); +my $run_state ='new'; + GetOptions( 'sky_id=s' => \$sky_id, # Diff identifier + 'run-state=s' => \$run_state, 'camera|c=s' => \$camera, # Camera name 'dbname|d=s' => \$dbname, # Database name @@ -88,4 +92,8 @@ defined $camera; + +&my_die("unexpected run state: $run_state", $sky_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new') or ($run_state eq 'update'); +my $updatemode = $run_state eq 'update'; + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -126,5 +134,5 @@ my $nInputs = @$files; -if (1) { +{ my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); # Recipe to use for psphot my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); # Recipe to use for ppsub @@ -163,5 +171,33 @@ my $expnumRaw = $ipprc->filename("PPSTACK.UNCONV.EXPNUM", $path_base ); # Expnum name - my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name + my $sources; + my ($psfRaw, $backmdlRaw); + if (!$updatemode) { + # sources from stack run. + # XXX: We don't need this file anymore. + $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name + } else { + # name of output sources from this run when in new state + $require_sources = 1 unless $no_op; + $sources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $stack_id); + # During update we rename the sources file to this + my $originalSources = "$sources.original"; + if (!$ipprc->file_exists($originalSources)) { + if ($ipprc->file_exists($sources)) { + unless ($no_op) { + if (!$ipprc->file_rename($sources, $originalSources)) { + &my_die("failed to rename $sources $originalSources", $sky_id, $PS_EXIT_PROG_ERROR); + } + } + } else { + # maybe this should be a fault 2 + &my_die("failed to to find either sources file to update as either $sources or $originalSources", $sky_id, $PS_EXIT_PROG_ERROR); + } + } + $sources = $originalSources; + $psfRaw = $ipprc->filename("PSPHOT.STACK.PSF.SAVE", $outroot, $stack_id); + $backmdlRaw = $ipprc->filename("PSPHOT.STACK.BACKMDL", $outroot, $stack_id); + } + # XXX is this the correct PSF file? @@ -194,4 +230,10 @@ print $listFile " SOURCES STR " . $sources . "\n"; } + if ($updatemode) { + &my_die("Couldn't find input: $psfRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfRaw"); + &my_die("Couldn't find input: $backmdlRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$backmdlRaw"); + print $listFile " RAW:PSF STR " . $psfRaw . "\n"; + print $listFile " RAW:BACKMDL STR " . $backmdlRaw . "\n"; + } if ($needConvolvedImages) { print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; @@ -205,9 +247,10 @@ } - # my $cmdflags; + close $listFile; # Perform stack photometry analysis { my $command = "$psphotStack $outroot"; + $command .= " -updatemode" if $updatemode; $command .= " -input $listName"; $command .= " -threads $threads" if defined $threads; @@ -215,5 +258,5 @@ $command .= " -recipe PPSUB $recipe_ppsub"; $command .= " -recipe PPSTACK $recipe_ppstack"; - $command .= " -dumpconfig $configuration"; + $command .= " -dumpconfig $configuration" if !$updatemode; $command .= " -tracedest $traceDest -log $logDest"; # $command .= " -dbname $dbname" if defined $dbname; @@ -275,4 +318,8 @@ # &my_die("Couldn't find expected output file: $chisqMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask); # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance); + + if (!$updatemode) { + # XXX: should be checking for existence of psf and backmdl output files + } } } else { @@ -280,94 +327,4 @@ } } -} else { - # XXX This mode is no longer used. We run psphotStack even if there is only one filter - # single input. Run psphot - # find the recipe - my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_SINGLE_PSPHOT'); # Recipe to use for psphot - unless ($recipe_psphot) { - &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); - } - - print "reduction: $reduction\n"; - print "recipe_psphot: $recipe_psphot\n"; - - my $configuration = $ipprc->filename("PSPHOT.SKY.CONFIG", $outroot); - - my $file = $files->[0]; - - # XXX if we take the input from 'warp', we will need to make different selections here - my $path_base = $file->{path_base}; - print "input: $path_base\n"; - - # examine the recipe to determine whether to analyze the "raw" or convolved images - my $command = "$ppConfigDump -camera $camera -dump-recipe PSPHOT -recipe PSPHOT $recipe_psphot -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $sky_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $sky_id, $PS_EXIT_SYS_ERROR); - - my $use_raw = metadataLookupBool($recipeData, 'PSPHOT.STACK.USE.RAW'); - - my ($image, $mask, $variance, $expnum); - if ($use_raw) { - $image = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name - $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $path_base ); # Mask name - $variance = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name - $expnum = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name - } else { - $image = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name - $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name - $variance = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name - $expnum = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name - } - - my $output_sources_filerule = "PSPHOT.OUT.CMF.MEF"; - my $output_psf_filerule = "PSPHOT.PSF.SKY.SAVE"; - - # XXX we could make some different choices if some inputs do not exist... - &my_die("Couldn't find input: $image", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$image"); - &my_die("Couldn't find input: $mask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$mask"); - &my_die("Couldn't find input: $variance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$variance"); - &my_die("Couldn't find input: $expnum", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnum"); - - # Perform stack photometry analysis - { - my $command = "$psphot $outroot"; - $command .= " -file $image"; - $command .= " -mask $mask"; - $command .= " -variance $variance"; - $command .= " -expnum $expnum"; - $command .= " -threads $threads" if defined $threads; - $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -dumpconfig $configuration" if $configuration; - $command .= " -tracedest $traceDest -log $logDest"; - $command .= " -F PSPHOT.OUTPUT $output_sources_filerule"; - $command .= " -F PSPHOT.PSF.SAVE $output_psf_filerule"; - - unless ($no_op) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform psphot: $error_code", $sky_id, $error_code); - } - - my $quality = 0; - if (!$quality) { - my $outputSources = $ipprc->filename($output_sources_filerule, $outroot); - &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - my $outputPSF = $ipprc->filename($output_psf_filerule, $outroot); - &my_die("Couldn't find expected output file: $outputPSF", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); - } - - &my_die("Couldn't find expected output file: $configuration", $sky_id, $PS_EXIT_SYS_ERROR) - unless $ipprc->file_exists($configuration); - } else { - print "Not executing: $command\n"; - } - } } @@ -377,10 +334,13 @@ { my $command = "$staticskytool -sky_id $sky_id"; - $command .= " -addresult -path_base $outroot"; - $command .= " -num_inputs $nInputs"; - # $command .= " $cmdflags"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -hostname $host" if defined $host; - $command .= " -dbname $dbname" if defined $dbname; + if ($updatemode) { + $command .= " -updaterun -set_state full"; + } else { + $command .= " -addresult -path_base $outroot"; + $command .= " -num_inputs $nInputs"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + } my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -388,5 +348,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - my $err_message = "Unable to perform staticskytool -addresult"; + my $err_message = "Unable to perform: $command\n"; warn($err_message); exit $error_code; @@ -406,10 +366,15 @@ warn($msg); if (defined $sky_id and not $no_update) { + my $command = "$staticskytool -sky_id $sky_id -fault $exit_code"; - $command .= " -addresult"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -hostname $host" if defined $host; - $command .= " -path_base $outroot" if defined $outroot; - $command .= " -dbname $dbname" if defined $dbname; + if ($updatemode) { + $command .= " -updateresult"; + } else { + $command .= " -addresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + } run(command => $command, verbose => $verbose); } Index: branches/eam_branches/ipp-20150616/ippTasks/staticsky.pro =================================================================== --- branches/eam_branches/ipp-20150616/ippTasks/staticsky.pro (revision 38551) +++ branches/eam_branches/ipp-20150616/ippTasks/staticsky.pro (revision 38552) @@ -172,10 +172,11 @@ # set.host.for.skycell $SKYCELL_ID host anyhost - set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR - - basename $TESS_DIR -var TESS_ID + if ("$PATH_BASE" == "NULL") + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + basename $TESS_DIR -var TESS_ID sprintf outroot "%s/%s/%s/%s.%s.sky.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKY_ID else + # run-state must be update use existing path_base $outroot = $PATH_BASE end @@ -184,5 +185,5 @@ stderr $LOGDIR/staticsky.log - $run = staticsky.pl --threads @MAX_THREADS@ --sky_id $SKY_ID --outroot $outroot --redirect-output --camera $CAMERA + $run = staticsky.pl --sky_id $SKY_ID --run-state $RUN_STATE --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION Index: branches/eam_branches/ipp-20150616/psLib/src/fits/psFitsImage.c =================================================================== --- branches/eam_branches/ipp-20150616/psLib/src/fits/psFitsImage.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psLib/src/fits/psFitsImage.c (revision 38552) @@ -751,35 +751,35 @@ // Remove any BOFFSET values that exist in the header if we are not using that scaling anymore - if (options&&(!((options->scaling == PS_FITS_SCALE_LOG_RANGE)|| - (options->scaling == PS_FITS_SCALE_LOG_MANUAL)|| - (options->scaling == PS_FITS_SCALE_LOG_STDEV_POSITIVE)|| - (options->scaling == PS_FITS_SCALE_LOG_STDEV_NEGATIVE)|| - (options->scaling == PS_FITS_SCALE_LOG_STDEV_BOTH)|| - (options->scaling == PS_FITS_SCALE_ASINH_RANGE)|| - (options->scaling == PS_FITS_SCALE_ASINH_MANUAL)|| - (options->scaling == PS_FITS_SCALE_ASINH_STDEV_POSITIVE)|| - (options->scaling == PS_FITS_SCALE_ASINH_STDEV_NEGATIVE)|| - (options->scaling == PS_FITS_SCALE_ASINH_STDEV_BOTH)))) { - if (psMetadataLookup(header,"BOFFSET")) { + if (options && (!((options->scaling == PS_FITS_SCALE_LOG_RANGE)|| + (options->scaling == PS_FITS_SCALE_LOG_MANUAL)|| + (options->scaling == PS_FITS_SCALE_LOG_STDEV_POSITIVE)|| + (options->scaling == PS_FITS_SCALE_LOG_STDEV_NEGATIVE)|| + (options->scaling == PS_FITS_SCALE_LOG_STDEV_BOTH)|| + (options->scaling == PS_FITS_SCALE_ASINH_RANGE)|| + (options->scaling == PS_FITS_SCALE_ASINH_MANUAL)|| + (options->scaling == PS_FITS_SCALE_ASINH_STDEV_POSITIVE)|| + (options->scaling == PS_FITS_SCALE_ASINH_STDEV_NEGATIVE)|| + (options->scaling == PS_FITS_SCALE_ASINH_STDEV_BOTH)))) { + if (header && psMetadataLookup(header,"BOFFSET")) { psMetadataRemoveKey(header,"BOFFSET"); } } // Remove any BSOFTENvalues that exist in the header if we are not using that scaling anymore - if (options&&(!((options->scaling == PS_FITS_SCALE_ASINH_RANGE)|| - (options->scaling == PS_FITS_SCALE_ASINH_MANUAL)|| - (options->scaling == PS_FITS_SCALE_ASINH_STDEV_POSITIVE)|| - (options->scaling == PS_FITS_SCALE_ASINH_STDEV_NEGATIVE)|| - (options->scaling == PS_FITS_SCALE_ASINH_STDEV_BOTH)))) { - if (psMetadataLookup(header,"BSOFTEN")) { + if (options && (!((options->scaling == PS_FITS_SCALE_ASINH_RANGE)|| + (options->scaling == PS_FITS_SCALE_ASINH_MANUAL)|| + (options->scaling == PS_FITS_SCALE_ASINH_STDEV_POSITIVE)|| + (options->scaling == PS_FITS_SCALE_ASINH_STDEV_NEGATIVE)|| + (options->scaling == PS_FITS_SCALE_ASINH_STDEV_BOTH)))) { + if (header && psMetadataLookup(header,"BSOFTEN")) { psMetadataRemoveKey(header,"BSOFTEN"); } } - // If there is no options set, remove all our non-standard keywords, because no one asked for them. + // If there are no options set, remove all our non-standard keywords, because no one asked for them. if (!options) { - if (psMetadataLookup(header,"BOFFSET")) { + if (header && psMetadataLookup(header,"BOFFSET")) { psMetadataRemoveKey(header,"BOFFSET"); } - if (psMetadataLookup(header,"BSOFTEN")) { + if (header && psMetadataLookup(header,"BSOFTEN")) { psMetadataRemoveKey(header,"BSOFTEN"); } Index: branches/eam_branches/ipp-20150616/psphot/src/psphot.h =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphot.h (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphot.h (revision 38552) @@ -513,5 +513,5 @@ bool psphotSourceChildren (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); bool psphotSourceChildrenReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); -psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objectsSrc, bool sourcesSubtracted); +psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *fileruleOut, const char *fileruleSrc, psArray *objectsSrc, bool sourcesSubtracted); bool psphotSourceParents (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); Index: branches/eam_branches/ipp-20150616/psphot/src/psphotExtendedSourceFits.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotExtendedSourceFits.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotExtendedSourceFits.c (revision 38552) @@ -149,4 +149,9 @@ fitOptions->poissonErrors = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS_POISSON"); if (!status) fitOptions->poissonErrors = true; + + // Save pointer to fitOptions for possible use later reinstantiating the pcm models on the output readout + // and during radial apertures measurements. + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PCM_FIT_OPTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "pcm fit options", + fitOptions); // maskVal is used to test for rejected pixels, and must include markVal Index: branches/eam_branches/ipp-20150616/psphot/src/psphotFitSourcesLinear.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotFitSourcesLinear.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotFitSourcesLinear.c (revision 38552) @@ -269,16 +269,16 @@ continue; } - if (modelSum < 0.5) continue; // skip sources with no model constraint (somewhat arbitrary limit) - if (modelSum < 0.8) { - psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "low-sig model @ %f, %f (%f masked sum, %f sum, %f peak)\n", - source->peak->xf, source->peak->yf, maskedSum, modelSum, source->peak->rawFlux); - } - if (maskedSum < 0.5) { - psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "worrying model @ %f, %f (%f masked sum, %f sum, %f peak)\n", - source->peak->xf, source->peak->yf, maskedSum, modelSum, source->peak->rawFlux); - } - bool isPSF = false; pmModel *model = pmSourceGetModel (&isPSF, source); + + float Io = model->params->data.F32[PM_PAR_I0]; + model->params->data.F32[PM_PAR_I0] = 1.0; + float normFlux = model->class->modelFlux(model->params); + model->params->data.F32[PM_PAR_I0] = Io; + + // printf("%5d %4.3f %4.3f %4.3f\n", source->seq, normFlux, modelSum, maskedSum); + float cut = .85; + if (modelSum < cut * normFlux) continue; + if (maskedSum < cut * normFlux) continue; // clear the 'mark' pixels and remask on the fit aperture Index: branches/eam_branches/ipp-20150616/psphot/src/psphotGalaxyParams.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotGalaxyParams.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotGalaxyParams.c (revision 38552) @@ -581,5 +581,5 @@ if (bgPixels->n < numPixels) { - fprintf(stderr, "could not find enough background pixels for source %d found %ld need %d\n", source->id, bgPixels->n, numPixels); + // fprintf(stderr, "could not find enough background pixels for source %d found %ld need %d\n", source->id, bgPixels->n, numPixels); goto restoreModel; } Index: branches/eam_branches/ipp-20150616/psphot/src/psphotMergeSources.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotMergeSources.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotMergeSources.c (revision 38552) @@ -853,22 +853,37 @@ // array containing the child sources. XXX currently, this is only used by psphotStackReadout // (sources go on allSources so that psphotChoosePSF can be called repeatedly) -psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objectsSrc, bool sourcesSubtracted) { +psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *fileruleOut, const char *fileruleSrc, psArray *objectsSrc, bool sourcesSubtracted) { bool status; - int nImages = psphotFileruleCount(config, filerule); + int nImages = psphotFileruleCount(config, fileruleOut); // generate look-up arrays for detections and readouts psArray *detArrays = psArrayAlloc(nImages); psArray *readouts = psArrayAlloc(nImages); + psArray *fitOptionsArray = psArrayAlloc(nImages); + + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); for (int i = 0; i < nImages; i++) { // find the currently selected readout - pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + pmFPAfile *file = pmFPAfileSelectSingle(config->files, fileruleOut, i); // File of interest psAssert (file, "missing file?"); pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); psAssert (readout, "missing readout?"); + + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, fileruleSrc, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + // create DETECTIONS containers for each image, in case one lacks it @@ -886,8 +901,17 @@ psAssert (detections, "missing detections?"); } + pmSourceFitOptions *fitOptions = NULL; + if (psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_FITS")) { + fitOptions = psMetadataLookupPtr (&status, readoutSrc->analysis, "PCM_FIT_OPTIONS"); + psAssert (fitOptions, "missing pcm fit options"); + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PCM_FIT_OPTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "pcm fit options", fitOptions); + } // we need to save the new sources on the detection arrays of the appropriate image detArrays->data[i] = psMemIncrRefCounter(detections); readouts->data[i] = psMemIncrRefCounter(readout); + if (fitOptions) { + fitOptionsArray->data[i] = psMemIncrRefCounter(fitOptions); + } } @@ -932,12 +956,22 @@ // does this copy all model data? (NO) sourceOut->modelPSF = pmModelCopy(sourceSrc->modelPSF); - sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); - + + bool foundModelEXT = false; if (sourceSrc->modelFits) { sourceOut->modelFits = psArrayAlloc(sourceSrc->modelFits->n); for (int j = 0; j < sourceSrc->modelFits->n; j++) { - sourceOut->modelFits->data[j] = pmModelCopy(sourceSrc->modelFits->data[j]); - } + pmModel *modelSrc = sourceSrc->modelFits->data[j]; + pmModel *modelOut = sourceOut->modelFits->data[j] = pmModelCopy(modelSrc); + if (modelSrc == sourceSrc->modelEXT) { + foundModelEXT = true; + sourceOut->modelEXT = psMemIncrRefCounter (modelOut); + } + modelOut->isPCM = modelSrc->isPCM; + } } + if (!foundModelEXT && sourceSrc->modelEXT) { + // Will this ever happen? + sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); + } // drop the references to the original image pixels: @@ -949,8 +983,10 @@ pmReadout *readout = readouts->data[index]; + pmSourceFitOptions *fitOptions = fitOptionsArray->data[index]; + // allocate image, weight, mask for the new image for each peak if (sourceOut->modelPSF) { pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, - sourceOut->modelPSF->fitRadius); + sourceSrc->windowRadius); } else { // if we have no pixels we can't use it to determine the psf so make sure this bit is off @@ -965,5 +1001,23 @@ if (!sourcesSubtracted) { sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; - } + } else { + bool isPSF = false; + pmModel *model = pmSourceGetModel (&isPSF, sourceOut); + if (model && sourceSrc->modelFlux) { + if (model->isPCM) { + pmPCMdata *pcm = pmPCMinit (sourceOut, fitOptions, model, maskVal, psfSize); + if (pcm) { + // pmPCMMakeModel (sourceOut, model, pcm->nsigma, maskVal, psfSize); + pmPCMCacheModel (sourceOut, maskVal, psfSize, pcm->nsigma); + psFree(pcm); + } else { + // What to do here? + psAssert (pcm, "pmPCMinit failed!"); + } + } else { + pmSourceCacheModel (sourceOut, maskVal); + } + } + } // set the output detections: @@ -981,4 +1035,5 @@ psFree (detArrays); psFree (readouts); + psFree (fitOptionsArray); return objectsOut; Index: branches/eam_branches/ipp-20150616/psphot/src/psphotModelBackground.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotModelBackground.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotModelBackground.c (revision 38552) @@ -558,5 +558,5 @@ fprintf (stdout, "\n"); - psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Model Background ---"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Load Background Model ---"); // loop over the available readouts Index: branches/eam_branches/ipp-20150616/psphot/src/psphotReplaceUnfit.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotReplaceUnfit.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotReplaceUnfit.c (revision 38552) @@ -316,4 +316,9 @@ } + pmSourceFitOptions *fitOptions = NULL; + if (psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS")) { + fitOptions = psMetadataLookupPtr (&status, readout->analysis, "PCM_FIT_OPTIONS"); + psAssert (fitOptions, "missing pcm fit options"); + } // XXX the sources have already been copied (merge into here?) @@ -368,8 +373,9 @@ source->modelPSF = modelPSF; source->modelPSF->fitRadius = radius; - model = source->modelPSF; + // model = source->modelPSF; } if (model->isPCM) { +#if 0 psAssert(false, "this section is not complete"); @@ -389,10 +395,18 @@ psImageConvolveFFT (source->modelFlux, rawModelFlux, NULL, 0, psfKernel); - psFree (psfKernel); psFree (rawModelFlux); - } - - pmSourceCacheModel (source, maskVal); // ALLOC x14 (!) +#endif + pmPCMdata *pcm = pmPCMinit (source, fitOptions, model, maskVal, psfSize); + if (pcm) { + pmPCMCacheModel (source, maskVal, psfSize, pcm->nsigma); + psFree(pcm); + } else { + psAssert (pcm, "pmPCMinit failed!"); + } + + } else { + pmSourceCacheModel (source, maskVal); // ALLOC x14 (!) + } } Index: branches/eam_branches/ipp-20150616/psphot/src/psphotStackReadout.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotStackReadout.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotStackReadout.c (revision 38552) @@ -35,4 +35,6 @@ psphotSaveImage (NULL, readout->mask, line); } + // psphotSaveImage leaves an error on the stack + psErrorClear(); return true; } @@ -348,5 +350,8 @@ // XXX NOTE : if we use the pre-20130914 psphotStackReadout code, we need to use 'false' for the // sourcesSubtracted argument - psArray *objectsOut = psphotSourceChildrenByObject (config, view, STACK_OUT, objects, true); + + // do this here so that the variance image is available to reset the models + psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); + psArray *objectsOut = psphotSourceChildrenByObject (config, view, STACK_OUT, STACK_RAW, objects, true); if (!objectsOut) { psFree(objects); @@ -364,5 +369,4 @@ // set up the FWHM vector - psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); psphotDumpImages (config, view, STACK_RAW, "raw.t0"); psphotDumpImages (config, view, STACK_OUT, "out.t0"); Index: branches/eam_branches/ipp-20150616/psphot/src/psphotStackUpdateReadout.c =================================================================== --- branches/eam_branches/ipp-20150616/psphot/src/psphotStackUpdateReadout.c (revision 38551) +++ branches/eam_branches/ipp-20150616/psphot/src/psphotStackUpdateReadout.c (revision 38552) @@ -116,5 +116,7 @@ // XXX NOTE : if we use the pre-20130914 psphotStackReadout code, we need to use 'false' for the // sourcesSubtracted argument - psArray *objectsOut = psphotSourceChildrenByObject (config, view, STACK_OUT, objects, true); + // XXX: do this here so that the variance is availble to rebuild the models + psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); + psArray *objectsOut = psphotSourceChildrenByObject (config, view, STACK_OUT, STACK_RAW, objects, true); if (!objectsOut) { psFree(objects); @@ -135,6 +137,6 @@ // this forces photometry on the undetected sources from other images - // set up the FWHM vector - psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); +// Moved this up above +// psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); psphotDumpImages (config, view, STACK_RAW, "raw.t0"); psphotDumpImages (config, view, STACK_OUT, "out.t0"); @@ -177,4 +179,9 @@ // and to subtract the sources psphotFitSourcesLinear (config, view, STACK_OUT, false, false); + +#ifdef notyet + psphotRemoveAllSources (config, view, STACK_OUT, false); +#endif + snprintf (line, 256, "%s.%d", "out.t4", entry); psphotDumpImages (config, view, STACK_OUT, line); @@ -221,5 +228,5 @@ COPY_PARAM( "MSKY_NX" ); COPY_PARAM( "MSKY_NY" ); - COPY_PARAM( "CHIP_SEEING" ); + // COPY_PARAM( "CHIP_SEEING" ); } @@ -293,4 +300,28 @@ maskVal |= markVal; + // This code currently assumes that all extended source models are PCM. + // Examine the recipe and make sure that this is true. + psMetadata *allModels = psMetadataLookupMetadata (&status, recipe, "EXTENDED_SOURCE_MODELS"); + if (!status) { + psAssert (allModels, "failed to find list of extended source models\n"); + } + psMetadataIterator *iter = psMetadataIteratorAlloc (allModels, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + if (item->type != PS_DATA_METADATA) { + psAbort ("Invalid type for EXTENDED_SOURCE_MODEL entry %s, not a metadata folder", item->name); + } + psMetadata *model = (psMetadata *) item->data.md; + // char *modelName = psMetadataLookupStr (&status, model, "MODEL"); + + char *convolvedWord = psMetadataLookupStr (&status, model, "PSF_CONVOLVED"); + if (!status || (strcasecmp (convolvedWord, "true") && strcasecmp (convolvedWord, "false"))) { + psAbort ("PSF_CONVOLVED entry invalid or missing for EXTENDED_SOURCE_MODEL entry %s", item->name); + } + bool convolved = !strcasecmp (convolvedWord, "true"); + psAssert (convolved, "EXTENDED SOURCE model %s is not convolved. This code is not ready for that\n", item->name); + } + psFree (iter); + // setup the PSF fit radius details psphotInitRadiusPSF (recipe, readout); @@ -312,6 +343,10 @@ fitOptions->poissonErrors = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS_POISSON"); if (!status) fitOptions->poissonErrors = true; + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); assert (status); + + // Remember these data for later + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PCM_FIT_OPTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "pcm fit options", fitOptions); for (int i = 0; i < sources->n; i++) { @@ -331,11 +366,15 @@ // fprintf(stderr, "source %d %5d should not have a psf model 0x%08X%08X\n", index, source->seq, source->mode2, source->mode); psFree(source->modelPSF) ; + modelPSF = NULL; + // At least once I saw a source with an extended model but no psf model + psFree(source->modelEXT); + source->modelEXT = NULL; } // make sure that the window radius is large enough. Should we do this regardless of whether // or not the source is extended? - if (source->modelEXT) { - float fitRadius = NAN; - float windowRadius = NAN; + float fitRadius = NAN; + float windowRadius = NAN; + if (1 || source->modelEXT) { if (!psphotSetRadiusMoments (&fitRadius, &windowRadius, readout, source, markVal)) { psError (PSPHOT_ERR_UNKNOWN, false, "failed to set radius for ext source"); @@ -387,5 +426,8 @@ float modelFlux = pow(10., -0.4 * model->mag); - // XXX: We are assuming here that we only use PCM models. Should get that information from the recipe. + model->fitRadius = fitRadius; + + // XXX: We are assuming here that we only use PCM models. + // I Should be getting that information from the recipe. pmPCMdata *pcm = pmPCMinit (source, fitOptions, model, maskVal, psfSize); if (!pcm) { @@ -394,5 +436,6 @@ } model->params->data.F32[PM_PAR_I0] = 1.0; - pmPCMMakeModel (source, model, pcm->nsigma, maskVal, psfSize); + // XXX: this duplicates work that pmPCMCacheModel does + // pmPCMMakeModel (source, model, pcm->nsigma, maskVal, psfSize); float normFlux = model->class->modelFlux(model->params); float I0 = modelFlux / normFlux; Index: branches/eam_branches/ipp-20150616/pstamp/scripts/pstampparse.pl =================================================================== --- branches/eam_branches/ipp-20150616/pstamp/scripts/pstampparse.pl (revision 38551) +++ branches/eam_branches/ipp-20150616/pstamp/scripts/pstampparse.pl (revision 38552) @@ -142,6 +142,5 @@ if ($extver eq "1") { - print STDERR "WARNING WARNING WARNING\n"; - print STDERR "Very soon version 1 postage stamp requests will be rejected. Please update your request tables to version 2 format.\n"; + print STDERR "Version 1 postage stamp requests are no longer accepted. Please update your request tables to version 2 format.\n"; print STDERR "Note that EMAIL will not be optional\n"; @@ -247,5 +246,5 @@ # if any is known. Do we want to do this for any other channels? # requests from data store will have those values. -if ($label and $label eq 'WEB.UP') { +if ($label and ($label eq 'WEB.UP' or $label eq 'WEB')) { if ($labelForUser) { $label = $labelForUser; Index: branches/eam_branches/ipp-20150616/tools/croot =================================================================== --- branches/eam_branches/ipp-20150616/tools/croot (revision 38551) +++ branches/eam_branches/ipp-20150616/tools/croot (revision 38552) @@ -15,5 +15,5 @@ $setlast = 1; shift; - } elsif ($ARGV[0] eq '--camera') { + } elsif (($ARGV[0] eq '--camera') or ($ARGV[0] eq '-c')) { shift; $camera = shift;