Index: /branches/czw_branch/cleanup/pstamp/scripts/pstamp_finish.pl =================================================================== --- /branches/czw_branch/cleanup/pstamp/scripts/pstamp_finish.pl (revision 25142) +++ /branches/czw_branch/cleanup/pstamp/scripts/pstamp_finish.pl (revision 25143) @@ -8,4 +8,5 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); +use Carp; use Time::Local; @@ -47,5 +48,4 @@ my $ipprc = PS::IPP::Config->new(); # IPP Configuration - if ($redirect_output) { my $logDest = "$out_dir/psfinish.$req_id.log"; @@ -94,5 +94,5 @@ if (!mkdir $out_dir) { print STDERR "cannot create output directory $out_dir"; - stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR); + stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -102,10 +102,10 @@ # request print STDERR "output directory $out_dir exists but is not a directory"; - stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR); + stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); } if (! -e $req_file ) { print STDERR "request file $req_file is missing\n"; - stop_request($req_id, $PS_EXIT_CONFIG_ERROR); + stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR); } @@ -117,5 +117,5 @@ # point print STDERR "failed to read request_file $req_file" ; - stop_request($req_id, $PS_EXIT_CONFIG_ERROR); + stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR); } @@ -154,14 +154,6 @@ print STDERR "Request $req_id produced no jobs.\n" } - # No Jobs. - # XXXX Ouch. We need results for each rownum (each request specification) in the request file - # including those that produced no jobs. - # for now add an entry for rownum 1 and a phony error code. - # we've included parse_error.txt to the fileset if it exists - # - my $rownum = 0; - my $fault = 42; # get a real error code - print $tdf "$rownum|$fault|0|0|"; - print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n"; + # This should not happen. A fake job should be queued + stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR); } else { my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); @@ -182,4 +174,11 @@ my $exp_id = $job->{exp_id}; + if ($fault eq $PSTAMP_DUP_REQUEST) { + # this request had a duplicate request name. We can't put the results + # on the data store since the product name is already used + # maybe we could be more clever about this and choose a fileset name + stop_request_and_exit($req_id, $fault); + } + my ($row, $req_info, $project) = get_request_info($rows, $rownum); @@ -188,5 +187,7 @@ if (!$image_db) { carp("failed to find imagedb for project: $project"); - stop_request($req_id, $PS_EXIT_CONFIG_ERROR); + if (!$fault) { + stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR); + } } @@ -227,4 +228,5 @@ print $tdf "$exp_info|"; print $tdf "$req_info|"; + print $tdf "\n"; } } else { @@ -277,5 +279,5 @@ } -sub stop_request { +sub stop_request_and_exit { my $req_id = shift; my $fault = shift; @@ -299,8 +301,14 @@ my $row = $rows->{$rownum}; + # these may be set to null during processing + my $component = $row->{COMPONENT}; + $component = "null" if !$component; + my $tess_id = $row->{TESS_ID}; + $tess_id = "null" if !$tess_id; + # This is ugly, error prone and hard to change. # Create a results file module and provide a list of the names (we have the data in the columns) my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|"; - $rowinfo .= "$row->{ID}|$row->{TESS_ID}|$row->{COMPONENT}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|"; + $rowinfo .= "$row->{ID}|$tess_id|$component|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|"; $rowinfo .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|"; $rowinfo .= "$row->{WIDTH}|$row->{HEIGHT}"; @@ -313,5 +321,5 @@ my $exp_id = shift; - if (!$exp_id) { + if (!$exp_id or !$image_db) { # no exposure id just return zeros # XXX: we could put a value in for filter, but we don't have a good place to find it Index: /branches/czw_branch/cleanup/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /branches/czw_branch/cleanup/pstamp/scripts/pstamp_job_run.pl (revision 25142) +++ /branches/czw_branch/cleanup/pstamp/scripts/pstamp_job_run.pl (revision 25143) @@ -208,4 +208,6 @@ $command .= " -job_id $job_id"; $command .= " -fault $exit_code"; + # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided + $command .= " -state run"; $command .= " -dbname $dbname" if defined $dbname; $command .= " -dbserver $dbserver" if defined $dbserver; Index: /branches/czw_branch/cleanup/pstamp/scripts/pstamp_listjobs.pl =================================================================== --- /branches/czw_branch/cleanup/pstamp/scripts/pstamp_listjobs.pl (revision 25142) +++ /branches/czw_branch/cleanup/pstamp/scripts/pstamp_listjobs.pl (revision 25143) @@ -12,5 +12,5 @@ my $verbose; my $dbname; -my $dserver; +my $dbserver; GetOptions( Index: /branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl (revision 25142) +++ /branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl (revision 25143) @@ -14,4 +14,5 @@ use PS::IPP::PStamp::Job qw( :standard ); use File::Temp qw(tempfile); +use Carp; my $verbose; @@ -91,9 +92,31 @@ # make sure the file contains what we are expecting -die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST"; -die "REQ_NAME not found in $request_file_name" if (!$req_name); -die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1"); +# we shouldn't get here if this is the case so just die. No need to notify the client +my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST"; +my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR) if (!$req_name); +my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1"); + +# check for duplicate request name +if (!$no_update) { + my $command = "$pstamptool -listreq -name $req_name"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + # no verbose so that error message about request not found doesn't appear in parse_error.txt + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + my $exitStatus = $error_code >> 8; + if ($success) { + # -listreq succeeded duplicate request name + insertFakeJobForRow(undef, 0, $PSTAMP_DUP_REQUEST); + exit 0; + } elsif ($exitStatus ne $PS_EXIT_DATA_ERROR) { + # wrong error code something else is wrong + my_die("$command failed", $exitStatus); + } +} if ($req_id and !$no_update) { + # update the database with the request name. This will be used as the + # the output data store's product name my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; $command .= " -outProduct $product"; @@ -103,5 +126,5 @@ run(command => $command, verbose => $verbose); unless ($success) { - die "$command failed"; + my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); } } @@ -120,5 +143,5 @@ } my $table = $mdcParser->parse(join "", @$stdout_buf) or - die("Unable to parse metdata config doc"); + my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR); $rows = parse_md_list($table); @@ -143,4 +166,14 @@ # parameters that select the images of interest my $project = $row->{PROJECT}; + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); + if (!$proj_hash) { + print STDERR "project $project not found\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + $num_jobs++; + next; + } my $req_type = $row->{REQ_TYPE}; $stage = $row->{IMG_TYPE}; @@ -153,18 +186,32 @@ my $option_mask= $row->{OPTION_MASK}; - - die "valid region of interest is required to make postage stamps" - if (($job_type eq "stamp") and ! validROI($row)); + my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0; + $row->{inverse} = $inverse; my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); - $component = "" if (defined($component) and ($component eq "null" or $component eq "all")); + + my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component; - die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri"); - - - # note: resolve_project avoids running pstamptool every time by remembering the - # last project resolved - my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); - die "project $project not found\n" unless $proj_hash; + if (!$skycenter and !$search_component) { + print STDERR "COMPONENT must be specified for pixel coordinate ROI\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + $num_jobs++; + next; + } + + $search_component = "" if $search_component eq "all"; + + if (($job_type eq "stamp") and ! validROI($row)) { + print STDERR "invalid ROI for row $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + $num_jobs++; + next; + } + + + # $mode list_uri is a debugging mode (it may used by the http interface) + # if this happens just croak + my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); + my $image_db = $proj_hash->{dbname}; @@ -173,14 +220,18 @@ # collect rows with the same images of interest in a list so that they - # can be processed optimially + # can be looked up together if (@rowList) { my $firstRow = $rowList[0]; - # XXX: the collecting might work with !$skycenter but I need to think about it - if ($skycenter and same_images_of_interest($row, $firstRow)) { + # note order of these parameters matters + if (same_images_of_interest($firstRow, $row)) { + + # add this row to the list and move on push @rowList, $row; - # On to the next row + next; + } else { - # queue the jobs for this set of rows + # this row has different selectors + # queue the jobs for the ones we've collected $num_jobs += queueJobs($mode, \@rowList, $imageList); @rowList = (); @@ -188,22 +239,31 @@ } + # look up images for the current row if ($req_type eq "bycoord") { - die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter); - die "lookup bycoord is not yet implemented"; + if (!$skycenter) { + print STDERR "center must be specified in sky coordintes for bycoord" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + $num_jobs++; + next; + } + print STDERR "lookup bycoord is not yet implemented" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); + $num_jobs++; + next; } else { - - # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this - # request specification. An array reference is returned + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this + # request specification. An array reference is returned. my ($x, $y); - $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter, - $x, $y, $mjd_min, $mjd_max, $filter, $verbose); - } - - if (!$imageList or !@$imageList) { - print STDERR "no matching images found for row $rownum\n"; - next; - } - $row->{need_magic} = $need_magic; - push @rowList, $row; + + $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $search_component, + $inverse, $skycenter, $x, $y, $mjd_min, $mjd_max, $filter, $verbose); + + if (!$imageList or !@$imageList) { + print STDERR "no matching images found for row $rownum\n" if $verbose; + # note in this case queueJobs inserts the fake job for these rows + } + $row->{need_magic} = $need_magic; + push @rowList, $row; + } } @@ -212,15 +272,12 @@ } -if ($mode eq "queue_jobs") { - if ($num_jobs) { - exit 0; - } else { - # no jobs queued return error to get the request stopped - # TODO: need to define meaningful error codes - exit $PS_EXIT_UNKNOWN_ERROR; - } -} else { - exit 0; -} +if (($mode eq "queue_jobs") and ($num_jobs eq 0)) { + print STDERR "no jobs created for $req_name\n" if $verbose; + insertFakeJobForRow(undef, 0, $PSTAMP_UNKNOWN_ERROR); +} + +# PAU + +exit 0; @@ -232,4 +289,30 @@ my $have_skycells = shift; my $need_magic = shift; + my $num_jobs = 0; + + my $rownum = $row->{ROWNUM}; + + my $components = $row->{components}; + + my $roi_string; + + # note values were insured to be numbers in validROI() + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $w = $row->{WIDTH}; + my $h = $row->{HEIGHT}; + my $coord_mask = $row->{COORD_MASK}; + if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { + if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { + $roi_string = "-pixcenter $x $y"; + } else { + $roi_string = "-skycenter $x $y"; + } + if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { + $roi_string .= " -pixrange $w $h"; + } else { + $roi_string .= " -arcrange $w $h"; + } + } # loop over images @@ -244,33 +327,17 @@ # skip this component if it is not in the list for this row - next if !$row->{components}->{$component}; - - my $rownum = $row->{ROWNUM}; - - my $roi_string; - - # note values were insured to be numbers in validROI() - my $x = $row->{CENTER_X}; - my $y = $row->{CENTER_Y}; - my $w = $row->{WIDTH}; - my $h = $row->{HEIGHT}; - my $coord_mask = $row->{COORD_MASK}; - if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { - $roi_string = "-pixcenter $x $y"; - } else { - $roi_string = "-skycenter $x $y"; - } - if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { - $roi_string .= " -pixrange $w $h"; - } else { - $roi_string .= " -arcrange $w $h"; - } - } + next if ! $components->{$component}; + + $job_num++; my $imagefile = $image->{image}; if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # XXX: should we add a faulted job so the client can know what happened? + # XXX: should we add a faulted job so the client can know what happened if no images come back? print STDERR "skippping non-magicked image $imagefile\n" if $verbose; + + # for now assume yes. + + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); + next; } @@ -282,6 +349,4 @@ } - $job_num++; - # add astrometry file for raw and chip images if one is available if (($stage eq "chip") || ($stage eq "raw")) { @@ -302,16 +367,27 @@ # copy the argument list to a file - open ARGSLIST, ">$argslist" or die "failed to open $argslist"; + open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); print ARGSLIST "$args\n"; - close ARGSLIST or die "failed to close $argslist"; - - # XXX: TODO: here is where we need to check whether or not the source inputs still exist - # and if not, queue an update job and set the job state appropriately. + close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); my $newState = "run"; + my $fault = 0; + + if (($stage ne 'stack') and ($stage ne 'raw')) { + if (($image->{state} eq 'goto_purged') or ($image->{data_state} eq 'purged')) { + $newState = 'stop'; + $fault = $PSTAMP_GONE; + } elsif (($image->{data_state} ne 'full') or ($image->{state} ne 'full' )) { + # XXX here is where we need to queue an update job + # for now just say that the image is not available + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } + } $num_jobs++; my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" . " -outputBase $output_base -rownum $rownum -state $newState"; + $command .= " -fault $fault" if $fault; $command .= " -exp_id $exp_id" if $exp_id; $command .= " -dbname $dbname" if $dbname; @@ -331,5 +407,5 @@ # should we keep going for other uris? If so how do we report that some # of the work that the request wanted isn't going to get done - die "failed to queue job for request $req_id"; + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); } } else { @@ -337,7 +413,13 @@ } } + if ( $num_jobs == 0 ) { + print STDERR "no jobs for row $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP); + $num_jobs = 1; + } return $num_jobs; } +# queue jobs for a collection of request specifications that have the same Images of Interest sub queueJobs { @@ -358,14 +440,19 @@ } } elsif ($job_type eq "get_image") { - die "get_image jobs not implemented yet"; + print STDERR "get_image jobs not implemented yet" if $verbose; + foreach my $row (@$rowList) { + insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); + $num_jobs++; + } } else { - my $thisRun; - - my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); - foreach my $row (@$rowList) { - print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; - $row->{components} = {}; - } - close $pointsList; + if (!$imageList or (scalar @$imageList eq 0)) { + # we didn't find any images for this set of rows. Insert a fake job to carry + # the status back to the requestor + foreach my $row (@$rowList) { + insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH); + $num_jobs++; + } + return $num_jobs; + } my $have_skycells; @@ -376,8 +463,30 @@ } + my $thisRun; + + my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); + my $npoints = 0; + foreach my $row (@$rowList) { + $row->{components} = {}; + if ($row->{skycenter}) { + print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; + $npoints++; + } else { + # this row's center is in pixel coordinates add all images to the component list for this row + foreach my $i (@$imageList) { + my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; + my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component; + $row->{components}->{$component} = 1; + } + } + } + close $pointsList; + my $tess_dir_abs; my $last_tess_id = ""; while ($thisRun = getOneRun($stage, $imageList)) { - { + if ($npoints) { + # we collected a set of sky coordintates above filter the images so that only + # those tat contain the centers are processed my $command = "$dvoImagesAtCoords $pointsListName"; if ($have_skycells) { @@ -391,5 +500,5 @@ } else { my $astrom = $thisRun->[0]->{astrom}; - die "no astrometry file found" if !$astrom; + my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom; my $astrom_resolved = $ipprc->file_resolve($astrom); $command .= " -astrom $astrom_resolved"; @@ -400,5 +509,5 @@ print STDERR @$stderr_buf; my $rc = $error_code >> 8; - die "dvoImagesAtCoords failed: $rc"; + my_die( "dvoImagesAtCoords failed: $rc", $PS_EXIT_UNKNOWN_ERROR); } # now we have a list of row numbers and components @@ -422,4 +531,43 @@ } return $num_jobs; +} +sub insertFakeJobForRow +{ + my $row = shift; + my $job_num = shift; + my $fault = shift; + + my ($job_type, $rownum); + if ($row) { + $job_type = $row->{JOB_TYPE}; + $rownum = $row->{ROWNUM}; + } else { + $job_type = "none"; + $rownum = 0; + } + + my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" + . " -rownum $rownum -state stop -fault $fault"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + + if ($mode eq "list_job") { + # this is a debugging mode, just print the pstamptool that would have run + # this is sort of like the mode -noupdate that some other tools support + print "$command\n"; + } elsif (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping command: $command\n"; + } } @@ -440,5 +588,5 @@ return $image->{diff_id}; } else { - die "unenexpected stage: $stage found"; + my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR); } } @@ -476,8 +624,12 @@ return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE}); return 0 if ($r1->{ID} ne $r2->{ID}); + return 0 if ($r1->{inverse} ne $r2->{inverse}); if (defined($r1->{COMPONENT})) { return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT}); } elsif (defined($r2->{COMPONENT})) { + # if first row has no component all of the images will be retrieved, so + # the fact that this row has a component is ok. Fall through to return 1 + # XXX Nope that doesn't work (yet) return 0; } @@ -514,2 +666,14 @@ return undef; } + +sub my_die +{ + my $msg = shift; + my $fault = shift; + + carp $msg; + + # we don't fault the request here pstamp_parser_run.pl handles that if necessary + + return $fault; +} Index: /branches/czw_branch/cleanup/pstamp/src/ppstampMakeStamp.c =================================================================== --- /branches/czw_branch/cleanup/pstamp/src/ppstampMakeStamp.c (revision 25142) +++ /branches/czw_branch/cleanup/pstamp/src/ppstampMakeStamp.c (revision 25143) @@ -17,4 +17,5 @@ static void skyToChip(pmAstromObj *pt, pmFPA *fpa, pmChip *chip); +static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance); // convert the input chip's transforms to the output @@ -325,4 +326,12 @@ break; } + if (readout->variance) { + outReadout->variance = extractStamp(readout->variance, extractRegion, 0); + if (!outReadout->variance) { + psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp weight image\n"); + status = false; + break; + } + } if (readout->mask) { outReadout->mask = extractStamp(readout->mask, extractRegion, 0); @@ -332,11 +341,9 @@ break; } - } - if (readout->variance) { - outReadout->variance = extractStamp(readout->variance, extractRegion, 0); - if (!outReadout->variance) { - psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp weight image\n"); - status = false; - break; + + if (!setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) { + psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp mask image\n"); + status = false; + break; } } @@ -668,2 +675,48 @@ +static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance) +{ + bool status; + psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS"); + if (!status) { + psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n"); + return false; + } + // we set anything masked to NAN except if CONV.POOR is the only bit set + // First check the old value + psU32 convPoor = psMetadataLookupU32(&status, masks, "POOR.WARP"); + if (!status) { + convPoor = psMetadataLookupU32(&status, masks, "CONV.POOR"); + if (!status) { + psError(PM_ERR_CONFIG, false, "failed to lookup mask value for CONV.POOR in recipes\n"); + return false; + } + } + psU32 maskMask = ~convPoor; + + double exciseValue; + if (image->type.type == PS_TYPE_U16) { + exciseValue = 0xffff; + } else if (image->type.type == PS_TYPE_F32) { + exciseValue = NAN; + } else { + psError(PS_ERR_PROGRAMMING, true, "unexpected image type: %d\n", image->type.type); + return false; + } + long numExcised = 0; + for (int y=0; ynumRows; y++) { + for (int x=0; xnumCols; x++) { + psU16 maskVal = psImageGet(mask, x, y); + if (maskVal & maskMask) { + numExcised++; + psImageSet(image, x, y, exciseValue); + if (variance) { + psImageSet(variance, x, y, exciseValue); + } + } + } + } + fprintf(stderr, "excised %ld masked pixels\n", numExcised); + + return true; +}