Index: branches/simmosaic_branches/pstamp/scripts/pstampparse.pl =================================================================== --- branches/simmosaic_branches/pstamp/scripts/pstampparse.pl (revision 24860) +++ branches/simmosaic_branches/pstamp/scripts/pstampparse.pl (revision 27839) @@ -13,4 +13,9 @@ use PS::IPP::PStamp::RequestFile qw( :standard ); use PS::IPP::PStamp::Job qw( :standard ); +use File::Temp qw(tempfile); +use File::Basename qw(basename); +use Carp; +use POSIX; +use DateTime; my $verbose; @@ -22,4 +27,7 @@ my $out_dir; my $product; +my $label; +my $save_temps; +my $no_update; GetOptions( @@ -28,11 +36,14 @@ 'out_dir=s' => \$out_dir, 'product=s' => \$product, + 'label=s' => \$label, 'mode=s' => \$mode, 'dbname=s' => \$dbname, 'dbserver=s'=> \$dbserver, 'verbose' => \$verbose, + 'save-temps'=> \$save_temps, + 'no-update' => \$no_update, ); -die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job"); +die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job"); die "--file is required" if !defined($request_file_name); @@ -62,4 +73,11 @@ exit ($PS_EXIT_CONFIG_ERROR); } + +# just deal with these arguments once and for all +$pstamptool .= " -dbname $dbname" if $dbname; +$pstamptool .= " -dbserver $dbserver" if $dbserver; + +# list_job is a deugging mode +$no_update = 1 if $mode eq "list_job"; my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -82,19 +100,43 @@ # make sure the file contains what we are expecting - -die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST"; -die "REQ_NAME not found in $request_file_name" if (!$req_name); -die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1"); - -if ($req_id) { +# This program shouldn't have been run if the request file is bogus. +# No need to notify the client +my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST"; +my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR) if (!$req_name); +my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1"); + + +# check for duplicate request name +my $duplicate_req_name = 0; +if ($req_id and !$no_update) { + my $command = "$pstamptool -listreq -name $req_name -not_req_id $req_id"; + # no verbose so that error message about request not found doesn't appear in parse_error.txt + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + my $exitStatus = $error_code >> 8; + if ($success) { + # -listreq succeeded duplicate request name + print STDERR "REQ_NAME $req_name has already been used\n"; + insertFakeJobForRow(undef, 0, $PSTAMP_DUP_REQUEST); + $duplicate_req_name = 1; + my $datestr = strftime "%Y%m%d%H%M%S.$req_id", gmtime; + $req_name = "ERROR.$datestr"; + #exit 0; + } +} + +if ($req_id and !$no_update) { + # update the database with the request name. This will be used as the + # the output data store's product name my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; $command .= " -outProduct $product"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - die "$command failed"; - } + my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); + } +} +if ($duplicate_req_name) { + exit 0; } @@ -111,22 +153,295 @@ print STDERR @$stderr_buf; } - my $table = $mdcParser->parse(join "", @$stdout_buf) or - die("Unable to parse metdata config doc"); - - $rows = parse_md_list($table); -} - + if (@$stdout_buf) { + my $table = $mdcParser->parse(join "", @$stdout_buf) or + my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR); + $rows = parse_md_list($table); + } + +} + +# +# Loop over rows in the request file collecting consecutive rows that have the "same images of interest" +# in the sense that their selection parameters will yield the same "Runs". +# Process the groups of rows together to reduce lookup time and to potentially make multiple +# stamps from the same ppstamp process. +# +my @rowList; my $num_jobs = 0; +my $imageList; +my $stage; +my $need_magic; foreach my $row (@$rows) { + # santiy check the paramaters + if (!checkRow($row)) { + # when it enconters an error checkRow adds a fake job with an appropriate error code to the database + $num_jobs++; + next; + } + # initialize counter for "job number" + $row->{job_num} = 0; + $row->{error_code} = 0; + + if (scalar @rowList == 0) { + push @rowList, $row; + next; + } + + my $firstRow = $rowList[0]; + if (same_images_of_interest($firstRow, $row)) { + # add this row to the list and move on + push @rowList, $row; + next; + } + + # the images of interest for this new row doesn't match the list. + # process the list ... + $num_jobs += processRows(\@rowList); + + # and reset the list to contain just the new row + @rowList = ($row); +} + +# out of rows process the list +if (scalar @rowList > 0) { + $num_jobs += processRows(\@rowList); +} + +if (($mode eq "queue_job") and ($num_jobs eq 0)) { + print STDERR "no jobs created for $req_name\n" if $verbose; + insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); +} + +exit 0; + +sub checkRow { + + my $row = shift; + + # If we encounter an error for a particular row add a job with the proper fault code. + my $rownum = $row->{ROWNUM}; + if (!validID($rownum)) { + print STDERR "$rownum is not a valid ROWNUM\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $job_type = $row->{JOB_TYPE}; + if (($job_type ne "stamp") and ($job_type ne "get_image")) { + print STDERR "$job_type is not a valid JOB_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + my $req_type = $row->{REQ_TYPE}; + if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and + ($req_type ne "byskycell") and ($req_type ne "bydiff")) { + print STDERR "$req_type is not a valid REQ_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + + my $component = $row->{COMPONENT}; + if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) { + $row->{COMPONENT} = $component = ""; + } + $row->{TESS_ID} = "" if !defined $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + if ($filter) { + if (length($filter) == 1) { + # allow single character filter cuts to work + $row->{REQFILT} .= '%'; + } + } + my $mjd_min = $row->{MJD_MIN}; + if (defined($mjd_min) and !validNumber($mjd_min)) { + print STDERR "$mjd_min is not a valid MJD_MIN\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $mjd_max = $row->{MJD_MAX}; + if (defined($mjd_max) and !validNumber($mjd_max)) { + print STDERR "$mjd_max is not a valid MJD_MAX\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $data_group = $row->{DATA_GROUP}; + if (!defined $data_group) { + # backwards compatability hook + $data_group = $row->{LABEL}; + $data_group = "null" if !defined $data_group; + $row->{DATA_GROUP} = $data_group; + } + + # req_finish doesn't work if bit zero of option mask is not set; + $row->{OPTION_MASK} |= 1; + + my $option_mask= $row->{OPTION_MASK}; + my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0; + $row->{inverse} = $inverse; + my $unconvolved = ($option_mask & $PSTAMP_SELECT_UNCONV) ? 1 : 0; + $row->{unconvolved} = $unconvolved; + + my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + + if (!$skycenter and !$component) { + print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + my $stage = $row->{IMG_TYPE}; + if (!check_image_type($stage)) { + print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + if ((($job_type eq "stamp") or ($req_type eq "bycoord")) and ! validROI($row)) { + print STDERR "invalid ROI for row $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + if (($req_type eq "byexp") and ($stage eq "stack")) { + print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); + return 0; + } + + # $mode list_uri is a debugging mode (it may used by the http interface) + # if this happens just croak + # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); + + + if ($req_type eq "bycoord") { + if (!$skycenter) { + print STDERR "center must be specified in sky coordintes for bycoord" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + } + + if (($req_type eq "byid") or ($req_type eq "bydiff")) { + if (!validID($row->{ID})) { + print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0 + } + } + + return 1; +} + +sub processRows { + my $rowList = shift; + my $num_jobs = 0; + + # all rows in the list are compatible + my $row = $rowList->[0]; + my $project = $row->{PROJECT}; - my $req_type = $row->{REQ_TYPE}; - my $img_type = $row->{IMG_TYPE}; - my $id = $row->{ID}; - my $class_id = $row->{CLASS_ID}; - my $filter = $row->{REQFILT}; - my $mjd_min = $row->{MJD_MIN}; - my $mjd_max = $row->{MJD_MAX}; + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); + if (!$proj_hash) { + print STDERR "project $project not found\n" if $verbose; + foreach $row (@$rowList) { + insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + $num_jobs++; + } + return $num_jobs; + } + my $req_type = $row->{REQ_TYPE}; + $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $component = $row->{COMPONENT}; + my $tess_id = $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $data_group = $row->{DATA_GROUP}; + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + my $option_mask= $row->{OPTION_MASK}; + + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + $need_magic = $proj_hash->{need_magic}; + + # Temporary hack so that MOPS can get at non-magicked data + my $allow_mops_unmagicked = 1; + if ($allow_mops_unmagicked) { + if ($product and (($product eq "mops-pstamp-results") or + ($product eq "mops-pstamp-results2"))) { + $need_magic = 0; + } + } + + my $numRows = scalar @$rowList; + +# $tess_id = "" if !defined $tess_id; +# $component = "" if !defined $component; + + print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n"; + + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this + # request specification. An array reference is returned. + my $start_locate = DateTime->now->mjd; + + # XXX: perhaps we should get rid of most of this argument list. + # Now that we are passing down compatible rows all of the + # information required is contained there + + $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component, + $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + + # XXX: why use mjd? It doesn't have great precision. + my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.; + print "Time to locate_images for row $rownum $dtime_locate\n"; + + if (!$imageList or !@$imageList) { + print STDERR "no matching images found for row $rownum\n" if $verbose; + # note in this case queueJobs inserts the fake job for these rows + } + # handle this + $row->{need_magic} = $need_magic; + + $num_jobs += queueJobs($mode, \@rowList, $imageList); + + # if a row slipped through with no jobs add one + foreach my $row (@rowList) { + if ($row->{job_num} == 0) { + print "row $row->{ROWNUM} produced no jobs\n"; + print STDERR "row $row->{ROWNUM} produced no jobs\n"; + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + } + } + + return $num_jobs; +} + +sub queueJobForImage +{ + my $row = shift; + my $stage = shift; + my $image = shift; + my $need_magic = shift; + my $mode = shift; + + my $rownum = $row->{ROWNUM}; + my $option_mask = $row->{OPTION_MASK}; + my $components = $row->{components}; + + my $roi_string; + + # note values were checked by the function validROI() my $x = $row->{CENTER_X}; my $y = $row->{CENTER_Y}; @@ -134,94 +449,305 @@ my $h = $row->{HEIGHT}; my $coord_mask = $row->{COORD_MASK}; - my $option_mask= $row->{OPTION_MASK}; - - $class_id = "" if ($class_id eq "null" or $class_id eq "all"); - - # XXX: TODO: sanity check all parameters - - # XXX: TODO: We shouldn't just die in this loop. - # If we encounter an error for a particular row - # add a job with the proper fault code. If there is a db or config error we should probably just - # trash the request. - die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri"); - - my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); - die "project $project not found\n" unless $proj_hash; - - my $image_db = $proj_hash->{dbname}; - my $camera = $proj_hash->{camera}; - my $need_magic = $proj_hash->{need_magic}; - - my $roi_string; - # XXX we're depending on other code to insure valid values for roi components - # this is checked in and ppstamp but I should check here so that we don't get that far, - # but not today - if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { - $roi_string = "-pixcenter $x $y"; - } else { - $roi_string = "-skycenter $x $y"; - } - if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { - $roi_string .= " -pixrange $w $h"; - } else { - $roi_string .= " -arcrange $w $h"; - } - } - - die "region of interest is required to make postage stamps" - if (($job_type eq "stamp") && !defined($roi_string)); - - # find the uris for this request - - if ($req_type eq "bycoord") { - die "region of interest is required for request type bycoord" if !defined($roi_string) ; - die "center must be specified in sky coordintes for bycoord" - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS); - } - - # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification - my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id, - $x, $y, $mjd_min, $mjd_max, $filter, $verbose); - - if (!$images) { - print STDERR "no matching images found for row $rownum\n"; - next; - } + if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { + $roi_string = "-pixcenter $x $y"; + } else { + $roi_string = "-skycenter $x $y"; + } + if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { + $roi_string .= " -pixrange $w $h"; + } else { + $roi_string .= " -arcrange $w $h"; + } + + my $component = $image->{component}; + + my $job_num = ++($row->{job_num}); + + my $imagefile = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + # XXX: should we add a faulted job so the client can know what happened if no images come back? + # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback + # to users, but speeds up processing significantly + print STDERR "skipping non-magicked image $imagefile\n" if $verbose; + + # for now assume yes. + + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); + return 1; + } elsif ($stage eq "stack") { + # unconvolved stack images weren't available prior to some point in time. + # XXX: handle this more correctly by examining the stack run's config dump file. + # It looks like # the feature was turned on sometime around November 11, 2009. stackRun 30067 is the lowest + # one that I found with an unconvolved image. + my $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES = 30067; + if ($row->{unconvolved} and ($row->{PROJECT} eq 'gpc1') and + ($image->{stack_id} < $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES)) { + print STDERR "Unconvolved stack image is not available for stackRun.stack_id: $image->{stack_id}\n"; + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE); + return 1; + } + } + my $exp_id = $image->{exp_id}; + + my $args = $roi_string ? $roi_string : ""; + if ($stage eq "raw" or $stage eq "chip") { + $args .= " -class_id $component" if $component; + } + + # add astrometry file for raw and chip images if one is available + if (($stage eq "chip") || ($stage eq "raw")) { + $args .= " -astrom $image->{astrom}" if $image->{astrom}; + } + + $image->{job_args} = $args; + + # XXX: we can get rid of the following everything that we need is + # in the params file + + $args .= " -file $imagefile"; + + if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { + $args .= " -mask $image->{mask}"; + } + if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { + $args .= " -variance $image->{weight}"; + } + + my $base = basename($image->{image}); + if (! $base =~ /.fits$/ ) { + my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); + } + $base =~ s/.fits$//; + + my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; + my $argslist = "${output_base}.args"; + + # copy the argument list to a file + open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); + print ARGSLIST "$args\n"; + close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + + write_params($output_base, $image); + + my $newState = "run"; + my $fault = 0; + my $dep_id; + + # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it + if ($stage ne 'raw') { + # updates for stack stage not supported yet + my $allow_wait_for_update = ($stage ne 'stack'); + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + $data_state = $run_state if $stage eq 'stack'; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'drop') or + ($run_state eq 'error_cleaned') or ($data_state eq 'error_cleaned') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back + $newState = 'stop'; + $fault = $PSTAMP_GONE; + } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) { + if ($stage eq 'chip') { + my $burntool_state = $image->{burntool_state}; + if ($burntool_state and (abs($burntool_state) < 14)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } + } + if (!$allow_wait_for_update) { + print STDERR "wait for update not supported for stage $stage yet\n"; + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } + if (!$fault) { + # wait for update unless the customer asks us not to + if (($option_mask & $PSTAMP_NO_WAIT_FOR_UPDATE)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } elsif (!$image->{magicked}) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_DESTREAKED; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); + } + } + } + } + + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; + $command .= " -fault $fault" if $fault; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dep_id $dep_id" if $dep_id; + + if (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping command: $command\n"; + } + + return 1; +} + +# queue jobs for a collection of request specifications that have the same Images of Interest +sub queueJobs +{ + my $mode = shift; + my $rowList = shift; + my $imageList = shift; + + my $firstRow = $rowList[0]; + my $stage = $firstRow->{IMG_TYPE}; + my $job_type = $firstRow->{JOB_TYPE}; + my $need_magic = $firstRow->{need_magic}; + + my $num_jobs = 0; + if ($mode eq "list_uri") { - foreach my $image (@$images) { + foreach my $image (@$imageList) { print "$image->{image}\n"; } } elsif ($job_type eq "get_image") { - # XXX TODO: Get rid of this block and use the same code as the stamp jobs - # XXX This doesn't work to get the mask and weight images - my $listfile = "$out_dir/filelist"; - - - # map our img_type to the Data Store file types. - my %filelist_img_types = ( "raw" => "chip", - "chip" => "chipproc", - "warp" => "warp", - "stack"=>"stack", - "diff" => "diff"); - - my $filelist_img_type = $filelist_img_types{$img_type}; - - open LISTFILE, ">$listfile" - or die "failed to open file list: $listfile while parsing get_image request $req_id"; - - foreach my $image (@$images) { - my $class_id = $image->{class_id} ? $image->{class_id} : ""; - print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n"; - } - close LISTFILE; + my $n = scalar @$rowList; + + my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; + + $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); + + } else { + if (!$imageList or (scalar @$imageList eq 0)) { + # We didn't find any images for this set of rows. Insert a fake job to carry + # the status back to the requestor. + foreach my $row (@$rowList) { + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + $num_jobs++; + } + return $num_jobs; + } + + foreach my $image (@$imageList) { + # get the array of row indices that touch this image + my $row_index = $image->{row_index}; + if (!$row_index or scalar @$row_index == 0) { + # XXX should this happen? Why did something get returned. + print "image ${stage}_id: $image->{stage_id} component: $image->{component} matched no rows\n"; + next; + } + # XXX: TODO: eventually we may change ppstamp to be able to make multiple stamps per invocation + + foreach my $i (@$row_index) { + my $row = $rowList->[$i]; + + $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode); + } + } + } + + return $num_jobs; +} + +# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); +sub queueGetImageJobs +{ + my $row = shift; + my $imageList = shift; + my $stage = shift; + my $need_magic = shift; + my $mode = shift; + + my $num_jobs = 0; + my $rownum = $row->{ROWNUM}; + my $option_mask = $row->{OPTION_MASK}; + + # For dist_bundle we need + # --camera from $image + # --stage + # --stage_id from $image + # --component from $image + # --path_base + # --outdir global to this script + + # loop over images + foreach my $image (@$imageList) { + my $stage_id = $image->{stage_id}; + my $component = $image->{component}; + + # skip faulted components for now. Should we even be here? + if ($image->{fault} > 0) { + printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; + next; + } + + my $job_num = ++($row->{job_num}); + + my $imagefile = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed + # in locate_images because it's much more efficient to do the test in the database. + # For these two modes we fall through to here in order to give feedback to the requestor as + # to why the request failed to queue jobs. + print STDERR "skipping non-magicked image $imagefile\n" if $verbose; + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); + $num_jobs++; + + next; + } + my $exp_id = $image->{exp_id}; + + my $output_base = "$out_dir/${rownum}_${job_num}"; + + write_params($output_base, $image); + + my $newState = "run"; + my $fault = 0; + my $dep_id; + + if ($stage ne 'raw') { + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + $data_state = $run_state if $stage eq "stack"; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back + $newState = 'stop'; + $fault = $PSTAMP_GONE; + } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) { + # wait for update unless the customer asks us to not to + if ($option_mask & $PSTAMP_NO_WAIT_FOR_UPDATE) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); + } + } + } + $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image" - . " -uri $listfile -outputBase $out_dir -rownum $rownum"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; - if ($mode eq "list_job") { - print "$command\n"; - } else { + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; + $command .= " -fault $fault" if $fault; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dep_id $dep_id" if $dep_id; + + if (!$no_update) { + # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -229,82 +755,217 @@ print STDERR @$stderr_buf; # XXX TODO: now what? Should we mark the error state for the request? - die "failed to queue job for request $req_id: rownum: $rownum"; + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); } + } else { + print "skipping command: $command\n"; + } + } + if ( $num_jobs == 0 ) { + print STDERR "no jobs for row $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_NO_JOBS_QUEUED); + $num_jobs = 1; + } + return $num_jobs; +} +sub insertFakeJobForRow +{ + my $row = shift; + my $job_num = shift; + my $fault = shift; + + my ($job_type, $rownum); + if ($row) { + $job_type = $row->{JOB_TYPE}; + $rownum = $row->{ROWNUM}; + $rownum = 0 if !defined $rownum; + if ($job_type) { + if (($job_type ne "stamp") and ($job_type ne "get_image")) { + print STDERR "invalid job type: $job_type found in row $rownum\n"; + $job_type = "none"; + } + } else { + print STDERR "undefined job type found in row $rownum\n"; + $job_type = "none"; } } else { - # sequence number for the the job for a request spec. Not to be confused with job_id - my $job_num = 0; - - # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images - # use -list instead of -file - foreach my $image (@$images) { - my $uri = $image->{image}; - if (($img_type ne "stack") and ($need_magic and !$image->{magicked})) { - print STDERR "skippping non-magicked image $uri\n" if $verbose; - next; - } - my $exp_id = $image->{exp_id}; - - my $args = $roi_string ? $roi_string : ""; - $args .= " -class_id $class_id" if $class_id; - - $job_num++; - - my $output_base = "$out_dir/${rownum}_${job_num}"; - - # add astrometry file for raw and chip images if one is available - if (($img_type eq "chip") || ($img_type eq "raw")) { - $args .= " -astrom $image->{astrom}" if $image->{astrom}; - } - - if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { - $args .= " -mask $image->{mask}"; - } - if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { - $args .= " -weight $image->{weight}"; - } - - # XXX: TODO: here is where we need to check whether or not the source inputs still exist - # and if not, queue an update job and set the job state appropriately. - - my $newState = "run"; - - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" - . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum" - . " -state $newState"; - $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; - - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } else { - # mode eq "queue_job" - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - # should we keep going for other uris? If so how do we report that some - # of the work that the request wanted isn't going to get done - die "failed to queue job for request $req_id"; - } - } - } - } -} - -if ($mode eq "queue_jobs") { - if ($num_jobs) { - exit 0; + $job_type = "none"; + $rownum = 0; + } + + my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" + . " -rownum $rownum -state stop -fault $fault"; + + if (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); + } } else { - # no jobs queued return error to get the request stopped - # TODO: need to define meaningful error codes - exit $PS_EXIT_UNKNOWN_ERROR; - } -} else { - exit 0; -} + print "skipping command: $command\n"; + } +} + +sub same_images_of_interest { + my $r1 = shift; + my $r2 = shift; + + return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); + return 0 if (($r1->{JOB_TYPE} eq "get_image") or ($r2->{JOB_TYPE} eq "get_image")); + return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE}); + return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE}); + return 0 if ($r1->{ID} ne $r2->{ID}); + return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID}); + return 0 if ($r1->{COMPONENT} ne $r2->{COMPONENT}); + return 0 if ($r1->{REQFILT} ne $r2->{REQFILT}); + return 0 if ($r1->{DATA_GROUP} ne $r2->{DATA_GROUP}); + return 0 if ($r1->{MJD_MIN} ne $r2->{MJD_MAX}); + return 0 if ($r1->{MJD_MAX} ne $r2->{MJD_MAX}); + return 0 if ($r1->{OPTION_MASK} ne $r2->{OPTION_MASK}); + return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); + return 0 if ($r1->{inverse} ne $r2->{inverse}); + return 0 if ($r1->{unconvolved} ne $r2->{unconvolved}); + # don't combine requests in pixel coordinates + return 0 if (($r1->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS) || ($r2->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS)); + + return 1; +} + +sub validNumber +{ + my $val = shift; + + return 0 if !defined $val; + + return ($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/); +} +sub validID +{ + my $val = shift; + + return 0 if !$val; + + return ! ($val =~ /\D/); +} + +sub validROI +{ + my $row = shift; + return 0 if !validNumber($row->{CENTER_X}); + return 0 if !validNumber($row->{CENTER_Y}); + return 0 if !validNumber($row->{WIDTH}); + return 0 if !validNumber($row->{HEIGHT}); + + return 1; +} + +sub findRow +{ + my $rownum = shift; + my $rowList = shift; + + foreach my $row (@$rowList) { + return $row if $row->{ROWNUM} eq $rownum; + } + return undef; +} + +sub queue_update_run +{ + my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_; + + if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { + my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR); + } + + my $dep_id; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component"; + $command .= " -need_magic" if $need_magic; + + # compute rlabel for the run. + # XXX: This bit of policy shouldn't be buried so deeply in the code + # For now use one that implies 'postage stamp server' 'update' 'request_label" + my $rlabel = "ps_ud_" . $label if $label; + $command .= " -rlabel $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); + } + my $output = join "", @$stdout_buf; + chomp $output; + $dep_id = $output; + # + # XXX: need to fault the request or something + my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id; + } else { + print STDERR "skipping $command\n"; + $dep_id = 42; + } + + $$r_dep_id = $dep_id; + $$r_fault = 0; + $$r_jobState = 'run'; +} + +sub write_params { + my $output_base = shift; + my $image = shift; + + # write the contents of this "image" as a metadata config doc + # Simply treat all values as strings. This is ok since we are only going to + # read it from another perl script + my $mdc_file = "${output_base}.mdc"; + open P, ">$mdc_file" or my_die("failed to open $mdc_file", $PS_EXIT_UNKNOWN_ERROR); + + print P "params METADATA\n"; + + foreach my $key (keys %$image) { + my $value = $image->{$key}; + if (defined $value) { + printf P " %-20s STR %s\n", $key, $value; + } else { + printf P " %-20s STR NULL\n", $key; + } + } + + print P "END\n"; + close P or my_die("failed to close $mdc_file", $PS_EXIT_UNKNOWN_ERROR); +} + +sub check_image_type +{ + my $img_type = shift; + if (!$img_type) { + print STDERR "NULL IMG_TYPE supplied\n"; + return 0; + } + if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or + ($img_type eq "stack") or ($img_type eq "diff")) { + return 1; + } else { + print STDERR "$img_type is not a valid IMG_TYPE\n"; + return 0; + } +} + +sub my_die +{ + my $msg = shift; + my $fault = shift; + + carp $msg; + + # we don't fault the request here pstamp_parser_run.pl handles that if necessary + + exit $fault; +}