Index: trunk/pstamp/scripts/pstampparse.pl =================================================================== --- trunk/pstamp/scripts/pstampparse.pl (revision 27524) +++ trunk/pstamp/scripts/pstampparse.pl (revision 27751) @@ -17,4 +17,5 @@ use Carp; use POSIX; +use DateTime; my $verbose; @@ -26,4 +27,5 @@ my $out_dir; my $product; +my $label; my $save_temps; my $no_update; @@ -34,4 +36,5 @@ 'out_dir=s' => \$out_dir, 'product=s' => \$product, + 'label=s' => \$label, 'mode=s' => \$mode, 'dbname=s' => \$dbname, @@ -42,5 +45,5 @@ ); -die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job"); +die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job"); die "--file is required" if !defined($request_file_name); @@ -64,5 +67,4 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1); -my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); @@ -159,4 +161,10 @@ } +# +# Loop over rows in the request file collecting consecutive rows that have the "same images of interest" +# in the sense that their selection parameters will yield the same "Runs". +# Process the groups of rows together to reduce lookup time and to potentially make multiple +# stamps from the same ppstamp process. +# my @rowList; my $num_jobs = 0; @@ -165,30 +173,79 @@ my $need_magic; foreach my $row (@$rows) { - # XXX: TODO: sanity check all parameters - - # If we encounter an error for a particular row add a job with the proper fault code. - # If we encounter an error in this loop we shouldn't really just die. - # We only do that now in the case of I/O or DB errors or the like. - - my $rownum = $row->{ROWNUM}; - my $job_type = $row->{JOB_TYPE}; - - # parameters that select the images of interest - my $project = $row->{PROJECT}; - - # note: resolve_project avoids running pstamptool every time by remembering the - # last project resolved - my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); - if (!$proj_hash) { - print STDERR "project $project not found\n" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + # santiy check the paramaters + if (!checkRow($row)) { + # when it enconters an error checkRow adds a fake job with an appropriate error code to the database $num_jobs++; next; } + # initialize counter for "job number" + $row->{job_num} = 0; + $row->{error_code} = 0; + + if (scalar @rowList == 0) { + push @rowList, $row; + next; + } + + my $firstRow = $rowList[0]; + if (same_images_of_interest($firstRow, $row)) { + # add this row to the list and move on + push @rowList, $row; + next; + } + + # the images of interest for this new row doesn't match the list. + # process the list ... + $num_jobs += processRows(\@rowList); + + # and reset the list to contain just the new row + @rowList = ($row); +} + +# out of rows process the list +if (scalar @rowList > 0) { + $num_jobs += processRows(\@rowList); +} + +if (($mode eq "queue_job") and ($num_jobs eq 0)) { + print STDERR "no jobs created for $req_name\n" if $verbose; + insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); +} + +exit 0; + +sub checkRow { + + my $row = shift; + + # If we encounter an error for a particular row add a job with the proper fault code. + + my $rownum = $row->{ROWNUM}; + if (!validID($rownum)) { + print STDERR "$rownum is not a valid ROWNUM\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $job_type = $row->{JOB_TYPE}; + if (($job_type ne "stamp") and ($job_type ne "get_image")) { + print STDERR "$job_type is not a valid JOB_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $req_type = $row->{REQ_TYPE}; - $stage = $row->{IMG_TYPE}; - my $id = $row->{ID}; + if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and + ($req_type ne "byskycell") and ($req_type ne "bydiff")) { + print STDERR "$req_type is not a valid REQ_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + my $component = $row->{COMPONENT}; - my $tess_id = $row->{TESS_ID}; + if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) { + $row->{COMPONENT} = $component = ""; + } + $row->{TESS_ID} = "" if !defined $row->{TESS_ID}; my $filter = $row->{REQFILT}; @@ -196,9 +253,19 @@ if (length($filter) == 1) { # allow single character filter cuts to work - $filter .= '%'; + $row->{REQFILT} .= '%'; } } my $mjd_min = $row->{MJD_MIN}; + if (defined($mjd_min) and !validNumber($mjd_min)) { + print STDERR "$mjd_min is not a valid MJD_MIN\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $mjd_max = $row->{MJD_MAX}; + if (defined($mjd_max) and !validNumber($mjd_max)) { + print STDERR "$mjd_max is not a valid MJD_MAX\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $data_group = $row->{DATA_GROUP}; if (!defined $data_group) { @@ -209,9 +276,7 @@ } - my $x = $row->{CENTER_X}; - my $y = $row->{CENTER_Y}; - - # XXX things don't work if bit zero of option mask is not set; + # req_finish doesn't work if bit zero of option mask is not set; $row->{OPTION_MASK} |= 1; + my $option_mask= $row->{OPTION_MASK}; my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0; @@ -222,21 +287,15 @@ my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); - my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component; - - if (!$skycenter and !$search_component) { + if (!$skycenter and !$component) { print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; - } - - # user requested us to search all components. Set to "" - $search_component = "" if $search_component eq "all"; - + return 0; + } + + my $stage = $row->{IMG_TYPE}; if (!check_image_type($stage)) { print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; + return 0; } @@ -244,6 +303,5 @@ print STDERR "invalid ROI for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; + return 0; } @@ -251,62 +309,100 @@ print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); - $num_jobs++; - next; - } - - + return 0; + } + # $mode list_uri is a debugging mode (it may used by the http interface) # if this happens just croak - my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); - - my $image_db = $proj_hash->{dbname}; - my $camera = $proj_hash->{camera}; - $need_magic = $proj_hash->{need_magic}; - - # Temporary hack so that MOPS can get at non-magicked data -# if ($product and ($product eq "mops-pstamp-results")) { -# $need_magic = 0; -# } - - # For "stamp" and "list_uri" jobs collect rows with the same images of interest in a list so that they - # can be looked up together. - if (@rowList) { - my $firstRow = $rowList[0]; - if (($firstRow->{JOB_TYPE} ne "get_image") and same_images_of_interest($firstRow, $row)) { - - # add this row to the list and move on - push @rowList, $row; - - next; - - } else { - # this row has different selectors - # queue the jobs for the ones we've collected - $num_jobs += queueJobs($mode, \@rowList, $imageList); - @rowList = (); - } - } - - # look up images for the current row + # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); + + if ($req_type eq "bycoord") { if (!$skycenter) { print STDERR "center must be specified in sky coordintes for bycoord" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; - } - } + return 0; + } + } + if (($req_type eq "byid") or ($req_type eq "bydiff")) { - if (!validID($id)) { + if (!validID($row->{ID})) { print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0 + } + } + + return 1; +} + +sub processRows { + my $rowList = shift; + my $num_jobs = 0; + + # all rows in the list are compatible + my $row = $rowList->[0]; + + my $project = $row->{PROJECT}; + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); + if (!$proj_hash) { + print STDERR "project $project not found\n" if $verbose; + foreach $row (@$rowList) { + insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); $num_jobs++; - next; - } - } + } + return $num_jobs; + } + my $req_type = $row->{REQ_TYPE}; + $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $component = $row->{COMPONENT}; + my $tess_id = $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $data_group = $row->{DATA_GROUP}; + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + my $option_mask= $row->{OPTION_MASK}; + + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + $need_magic = $proj_hash->{need_magic}; + + # Temporary hack so that MOPS can get at non-magicked data + my $allow_mops_unmagicked = 1; + if ($allow_mops_unmagicked) { + if ($product and (($product eq "mops-pstamp-results") or + ($product eq "mops-pstamp-results2"))) { + $need_magic = 0; + } + } + + my $numRows = scalar @$rowList; + +# $tess_id = "" if !defined $tess_id; +# $component = "" if !defined $component; + + print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n"; + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this # request specification. An array reference is returned. - $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component, - $option_mask, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + my $start_locate = DateTime->now->mjd; + + # XXX: perhaps we should get rid of most of this argument list. + # Now that we are passing down compatible rows all of the + # information required is contained there + + $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component, + $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + + # XXX: why use mjd? It doesn't have great precision. + my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.; + print "Time to locate_images for row $rownum $dtime_locate\n"; if (!$imageList or !@$imageList) { @@ -314,32 +410,31 @@ # note in this case queueJobs inserts the fake job for these rows } + # handle this $row->{need_magic} = $need_magic; - push @rowList, $row; -} - -if (@rowList) { + $num_jobs += queueJobs($mode, \@rowList, $imageList); -} - -if (($mode eq "queue_job") and ($num_jobs eq 0)) { - print STDERR "no jobs created for $req_name\n" if $verbose; - insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); -} - -# PAU - -exit 0; - - -sub queueJobsForRow + + # if a row slipped through with no jobs add one + foreach my $row (@rowList) { + if ($row->{job_num} == 0) { + print "row $row->{ROWNUM} produced no jobs\n"; + print STDERR "row $row->{ROWNUM} produced no jobs\n"; + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + } + } + + return $num_jobs; +} + +sub queueJobForImage { my $row = shift; my $stage = shift; - my $imageList = shift; - my $have_skycells = shift; + my $image = shift; my $need_magic = shift; my $mode = shift; - my $num_jobs = 0; my $rownum = $row->{ROWNUM}; my $option_mask = $row->{OPTION_MASK}; @@ -365,28 +460,243 @@ } + my $component = $image->{component}; + + my $job_num = ++($row->{job_num}); + + my $imagefile = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + # XXX: should we add a faulted job so the client can know what happened if no images come back? + # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback + # to users, but speeds up processing significantly + print STDERR "skipping non-magicked image $imagefile\n" if $verbose; + + # for now assume yes. + + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); + return 1; + } elsif ($stage eq "stack") { + # unconvolved stack images weren't available prior to some point in time. + # XXX: handle this more correctly by examining the stack run's config dump file. + # It looks like # the feature was turned on sometime around November 11, 2009. stackRun 30067 is the lowest + # one that I found with an unconvolved image. + my $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES = 30067; + if ($row->{unconvolved} and ($row->{PROJECT} eq 'gpc1') and + ($image->{stack_id} < $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES)) { + print STDERR "Unconvolved stack image is not available for stackRun.stack_id: $image->{stack_id}\n"; + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE); + return 1; + } + } + my $exp_id = $image->{exp_id}; + + my $args = $roi_string ? $roi_string : ""; + if ($stage eq "raw" or $stage eq "chip") { + $args .= " -class_id $component" if $component; + } + + # add astrometry file for raw and chip images if one is available + if (($stage eq "chip") || ($stage eq "raw")) { + $args .= " -astrom $image->{astrom}" if $image->{astrom}; + } + + $image->{job_args} = $args; + + # XXX: we can get rid of the following everything that we need is + # in the params file + + $args .= " -file $imagefile"; + + if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { + $args .= " -mask $image->{mask}"; + } + if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { + $args .= " -variance $image->{weight}"; + } + + my $base = basename($image->{image}); + if (! $base =~ /.fits$/ ) { + my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); + } + $base =~ s/.fits$//; + + my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; + my $argslist = "${output_base}.args"; + + # copy the argument list to a file + open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); + print ARGSLIST "$args\n"; + close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + + write_params($output_base, $image); + + my $newState = "run"; + my $fault = 0; + my $dep_id; + + # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it + if ($stage ne 'raw') { + # not ready to handle diff update yet. May never support stack + my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff')); + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + $data_state = $run_state if $stage eq 'stack'; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'drop') or + ($run_state eq 'error_cleaned') or ($data_state eq 'error_cleaned') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back + $newState = 'stop'; + $fault = $PSTAMP_GONE; + } elsif ($allow_wait_for_update and + (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) { + if ($stage eq 'chip') { + my $burntool_state = $image->{burntool_state}; + if ($burntool_state and (abs($burntool_state) < 14)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } + } + if (!$fault) { + # don't wait for update unless the caller asks us to + if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } elsif (!$image->{magicked}) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_DESTREAKED; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); + } + } + } + } + + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; + $command .= " -fault $fault" if $fault; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dep_id $dep_id" if $dep_id; + + if (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping command: $command\n"; + } + + return 1; +} + +# queue jobs for a collection of request specifications that have the same Images of Interest +sub queueJobs +{ + my $mode = shift; + my $rowList = shift; + my $imageList = shift; + + my $firstRow = $rowList[0]; + my $stage = $firstRow->{IMG_TYPE}; + my $job_type = $firstRow->{JOB_TYPE}; + my $need_magic = $firstRow->{need_magic}; + + my $num_jobs = 0; + + if ($mode eq "list_uri") { + foreach my $image (@$imageList) { + print "$image->{image}\n"; + } + } elsif ($job_type eq "get_image") { + my $n = scalar @$rowList; + + my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; + + $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); + + } else { + if (!$imageList or (scalar @$imageList eq 0)) { + # We didn't find any images for this set of rows. Insert a fake job to carry + # the status back to the requestor. + foreach my $row (@$rowList) { + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + $num_jobs++; + } + return $num_jobs; + } + + foreach my $image (@$imageList) { + # get the array of row indices that touch this image + my $row_index = $image->{row_index}; + if (!$row_index or scalar @$row_index == 0) { + # XXX should this happen? Why did something get returned. + print "image ${stage}_id: $image->{stage_id} component: $image->{component} matched no rows\n"; + next; + } + # XXX: TODO: eventually we may change ppstamp to be able to make multiple stamps per invocation + + foreach my $i (@$row_index) { + my $row = $rowList->[$i]; + + $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode); + } + } + } + + return $num_jobs; +} + +# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); +sub queueGetImageJobs +{ + my $row = shift; + my $imageList = shift; + my $stage = shift; + my $need_magic = shift; + my $mode = shift; + + my $num_jobs = 0; + my $rownum = $row->{ROWNUM}; + my $option_mask = $row->{OPTION_MASK}; + + # For dist_bundle we need + # --camera from $image + # --stage + # --stage_id from $image + # --component from $image + # --path_base + # --outdir global to this script + # loop over images - my $job_num = 0; foreach my $image (@$imageList) { - my $component; - if ($have_skycells) { - $component = $image->{skycell_id}; - } else { - $component = $image->{class_id}; - } - - # skip this component if it is not in the list for this row - next if ! $components->{$component}; - - $job_num++; + my $stage_id = $image->{stage_id}; + my $component = $image->{component}; + + # skip faulted components for now. Should we even be here? + if ($image->{fault} > 0) { + printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; + next; + } + + my $job_num = ++($row->{job_num}); my $imagefile = $image->{image}; if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # XXX: should we add a faulted job so the client can know what happened if no images come back? - # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback - # to users, but speeds up processing significantly + # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed + # in locate_images because it's much more efficient to do the test in the database. + # For these two modes we fall through to here in order to give feedback to the requestor as + # to why the request failed to queue jobs. print STDERR "skipping non-magicked image $imagefile\n" if $verbose; - - # for now assume yes. - insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); $num_jobs++; @@ -396,46 +706,5 @@ my $exp_id = $image->{exp_id}; - my $args = $roi_string ? $roi_string : ""; - if ($stage eq "raw" or $stage eq "chip") { - $args .= " -class_id $component" if $component; - } - - # add astrometry file for raw and chip images if one is available - if (($stage eq "chip") || ($stage eq "raw")) { - $args .= " -astrom $image->{astrom}" if $image->{astrom}; - } - - $image->{job_args} = $args; - - # XXX: we can get rid of the following everything that we need is - # in the params file - - $args .= " -file $imagefile"; - - if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { - $args .= " -mask $image->{mask}"; - } - if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { - $args .= " -variance $image->{weight}"; - } - - my $base = basename($image->{image}); - if (! $base =~ /.fits$/ ) { - my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); - } - $base =~ s/.fits$//; - - # XXX: TODO use filerule for this. I don't have a camera defined here - if (($stage eq 'chip') and ($image->{camera} eq 'GPC1')) { - $base = "${base}.${component}"; - } - - my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; - my $argslist = "${output_base}.args"; - - # copy the argument list to a file - open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); - print ARGSLIST "$args\n"; - close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + my $output_base = "$out_dir/${rownum}_${job_num}"; write_params($output_base, $image); @@ -454,5 +723,5 @@ $newState = 'stop'; $fault = $PSTAMP_GONE; - } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { + } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) { # don't wait for update unless the caller asks us to if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { @@ -463,5 +732,5 @@ # set up to queue an update run queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, - $run_state, $stage, $image->{stage_id}, $need_magic); + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); } } @@ -475,256 +744,5 @@ $command .= " -dep_id $dep_id" if $dep_id; - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { - # mode eq "queue_job" - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - # should we keep going for other uris? If so how do we report that some - # of the work that the request wanted isn't going to get done - my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping command: $command\n"; - } - } - if ( $num_jobs == 0 ) { - print STDERR "no jobs for row $rownum\n" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP); - $num_jobs = 1; - } - return $num_jobs; -} - -# queue jobs for a collection of request specifications that have the same Images of Interest -sub queueJobs -{ - my $mode = shift; - my $rowList = shift; - my $imageList = shift; - - my $firstRow = $rowList[0]; - my $stage = $firstRow->{IMG_TYPE}; - my $job_type = $firstRow->{JOB_TYPE}; - my $need_magic = $firstRow->{need_magic}; - - my $num_jobs = 0; - - if ($mode eq "list_uri") { - foreach my $image (@$imageList) { - print "$image->{image}\n"; - } - } elsif ($job_type eq "get_image") { - my $n = scalar @$rowList; - - my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; - - $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); - - } else { - if (!$imageList or (scalar @$imageList eq 0)) { - # we didn't find any images for this set of rows. Insert a fake job to carry - # the status back to the requestor - foreach my $row (@$rowList) { - insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH); - $num_jobs++; - } - return $num_jobs; - } - - my $have_skycells; - if (($stage eq "raw") or ($stage eq "chip")) { - $have_skycells = 0; - } else { - $have_skycells = 1; - } - - my $thisRun; - - my $npoints = 0; - my ($pointsList, $pointsListName); - if (scalar @$imageList > 1) { - ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); - foreach my $row (@$rowList) { - $row->{components} = {}; - if ($row->{skycenter}) { - print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; - $npoints++; - } else { - # this row's center is in pixel coordinates add all images to the component list for this row - foreach my $i (@$imageList) { - my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; - my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) - if !$component; - $row->{components}->{$component} = 1; - } - } - } - close $pointsList; - } else { - # only one image. Avoid the expense of dvoImagesAtCoords. - # queue the job and let ppstamp figure out if the center is valid for the image - # the resulting result code will be the same. - foreach my $row (@$rowList) { - $row->{components} = {}; - my $i = $imageList->[0]; - my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; - my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component; - $row->{components}->{$component} = 1; - } - } - - my $tess_dir_abs; - my $last_tess_id = ""; - while ($thisRun = getOneRun($stage, $imageList)) { - if ($npoints) { - my_die( "pointsListName is not defined", $PS_EXIT_PROG_ERROR) if !$pointsListName; - # we collected a set of sky coordinates above. - # filter the images so that only those that contain the centers of the ROIs are processed - my $command = "$dvoImagesAtCoords -coords $pointsListName"; - if ($have_skycells) { - my $tess_id = $thisRun->[0]->{tess_id}; - if ($tess_id ne $last_tess_id) { - $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); - $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); - $last_tess_id = $tess_id; - } - $command .= " -D CATDIR $tess_dir_abs"; - } else { - my $astrom = $thisRun->[0]->{astrom}; - my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom; - my $astrom_resolved = $ipprc->file_resolve($astrom); - $command .= " -astrom $astrom_resolved"; - } - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP - # That just means that the coordinate didn't match any image/skycell - if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) { - print STDERR @$stderr_buf; - my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR; - my_die( "dvoImagesAtCoords failed: $rc", $rc); - } - } - # now we have a list of row numbers and components - # eventually we might want to multiple stamp requests for the same image - # into the same ppstamp job but not yet. For now we will queue a new - my @lines = split "\n", join "", @$stdout_buf; - foreach my $line (@lines) { - # parse the line, ignoring the ra and dec - my ($rownum, undef, undef, $component) = split " ", $line; - - # I guess since we need this function we should be using a hash for rowList - my $row = findRow($rownum, $rowList); - $row->{components}->{$component} = 1; - } - } - - foreach my $row (@$rowList) { - $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic, $mode); - } - } - } - return $num_jobs; -} - -# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); -sub queueGetImageJobs -{ - my $row = shift; - my $imageList = shift; - my $stage = shift; - my $need_magic = shift; - my $mode = shift; - - my $num_jobs = 0; - my $rownum = $row->{ROWNUM}; - my $option_mask = $row->{OPTION_MASK}; - - # For dist_bundle we need - # --camera from $image - # --stage - # --stage_id from $image - # --component from $image - # --path_base - # --outdir global to this script - - # loop over images - my $job_num = 0; - foreach my $image (@$imageList) { - my $stage_id = $image->{stage_id}; - my $component = $image->{component}; - - # skip faulted components for now. Should we even be here? - if ($image->{fault} > 0) { - printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; - next; - } - - $job_num++; - - my $imagefile = $image->{image}; - if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed - # in locate_images because it's much more efficient to do the test in the database. - # For these two modes we fall through to here in order to give feedback to the requestor as - # to why the request failed to queue jobs. - print STDERR "skipping non-magicked image $imagefile\n" if $verbose; - insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); - $num_jobs++; - - next; - } - my $exp_id = $image->{exp_id}; - - my $output_base = "$out_dir/${rownum}_${job_num}"; - - write_params($output_base, $image); - - my $newState = "run"; - my $fault = 0; - my $dep_id; - - if ($stage ne 'raw') { - my $run_state = $image->{state}; - my $data_state = $image->{data_state}; - $data_state = $run_state if $stage eq "stack"; - if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or - ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { - # image is gone and it's not coming back - $newState = 'stop'; - $fault = $PSTAMP_GONE; - } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { - # don't wait for update unless the caller asks us to - if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { - $newState = 'stop'; - $fault = $PSTAMP_NOT_AVAILABLE; - } else { - # cause the image to be re-made - # set up to queue an update run - queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, - $run_state, $stage, $image->{stage_id}, $need_magic); - } - } - } - - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" - . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; - $command .= " -fault $fault" if $fault; - $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dep_id $dep_id" if $dep_id; - - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { + if (!$no_update) { # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -776,9 +794,5 @@ . " -rownum $rownum -state stop -fault $fault"; - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { + if (!$no_update) { # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -796,74 +810,25 @@ } -sub get_run_id -{ - my $stage = shift; - my $image = shift; - - if ($stage eq "raw") { - return $image->{exp_id}; - } elsif ($stage eq "chip") { - return $image->{chip_id}; - } elsif ($stage eq "warp") { - return $image->{warp_id}; - } elsif ($stage eq "stack") { - return $image->{stack_id}; - } elsif ($stage eq "diff") { - return $image->{diff_id}; - } else { - my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR); - } -} - -# extract components from the imageList that have the same run id and return the list -sub getOneRun { - my $stage = shift; - my $imageList = shift; - - # return if array is empty - return undef if ! @$imageList; - - my $last_run_id = 0; - my @runList; - while ($imageList->[0]) { - my $run_id = get_run_id($stage, $imageList->[0]); - - last if ($last_run_id and ($run_id ne $last_run_id)); - - my $image = shift @$imageList; - $image->{stage} = $stage; - push @runList, $image; - $last_run_id = $run_id; - } - return \@runList; -} - sub same_images_of_interest { my $r1 = shift; my $r2 = shift; - return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); - return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); - return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE}); + return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); + return 0 if (($r1->{JOB_TYPE} eq "get_image") or ($r2->{JOB_TYPE} eq "get_image")); return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE}); return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE}); return 0 if ($r1->{ID} ne $r2->{ID}); return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID}); + return 0 if ($r1->{COMPONENT} ne $r2->{COMPONENT}); return 0 if ($r1->{REQFILT} ne $r2->{REQFILT}); return 0 if ($r1->{DATA_GROUP} ne $r2->{DATA_GROUP}); return 0 if ($r1->{MJD_MIN} ne $r2->{MJD_MAX}); return 0 if ($r1->{MJD_MAX} ne $r2->{MJD_MAX}); + return 0 if ($r1->{OPTION_MASK} ne $r2->{OPTION_MASK}); + return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); return 0 if ($r1->{inverse} ne $r2->{inverse}); return 0 if ($r1->{unconvolved} ne $r2->{unconvolved}); - - if (defined($r1->{COMPONENT})) { - return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT}); - } elsif (defined($r2->{COMPONENT})) { - # if first row has no component all of the images will be retrieved, so - # the fact that this row has a component is ok. Fall through to return 1 - # XXX Nope this doesn't work. It is consistent with the other logic in some way - # that i haven't fully thought through - return 0; - } + # don't combine requests in pixel coordinates + return 0 if (($r1->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS) || ($r2->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS)); return 1; @@ -911,5 +876,5 @@ sub queue_update_run { - my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $need_magic) = @_; + my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_; if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { @@ -918,6 +883,13 @@ my $dep_id; - my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb"; - $command .= " -no_magic" if !$need_magic; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component"; + $command .= " -need_magic" if $need_magic; + + # compute rlabel for the run. + # XXX: This bit of policy shouldn't be buried so deeply in the code + # For now use one that implies 'postage stamp server' 'update' 'request_label" + my $rlabel = "ps_ud_" . $label if $label; + $command .= " -rlabel $rlabel" if $rlabel; + if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -929,4 +901,6 @@ chomp $output; $dep_id = $output; + # + # XXX: need to fault the request or something my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id; } else { @@ -936,6 +910,6 @@ $$r_dep_id = $dep_id; - $r_fault = 0; - $r_jobState = 'blocked'; + $$r_fault = 0; + $$r_jobState = 'run'; } @@ -990,4 +964,4 @@ # we don't fault the request here pstamp_parser_run.pl handles that if necessary - return $fault; -} + exit $fault; +}