Index: trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27704) +++ trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27751) @@ -48,4 +48,5 @@ my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); @@ -86,5 +87,5 @@ } else { if ($stage eq "chip") { - $cmd = "$chiptool -processedimfile -chip_id $stage_id -class_id $component"; + $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component"; } elsif ($stage eq "warp") { $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; @@ -96,11 +97,9 @@ } -my $components = runToolAndParse($cmd, $verbose); - -my $n = scalar @$components; -my_die("Unexpected number of components found $n", $PS_EXIT_PROG_ERROR) if ($n != 1); +my $it = runToolAndParseExpectOne($cmd, $verbose); + +my_die("no components found", $PS_EXIT_PROG_ERROR) if ( !$it); # Got "it" -my $it = $components->[0]; my $status = 0; @@ -130,6 +129,5 @@ # need to queue the updates if ($stage eq 'chip') { - # queue_update_chip takes an array so that queue_update_warp can pass a set of - # chips to it + # queue_update_chip takes an array so that queue_update_warp can pass it set of chips my $chips = [$it]; $status = queue_update_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); @@ -145,8 +143,22 @@ # detect states that cannot be updated and update the job state - # XXX: we should be more assertive here and check for the specific states where we can continue + # XXX: Perhaps I should be more assertive here and check for the specific states that we know + # that we can continue. + my $state = $it->{state}; my $job_fault = 0; - if (($state =~ /scrub/) or ($state =~ /purge/)) { + + # temporary hack to deal with data with old burntool tables. This check is now done by pstampparse.pl + # so runs in this state will not get queued + if ($stage eq 'chip') { + my $burntool_state = $it->{burntool_state}; + if ($burntool_state and (abs($burntool_state) < 14)) { + print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n"; + $job_fault = $PSTAMP_NOT_AVAILABLE; + } + } + if ($state eq 'error_cleaned') { + $job_fault = $PSTAMP_NOT_AVAILABLE; + } elsif (($state =~ /scrub/) or ($state =~ /purge/)) { # jobs must have changed state since depenency was made print STDERR "Dependency cannot be satisfied\n"; @@ -155,4 +167,7 @@ # Dependency never should have been inserted my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR); + } + if (!$job_fault and ($stage eq 'chip')) { + $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $it->{raw_magicked}, $it->{dsRun_state}); } if ($job_fault) { @@ -213,7 +228,334 @@ } + my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $raw_all_magicked, $dsRun_state); + + return $status; +} + +sub queue_update_warp { + # check status of input chips. If state is not updatable set error code for job + + # if chipProcessedImfile.state is cleaned call queue_update_chip + + # need to code warptool -setskyfiletoupdate + my $metadata = shift; + my $whole_run = shift; # if true queue entire run for update + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + + my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + + my $warp_id = $metadata->{warp_id}; + if (!$whole_run) { + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data == 0; + + my $good_to_go = 1; + my @chipsToUpdate; + my $chip_id; + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $good_to_go = 0; + push @chipsToUpdate, $chip; + } else { + # this chip is good to go + } + } + + if ($good_to_go) { + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); + } + } else { + my $warpRun = $metadata; + my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data != 1; + + my $chipRun = $data->[0]; + + my $chipRunState = $chipRun->{state}; + if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { + print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; + faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); + return 0; + } + my $warpRunState = $warpRun->{state}; + if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { + # The inputs and outputs are ready. Queue the warpRun for update. + + # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif ($chipRunState eq 'cleaned' or + (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { + my $data = [$chipRun]; + return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); + } + } + + # return value may be the return status of script so zero is good + return 0; +} +sub queue_update_diff { + my $metadata = shift; + my $whole_run = shift; # if true queue entire run for update + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + +# XXXX This is not ready to go yet XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXxx + + my $diff_id = $metadata->{diff_id}; + my $diff_mode = $metadata->{diff_mode}; + if (!$whole_run) { + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + # put this in a module somewhere + my $IPP_DIFF_MODE_WARP_WARP = 1; + my $IPP_DIFF_MODE_WARP_STACK = 2; + my $IPP_DIFF_MODE_STACK_WARP = 2; + my $IPP_DIFF_MODE_STACK_STACK = 4; + if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { + # check the state of the stack + my $command = "$stacktool -sumkskyfile -stack_id $skycell->{stack2}"; + my $stack = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; + + if ($stack->{state} ne 'full') { + print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + # this faults all jobs depending on this dep_id + faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # now check the warp + $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; + my $warp = runToolAndExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) + if !$warp; + } + + my $good_to_go = 1; + my @chipsToUpdate; + my $chip_id; + my $warp_id; + my $data ; #XXXX + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $good_to_go = 0; + push @chipsToUpdate, $chip; + } else { + # this chip is good to go + } + } + + if ($good_to_go) { + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); + } + } else { + my $warpRun = $metadata; + my $warp_id = $warpRun->{warp_id}; + my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data != 1; + + my $chipRun = $data->[0]; + + my $chipRunState = $chipRun->{state}; + if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { + print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; + faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); + return 0; + } + my $warpRunState = $warpRun->{state}; + if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { + # The inputs and outputs are ready. Queue the warpRun for update. + + # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif ($chipRunState eq 'cleaned' or + (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { + my $data = [$chipRun]; + return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); + } + } + + # return value may be the return status of script so zero is good + return 0; +} + + +# run a command that produces metadata output and parse the results into an array of objects +sub runToolAndParse { + my $command = shift; + my $verbose = shift; + + my ($program) = split " ", $command; + $program = basename($program); + + print "Running $command\n" if !$verbose; + my $start_tool = DateTime->now->mjd; + # run the command and parse the output + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf if !$verbose; + return undef; + } + + my $now = DateTime->now->mjd; + my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; + print "Time to run $program: $dtime_tool\n"; + + my $buf = join "", @$stdout_buf; + if (!$buf) { + return undef; + } + + my $start_parse = DateTime->now->mjd; + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results = parse_md_fast($mdcParser, $buf) + or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR); + + my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; + print "Time to parse results from $program: $dtime_parse\n"; + + return $results; +} + +# run an command returning metadata where we expect 1 entry +sub runToolAndParseExpectOne { + my $command = shift; + my $verbose = shift; + + my $data = runToolAndParse($command, $verbose); + + if (!$data) { + return undef; + } + + my $n = scalar @$data; + if ($n > 1) { + my_die("Unexpected number of components $n returned by $command", $PS_EXIT_PROG_ERROR); + } + + return $data->[0]; +} + +# splits meta data config input stream into single units to work around the pathalogically +# slow parser. This is similar to and adapted from code in various ippScripts. +sub parse_md_fast { + my $mdcParser = shift; + my $input = shift; + my $output = (); + + my @whole = split /\n/, $input; + my @single = (); + + my $n; + while ( ($n = @whole) > 0) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or + print STDERR "Unable to parse metdata config doc" and return undef; + push @$output, $list->[0]; + + @single = (); + } + } + return $output; +} + +sub faultJobs { + my ($state, $stage, $stage_id, $job_fault) = @_; + + my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set pstampJob.fault for dep_id: $dep_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } +} + +sub queue_update_magicDSRun { + my $stage = shift; + my $stage_id = shift; + my $rlabel = shift; + my $input_magicked = shift; + my $dsRun_state = shift; + + # XXX: this code is only fully coded and tested for chip stage + my_die ("queue_update_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; + # if called from queue_update_warp dsRun_state is unknown. Go find it. if (!$dsRun_state) { - my $command = "$chiptool -listrun -chip_id $chip_id"; + my $command = "$chiptool -listrun -chip_id $stage_id"; my $data = runToolAndParse($command, $verbose); my $chipRun = $data->[0]; @@ -222,5 +564,5 @@ # if the input file is already magicked no need to queue destreaking for this chipRun - if ($need_magic and !$raw_all_magicked) { + if ($need_magic and !$input_magicked) { if ($dsRun_state eq 'cleaned') { my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id"; @@ -242,177 +584,14 @@ } } - - # return value may be the return status of script so zero is good return 0; } -sub queue_update_warp { - # check status of input chips. If state is not updatable set error code for job - - # if chipProcessedImfile.state is cleaned call queue_update_chip - - # need to code warptool -setskyfiletoupdate - my $metadata = shift; - my $whole_run = shift; # if true queue entire run for update - my $rlabel = shift; # if defined a new label for the chipRun - my $need_magic = shift; - - my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked - - my $warp_id = $metadata->{warp_id}; - if (!$whole_run) { - my $skycell = $metadata; - my $skycell_id = $skycell->{skycell_id}; - - my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; - my $data = runToolAndParse($command, $verbose); - my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) - if !$data or scalar @$data == 0; - - my $good_to_go = 1; - my @chipsToUpdate; - my $chip_id; - foreach my $chip (@$data) { - $chip_id = $chip->{chip_id}; - if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { - $good_to_go = 0; - push @chipsToUpdate, $chip; - } else { - # this chip is good to go - } - } - - if ($good_to_go) { - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif (scalar @chipsToUpdate > 0) { - return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); - } - } else { - my $warpRun = $metadata; - my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; - my $data = runToolAndParse($command, $verbose); - my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) - if !$data or scalar @$data != 1; - - my $chipRun = $data->[0]; - - my $chipRunState = $chipRun->{state}; - if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { - print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; - faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); - return 0; - } - my $warpRunState = $warpRun->{state}; - if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { - # The inputs and outputs are ready. Queue the warpRun for update. - - # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif ($chipRunState eq 'cleaned' or - (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { - my $data = [$chipRun]; - return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); - } - } - - # return value may be the return status of script so zero is good - return 0; -} -sub queue_update_diff { - return 0; -} - - -# run a command that produces metadata output and parse the results into an array of objects -sub runToolAndParse { - my $command = shift; - my $verbose = shift; - - my ($program) = split " ", $command; - $program = basename($program); - - print "Running $command\n" if !$verbose; - my $start_tool = DateTime->now->mjd; - # run the command and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf if !$verbose; - return undef; - } - - my $now = DateTime->now->mjd; - my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; - print "Time to run $program: $dtime_tool\n"; - - my $buf = join "", @$stdout_buf; - if (!$buf) { - return undef; - } - - my $start_parse = DateTime->now->mjd; - - my $mdcParser = PS::IPP::Metadata::Config->new; - my $results = parse_md_fast($mdcParser, $buf) - or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR); - - my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; - print "Time to parse results from $program: $dtime_parse\n"; - - return $results; -} - -# splits meta data config input stream into single units to work around the pathalogically -# slow parser. This is similar to and adapted from code in various ippScripts. -sub parse_md_fast { - my $mdcParser = shift; - my $input = shift; - my $output = (); - - my @whole = split /\n/, $input; - my @single = (); - - my $n; - while ( ($n = @whole) > 0) { - my $value = shift @whole; - push @single, $value; - if ($value =~ /^\s*END\s*$/) { - push @single, "\n"; - - my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or - print STDERR "Unable to parse metdata config doc" and return undef; - push @$output, $list->[0]; - - @single = (); - } - } - return $output; -} -sub faultJobs { - my ($state, $stage, $stage_id, $job_fault) = @_; - - my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id"; +sub my_die +{ + my $msg = shift; + my $fault = shift; + carp $msg; + + my $command = "$pstamptool -updatedepedent -fault $fault -dep_id $dep_id"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -421,5 +600,5 @@ run(command => $command, verbose => $verbose); unless ($success) { - my_die("failed to set pstamJob.fault for dep_id: $dep_id", + my_die("failed to set pstampDependent.fault for dep_id: $dep_id", $PS_EXIT_UNKNOWN_ERROR); } @@ -427,13 +606,5 @@ print "skipping $command\n"; } -} - -sub my_die -{ - my $msg = shift; - my $fault = shift; - - carp $msg; - - return $fault; -} + + exit $fault; +} Index: trunk/pstamp/scripts/pstamp_finish.pl =================================================================== --- trunk/pstamp/scripts/pstamp_finish.pl (revision 27704) +++ trunk/pstamp/scripts/pstamp_finish.pl (revision 27751) @@ -124,5 +124,5 @@ my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); print $rlf "results.fits|||table|\n"; - print $rlf "results.mdc|||txt|\n"; + print $rlf "results.mdc|||text|\n"; my $err_file = "parse_error.txt"; @@ -171,4 +171,5 @@ my $exp_id = $job->{exp_id}; + if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { # this request had a duplicate request name. We can't put the results @@ -188,9 +189,20 @@ } - if ($exp_id ne $last_exp_id) { + my $job_params = get_job_parameters($job); + my $stage = ""; + if ($job_params) { + $stage = $job_params->{stage}; + } + + if ($stage ne 'stack') { # get the metadata for the exposure (if any i.e. stack) # returns an appropriate string if !$exp_id $exp_info = get_exposure_info($image_db, $exp_id); $last_exp_id = $exp_id; + } else { + my $filter = $job_params->{filter}; + $filter = "0" if !$filter; + $exp_info = "0|0|0|$filter|0|0"; + $last_exp_id = -1; } @@ -344,4 +356,32 @@ } +sub get_job_parameters { + my $job = shift; + if (!$job->{outputBase}) { + print "get_job_parameters: $job->{job_id} has no outputBase\n"; + return undef; + } + my $params_file = $job->{outputBase} . '.mdc'; + if (! -e $params_file ) { + print "get_job_parameters: $job->{job_id} has no parameters file\n"; + return undef; + } + open IN, "<$params_file" or die "unable to open $params_file"; + my $data = join "", (); + close IN; + if (! $data ) { + print "get_job_parameters: parameters file is empty\n"; + return undef; + } + my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc"); + + my $results = parse_md_list($metadata); + if (scalar @$results != 1) { + print STDERR "get_job_params: failed to parse_md_list\n"; + return undef; + } + return $results->[0]; +} + sub get_exposure_info { my $image_db= shift; Index: trunk/pstamp/scripts/pstamp_job_run.pl =================================================================== --- trunk/pstamp/scripts/pstamp_job_run.pl (revision 27704) +++ trunk/pstamp/scripts/pstamp_job_run.pl (revision 27751) @@ -250,5 +250,5 @@ copy_and_register_file($f, $backmdl_file, $outdir, $prefix); } - if (0) { + if (0) { # don't enable this yet if ($pattern_file) { @@ -256,5 +256,5 @@ copy_and_register_file($f, $pattern_file, $outdir, $prefix); } - } + } } } Index: trunk/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- trunk/pstamp/scripts/pstamp_parser_run.pl (revision 27704) +++ trunk/pstamp/scripts/pstamp_parser_run.pl (revision 27751) @@ -28,4 +28,5 @@ my $redirect_output; my $product; +my $label; my $verbose; my $dbname; @@ -36,4 +37,5 @@ 'uri=s' => \$uri, 'product=s' => \$product, + 'label=s' => \$label, 'redirect-output' => \$redirect_output, 'verbose' => \$verbose, @@ -146,5 +148,6 @@ if ($request_type eq "PS1_PS_REQUEST") { $reqType = 'pstamp'; - $parse_cmd = $pstampparse; + $parse_cmd = "$pstampparse"; + $parse_cmd .= " --label $label" if $label; } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") { $reqType = 'dquery'; Index: trunk/pstamp/scripts/pstamp_request_file =================================================================== --- trunk/pstamp/scripts/pstamp_request_file (revision 27704) +++ trunk/pstamp/scripts/pstamp_request_file (revision 27751) @@ -187,5 +187,11 @@ for (my $i = 0; $i < scalar @colNames; $i++) { my $writeType = $colWriteType[$i]; - $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status ); + my $data = $colData->[$i]; + if ($writeType == TULONG) { + die "invalid integer data found in column $i\n" unless validIntegers($data); + } elsif ($writeType == TDOUBLE) { + die "invalid numeric data found in column $i\n" unless validNumbers($data); + } + $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $data, $status ); check_fitsio( $status ); } @@ -312,4 +318,42 @@ } +sub checkValid { + my $aref = shift; + my $float = shift; + + return 0 if !defined $aref; + + my $valid = 0; + my $row = 0; + foreach my $val (@$aref) { + $row++; + if ($float) { + if (!($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/)) { + $valid = 0; + print STDERR "Error on row $row: '$val' is not a valid number\n"; + last; + } + } else { + if ($val =~ /\D/) { + $valid = 0; + print STDERR "Error on row $row: '$val' is not a valid integer\n"; + last; + } + } + $valid = 1; + } + + return $valid; +} +sub validIntegers { + return checkValid(@_, 0); +} + +sub validNumbers +{ + return checkValid(@_, 1); +} + + sub printhelp { Index: trunk/pstamp/scripts/pstampparse.pl =================================================================== --- trunk/pstamp/scripts/pstampparse.pl (revision 27704) +++ trunk/pstamp/scripts/pstampparse.pl (revision 27751) @@ -17,4 +17,5 @@ use Carp; use POSIX; +use DateTime; my $verbose; @@ -26,4 +27,5 @@ my $out_dir; my $product; +my $label; my $save_temps; my $no_update; @@ -34,4 +36,5 @@ 'out_dir=s' => \$out_dir, 'product=s' => \$product, + 'label=s' => \$label, 'mode=s' => \$mode, 'dbname=s' => \$dbname, @@ -42,5 +45,5 @@ ); -die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job"); +die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job"); die "--file is required" if !defined($request_file_name); @@ -64,5 +67,4 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1); -my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); @@ -159,4 +161,10 @@ } +# +# Loop over rows in the request file collecting consecutive rows that have the "same images of interest" +# in the sense that their selection parameters will yield the same "Runs". +# Process the groups of rows together to reduce lookup time and to potentially make multiple +# stamps from the same ppstamp process. +# my @rowList; my $num_jobs = 0; @@ -165,30 +173,79 @@ my $need_magic; foreach my $row (@$rows) { - # XXX: TODO: sanity check all parameters - - # If we encounter an error for a particular row add a job with the proper fault code. - # If we encounter an error in this loop we shouldn't really just die. - # We only do that now in the case of I/O or DB errors or the like. - - my $rownum = $row->{ROWNUM}; - my $job_type = $row->{JOB_TYPE}; - - # parameters that select the images of interest - my $project = $row->{PROJECT}; - - # note: resolve_project avoids running pstamptool every time by remembering the - # last project resolved - my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); - if (!$proj_hash) { - print STDERR "project $project not found\n" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + # santiy check the paramaters + if (!checkRow($row)) { + # when it enconters an error checkRow adds a fake job with an appropriate error code to the database $num_jobs++; next; } + # initialize counter for "job number" + $row->{job_num} = 0; + $row->{error_code} = 0; + + if (scalar @rowList == 0) { + push @rowList, $row; + next; + } + + my $firstRow = $rowList[0]; + if (same_images_of_interest($firstRow, $row)) { + # add this row to the list and move on + push @rowList, $row; + next; + } + + # the images of interest for this new row doesn't match the list. + # process the list ... + $num_jobs += processRows(\@rowList); + + # and reset the list to contain just the new row + @rowList = ($row); +} + +# out of rows process the list +if (scalar @rowList > 0) { + $num_jobs += processRows(\@rowList); +} + +if (($mode eq "queue_job") and ($num_jobs eq 0)) { + print STDERR "no jobs created for $req_name\n" if $verbose; + insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); +} + +exit 0; + +sub checkRow { + + my $row = shift; + + # If we encounter an error for a particular row add a job with the proper fault code. + + my $rownum = $row->{ROWNUM}; + if (!validID($rownum)) { + print STDERR "$rownum is not a valid ROWNUM\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $job_type = $row->{JOB_TYPE}; + if (($job_type ne "stamp") and ($job_type ne "get_image")) { + print STDERR "$job_type is not a valid JOB_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $req_type = $row->{REQ_TYPE}; - $stage = $row->{IMG_TYPE}; - my $id = $row->{ID}; + if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and + ($req_type ne "byskycell") and ($req_type ne "bydiff")) { + print STDERR "$req_type is not a valid REQ_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + my $component = $row->{COMPONENT}; - my $tess_id = $row->{TESS_ID}; + if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) { + $row->{COMPONENT} = $component = ""; + } + $row->{TESS_ID} = "" if !defined $row->{TESS_ID}; my $filter = $row->{REQFILT}; @@ -196,9 +253,19 @@ if (length($filter) == 1) { # allow single character filter cuts to work - $filter .= '%'; + $row->{REQFILT} .= '%'; } } my $mjd_min = $row->{MJD_MIN}; + if (defined($mjd_min) and !validNumber($mjd_min)) { + print STDERR "$mjd_min is not a valid MJD_MIN\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $mjd_max = $row->{MJD_MAX}; + if (defined($mjd_max) and !validNumber($mjd_max)) { + print STDERR "$mjd_max is not a valid MJD_MAX\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $data_group = $row->{DATA_GROUP}; if (!defined $data_group) { @@ -209,9 +276,7 @@ } - my $x = $row->{CENTER_X}; - my $y = $row->{CENTER_Y}; - - # XXX things don't work if bit zero of option mask is not set; + # req_finish doesn't work if bit zero of option mask is not set; $row->{OPTION_MASK} |= 1; + my $option_mask= $row->{OPTION_MASK}; my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0; @@ -222,21 +287,15 @@ my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); - my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component; - - if (!$skycenter and !$search_component) { + if (!$skycenter and !$component) { print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; - } - - # user requested us to search all components. Set to "" - $search_component = "" if $search_component eq "all"; - + return 0; + } + + my $stage = $row->{IMG_TYPE}; if (!check_image_type($stage)) { print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; + return 0; } @@ -244,6 +303,5 @@ print STDERR "invalid ROI for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; + return 0; } @@ -251,62 +309,100 @@ print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); - $num_jobs++; - next; - } - - + return 0; + } + # $mode list_uri is a debugging mode (it may used by the http interface) # if this happens just croak - my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); - - my $image_db = $proj_hash->{dbname}; - my $camera = $proj_hash->{camera}; - $need_magic = $proj_hash->{need_magic}; - - # Temporary hack so that MOPS can get at non-magicked data -# if ($product and ($product eq "mops-pstamp-results")) { -# $need_magic = 0; -# } - - # For "stamp" and "list_uri" jobs collect rows with the same images of interest in a list so that they - # can be looked up together. - if (@rowList) { - my $firstRow = $rowList[0]; - if (($firstRow->{JOB_TYPE} ne "get_image") and same_images_of_interest($firstRow, $row)) { - - # add this row to the list and move on - push @rowList, $row; - - next; - - } else { - # this row has different selectors - # queue the jobs for the ones we've collected - $num_jobs += queueJobs($mode, \@rowList, $imageList); - @rowList = (); - } - } - - # look up images for the current row + # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); + + if ($req_type eq "bycoord") { if (!$skycenter) { print STDERR "center must be specified in sky coordintes for bycoord" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; - } - } + return 0; + } + } + if (($req_type eq "byid") or ($req_type eq "bydiff")) { - if (!validID($id)) { + if (!validID($row->{ID})) { print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0 + } + } + + return 1; +} + +sub processRows { + my $rowList = shift; + my $num_jobs = 0; + + # all rows in the list are compatible + my $row = $rowList->[0]; + + my $project = $row->{PROJECT}; + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); + if (!$proj_hash) { + print STDERR "project $project not found\n" if $verbose; + foreach $row (@$rowList) { + insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); $num_jobs++; - next; - } - } + } + return $num_jobs; + } + my $req_type = $row->{REQ_TYPE}; + $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $component = $row->{COMPONENT}; + my $tess_id = $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $data_group = $row->{DATA_GROUP}; + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + my $option_mask= $row->{OPTION_MASK}; + + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + $need_magic = $proj_hash->{need_magic}; + + # Temporary hack so that MOPS can get at non-magicked data + my $allow_mops_unmagicked = 1; + if ($allow_mops_unmagicked) { + if ($product and (($product eq "mops-pstamp-results") or + ($product eq "mops-pstamp-results2"))) { + $need_magic = 0; + } + } + + my $numRows = scalar @$rowList; + +# $tess_id = "" if !defined $tess_id; +# $component = "" if !defined $component; + + print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n"; + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this # request specification. An array reference is returned. - $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component, - $option_mask, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + my $start_locate = DateTime->now->mjd; + + # XXX: perhaps we should get rid of most of this argument list. + # Now that we are passing down compatible rows all of the + # information required is contained there + + $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component, + $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + + # XXX: why use mjd? It doesn't have great precision. + my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.; + print "Time to locate_images for row $rownum $dtime_locate\n"; if (!$imageList or !@$imageList) { @@ -314,32 +410,31 @@ # note in this case queueJobs inserts the fake job for these rows } + # handle this $row->{need_magic} = $need_magic; - push @rowList, $row; -} - -if (@rowList) { + $num_jobs += queueJobs($mode, \@rowList, $imageList); -} - -if (($mode eq "queue_job") and ($num_jobs eq 0)) { - print STDERR "no jobs created for $req_name\n" if $verbose; - insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); -} - -# PAU - -exit 0; - - -sub queueJobsForRow + + # if a row slipped through with no jobs add one + foreach my $row (@rowList) { + if ($row->{job_num} == 0) { + print "row $row->{ROWNUM} produced no jobs\n"; + print STDERR "row $row->{ROWNUM} produced no jobs\n"; + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + } + } + + return $num_jobs; +} + +sub queueJobForImage { my $row = shift; my $stage = shift; - my $imageList = shift; - my $have_skycells = shift; + my $image = shift; my $need_magic = shift; my $mode = shift; - my $num_jobs = 0; my $rownum = $row->{ROWNUM}; my $option_mask = $row->{OPTION_MASK}; @@ -365,28 +460,243 @@ } + my $component = $image->{component}; + + my $job_num = ++($row->{job_num}); + + my $imagefile = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + # XXX: should we add a faulted job so the client can know what happened if no images come back? + # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback + # to users, but speeds up processing significantly + print STDERR "skipping non-magicked image $imagefile\n" if $verbose; + + # for now assume yes. + + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); + return 1; + } elsif ($stage eq "stack") { + # unconvolved stack images weren't available prior to some point in time. + # XXX: handle this more correctly by examining the stack run's config dump file. + # It looks like # the feature was turned on sometime around November 11, 2009. stackRun 30067 is the lowest + # one that I found with an unconvolved image. + my $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES = 30067; + if ($row->{unconvolved} and ($row->{PROJECT} eq 'gpc1') and + ($image->{stack_id} < $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES)) { + print STDERR "Unconvolved stack image is not available for stackRun.stack_id: $image->{stack_id}\n"; + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE); + return 1; + } + } + my $exp_id = $image->{exp_id}; + + my $args = $roi_string ? $roi_string : ""; + if ($stage eq "raw" or $stage eq "chip") { + $args .= " -class_id $component" if $component; + } + + # add astrometry file for raw and chip images if one is available + if (($stage eq "chip") || ($stage eq "raw")) { + $args .= " -astrom $image->{astrom}" if $image->{astrom}; + } + + $image->{job_args} = $args; + + # XXX: we can get rid of the following everything that we need is + # in the params file + + $args .= " -file $imagefile"; + + if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { + $args .= " -mask $image->{mask}"; + } + if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { + $args .= " -variance $image->{weight}"; + } + + my $base = basename($image->{image}); + if (! $base =~ /.fits$/ ) { + my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); + } + $base =~ s/.fits$//; + + my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; + my $argslist = "${output_base}.args"; + + # copy the argument list to a file + open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); + print ARGSLIST "$args\n"; + close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + + write_params($output_base, $image); + + my $newState = "run"; + my $fault = 0; + my $dep_id; + + # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it + if ($stage ne 'raw') { + # not ready to handle diff update yet. May never support stack + my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff')); + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + $data_state = $run_state if $stage eq 'stack'; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'drop') or + ($run_state eq 'error_cleaned') or ($data_state eq 'error_cleaned') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back + $newState = 'stop'; + $fault = $PSTAMP_GONE; + } elsif ($allow_wait_for_update and + (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) { + if ($stage eq 'chip') { + my $burntool_state = $image->{burntool_state}; + if ($burntool_state and (abs($burntool_state) < 14)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } + } + if (!$fault) { + # don't wait for update unless the caller asks us to + if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } elsif (!$image->{magicked}) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_DESTREAKED; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); + } + } + } + } + + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; + $command .= " -fault $fault" if $fault; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dep_id $dep_id" if $dep_id; + + if (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping command: $command\n"; + } + + return 1; +} + +# queue jobs for a collection of request specifications that have the same Images of Interest +sub queueJobs +{ + my $mode = shift; + my $rowList = shift; + my $imageList = shift; + + my $firstRow = $rowList[0]; + my $stage = $firstRow->{IMG_TYPE}; + my $job_type = $firstRow->{JOB_TYPE}; + my $need_magic = $firstRow->{need_magic}; + + my $num_jobs = 0; + + if ($mode eq "list_uri") { + foreach my $image (@$imageList) { + print "$image->{image}\n"; + } + } elsif ($job_type eq "get_image") { + my $n = scalar @$rowList; + + my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; + + $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); + + } else { + if (!$imageList or (scalar @$imageList eq 0)) { + # We didn't find any images for this set of rows. Insert a fake job to carry + # the status back to the requestor. + foreach my $row (@$rowList) { + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + $num_jobs++; + } + return $num_jobs; + } + + foreach my $image (@$imageList) { + # get the array of row indices that touch this image + my $row_index = $image->{row_index}; + if (!$row_index or scalar @$row_index == 0) { + # XXX should this happen? Why did something get returned. + print "image ${stage}_id: $image->{stage_id} component: $image->{component} matched no rows\n"; + next; + } + # XXX: TODO: eventually we may change ppstamp to be able to make multiple stamps per invocation + + foreach my $i (@$row_index) { + my $row = $rowList->[$i]; + + $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode); + } + } + } + + return $num_jobs; +} + +# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); +sub queueGetImageJobs +{ + my $row = shift; + my $imageList = shift; + my $stage = shift; + my $need_magic = shift; + my $mode = shift; + + my $num_jobs = 0; + my $rownum = $row->{ROWNUM}; + my $option_mask = $row->{OPTION_MASK}; + + # For dist_bundle we need + # --camera from $image + # --stage + # --stage_id from $image + # --component from $image + # --path_base + # --outdir global to this script + # loop over images - my $job_num = 0; foreach my $image (@$imageList) { - my $component; - if ($have_skycells) { - $component = $image->{skycell_id}; - } else { - $component = $image->{class_id}; - } - - # skip this component if it is not in the list for this row - next if ! $components->{$component}; - - $job_num++; + my $stage_id = $image->{stage_id}; + my $component = $image->{component}; + + # skip faulted components for now. Should we even be here? + if ($image->{fault} > 0) { + printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; + next; + } + + my $job_num = ++($row->{job_num}); my $imagefile = $image->{image}; if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # XXX: should we add a faulted job so the client can know what happened if no images come back? - # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback - # to users, but speeds up processing significantly + # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed + # in locate_images because it's much more efficient to do the test in the database. + # For these two modes we fall through to here in order to give feedback to the requestor as + # to why the request failed to queue jobs. print STDERR "skipping non-magicked image $imagefile\n" if $verbose; - - # for now assume yes. - insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); $num_jobs++; @@ -396,46 +706,5 @@ my $exp_id = $image->{exp_id}; - my $args = $roi_string ? $roi_string : ""; - if ($stage eq "raw" or $stage eq "chip") { - $args .= " -class_id $component" if $component; - } - - # add astrometry file for raw and chip images if one is available - if (($stage eq "chip") || ($stage eq "raw")) { - $args .= " -astrom $image->{astrom}" if $image->{astrom}; - } - - $image->{job_args} = $args; - - # XXX: we can get rid of the following everything that we need is - # in the params file - - $args .= " -file $imagefile"; - - if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { - $args .= " -mask $image->{mask}"; - } - if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { - $args .= " -variance $image->{weight}"; - } - - my $base = basename($image->{image}); - if (! $base =~ /.fits$/ ) { - my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); - } - $base =~ s/.fits$//; - - # XXX: TODO use filerule for this. I don't have a camera defined here - if (($stage eq 'chip') and ($image->{camera} eq 'GPC1')) { - $base = "${base}.${component}"; - } - - my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; - my $argslist = "${output_base}.args"; - - # copy the argument list to a file - open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); - print ARGSLIST "$args\n"; - close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + my $output_base = "$out_dir/${rownum}_${job_num}"; write_params($output_base, $image); @@ -454,5 +723,5 @@ $newState = 'stop'; $fault = $PSTAMP_GONE; - } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { + } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) { # don't wait for update unless the caller asks us to if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { @@ -463,5 +732,5 @@ # set up to queue an update run queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, - $run_state, $stage, $image->{stage_id}, $need_magic); + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); } } @@ -475,256 +744,5 @@ $command .= " -dep_id $dep_id" if $dep_id; - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { - # mode eq "queue_job" - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - # should we keep going for other uris? If so how do we report that some - # of the work that the request wanted isn't going to get done - my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping command: $command\n"; - } - } - if ( $num_jobs == 0 ) { - print STDERR "no jobs for row $rownum\n" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP); - $num_jobs = 1; - } - return $num_jobs; -} - -# queue jobs for a collection of request specifications that have the same Images of Interest -sub queueJobs -{ - my $mode = shift; - my $rowList = shift; - my $imageList = shift; - - my $firstRow = $rowList[0]; - my $stage = $firstRow->{IMG_TYPE}; - my $job_type = $firstRow->{JOB_TYPE}; - my $need_magic = $firstRow->{need_magic}; - - my $num_jobs = 0; - - if ($mode eq "list_uri") { - foreach my $image (@$imageList) { - print "$image->{image}\n"; - } - } elsif ($job_type eq "get_image") { - my $n = scalar @$rowList; - - my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; - - $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); - - } else { - if (!$imageList or (scalar @$imageList eq 0)) { - # we didn't find any images for this set of rows. Insert a fake job to carry - # the status back to the requestor - foreach my $row (@$rowList) { - insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH); - $num_jobs++; - } - return $num_jobs; - } - - my $have_skycells; - if (($stage eq "raw") or ($stage eq "chip")) { - $have_skycells = 0; - } else { - $have_skycells = 1; - } - - my $thisRun; - - my $npoints = 0; - my ($pointsList, $pointsListName); - if (scalar @$imageList > 1) { - ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); - foreach my $row (@$rowList) { - $row->{components} = {}; - if ($row->{skycenter}) { - print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; - $npoints++; - } else { - # this row's center is in pixel coordinates add all images to the component list for this row - foreach my $i (@$imageList) { - my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; - my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) - if !$component; - $row->{components}->{$component} = 1; - } - } - } - close $pointsList; - } else { - # only one image. Avoid the expense of dvoImagesAtCoords. - # queue the job and let ppstamp figure out if the center is valid for the image - # the resulting result code will be the same. - foreach my $row (@$rowList) { - $row->{components} = {}; - my $i = $imageList->[0]; - my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; - my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component; - $row->{components}->{$component} = 1; - } - } - - my $tess_dir_abs; - my $last_tess_id = ""; - while ($thisRun = getOneRun($stage, $imageList)) { - if ($npoints) { - my_die( "pointsListName is not defined", $PS_EXIT_PROG_ERROR) if !$pointsListName; - # we collected a set of sky coordinates above. - # filter the images so that only those that contain the centers of the ROIs are processed - my $command = "$dvoImagesAtCoords -coords $pointsListName"; - if ($have_skycells) { - my $tess_id = $thisRun->[0]->{tess_id}; - if ($tess_id ne $last_tess_id) { - $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); - $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); - $last_tess_id = $tess_id; - } - $command .= " -D CATDIR $tess_dir_abs"; - } else { - my $astrom = $thisRun->[0]->{astrom}; - my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom; - my $astrom_resolved = $ipprc->file_resolve($astrom); - $command .= " -astrom $astrom_resolved"; - } - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP - # That just means that the coordinate didn't match any image/skycell - if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) { - print STDERR @$stderr_buf; - my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR; - my_die( "dvoImagesAtCoords failed: $rc", $rc); - } - } - # now we have a list of row numbers and components - # eventually we might want to multiple stamp requests for the same image - # into the same ppstamp job but not yet. For now we will queue a new - my @lines = split "\n", join "", @$stdout_buf; - foreach my $line (@lines) { - # parse the line, ignoring the ra and dec - my ($rownum, undef, undef, $component) = split " ", $line; - - # I guess since we need this function we should be using a hash for rowList - my $row = findRow($rownum, $rowList); - $row->{components}->{$component} = 1; - } - } - - foreach my $row (@$rowList) { - $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic, $mode); - } - } - } - return $num_jobs; -} - -# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); -sub queueGetImageJobs -{ - my $row = shift; - my $imageList = shift; - my $stage = shift; - my $need_magic = shift; - my $mode = shift; - - my $num_jobs = 0; - my $rownum = $row->{ROWNUM}; - my $option_mask = $row->{OPTION_MASK}; - - # For dist_bundle we need - # --camera from $image - # --stage - # --stage_id from $image - # --component from $image - # --path_base - # --outdir global to this script - - # loop over images - my $job_num = 0; - foreach my $image (@$imageList) { - my $stage_id = $image->{stage_id}; - my $component = $image->{component}; - - # skip faulted components for now. Should we even be here? - if ($image->{fault} > 0) { - printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; - next; - } - - $job_num++; - - my $imagefile = $image->{image}; - if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed - # in locate_images because it's much more efficient to do the test in the database. - # For these two modes we fall through to here in order to give feedback to the requestor as - # to why the request failed to queue jobs. - print STDERR "skipping non-magicked image $imagefile\n" if $verbose; - insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); - $num_jobs++; - - next; - } - my $exp_id = $image->{exp_id}; - - my $output_base = "$out_dir/${rownum}_${job_num}"; - - write_params($output_base, $image); - - my $newState = "run"; - my $fault = 0; - my $dep_id; - - if ($stage ne 'raw') { - my $run_state = $image->{state}; - my $data_state = $image->{data_state}; - $data_state = $run_state if $stage eq "stack"; - if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or - ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { - # image is gone and it's not coming back - $newState = 'stop'; - $fault = $PSTAMP_GONE; - } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { - # don't wait for update unless the caller asks us to - if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { - $newState = 'stop'; - $fault = $PSTAMP_NOT_AVAILABLE; - } else { - # cause the image to be re-made - # set up to queue an update run - queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, - $run_state, $stage, $image->{stage_id}, $need_magic); - } - } - } - - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" - . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; - $command .= " -fault $fault" if $fault; - $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dep_id $dep_id" if $dep_id; - - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { + if (!$no_update) { # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -776,9 +794,5 @@ . " -rownum $rownum -state stop -fault $fault"; - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { + if (!$no_update) { # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -796,74 +810,25 @@ } -sub get_run_id -{ - my $stage = shift; - my $image = shift; - - if ($stage eq "raw") { - return $image->{exp_id}; - } elsif ($stage eq "chip") { - return $image->{chip_id}; - } elsif ($stage eq "warp") { - return $image->{warp_id}; - } elsif ($stage eq "stack") { - return $image->{stack_id}; - } elsif ($stage eq "diff") { - return $image->{diff_id}; - } else { - my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR); - } -} - -# extract components from the imageList that have the same run id and return the list -sub getOneRun { - my $stage = shift; - my $imageList = shift; - - # return if array is empty - return undef if ! @$imageList; - - my $last_run_id = 0; - my @runList; - while ($imageList->[0]) { - my $run_id = get_run_id($stage, $imageList->[0]); - - last if ($last_run_id and ($run_id ne $last_run_id)); - - my $image = shift @$imageList; - $image->{stage} = $stage; - push @runList, $image; - $last_run_id = $run_id; - } - return \@runList; -} - sub same_images_of_interest { my $r1 = shift; my $r2 = shift; - return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); - return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); - return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE}); + return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); + return 0 if (($r1->{JOB_TYPE} eq "get_image") or ($r2->{JOB_TYPE} eq "get_image")); return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE}); return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE}); return 0 if ($r1->{ID} ne $r2->{ID}); return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID}); + return 0 if ($r1->{COMPONENT} ne $r2->{COMPONENT}); return 0 if ($r1->{REQFILT} ne $r2->{REQFILT}); return 0 if ($r1->{DATA_GROUP} ne $r2->{DATA_GROUP}); return 0 if ($r1->{MJD_MIN} ne $r2->{MJD_MAX}); return 0 if ($r1->{MJD_MAX} ne $r2->{MJD_MAX}); + return 0 if ($r1->{OPTION_MASK} ne $r2->{OPTION_MASK}); + return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); return 0 if ($r1->{inverse} ne $r2->{inverse}); return 0 if ($r1->{unconvolved} ne $r2->{unconvolved}); - - if (defined($r1->{COMPONENT})) { - return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT}); - } elsif (defined($r2->{COMPONENT})) { - # if first row has no component all of the images will be retrieved, so - # the fact that this row has a component is ok. Fall through to return 1 - # XXX Nope this doesn't work. It is consistent with the other logic in some way - # that i haven't fully thought through - return 0; - } + # don't combine requests in pixel coordinates + return 0 if (($r1->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS) || ($r2->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS)); return 1; @@ -911,5 +876,5 @@ sub queue_update_run { - my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $need_magic) = @_; + my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_; if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { @@ -918,6 +883,13 @@ my $dep_id; - my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb"; - $command .= " -no_magic" if !$need_magic; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component"; + $command .= " -need_magic" if $need_magic; + + # compute rlabel for the run. + # XXX: This bit of policy shouldn't be buried so deeply in the code + # For now use one that implies 'postage stamp server' 'update' 'request_label" + my $rlabel = "ps_ud_" . $label if $label; + $command .= " -rlabel $rlabel" if $rlabel; + if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -929,4 +901,6 @@ chomp $output; $dep_id = $output; + # + # XXX: need to fault the request or something my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id; } else { @@ -936,6 +910,6 @@ $$r_dep_id = $dep_id; - $r_fault = 0; - $r_jobState = 'blocked'; + $$r_fault = 0; + $$r_jobState = 'run'; } @@ -990,4 +964,4 @@ # we don't fault the request here pstamp_parser_run.pl handles that if necessary - return $fault; -} + exit $fault; +}