Index: trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 27750) +++ trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 27751) @@ -27,4 +27,10 @@ use PS::IPP::Config qw( :standard ); use Carp; +use DateTime; +use File::Temp qw(tempfile); +use File::Basename; +use POSIX; + +my $save_temps = 0; # caches of camProcessedExp objects. @@ -34,10 +40,14 @@ my %camRunByCamIDCache; -### my @images = locate_images($image_db, $req_type, $img_type, $id, $component, -### $mjd_min, $mjd_max, $filter); +# cache of last project looked up +my $last_project = ""; + +my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool); +my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump); sub locate_images { my $ipprc = shift; # required - my $image_db = shift; # required + my $imagedb = shift; # required + my $rowList = shift; # required my $req_type = shift; # required my $img_type = shift; # required @@ -47,6 +57,4 @@ my $option_mask = shift; my $need_magic = shift; - my $x = shift; - my $y = shift; my $mjd_min = shift; my $mjd_max = shift; @@ -56,6 +64,7 @@ # we die in response to bad data in request files - # The wrapper script is responsible for updating the database - die "Unknown req_type: $req_type" + # The caller is responsible for updating the database + # pstampparse.pl error checks now so this shouldn't happen + &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) if ($req_type ne "byid") and ($req_type ne "byexp") and @@ -84,5 +93,10 @@ if ($req_type eq "bycoord") { - my $results = lookup_bycoord($ipprc, $image_db, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); + my $num_rows = scalar @$rowList; + die "Unexpected number of rows found in rowList: $num_rows" if $num_rows != 1; + my $row = $rowList->[0]; + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); return $results; } @@ -93,10 +107,12 @@ # in one place $req_type = "bydiff"; - my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $option_mask, $img_type, $verbose); + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose); return $results; } if ($req_type eq "bydiff") { - my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $option_mask, $img_type, $verbose); + # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or + # the image from the diffRun + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose); if (!$results) { return undef; @@ -130,5 +146,5 @@ # fall though and lookup by stack_id } else { - # shouldn't I check this elsewhere? + # This is checked this elsewhere? print STDERR "Error: $img_type is an unknown image type\n"; return undef; @@ -145,16 +161,20 @@ } - my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); + my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component, + $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); return $results; } + +# The subroutine lookup handles byexp, byid, and byskycell lookups including lookups that are +# triggered by a bydiff request sub lookup { my $ipprc = shift; - my $image_db = shift; + my $rowList = shift; + my $imagedb = shift; my $req_type = shift; - my $img_type = shift; + my $stage = shift; my $id = shift; my $tess_id = shift; @@ -172,14 +192,4 @@ my $use_imfile_id = ($req_type eq "byid") && ($option_mask & $PSTAMP_USE_IMFILE_ID); - my $missing_tools; - my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); - my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); - my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); - my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); - my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } my $command; my $id_opt; # option for the lookup @@ -203,8 +213,12 @@ my $magic_arg = $need_magic ? " -destreaked" : ""; + # all rows have the same center type + my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + my $component_args; - if ($img_type eq "raw") { + my $choose_components = 0; + if ($stage eq "raw") { $class_id = $component; - $command = "$regtool -processedimfile -dbname $image_db"; + $command = "$regtool -processedimfile -dbname $imagedb"; # XXX: for now restrict lookups to type object # are stamps of detrend exposures interesting? @@ -214,9 +228,20 @@ $want_astrom = 1; $set_class_id = 1; - } elsif ($img_type eq "chip") { + } elsif ($stage eq "chip") { $class_id = $component; - $command = "$chiptool -processedimfile -dbname $image_db"; - $component_args = " -class_id $class_id" if $class_id; + # if the request is such that it will yield a single image per "run" or the + # center is specified in pixel coordinates + # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then + # choose the chips containing the center pixel by calling selectComponets() below + if ($class_id or $use_imfile_id or !$skycenter) { + $command = "$chiptool -processedimfile -dbname $imagedb"; + $component_args = " -class_id $class_id" if $class_id; + } else { + $command = "$chiptool -listrun -pstamp_order -dbname $imagedb"; + $choose_components = 1; + } $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id"; + # XXX: perhaps we should stop resolving images to this level here and do it at job run time. + # With the mdc file it has all of the information that it needs. $image_name = "PPIMAGE.CHIP"; $mask_name = "PPIMAGE.CHIP.MASK"; @@ -228,8 +253,13 @@ $want_astrom = 1; $set_class_id = 1; - } elsif ($img_type eq "warp") { + } elsif ($stage eq "warp") { $skycell_id = $component; - $command = "$warptool -warped -dbname $image_db"; - $component_args = " -skycell_id $skycell_id" if $skycell_id; + if ($skycell_id or $use_imfile_id or !$skycenter) { + $command = "$warptool -warped -dbname $imagedb"; + $component_args = " -skycell_id $skycell_id" if $skycell_id; + } else { + $command = "$warptool -listrun -pstamp_order -dbname $imagedb"; + $choose_components = 1; + } $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id"; $image_name = "PSWARP.OUTPUT"; @@ -239,8 +269,13 @@ $psf_name = "PSPHOT.PSF.SKY.SAVE"; $base_name = "path_base"; # name of the field for the warptool output - } elsif ($img_type eq "diff") { + } elsif ($stage eq "diff") { $skycell_id = $component; - $command = "$difftool -diffskyfile -dbname $image_db"; - $component_args = " -skycell_id $skycell_id" if $skycell_id; + if ($skycell_id or $use_imfile_id or !$skycenter) { + $command = "$difftool -diffskyfile -dbname $imagedb"; + $component_args = " -skycell_id $skycell_id" if $skycell_id; + } else { + $command = "$difftool -listrun -pstamp_order -dbname $imagedb"; + $choose_components = 1; + } $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id"; $image_name = "PPSUB.OUTPUT"; @@ -250,7 +285,7 @@ $psf_name = "PSPHOT.PSF.SKY.SAVE"; $base_name = "path_base"; - } elsif ($img_type eq "stack") { + } elsif ($stage eq "stack") { $skycell_id = $component; - $command = "$stacktool -sumskyfile -dbname $image_db"; + $command = "$stacktool -sumskyfile -dbname $imagedb"; $id_opt = "-stack_id"; $component_args = " -skycell_id $skycell_id" if $skycell_id; @@ -272,7 +307,8 @@ $base_name = "path_base"; } else { - die "Unknown img_type supplied: $img_type"; - } - + die "Unknown IMG_TYPe supplied: $stage"; + } + + my $filter_runs = 0; if ($req_type eq "byid") { $command .= " $id_opt $id"; @@ -288,12 +324,15 @@ # the reason as 'not destreaked' $magic_arg = ""; + # remove duplicate runs for the same exposure. + $filter_runs = 1; } elsif ($req_type eq "byskycell") { die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id; $command .= " -tess_id $tess_id -skycell_id $skycell_id"; } else { - die "Unknown req_type supplied: $req_type"; - } - - if ($img_type ne "stack") { + # this should be caught by caller + &my_die("Unexpected req_type supplied: $req_type", $PS_EXIT_PROG_ERROR); + } + + if ($stage ne "stack") { $command .= $magic_arg; $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; @@ -304,30 +343,32 @@ $command .= " -data_group $data_group" if !isnull($data_group); - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # not sure if we should die here - print STDERR @$stderr_buf; + my $images = runToolAndParse($command, $verbose); + + if (!$images or scalar @$images == 0) { return undef; } - my $buf = join "", @$stdout_buf; - if (!$buf) { - return; - } - - my $mdcParser = PS::IPP::Metadata::Config->new; - my $images = parse_md_fast($mdcParser, $buf) - or die ("Unable to parse metadata config doc"); - + if ($filter_runs) { + # The image selectors are such that multiple runs my have be returned for the same exposure. + # Return only the latest one. + $images = filterRuns($stage, $need_magic, $images, $verbose); + } + if ($choose_components) { + # the list of "images" is actually a list of "Runs" + # match the coords in the rows to the components in the runs + # returns an actual list of images + $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose); + } else { + # put each of the rows into all of the the images + setRowRefs($rowList, $images); + } + + my $camera; my $output = []; - - my $camera; foreach my $image (@$images) { my $base; next if $image->{fault}; - next if ($img_type ne "raw") and $image->{quality}; + next if ($stage ne "raw") and $image->{quality}; if ($base_name) { @@ -361,11 +402,11 @@ } my $stage_id; - if ($img_type eq "raw") { + if ($stage eq "raw") { $stage_id = $image->{exp_id}; - } elsif ($img_type eq "chip") { + } elsif ($stage eq "chip") { $stage_id = $image->{chip_id}; - } elsif ($img_type eq "warp") { + } elsif ($stage eq "warp") { $stage_id = $image->{warp_id}; - } elsif ($img_type eq "diff") { + } elsif ($stage eq "diff") { $stage_id = $image->{diff_id}; if ($inverse && $image->{bothways}) { @@ -375,10 +416,10 @@ $cmf_name = "PPSUB.INVERSE.SOURCES"; } - } elsif ($img_type eq "stack") { + } elsif ($stage eq "stack") { $stage_id = $image->{stack_id}; } $out->{stage_id} = $stage_id; - $out->{stage} = $img_type; - $out->{image_db} = $image_db; + $out->{stage} = $stage; + $out->{imagedb} = $imagedb; if ($base) { @@ -390,5 +431,7 @@ $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name; } - $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom; + if ($want_astrom and ! defined $out->{astrom}) { + $out->{astrom} = find_astrometry($ipprc, $imagedb, $image, $verbose); + } push @$output, $out; @@ -397,7 +440,9 @@ return $output; } + sub lookup_diff { my $ipprc = shift; - my $image_db = shift; + my $rowList = shift; + my $imagedb = shift; my $id = shift; my $skycell_id = shift; @@ -409,12 +454,5 @@ my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE; - my $missing_tools; - my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - - my $command = "$difftool -diffskyfile -dbname $image_db"; + my $command = "$difftool -diffskyfile -dbname $imagedb"; if ($byid) { @@ -425,21 +463,5 @@ $command .= " -skycell_id $skycell_id" if $skycell_id; - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # not sure if we should die here - print STDERR @$stderr_buf; - return undef; - } - - my $buf = join "", @$stdout_buf; - if (!$buf) { - return undef; - } - - my $mdcParser = PS::IPP::Metadata::Config->new; - my $images = parse_md_fast($mdcParser, $buf) - or die ("Unable to parse metadata config doc"); + my $images = runToolAndParse($command, $verbose); my $n = @$images; @@ -527,4 +549,7 @@ if ($img_type eq "diff") { + my @imageList = ($image); + + setRowRefs($rowList, \@imageList); # the $image is going to be returned directly in this case so we need to duplicate # some of processing that lookup does for other img_types @@ -539,5 +564,5 @@ $image->{stage_id} = $image->{diff_id}; $image->{stage} = "diff"; - $image->{image_db} = $image_db; + $image->{imagedb} = $imagedb; $image->{component} = $image->{skycell_id}; } else { @@ -554,5 +579,6 @@ sub lookup_bycoord { my $ipprc = shift; - my $image_db = shift; + my $rowList = shift; + my $imagedb = shift; my $img_type = shift; my $tess_id = shift; @@ -568,23 +594,15 @@ my $verbose = shift; - my $missing_tools; - my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); - my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); - my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); - my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - my $results = (); if (($img_type eq "raw") or ($img_type eq "chip")) { - my $runs = lookup_runs_by_camid_and_coords($ipprc, $image_db, $img_type, + my $runs = lookup_runs_by_cam_id_and_coords($ipprc, $imagedb, $img_type, $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose); + # lookup is going to filter these we don't need to do it here +# $runs = filterRuns($img_type, $need_magic, $runs, $verbose); foreach my $run (@$runs) { next if $component and ($run->{component} ne $component); - my $these_results = lookup($ipprc, $image_db, "byid", $img_type, $run->{id}, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $run->{id}, $tess_id, $run->{component}, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); @@ -602,5 +620,5 @@ foreach my $skycell (@$skycells) { - my $these_results = lookup($ipprc, $image_db, "byskycell", $img_type, undef, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef, $skycell->{tess_id}, $skycell->{component}, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose); @@ -614,10 +632,10 @@ } -# lookup_runs_by_camid_and_coords +# lookup_runs_by_cam_id_and_coords # given an ra, dec, and optionally other paramters, find camera runs for exposures # that are within some distance of the coordinates -sub lookup_runs_by_camid_and_coords { +sub lookup_runs_by_cam_id_and_coords { my $ipprc = shift; - my $image_db = shift; + my $imagedb = shift; my $img_type = shift; my $ra = shift; @@ -631,14 +649,7 @@ my $camruns; - my $missing_tools; - my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); - my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } { - my $command = "$camtool -dbname $image_db -processedexp"; + my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order"; $command .= " -ra $ra -decl $dec -radius 1.6"; $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; @@ -652,19 +663,7 @@ $command .= " -data_group $data_group" if $data_group; $command .= " -destreaked" if $need_magic; + # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - return undef; - } - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - - my $output = join "", @$stdout_buf; - if (!$output) { - print STDERR "no output returned from $command\n" if $verbose; - return undef; - } - $camruns = parse_md_fast($mdcParser, $output); + $camruns = runToolAndParse($command, $verbose); } if (!$camruns) { @@ -672,11 +671,18 @@ } my $runs; + my $last_exp_id = 0; foreach my $camRun (@$camruns) { my $cam_id = $camRun->{cam_id}; - - updateCamRunCache($camRun); + my $exp_id = $camRun->{exp_id}; + + next if $exp_id eq $last_exp_id; next if $camRun->{quality}; next if $camRun->{fault}; + + updateCamRunCache($camRun); + + $last_exp_id = $exp_id; + # XXX Use file rule my $astrom = $camRun->{path_base} . ".smf"; @@ -684,12 +690,16 @@ next if !$astrom_resolved; + my $start_dvo = DateTime->now->mjd; my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec"; # run the tool and parse the output + my $dvo_verbose = 0; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => $dvo_verbose); + my $dtime_dvo = (DateTime->now->mjd - $start_dvo)* 86400.; +# print "Time to run dvoImagesAtCoords: $dtime_dvo\n"; unless ($success) { my $result_code = $error_code >> 8; if ($result_code == $PSTAMP_NO_OVERLAP) { - print STDERR "no overlap for $astrom\n" if $verbose; + print "no overlap for $astrom\n" if $verbose; next; } @@ -721,4 +731,7 @@ die "unexpected output from dvoImagesAtCoords: $lines[0]"; } + + print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n"; + # build the hash to return my $run = { @@ -732,7 +745,11 @@ }; if ($img_type eq "chip") { - $run->{id} = $run->{chip_id}; + $run->{id} = $camRun->{chip_id}; + $run->{state} = $camRun->{chip_state}; + $run->{magicked} = $camRun->{chip_magicked}; } else { - $run->{id} = $run->{exp_id}; + $run->{id} = $camRun->{exp_id}; + $run->{state} = 'full'; + $run->{magicked} = $camRun->{raw_magicked}; } push @$runs, $run; @@ -749,11 +766,4 @@ my $dec = shift; my $verbose = shift; - - my $missing_tools; - my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } $requested_tess_id = "" if isnull($requested_tess_id); @@ -817,5 +827,5 @@ sub find_astrometry { my $ipprc = shift; - my $image_db = shift; + my $imagedb = shift; my $image = shift; # hashref to output of the lookup tool my $verbose = shift; @@ -825,5 +835,5 @@ my $exp_id = $image->{exp_id}; if (($exp_id eq $last_exp_id) and $lastAstromFile) { - # running camtool 60 times is really expensive when the answer is the same + # running camtool 60 times is really expensive when the answer is the same every time return $lastAstromFile; } @@ -831,15 +841,7 @@ $lastAstromFile = undef; - my $missing_tools; - my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1); - my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - my $camRun = getCamRunByExpID($exp_id); if (!$camRun) { - my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id"; + my $command = "$camtool -dbname $imagedb -processedexp -exp_id $exp_id -pstamp_order"; # run the tool and parse the output my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -864,6 +866,6 @@ # If there are multiple cam runs for this exposure, take the last completed one with good quality # on the assumption that it has the best astrometry. - while ($camRun = pop @$camruns) { - last if (($camRun->{quality} eq 0) and ($camRun->{fault} eq 0)); + foreach $camRun (@$camruns) { + last if (($camRun->{state} eq 'full') and ($camRun->{quality} eq 0) and ($camRun->{fault} eq 0)); } # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT @@ -959,9 +961,10 @@ # resolve_project() # get project specific information -my $last_project = ""; sub resolve_project { my $ipprc = shift; my $project_name = shift; + findTools(); + if (!$project_name) { carp ("project is not defined"); @@ -978,30 +981,13 @@ my $verbose = 0; - my $missing_tools; - my $pstamptool = can_run("pstamptool") or (warn "Can't find pstamptool" and $missing_tools = 1); - if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); - } - my $command = "$pstamptool -project -name $project_name"; $command .= " -dbname $dbname" if defined $dbname; $command .= " -dbserver $dbserver" if defined $dbserver; - # run the tool and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; + + my $proj_hash = runToolAndParse($command, $verbose); + if (!$proj_hash or scalar @$proj_hash == 0) { + print STDERR "Project $project_name not found\n"; return undef; } - - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - - my $output = join "", @$stdout_buf; - if (!$output) { - print STDERR "no output returned from $command\n" if $verbose; - return undef; - } - my $proj_hash = parse_md_fast($mdcParser, $output); $last_project = $proj_hash->[0]; @@ -1035,13 +1021,280 @@ sub getCamRunByExpID { my $exp_id = shift; - die "getCamRun: exp_id is nil" if !$exp_id; + &my_die ("getCamRun: exp_id is nil", $PS_EXIT_PROG_ERROR) if !$exp_id; return $camRunByExpIDCache{$exp_id}; } + sub getCamRunByCamID { my $cam_id = shift; - die "getCamRun: cam_id is nil" if !$cam_id; + &my_die ("getCamRun: cam_id is nil", $PS_EXIT_PROG_ERROR) if !$cam_id; return $camRunByCamIDCache{$cam_id}; } + +sub selectComponents { + my $ipprc = shift; + my $imagedb = shift; + my $req_type = shift; + my $stage = shift; + my $rowList = shift; + my $runList = shift; + my $verbose = shift; + my $results = []; + + my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); + my $npoints = 0; + foreach my $row (@$rowList) { + print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n"; + $npoints++; + } + close $pointsList; + + # XXX: need to loop over these + # my $run = $runList->[0]; + if (($req_type eq "byid") or ($req_type eq "byexp")) { + my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", ""); + foreach my $run (@$runList) { + my $command = "$dvoImagesAtCoords -coords $pointsListName"; + if (($stage eq "chip") or ($stage eq "raw")) { + # XXX: use file rule and handle cameras where the astrometry is solved at + # the chip stage. + $astrom_file = $run->{cam_path_base} . ".smf"; + my $astrom_file_resolved = $ipprc->file_resolve($astrom_file); + $command .= " -astrom $astrom_file_resolved"; + } else { + my $tess_id = $run->{tess_id}; + if ($tess_id ne $last_tess_id) { + $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); + $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); + $last_tess_id = $tess_id; + } + $command .= " -D CATDIR $tess_dir_abs" + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP + # # That just means that the coordinate didn't match any image/skycell + if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) { + print STDERR @$stderr_buf; + my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR; + &my_die( "dvoImagesAtCoords failed: $rc", $rc); + } + } + my %components; + my @lines = split "\n", join "", @$stdout_buf; + foreach my $line (@lines) { + my ($ptnum, undef, undef, $component) = split " ", $line; + my $ref = $components{$component}; + if (!$ref) { + $ref = $components{$component} = []; + } + push @$ref, $ptnum; + } + if ($verbose) { + foreach my $c (keys %components) { + my $ref = $components{$c}; + print "component $c contains: "; + foreach my $i (@$ref) { + print "$i "; + } + print "\n"; + } + } + # now find the images for this run + foreach my $c (keys %components) { + my $command; + my $stage_id; + if ($stage eq 'raw') { + $stage_id = $run->{exp_id}; + $command = "$regtool -processedimfile -exp_id $stage_id -class_id $c"; + } elsif ($stage eq 'chip') { + $stage_id = $run->{chip_id}; + $command = "$chiptool -processedimfile -chip_id $stage_id -class_id $c"; + } elsif ($stage eq 'warp') { + $stage_id = $run->{warp_id}; + $command = "$warptool -warped -warp_id $stage_id -skycell_id $c"; + } elsif ($stage eq 'diff') { + $stage_id = $run->{diff_id}; + $command = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $c"; + } + $command .= " -dbname $imagedb"; + my $images = runToolAndParse($command, $verbose); + if (!defined $images) { + # XXX: need to get the set error_code to $PSTAMP_NO_OVERLAP + print "No components containing coordinates found for ${stage}_id $stage_id\n"; + foreach my $row (@$rowList) { + $row->{error_code} = $PSTAMP_NO_OVERLAP; + } + next; + } + if (scalar @$images != 1) { + my $num_images = scalar @$images; + &my_die ("unexpected number of images returned: $num_images\n", $PS_EXIT_PROG_ERROR); + } + my $image = $images->[0]; + + my $ref = $components{$c}; + $image->{row_index} = $ref; + if (($stage eq "raw") or ($stage eq 'chip')) { + $image->{astrom} = $astrom_file; + $image->{cam_id} = $run->{cam_id}; + $image->{cam_path_base} = $run->{cam_path_base}; + } + push @$results, $image; + } + } + } else { + &my_die ("sorry not done with $req_type\n", $PS_EXIT_PROG_ERROR); + } + return $results; +} + +sub filterRuns { + my $stage = shift; + my $need_magic = shift; + my $inputs = shift; + my $verbose = shift; + + if ($inputs and (scalar @$inputs) == 1) { + # one run nothing to do + return $inputs; + } + + my $output = []; + + return $output if (!$inputs or scalar @$inputs == 0); + + my $id_name = $stage . "_id"; + + # input list is "order by exp_id, run_id DESC" run_id is one of (chip_id, warp_id, diff_id) + print "Starting filterRuns\n"; + my $last_exp_id = 0; + my $last_run_id = 0; + my $printed = 0; + foreach my $input (@$inputs) { + my $exp_id = $input->{exp_id}; + my $run_id = $input->{$id_name}; + my $magicked = $input->{magicked}; # this will be either stageRun.magicked or stage%file.magicked + my $state = $input->{state}; + + # can't process run in these states + if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) { + print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n"; + next; + } + + $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id)); + + # skip if we need magicked run and this one has never been magicked + if ($need_magic and !$magicked) { + print "skipping ${stage}Run $run_id for exp_id $exp_id not magicked\n" if !$printed; + next; + } + + if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) { + print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed; + next; + } + print "Keeping ${stage}Run $run_id for $exp_id\n"; + push @$output, $input; + $last_exp_id = $exp_id; + $last_run_id = $run_id; + $printed = 0; + } + + my $num_runs = scalar @$output; + print "filterRuns returning $num_runs ${stage}Run\n"; + + return $output; +} + +# run a command that produces metadata output and parse the results into an array of objects +sub runToolAndParse { + my $command = shift; + my $verbose = shift; + + my ($program) = split " ", $command; + $program = basename($program); + + print "Running $command\n" if !$verbose; + my $start_tool = DateTime->now->mjd; + # run the command and parse the output + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + # not sure if we should die here + print STDERR @$stderr_buf; + return undef; + } + + my $now = DateTime->now->mjd; + my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; + print "Time to run $program: $dtime_tool\n"; + + my $buf = join "", @$stdout_buf; + if (!$buf) { + return undef; + } + + my $start_parse = DateTime->now->mjd; + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results = parse_md_fast($mdcParser, $buf) + or die ("Unable to parse metadata config doc"); + + my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; + print "Time to parse results from $program: $dtime_parse\n"; + + return $results; +} + +sub findTools { + return if ($regtool); + + my $missing_tools; + $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); + $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); + $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); + $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); + $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); + $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); + $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" + and $missing_tools = 1); + $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); + $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); + } +} + +# add a row_index array to a set of images with entries for each of the rows in a list +sub setRowRefs { + my $rowList = shift; + my $images = shift; + + my $row_index = []; + for (my $i = 0; $i < scalar @$rowList; $i++) { + push @$row_index, $i; + } + foreach my $image (@$images) { + $image->{row_index} = $row_index; + } +} + +sub my_die +{ + my $msg = shift; + my $fault = shift; + + carp $msg; + + # we don't fault the request here pstamp_parser_run.pl handles that if necessary + + return $fault; +} 1; Index: trunk/ippTasks/pstamp.pro =================================================================== --- trunk/ippTasks/pstamp.pro (revision 27750) +++ trunk/ippTasks/pstamp.pro (revision 27751) @@ -13,4 +13,5 @@ $pstampFin_DB = 0 $pstampRev_DB = 0 +$pstampDep_DB = 0 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config @@ -28,8 +29,8 @@ book init pstampJob book init pstampFinish + book init pstampDependent end pstamp.reset - macro pstamp.on @@ -60,4 +61,7 @@ active true end + task pstamp.dependent.load + active true + end end @@ -88,4 +92,7 @@ task pstamp.job.revert active false + end + task pstamp.dependent.load + active true end end @@ -208,8 +215,9 @@ book getword pstampRequest $pageName uri -var URI book getword pstampRequest $pageName ds_outProduct -var PRODUCT + book getword pstampRequest $pageName label -var LABEL host anyhost - $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --redirect-output + $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --label $LABEL --redirect-output add_standard_args run @@ -512,2 +520,146 @@ end +task pstamp.dependent.load + host local + + periods -poll $LOADPOLL + # XXX: create a macro for this + periods -exec 30 +# periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout $LOGSUBDIR/pstamp.dependent.load.log + stderr $LOGSUBDIR/pstamp.dependent.load.log + $run = pstamptool -pendingdependent + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampDep_DB + $run = $run -dbname $DB:$pstampDep_DB $PS_DBSERVER + $pstampDep_DB ++ + if ($pstampDep_DB >= $DB:n) set pstampDep_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampDependent -key dep_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + book npages pstampDependent -var N + if ($VERBOSE > 2) + book listbook pstampDependent + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pstampDependent + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end + + task.exit default + showcommand failure + end + +end + +task pstamp.dependent.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec +# stdout $LOGSUBDIR/pstamp.dependent.run.log +# stderr $LOGSUBDIR/pstamp.dependent.run.log + book npages pstampDependent -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + book getpage pstampDependent 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + #echo pageName: $pageName + + book setword pstampDependent $pageName pantaskState RUN + book getword pstampDependent $pageName dep_id -var DEP_ID + book getword pstampDependent $pageName state -var STATE + book getword pstampDependent $pageName stage -var STAGE + book getword pstampDependent $pageName stage_id -var STAGE_ID + book getword pstampDependent $pageName component -var COMPONENT + book getword pstampDependent $pageName imagedb -var IMAGEDB + book getword pstampDependent $pageName rlabel -var RLABEL +# book getword pstampDependent $pageName output_base -var OUTPUT_BASE + book getword pstampDependent $pageName need_magic -var NEED_MAGIC + book getword pstampDependent $pageName dbname -var DBNAME + + if ($VERBOSE > 1) + book listpage pstampDependent $pageName + end + + host anyhost + + if ("$NEED_MAGIC" == "T") + $NEED_MAGIC="--need_magic" + else + $NEED_MAGIC="" + end + + # XXX: use OUTPUT_BASE for this + $LOGFILE = /data/ipp053.0/home/bills/pstamp/logs/pstampdep.$DEP_ID.log +# echo logfile is $LOGFILE +# stdout $LOGFILE +# stderr $LOGFILE + + $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit $EXIT_SUCCESS + if ($VERBOSE > 1) + echo pstamp.dependent.run task.exit $DEP_ID status: $JOB_STATUS + end + process_exit pstampDependent $options:0 $JOB_STATUS + end + task.exit default + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS + end + showcommand failure + process_exit pstampDependent $options:0 $JOB_STATUS + end + + task.exit crash + echo pstamp.job.run task.crash $DEP_ID status: $JOB_STATUS + process_exit pstampDependent $options:0 $JOB_STATUS + showcommand crash + book setword pstampDependent $options:0 pantaskState CRASH + end + + task.exit timeout + echo pstamp.job.run task.timeout $DEP_ID status: $JOB_STATUS + process_exit pstampDependent $options:0 $JOB_STATUS + showcommand timeout + book setword pstampDependent $options:0 pantaskState TIMEOUT + end +end + Index: trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27750) +++ trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27751) @@ -48,4 +48,5 @@ my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); @@ -86,5 +87,5 @@ } else { if ($stage eq "chip") { - $cmd = "$chiptool -processedimfile -chip_id $stage_id -class_id $component"; + $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component"; } elsif ($stage eq "warp") { $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; @@ -96,11 +97,9 @@ } -my $components = runToolAndParse($cmd, $verbose); - -my $n = scalar @$components; -my_die("Unexpected number of components found $n", $PS_EXIT_PROG_ERROR) if ($n != 1); +my $it = runToolAndParseExpectOne($cmd, $verbose); + +my_die("no components found", $PS_EXIT_PROG_ERROR) if ( !$it); # Got "it" -my $it = $components->[0]; my $status = 0; @@ -130,6 +129,5 @@ # need to queue the updates if ($stage eq 'chip') { - # queue_update_chip takes an array so that queue_update_warp can pass a set of - # chips to it + # queue_update_chip takes an array so that queue_update_warp can pass it set of chips my $chips = [$it]; $status = queue_update_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); @@ -145,8 +143,22 @@ # detect states that cannot be updated and update the job state - # XXX: we should be more assertive here and check for the specific states where we can continue + # XXX: Perhaps I should be more assertive here and check for the specific states that we know + # that we can continue. + my $state = $it->{state}; my $job_fault = 0; - if (($state =~ /scrub/) or ($state =~ /purge/)) { + + # temporary hack to deal with data with old burntool tables. This check is now done by pstampparse.pl + # so runs in this state will not get queued + if ($stage eq 'chip') { + my $burntool_state = $it->{burntool_state}; + if ($burntool_state and (abs($burntool_state) < 14)) { + print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n"; + $job_fault = $PSTAMP_NOT_AVAILABLE; + } + } + if ($state eq 'error_cleaned') { + $job_fault = $PSTAMP_NOT_AVAILABLE; + } elsif (($state =~ /scrub/) or ($state =~ /purge/)) { # jobs must have changed state since depenency was made print STDERR "Dependency cannot be satisfied\n"; @@ -155,4 +167,7 @@ # Dependency never should have been inserted my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR); + } + if (!$job_fault and ($stage eq 'chip')) { + $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $it->{raw_magicked}, $it->{dsRun_state}); } if ($job_fault) { @@ -213,7 +228,334 @@ } + my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $raw_all_magicked, $dsRun_state); + + return $status; +} + +sub queue_update_warp { + # check status of input chips. If state is not updatable set error code for job + + # if chipProcessedImfile.state is cleaned call queue_update_chip + + # need to code warptool -setskyfiletoupdate + my $metadata = shift; + my $whole_run = shift; # if true queue entire run for update + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + + my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + + my $warp_id = $metadata->{warp_id}; + if (!$whole_run) { + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data == 0; + + my $good_to_go = 1; + my @chipsToUpdate; + my $chip_id; + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $good_to_go = 0; + push @chipsToUpdate, $chip; + } else { + # this chip is good to go + } + } + + if ($good_to_go) { + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); + } + } else { + my $warpRun = $metadata; + my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data != 1; + + my $chipRun = $data->[0]; + + my $chipRunState = $chipRun->{state}; + if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { + print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; + faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); + return 0; + } + my $warpRunState = $warpRun->{state}; + if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { + # The inputs and outputs are ready. Queue the warpRun for update. + + # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif ($chipRunState eq 'cleaned' or + (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { + my $data = [$chipRun]; + return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); + } + } + + # return value may be the return status of script so zero is good + return 0; +} +sub queue_update_diff { + my $metadata = shift; + my $whole_run = shift; # if true queue entire run for update + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + +# XXXX This is not ready to go yet XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXxx + + my $diff_id = $metadata->{diff_id}; + my $diff_mode = $metadata->{diff_mode}; + if (!$whole_run) { + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + # put this in a module somewhere + my $IPP_DIFF_MODE_WARP_WARP = 1; + my $IPP_DIFF_MODE_WARP_STACK = 2; + my $IPP_DIFF_MODE_STACK_WARP = 2; + my $IPP_DIFF_MODE_STACK_STACK = 4; + if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { + # check the state of the stack + my $command = "$stacktool -sumkskyfile -stack_id $skycell->{stack2}"; + my $stack = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; + + if ($stack->{state} ne 'full') { + print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + # this faults all jobs depending on this dep_id + faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # now check the warp + $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; + my $warp = runToolAndExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) + if !$warp; + } + + my $good_to_go = 1; + my @chipsToUpdate; + my $chip_id; + my $warp_id; + my $data ; #XXXX + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $good_to_go = 0; + push @chipsToUpdate, $chip; + } else { + # this chip is good to go + } + } + + if ($good_to_go) { + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); + } + } else { + my $warpRun = $metadata; + my $warp_id = $warpRun->{warp_id}; + my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data != 1; + + my $chipRun = $data->[0]; + + my $chipRunState = $chipRun->{state}; + if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { + print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; + faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); + return 0; + } + my $warpRunState = $warpRun->{state}; + if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { + # The inputs and outputs are ready. Queue the warpRun for update. + + # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif ($chipRunState eq 'cleaned' or + (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { + my $data = [$chipRun]; + return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); + } + } + + # return value may be the return status of script so zero is good + return 0; +} + + +# run a command that produces metadata output and parse the results into an array of objects +sub runToolAndParse { + my $command = shift; + my $verbose = shift; + + my ($program) = split " ", $command; + $program = basename($program); + + print "Running $command\n" if !$verbose; + my $start_tool = DateTime->now->mjd; + # run the command and parse the output + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf if !$verbose; + return undef; + } + + my $now = DateTime->now->mjd; + my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; + print "Time to run $program: $dtime_tool\n"; + + my $buf = join "", @$stdout_buf; + if (!$buf) { + return undef; + } + + my $start_parse = DateTime->now->mjd; + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results = parse_md_fast($mdcParser, $buf) + or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR); + + my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; + print "Time to parse results from $program: $dtime_parse\n"; + + return $results; +} + +# run an command returning metadata where we expect 1 entry +sub runToolAndParseExpectOne { + my $command = shift; + my $verbose = shift; + + my $data = runToolAndParse($command, $verbose); + + if (!$data) { + return undef; + } + + my $n = scalar @$data; + if ($n > 1) { + my_die("Unexpected number of components $n returned by $command", $PS_EXIT_PROG_ERROR); + } + + return $data->[0]; +} + +# splits meta data config input stream into single units to work around the pathalogically +# slow parser. This is similar to and adapted from code in various ippScripts. +sub parse_md_fast { + my $mdcParser = shift; + my $input = shift; + my $output = (); + + my @whole = split /\n/, $input; + my @single = (); + + my $n; + while ( ($n = @whole) > 0) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or + print STDERR "Unable to parse metdata config doc" and return undef; + push @$output, $list->[0]; + + @single = (); + } + } + return $output; +} + +sub faultJobs { + my ($state, $stage, $stage_id, $job_fault) = @_; + + my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set pstampJob.fault for dep_id: $dep_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } +} + +sub queue_update_magicDSRun { + my $stage = shift; + my $stage_id = shift; + my $rlabel = shift; + my $input_magicked = shift; + my $dsRun_state = shift; + + # XXX: this code is only fully coded and tested for chip stage + my_die ("queue_update_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; + # if called from queue_update_warp dsRun_state is unknown. Go find it. if (!$dsRun_state) { - my $command = "$chiptool -listrun -chip_id $chip_id"; + my $command = "$chiptool -listrun -chip_id $stage_id"; my $data = runToolAndParse($command, $verbose); my $chipRun = $data->[0]; @@ -222,5 +564,5 @@ # if the input file is already magicked no need to queue destreaking for this chipRun - if ($need_magic and !$raw_all_magicked) { + if ($need_magic and !$input_magicked) { if ($dsRun_state eq 'cleaned') { my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id"; @@ -242,177 +584,14 @@ } } - - # return value may be the return status of script so zero is good return 0; } -sub queue_update_warp { - # check status of input chips. If state is not updatable set error code for job - - # if chipProcessedImfile.state is cleaned call queue_update_chip - - # need to code warptool -setskyfiletoupdate - my $metadata = shift; - my $whole_run = shift; # if true queue entire run for update - my $rlabel = shift; # if defined a new label for the chipRun - my $need_magic = shift; - - my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked - - my $warp_id = $metadata->{warp_id}; - if (!$whole_run) { - my $skycell = $metadata; - my $skycell_id = $skycell->{skycell_id}; - - my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; - my $data = runToolAndParse($command, $verbose); - my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) - if !$data or scalar @$data == 0; - - my $good_to_go = 1; - my @chipsToUpdate; - my $chip_id; - foreach my $chip (@$data) { - $chip_id = $chip->{chip_id}; - if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { - $good_to_go = 0; - push @chipsToUpdate, $chip; - } else { - # this chip is good to go - } - } - - if ($good_to_go) { - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif (scalar @chipsToUpdate > 0) { - return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); - } - } else { - my $warpRun = $metadata; - my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; - my $data = runToolAndParse($command, $verbose); - my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) - if !$data or scalar @$data != 1; - - my $chipRun = $data->[0]; - - my $chipRunState = $chipRun->{state}; - if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { - print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; - faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); - return 0; - } - my $warpRunState = $warpRun->{state}; - if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { - # The inputs and outputs are ready. Queue the warpRun for update. - - # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif ($chipRunState eq 'cleaned' or - (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { - my $data = [$chipRun]; - return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); - } - } - - # return value may be the return status of script so zero is good - return 0; -} -sub queue_update_diff { - return 0; -} - - -# run a command that produces metadata output and parse the results into an array of objects -sub runToolAndParse { - my $command = shift; - my $verbose = shift; - - my ($program) = split " ", $command; - $program = basename($program); - - print "Running $command\n" if !$verbose; - my $start_tool = DateTime->now->mjd; - # run the command and parse the output - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf if !$verbose; - return undef; - } - - my $now = DateTime->now->mjd; - my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; - print "Time to run $program: $dtime_tool\n"; - - my $buf = join "", @$stdout_buf; - if (!$buf) { - return undef; - } - - my $start_parse = DateTime->now->mjd; - - my $mdcParser = PS::IPP::Metadata::Config->new; - my $results = parse_md_fast($mdcParser, $buf) - or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR); - - my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; - print "Time to parse results from $program: $dtime_parse\n"; - - return $results; -} - -# splits meta data config input stream into single units to work around the pathalogically -# slow parser. This is similar to and adapted from code in various ippScripts. -sub parse_md_fast { - my $mdcParser = shift; - my $input = shift; - my $output = (); - - my @whole = split /\n/, $input; - my @single = (); - - my $n; - while ( ($n = @whole) > 0) { - my $value = shift @whole; - push @single, $value; - if ($value =~ /^\s*END\s*$/) { - push @single, "\n"; - - my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or - print STDERR "Unable to parse metdata config doc" and return undef; - push @$output, $list->[0]; - - @single = (); - } - } - return $output; -} -sub faultJobs { - my ($state, $stage, $stage_id, $job_fault) = @_; - - my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id"; +sub my_die +{ + my $msg = shift; + my $fault = shift; + carp $msg; + + my $command = "$pstamptool -updatedepedent -fault $fault -dep_id $dep_id"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -421,5 +600,5 @@ run(command => $command, verbose => $verbose); unless ($success) { - my_die("failed to set pstamJob.fault for dep_id: $dep_id", + my_die("failed to set pstampDependent.fault for dep_id: $dep_id", $PS_EXIT_UNKNOWN_ERROR); } @@ -427,13 +606,5 @@ print "skipping $command\n"; } -} - -sub my_die -{ - my $msg = shift; - my $fault = shift; - - carp $msg; - - return $fault; -} + + exit $fault; +} Index: trunk/pstamp/scripts/pstamp_finish.pl =================================================================== --- trunk/pstamp/scripts/pstamp_finish.pl (revision 27750) +++ trunk/pstamp/scripts/pstamp_finish.pl (revision 27751) @@ -124,5 +124,5 @@ my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); print $rlf "results.fits|||table|\n"; - print $rlf "results.mdc|||txt|\n"; + print $rlf "results.mdc|||text|\n"; my $err_file = "parse_error.txt"; @@ -171,4 +171,5 @@ my $exp_id = $job->{exp_id}; + if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { # this request had a duplicate request name. We can't put the results @@ -188,9 +189,20 @@ } - if ($exp_id ne $last_exp_id) { + my $job_params = get_job_parameters($job); + my $stage = ""; + if ($job_params) { + $stage = $job_params->{stage}; + } + + if ($stage ne 'stack') { # get the metadata for the exposure (if any i.e. stack) # returns an appropriate string if !$exp_id $exp_info = get_exposure_info($image_db, $exp_id); $last_exp_id = $exp_id; + } else { + my $filter = $job_params->{filter}; + $filter = "0" if !$filter; + $exp_info = "0|0|0|$filter|0|0"; + $last_exp_id = -1; } @@ -344,4 +356,32 @@ } +sub get_job_parameters { + my $job = shift; + if (!$job->{outputBase}) { + print "get_job_parameters: $job->{job_id} has no outputBase\n"; + return undef; + } + my $params_file = $job->{outputBase} . '.mdc'; + if (! -e $params_file ) { + print "get_job_parameters: $job->{job_id} has no parameters file\n"; + return undef; + } + open IN, "<$params_file" or die "unable to open $params_file"; + my $data = join "", (); + close IN; + if (! $data ) { + print "get_job_parameters: parameters file is empty\n"; + return undef; + } + my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc"); + + my $results = parse_md_list($metadata); + if (scalar @$results != 1) { + print STDERR "get_job_params: failed to parse_md_list\n"; + return undef; + } + return $results->[0]; +} + sub get_exposure_info { my $image_db= shift; Index: trunk/pstamp/scripts/pstamp_job_run.pl =================================================================== --- trunk/pstamp/scripts/pstamp_job_run.pl (revision 27750) +++ trunk/pstamp/scripts/pstamp_job_run.pl (revision 27751) @@ -250,5 +250,5 @@ copy_and_register_file($f, $backmdl_file, $outdir, $prefix); } - if (0) { + if (0) { # don't enable this yet if ($pattern_file) { @@ -256,5 +256,5 @@ copy_and_register_file($f, $pattern_file, $outdir, $prefix); } - } + } } } Index: trunk/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- trunk/pstamp/scripts/pstamp_parser_run.pl (revision 27750) +++ trunk/pstamp/scripts/pstamp_parser_run.pl (revision 27751) @@ -28,4 +28,5 @@ my $redirect_output; my $product; +my $label; my $verbose; my $dbname; @@ -36,4 +37,5 @@ 'uri=s' => \$uri, 'product=s' => \$product, + 'label=s' => \$label, 'redirect-output' => \$redirect_output, 'verbose' => \$verbose, @@ -146,5 +148,6 @@ if ($request_type eq "PS1_PS_REQUEST") { $reqType = 'pstamp'; - $parse_cmd = $pstampparse; + $parse_cmd = "$pstampparse"; + $parse_cmd .= " --label $label" if $label; } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") { $reqType = 'dquery'; Index: trunk/pstamp/scripts/pstamp_request_file =================================================================== --- trunk/pstamp/scripts/pstamp_request_file (revision 27750) +++ trunk/pstamp/scripts/pstamp_request_file (revision 27751) @@ -187,5 +187,11 @@ for (my $i = 0; $i < scalar @colNames; $i++) { my $writeType = $colWriteType[$i]; - $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status ); + my $data = $colData->[$i]; + if ($writeType == TULONG) { + die "invalid integer data found in column $i\n" unless validIntegers($data); + } elsif ($writeType == TDOUBLE) { + die "invalid numeric data found in column $i\n" unless validNumbers($data); + } + $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $data, $status ); check_fitsio( $status ); } @@ -312,4 +318,42 @@ } +sub checkValid { + my $aref = shift; + my $float = shift; + + return 0 if !defined $aref; + + my $valid = 0; + my $row = 0; + foreach my $val (@$aref) { + $row++; + if ($float) { + if (!($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/)) { + $valid = 0; + print STDERR "Error on row $row: '$val' is not a valid number\n"; + last; + } + } else { + if ($val =~ /\D/) { + $valid = 0; + print STDERR "Error on row $row: '$val' is not a valid integer\n"; + last; + } + } + $valid = 1; + } + + return $valid; +} +sub validIntegers { + return checkValid(@_, 0); +} + +sub validNumbers +{ + return checkValid(@_, 1); +} + + sub printhelp { Index: trunk/pstamp/scripts/pstampparse.pl =================================================================== --- trunk/pstamp/scripts/pstampparse.pl (revision 27750) +++ trunk/pstamp/scripts/pstampparse.pl (revision 27751) @@ -17,4 +17,5 @@ use Carp; use POSIX; +use DateTime; my $verbose; @@ -26,4 +27,5 @@ my $out_dir; my $product; +my $label; my $save_temps; my $no_update; @@ -34,4 +36,5 @@ 'out_dir=s' => \$out_dir, 'product=s' => \$product, + 'label=s' => \$label, 'mode=s' => \$mode, 'dbname=s' => \$dbname, @@ -42,5 +45,5 @@ ); -die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job"); +die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job"); die "--file is required" if !defined($request_file_name); @@ -64,5 +67,4 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1); -my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); @@ -159,4 +161,10 @@ } +# +# Loop over rows in the request file collecting consecutive rows that have the "same images of interest" +# in the sense that their selection parameters will yield the same "Runs". +# Process the groups of rows together to reduce lookup time and to potentially make multiple +# stamps from the same ppstamp process. +# my @rowList; my $num_jobs = 0; @@ -165,30 +173,79 @@ my $need_magic; foreach my $row (@$rows) { - # XXX: TODO: sanity check all parameters - - # If we encounter an error for a particular row add a job with the proper fault code. - # If we encounter an error in this loop we shouldn't really just die. - # We only do that now in the case of I/O or DB errors or the like. - - my $rownum = $row->{ROWNUM}; - my $job_type = $row->{JOB_TYPE}; - - # parameters that select the images of interest - my $project = $row->{PROJECT}; - - # note: resolve_project avoids running pstamptool every time by remembering the - # last project resolved - my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); - if (!$proj_hash) { - print STDERR "project $project not found\n" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + # santiy check the paramaters + if (!checkRow($row)) { + # when it enconters an error checkRow adds a fake job with an appropriate error code to the database $num_jobs++; next; } + # initialize counter for "job number" + $row->{job_num} = 0; + $row->{error_code} = 0; + + if (scalar @rowList == 0) { + push @rowList, $row; + next; + } + + my $firstRow = $rowList[0]; + if (same_images_of_interest($firstRow, $row)) { + # add this row to the list and move on + push @rowList, $row; + next; + } + + # the images of interest for this new row doesn't match the list. + # process the list ... + $num_jobs += processRows(\@rowList); + + # and reset the list to contain just the new row + @rowList = ($row); +} + +# out of rows process the list +if (scalar @rowList > 0) { + $num_jobs += processRows(\@rowList); +} + +if (($mode eq "queue_job") and ($num_jobs eq 0)) { + print STDERR "no jobs created for $req_name\n" if $verbose; + insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); +} + +exit 0; + +sub checkRow { + + my $row = shift; + + # If we encounter an error for a particular row add a job with the proper fault code. + + my $rownum = $row->{ROWNUM}; + if (!validID($rownum)) { + print STDERR "$rownum is not a valid ROWNUM\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $job_type = $row->{JOB_TYPE}; + if (($job_type ne "stamp") and ($job_type ne "get_image")) { + print STDERR "$job_type is not a valid JOB_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + my $req_type = $row->{REQ_TYPE}; - $stage = $row->{IMG_TYPE}; - my $id = $row->{ID}; + if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and + ($req_type ne "byskycell") and ($req_type ne "bydiff")) { + print STDERR "$req_type is not a valid REQ_TYPE\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } + + my $component = $row->{COMPONENT}; - my $tess_id = $row->{TESS_ID}; + if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) { + $row->{COMPONENT} = $component = ""; + } + $row->{TESS_ID} = "" if !defined $row->{TESS_ID}; my $filter = $row->{REQFILT}; @@ -196,9 +253,19 @@ if (length($filter) == 1) { # allow single character filter cuts to work - $filter .= '%'; + $row->{REQFILT} .= '%'; } } my $mjd_min = $row->{MJD_MIN}; + if (defined($mjd_min) and !validNumber($mjd_min)) { + print STDERR "$mjd_min is not a valid MJD_MIN\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $mjd_max = $row->{MJD_MAX}; + if (defined($mjd_max) and !validNumber($mjd_max)) { + print STDERR "$mjd_max is not a valid MJD_MAX\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0; + } my $data_group = $row->{DATA_GROUP}; if (!defined $data_group) { @@ -209,9 +276,7 @@ } - my $x = $row->{CENTER_X}; - my $y = $row->{CENTER_Y}; - - # XXX things don't work if bit zero of option mask is not set; + # req_finish doesn't work if bit zero of option mask is not set; $row->{OPTION_MASK} |= 1; + my $option_mask= $row->{OPTION_MASK}; my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0; @@ -222,21 +287,15 @@ my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); - my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component; - - if (!$skycenter and !$search_component) { + if (!$skycenter and !$component) { print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; - } - - # user requested us to search all components. Set to "" - $search_component = "" if $search_component eq "all"; - + return 0; + } + + my $stage = $row->{IMG_TYPE}; if (!check_image_type($stage)) { print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; + return 0; } @@ -244,6 +303,5 @@ print STDERR "invalid ROI for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; + return 0; } @@ -251,62 +309,100 @@ print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); - $num_jobs++; - next; - } - - + return 0; + } + # $mode list_uri is a debugging mode (it may used by the http interface) # if this happens just croak - my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); - - my $image_db = $proj_hash->{dbname}; - my $camera = $proj_hash->{camera}; - $need_magic = $proj_hash->{need_magic}; - - # Temporary hack so that MOPS can get at non-magicked data -# if ($product and ($product eq "mops-pstamp-results")) { -# $need_magic = 0; -# } - - # For "stamp" and "list_uri" jobs collect rows with the same images of interest in a list so that they - # can be looked up together. - if (@rowList) { - my $firstRow = $rowList[0]; - if (($firstRow->{JOB_TYPE} ne "get_image") and same_images_of_interest($firstRow, $row)) { - - # add this row to the list and move on - push @rowList, $row; - - next; - - } else { - # this row has different selectors - # queue the jobs for the ones we've collected - $num_jobs += queueJobs($mode, \@rowList, $imageList); - @rowList = (); - } - } - - # look up images for the current row + # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri"); + + if ($req_type eq "bycoord") { if (!$skycenter) { print STDERR "center must be specified in sky coordintes for bycoord" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); - $num_jobs++; - next; - } - } + return 0; + } + } + if (($req_type eq "byid") or ($req_type eq "bydiff")) { - if (!validID($id)) { + if (!validID($row->{ID})) { print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + return 0 + } + } + + return 1; +} + +sub processRows { + my $rowList = shift; + my $num_jobs = 0; + + # all rows in the list are compatible + my $row = $rowList->[0]; + + my $project = $row->{PROJECT}; + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); + if (!$proj_hash) { + print STDERR "project $project not found\n" if $verbose; + foreach $row (@$rowList) { + insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); $num_jobs++; - next; - } - } + } + return $num_jobs; + } + my $req_type = $row->{REQ_TYPE}; + $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $component = $row->{COMPONENT}; + my $tess_id = $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $data_group = $row->{DATA_GROUP}; + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + my $option_mask= $row->{OPTION_MASK}; + + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + $need_magic = $proj_hash->{need_magic}; + + # Temporary hack so that MOPS can get at non-magicked data + my $allow_mops_unmagicked = 1; + if ($allow_mops_unmagicked) { + if ($product and (($product eq "mops-pstamp-results") or + ($product eq "mops-pstamp-results2"))) { + $need_magic = 0; + } + } + + my $numRows = scalar @$rowList; + +# $tess_id = "" if !defined $tess_id; +# $component = "" if !defined $component; + + print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n"; + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this # request specification. An array reference is returned. - $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component, - $option_mask, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + my $start_locate = DateTime->now->mjd; + + # XXX: perhaps we should get rid of most of this argument list. + # Now that we are passing down compatible rows all of the + # information required is contained there + + $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component, + $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose); + + # XXX: why use mjd? It doesn't have great precision. + my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.; + print "Time to locate_images for row $rownum $dtime_locate\n"; if (!$imageList or !@$imageList) { @@ -314,32 +410,31 @@ # note in this case queueJobs inserts the fake job for these rows } + # handle this $row->{need_magic} = $need_magic; - push @rowList, $row; -} - -if (@rowList) { + $num_jobs += queueJobs($mode, \@rowList, $imageList); -} - -if (($mode eq "queue_job") and ($num_jobs eq 0)) { - print STDERR "no jobs created for $req_name\n" if $verbose; - insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST); -} - -# PAU - -exit 0; - - -sub queueJobsForRow + + # if a row slipped through with no jobs add one + foreach my $row (@rowList) { + if ($row->{job_num} == 0) { + print "row $row->{ROWNUM} produced no jobs\n"; + print STDERR "row $row->{ROWNUM} produced no jobs\n"; + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + } + } + + return $num_jobs; +} + +sub queueJobForImage { my $row = shift; my $stage = shift; - my $imageList = shift; - my $have_skycells = shift; + my $image = shift; my $need_magic = shift; my $mode = shift; - my $num_jobs = 0; my $rownum = $row->{ROWNUM}; my $option_mask = $row->{OPTION_MASK}; @@ -365,28 +460,243 @@ } + my $component = $image->{component}; + + my $job_num = ++($row->{job_num}); + + my $imagefile = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + # XXX: should we add a faulted job so the client can know what happened if no images come back? + # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback + # to users, but speeds up processing significantly + print STDERR "skipping non-magicked image $imagefile\n" if $verbose; + + # for now assume yes. + + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); + return 1; + } elsif ($stage eq "stack") { + # unconvolved stack images weren't available prior to some point in time. + # XXX: handle this more correctly by examining the stack run's config dump file. + # It looks like # the feature was turned on sometime around November 11, 2009. stackRun 30067 is the lowest + # one that I found with an unconvolved image. + my $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES = 30067; + if ($row->{unconvolved} and ($row->{PROJECT} eq 'gpc1') and + ($image->{stack_id} < $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES)) { + print STDERR "Unconvolved stack image is not available for stackRun.stack_id: $image->{stack_id}\n"; + insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE); + return 1; + } + } + my $exp_id = $image->{exp_id}; + + my $args = $roi_string ? $roi_string : ""; + if ($stage eq "raw" or $stage eq "chip") { + $args .= " -class_id $component" if $component; + } + + # add astrometry file for raw and chip images if one is available + if (($stage eq "chip") || ($stage eq "raw")) { + $args .= " -astrom $image->{astrom}" if $image->{astrom}; + } + + $image->{job_args} = $args; + + # XXX: we can get rid of the following everything that we need is + # in the params file + + $args .= " -file $imagefile"; + + if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { + $args .= " -mask $image->{mask}"; + } + if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { + $args .= " -variance $image->{weight}"; + } + + my $base = basename($image->{image}); + if (! $base =~ /.fits$/ ) { + my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); + } + $base =~ s/.fits$//; + + my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; + my $argslist = "${output_base}.args"; + + # copy the argument list to a file + open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); + print ARGSLIST "$args\n"; + close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + + write_params($output_base, $image); + + my $newState = "run"; + my $fault = 0; + my $dep_id; + + # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it + if ($stage ne 'raw') { + # not ready to handle diff update yet. May never support stack + my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff')); + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + $data_state = $run_state if $stage eq 'stack'; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'drop') or + ($run_state eq 'error_cleaned') or ($data_state eq 'error_cleaned') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back + $newState = 'stop'; + $fault = $PSTAMP_GONE; + } elsif ($allow_wait_for_update and + (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) { + if ($stage eq 'chip') { + my $burntool_state = $image->{burntool_state}; + if ($burntool_state and (abs($burntool_state) < 14)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } + } + if (!$fault) { + # don't wait for update unless the caller asks us to + if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } elsif (!$image->{magicked}) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_DESTREAKED; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); + } + } + } + } + + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; + $command .= " -fault $fault" if $fault; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dep_id $dep_id" if $dep_id; + + if (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping command: $command\n"; + } + + return 1; +} + +# queue jobs for a collection of request specifications that have the same Images of Interest +sub queueJobs +{ + my $mode = shift; + my $rowList = shift; + my $imageList = shift; + + my $firstRow = $rowList[0]; + my $stage = $firstRow->{IMG_TYPE}; + my $job_type = $firstRow->{JOB_TYPE}; + my $need_magic = $firstRow->{need_magic}; + + my $num_jobs = 0; + + if ($mode eq "list_uri") { + foreach my $image (@$imageList) { + print "$image->{image}\n"; + } + } elsif ($job_type eq "get_image") { + my $n = scalar @$rowList; + + my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; + + $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); + + } else { + if (!$imageList or (scalar @$imageList eq 0)) { + # We didn't find any images for this set of rows. Insert a fake job to carry + # the status back to the requestor. + foreach my $row (@$rowList) { + my $error_code = $row->{error_code}; + $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code; + insertFakeJobForRow($row, ++$row->{job_num}, $error_code); + $num_jobs++; + } + return $num_jobs; + } + + foreach my $image (@$imageList) { + # get the array of row indices that touch this image + my $row_index = $image->{row_index}; + if (!$row_index or scalar @$row_index == 0) { + # XXX should this happen? Why did something get returned. + print "image ${stage}_id: $image->{stage_id} component: $image->{component} matched no rows\n"; + next; + } + # XXX: TODO: eventually we may change ppstamp to be able to make multiple stamps per invocation + + foreach my $i (@$row_index) { + my $row = $rowList->[$i]; + + $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode); + } + } + } + + return $num_jobs; +} + +# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); +sub queueGetImageJobs +{ + my $row = shift; + my $imageList = shift; + my $stage = shift; + my $need_magic = shift; + my $mode = shift; + + my $num_jobs = 0; + my $rownum = $row->{ROWNUM}; + my $option_mask = $row->{OPTION_MASK}; + + # For dist_bundle we need + # --camera from $image + # --stage + # --stage_id from $image + # --component from $image + # --path_base + # --outdir global to this script + # loop over images - my $job_num = 0; foreach my $image (@$imageList) { - my $component; - if ($have_skycells) { - $component = $image->{skycell_id}; - } else { - $component = $image->{class_id}; - } - - # skip this component if it is not in the list for this row - next if ! $components->{$component}; - - $job_num++; + my $stage_id = $image->{stage_id}; + my $component = $image->{component}; + + # skip faulted components for now. Should we even be here? + if ($image->{fault} > 0) { + printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; + next; + } + + my $job_num = ++($row->{job_num}); my $imagefile = $image->{image}; if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # XXX: should we add a faulted job so the client can know what happened if no images come back? - # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback - # to users, but speeds up processing significantly + # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed + # in locate_images because it's much more efficient to do the test in the database. + # For these two modes we fall through to here in order to give feedback to the requestor as + # to why the request failed to queue jobs. print STDERR "skipping non-magicked image $imagefile\n" if $verbose; - - # for now assume yes. - insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); $num_jobs++; @@ -396,46 +706,5 @@ my $exp_id = $image->{exp_id}; - my $args = $roi_string ? $roi_string : ""; - if ($stage eq "raw" or $stage eq "chip") { - $args .= " -class_id $component" if $component; - } - - # add astrometry file for raw and chip images if one is available - if (($stage eq "chip") || ($stage eq "raw")) { - $args .= " -astrom $image->{astrom}" if $image->{astrom}; - } - - $image->{job_args} = $args; - - # XXX: we can get rid of the following everything that we need is - # in the params file - - $args .= " -file $imagefile"; - - if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { - $args .= " -mask $image->{mask}"; - } - if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { - $args .= " -variance $image->{weight}"; - } - - my $base = basename($image->{image}); - if (! $base =~ /.fits$/ ) { - my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR); - } - $base =~ s/.fits$//; - - # XXX: TODO use filerule for this. I don't have a camera defined here - if (($stage eq 'chip') and ($image->{camera} eq 'GPC1')) { - $base = "${base}.${component}"; - } - - my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; - my $argslist = "${output_base}.args"; - - # copy the argument list to a file - open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR); - print ARGSLIST "$args\n"; - close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR); + my $output_base = "$out_dir/${rownum}_${job_num}"; write_params($output_base, $image); @@ -454,5 +723,5 @@ $newState = 'stop'; $fault = $PSTAMP_GONE; - } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { + } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) { # don't wait for update unless the caller asks us to if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { @@ -463,5 +732,5 @@ # set up to queue an update run queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, - $run_state, $stage, $image->{stage_id}, $need_magic); + $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic); } } @@ -475,256 +744,5 @@ $command .= " -dep_id $dep_id" if $dep_id; - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { - # mode eq "queue_job" - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - # should we keep going for other uris? If so how do we report that some - # of the work that the request wanted isn't going to get done - my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping command: $command\n"; - } - } - if ( $num_jobs == 0 ) { - print STDERR "no jobs for row $rownum\n" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP); - $num_jobs = 1; - } - return $num_jobs; -} - -# queue jobs for a collection of request specifications that have the same Images of Interest -sub queueJobs -{ - my $mode = shift; - my $rowList = shift; - my $imageList = shift; - - my $firstRow = $rowList[0]; - my $stage = $firstRow->{IMG_TYPE}; - my $job_type = $firstRow->{JOB_TYPE}; - my $need_magic = $firstRow->{need_magic}; - - my $num_jobs = 0; - - if ($mode eq "list_uri") { - foreach my $image (@$imageList) { - print "$image->{image}\n"; - } - } elsif ($job_type eq "get_image") { - my $n = scalar @$rowList; - - my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1; - - $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); - - } else { - if (!$imageList or (scalar @$imageList eq 0)) { - # we didn't find any images for this set of rows. Insert a fake job to carry - # the status back to the requestor - foreach my $row (@$rowList) { - insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH); - $num_jobs++; - } - return $num_jobs; - } - - my $have_skycells; - if (($stage eq "raw") or ($stage eq "chip")) { - $have_skycells = 0; - } else { - $have_skycells = 1; - } - - my $thisRun; - - my $npoints = 0; - my ($pointsList, $pointsListName); - if (scalar @$imageList > 1) { - ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); - foreach my $row (@$rowList) { - $row->{components} = {}; - if ($row->{skycenter}) { - print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; - $npoints++; - } else { - # this row's center is in pixel coordinates add all images to the component list for this row - foreach my $i (@$imageList) { - my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; - my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) - if !$component; - $row->{components}->{$component} = 1; - } - } - } - close $pointsList; - } else { - # only one image. Avoid the expense of dvoImagesAtCoords. - # queue the job and let ppstamp figure out if the center is valid for the image - # the resulting result code will be the same. - foreach my $row (@$rowList) { - $row->{components} = {}; - my $i = $imageList->[0]; - my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id}; - my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component; - $row->{components}->{$component} = 1; - } - } - - my $tess_dir_abs; - my $last_tess_id = ""; - while ($thisRun = getOneRun($stage, $imageList)) { - if ($npoints) { - my_die( "pointsListName is not defined", $PS_EXIT_PROG_ERROR) if !$pointsListName; - # we collected a set of sky coordinates above. - # filter the images so that only those that contain the centers of the ROIs are processed - my $command = "$dvoImagesAtCoords -coords $pointsListName"; - if ($have_skycells) { - my $tess_id = $thisRun->[0]->{tess_id}; - if ($tess_id ne $last_tess_id) { - $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); - $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); - $last_tess_id = $tess_id; - } - $command .= " -D CATDIR $tess_dir_abs"; - } else { - my $astrom = $thisRun->[0]->{astrom}; - my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom; - my $astrom_resolved = $ipprc->file_resolve($astrom); - $command .= " -astrom $astrom_resolved"; - } - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP - # That just means that the coordinate didn't match any image/skycell - if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) { - print STDERR @$stderr_buf; - my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR; - my_die( "dvoImagesAtCoords failed: $rc", $rc); - } - } - # now we have a list of row numbers and components - # eventually we might want to multiple stamp requests for the same image - # into the same ppstamp job but not yet. For now we will queue a new - my @lines = split "\n", join "", @$stdout_buf; - foreach my $line (@lines) { - # parse the line, ignoring the ra and dec - my ($rownum, undef, undef, $component) = split " ", $line; - - # I guess since we need this function we should be using a hash for rowList - my $row = findRow($rownum, $rowList); - $row->{components}->{$component} = 1; - } - } - - foreach my $row (@$rowList) { - $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic, $mode); - } - } - } - return $num_jobs; -} - -# $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode); -sub queueGetImageJobs -{ - my $row = shift; - my $imageList = shift; - my $stage = shift; - my $need_magic = shift; - my $mode = shift; - - my $num_jobs = 0; - my $rownum = $row->{ROWNUM}; - my $option_mask = $row->{OPTION_MASK}; - - # For dist_bundle we need - # --camera from $image - # --stage - # --stage_id from $image - # --component from $image - # --path_base - # --outdir global to this script - - # loop over images - my $job_num = 0; - foreach my $image (@$imageList) { - my $stage_id = $image->{stage_id}; - my $component = $image->{component}; - - # skip faulted components for now. Should we even be here? - if ($image->{fault} > 0) { - printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose; - next; - } - - $job_num++; - - my $imagefile = $image->{image}; - if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { - # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed - # in locate_images because it's much more efficient to do the test in the database. - # For these two modes we fall through to here in order to give feedback to the requestor as - # to why the request failed to queue jobs. - print STDERR "skipping non-magicked image $imagefile\n" if $verbose; - insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED); - $num_jobs++; - - next; - } - my $exp_id = $image->{exp_id}; - - my $output_base = "$out_dir/${rownum}_${job_num}"; - - write_params($output_base, $image); - - my $newState = "run"; - my $fault = 0; - my $dep_id; - - if ($stage ne 'raw') { - my $run_state = $image->{state}; - my $data_state = $image->{data_state}; - $data_state = $run_state if $stage eq "stack"; - if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or - ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { - # image is gone and it's not coming back - $newState = 'stop'; - $fault = $PSTAMP_GONE; - } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { - # don't wait for update unless the caller asks us to - if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { - $newState = 'stop'; - $fault = $PSTAMP_NOT_AVAILABLE; - } else { - # cause the image to be re-made - # set up to queue an update run - queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, - $run_state, $stage, $image->{stage_id}, $need_magic); - } - } - } - - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" - . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask"; - $command .= " -fault $fault" if $fault; - $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dep_id $dep_id" if $dep_id; - - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { + if (!$no_update) { # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -776,9 +794,5 @@ . " -rownum $rownum -state stop -fault $fault"; - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } elsif (!$no_update) { + if (!$no_update) { # mode eq "queue_job" my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -796,74 +810,25 @@ } -sub get_run_id -{ - my $stage = shift; - my $image = shift; - - if ($stage eq "raw") { - return $image->{exp_id}; - } elsif ($stage eq "chip") { - return $image->{chip_id}; - } elsif ($stage eq "warp") { - return $image->{warp_id}; - } elsif ($stage eq "stack") { - return $image->{stack_id}; - } elsif ($stage eq "diff") { - return $image->{diff_id}; - } else { - my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR); - } -} - -# extract components from the imageList that have the same run id and return the list -sub getOneRun { - my $stage = shift; - my $imageList = shift; - - # return if array is empty - return undef if ! @$imageList; - - my $last_run_id = 0; - my @runList; - while ($imageList->[0]) { - my $run_id = get_run_id($stage, $imageList->[0]); - - last if ($last_run_id and ($run_id ne $last_run_id)); - - my $image = shift @$imageList; - $image->{stage} = $stage; - push @runList, $image; - $last_run_id = $run_id; - } - return \@runList; -} - sub same_images_of_interest { my $r1 = shift; my $r2 = shift; - return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); - return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); - return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE}); + return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord")); + return 0 if (($r1->{JOB_TYPE} eq "get_image") or ($r2->{JOB_TYPE} eq "get_image")); return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE}); return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE}); return 0 if ($r1->{ID} ne $r2->{ID}); return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID}); + return 0 if ($r1->{COMPONENT} ne $r2->{COMPONENT}); return 0 if ($r1->{REQFILT} ne $r2->{REQFILT}); return 0 if ($r1->{DATA_GROUP} ne $r2->{DATA_GROUP}); return 0 if ($r1->{MJD_MIN} ne $r2->{MJD_MAX}); return 0 if ($r1->{MJD_MAX} ne $r2->{MJD_MAX}); + return 0 if ($r1->{OPTION_MASK} ne $r2->{OPTION_MASK}); + return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); return 0 if ($r1->{inverse} ne $r2->{inverse}); return 0 if ($r1->{unconvolved} ne $r2->{unconvolved}); - - if (defined($r1->{COMPONENT})) { - return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT}); - } elsif (defined($r2->{COMPONENT})) { - # if first row has no component all of the images will be retrieved, so - # the fact that this row has a component is ok. Fall through to return 1 - # XXX Nope this doesn't work. It is consistent with the other logic in some way - # that i haven't fully thought through - return 0; - } + # don't combine requests in pixel coordinates + return 0 if (($r1->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS) || ($r2->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS)); return 1; @@ -911,5 +876,5 @@ sub queue_update_run { - my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $need_magic) = @_; + my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_; if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { @@ -918,6 +883,13 @@ my $dep_id; - my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb"; - $command .= " -no_magic" if !$need_magic; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component"; + $command .= " -need_magic" if $need_magic; + + # compute rlabel for the run. + # XXX: This bit of policy shouldn't be buried so deeply in the code + # For now use one that implies 'postage stamp server' 'update' 'request_label" + my $rlabel = "ps_ud_" . $label if $label; + $command .= " -rlabel $rlabel" if $rlabel; + if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -929,4 +901,6 @@ chomp $output; $dep_id = $output; + # + # XXX: need to fault the request or something my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id; } else { @@ -936,6 +910,6 @@ $$r_dep_id = $dep_id; - $r_fault = 0; - $r_jobState = 'blocked'; + $$r_fault = 0; + $$r_jobState = 'run'; } @@ -990,4 +964,4 @@ # we don't fault the request here pstamp_parser_run.pl handles that if necessary - return $fault; -} + exit $fault; +}