Changeset 35398 for trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

Timestamp:

Apr 12, 2013, 11:03:47 AM (13 years ago)

Author:

bills

Message:

If user specifies RELEASE_NAME or SURVEY use releasetool
to look up images.
When sources are selected extract them from the region of interest
using ppstamp. Use skycal cmf if available.

File:

• trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (28 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -15 +15 @@
 our @EXPORT_OK = qw( 
                     locate_images
-                    new_locate_images
+                    locate_images_for_row
                     resolve_project
                     getCamRunByCamID
@@ -79 +79 @@
            ($req_type ne "byskycell");
 
-    if (!$default_data_groups) {
+    if (!$data_group and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
@@ -171 +171 @@
 
     my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
-        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+        undef, undef, undef, $verbose);
 
     return $results;
 }
 
-sub new_locate_images {
+sub locate_images_for_row {
     my $ipprc    = shift; 
     my $imagedb  = shift;
@@ -195 +196 @@
            ($req_type ne "byskycell");
 
-    if (!$default_data_groups) {
-        $default_data_groups = load_data_groups($verbose);
-    }
-
     my $stage     = $row->{IMG_TYPE};
     my $id        = $row->{ID};
@@ -208 +205 @@
     my $mjd_max   = $row->{MJD_MAX};
     my $data_group = $row->{DATA_GROUP};
+    if (isnull($data_group) and !$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
 
     my $rownum     = $row->{ROWNUM};
@@ -230 +230 @@
         $filter = undef;
     }
-
-
 
     if ($req_type eq "bycoord") {
@@ -240 +238 @@
     }
 
-    # for compatability with the lower level functions
+    # for compatability with the lower level functions that haven't been converted to take a single row
     my $rowList = [$row];
 
@@ -251 +249 @@
         return $results;
     }
+
+    my ($release_name, $survey) = get_release_info($row);
 
     if ($req_type eq "bydiff") {
@@ -303 +303 @@
 
     my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
-        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+        undef, $release_name, $survey,
+        $verbose);
 
     return $results;
@@ -328 +330 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $selectedAstrom = shift;
+    my $release_name = shift;
+    my $survey = shift;
     my $verbose  = shift;
 
@@ -352 +357 @@
     my $magic_arg = $need_magic ? " -destreaked" : "";
 
+    my $row = $rowList->[0];
+
     # all rows have the same center type
-    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    my $skycenter = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
+    my $use_releasetool = 0;
+    my $release_args;
+    if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) {
+        unless (isnull($release_name) and isnull($survey)) {
+            $use_releasetool = 1;
+            $release_args = " -release_name $release_name" if $release_name;
+            $release_args .= " -surveyName $survey" if $survey;
+        }
+    }
 
     my $component_args;
@@ -384 +401 @@
         # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
         # choose the chips containing the center pixel by calling selectComponets() below
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $class_id = $component;
-                $component_args = " -class_id $class_id";
-            }
-        } else {
-            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        } else {
+            $command = "$chiptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -processedimfile";
+                if ($component and $component ne 'all') {
+                    $class_id = $component;
+                    $component_args = " -class_id $class_id";
+                }
+            } else {
+                $command .= " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        }
         # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
         # With the mdc file it has all of the information that it needs.
@@ -408 +431 @@
         $want_astrom  = 1;
         $set_class_id = 1;
-    } elsif ($stage eq "warp") {
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$warptool -warped -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $skycell_id = $component;
-                $component_args = " -skycell_id $skycell_id";
-            }
-        } else {
-            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
+    } elsif ($stage eq 'warp') {
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        } else {
+            $command = "$warptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -warped";
+                if ($component and $component ne 'all') {
+                    $skycell_id = $component;
+                    $component_args = " -skycell_id $skycell_id";
+                }
+            } else {
+                $command = " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        }
         $image_name   = "PSWARP.OUTPUT";
         $mask_name    = "PSWARP.OUTPUT.MASK";
@@ -427 +456 @@
         $base_name    = "path_base"; # name of the field for the warptool output
     } elsif ($stage eq "diff") {
+        # XXX: is this path ever used for diff images anymore?
         if ($component or $use_imfile_id or !$skycenter) {
             $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb";
@@ -447 +477 @@
     } elsif ($stage eq "stack") {
         $skycell_id = $component;
-        $command = "$stacktool -sumskyfile -dbname $imagedb";
-        # only consider stacks in full state.
-        $command .= " -state full";
-        $id_opt = "-stack_id";
+        if ($use_releasetool) {
+            $command = "$releasetool -listrelstack -dbname $imagedb $release_args";
+            $base_name = 'stack_path_base';
+        } else {
+            $command = "$stacktool -sumskyfile -dbname $imagedb";
+            # only consider stacks in full state.
+            $command .= " -state full";
+            $id_opt = "-stack_id";
+            $base_name = 'path_base';
+        }
         $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne  'all';
 
@@ -464 +500 @@
         # this is wrong but gets the right answer. Need to figure out how to find the
         # rule properly
-        #$cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $cmf_name     = "PSWARP.OUTPUT.SOURCES";
+        # $cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $psf_name    = "PPSTACK.TARGET.PSF";
-        $base_name   = "path_base";
     } else {
-        die "Unknown IMG_TYPe supplied: $stage";
+        die "Unknown IMG_TYPE supplied: $stage";
     }
 
@@ -512 +547 @@
     if (!isnull($data_group)) {
         $command .= " -data_group $data_group";
-    } elsif ($req_type eq 'byskycell') {
+    } elsif (!$use_releasetool and $req_type eq 'byskycell') {
+        # XXX: Why am I using default data_groups only for byskycell requests?
         $command .= $default_data_groups;
     }
@@ -527 +563 @@
         $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
     }
+
     if ($choose_components) {
         # the list of "images" is actually a list of "Runs"
         # match the coords in the rows to the components in the runs
         # returns an actual list of images
-        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        if ($skycenter) {
+            $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        } else {
+            $images = selectComponentsByName($ipprc, $imagedb, $req_type, $stage, $component, $rowList, $images, $verbose);
+        }
     } else {
-        # put index for each of the rows into all of the the images
+        # put index for each of the rows into all of the the selected images
         setRowRefs($rowList, $images);
     }
@@ -597 +638 @@
             # dquery wants these set
             $out->{magicked} = 0;
-            $out->{data_state} = $out->{state};
+            if ($use_releasetool) {
+                if ($image->{skycal_path_base}) {
+                    $out->{astrom} = $image->{skycal_path_base} . ".cmf";
+                }
+                $out->{data_state} = $out->{state} = $out->{stack_state};
+            } else {
+                $out->{data_state} = $out->{state};
+            }
         }
         $out->{stage_id} = $stage_id;
@@ -605 +653 @@
         my $mask_base;
         if ($want_astrom) {
-            if (! defined $out->{astrom}) {
+            if ($selectedAstrom) {
+                $out->{astrom} = $selectedAstrom;
+                $out->{cam_path_base} = basename($selectedAstrom);
+            } elsif (! defined $out->{astrom}) {
                 if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) {
                     print STDERR "failed to find astrometry for $stage $stage_id\n";
@@ -806 +857 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $astrom = shift;
     my $verbose    = shift;
 
-    my ($release_name, $survey) = get_release($row);
+    my ($release_name, $survey) = get_release_info($row);
 
     my $rowList = [$row];
@@ -825 +877 @@
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
                     $tess_id, $chip->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+                    $chip->{astrom}, undef, undef,
+                    $verbose);
 
                 next if !$these_results;
@@ -845 +899 @@
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
                     $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+                    undef, $release_name, $survey,
+                    $verbose);
 
                 next if !$these_results;
@@ -882 +938 @@
     my $using_camtool = 0;
     if ($release_name or $survey) {
-        my $command = "$releasetool -dbname $imagedb -listrelexp";
-        if ($survey) {
+        $command = "$releasetool -dbname $imagedb -listrelexp";
+        if (!isnull($survey)) {
             $command .= " -survey_name $survey";
         }
-        if ($release_name) {
+        if (!isnull($release_name)) {
             $command .= " -release_name $release_name";
         } else {
@@ -935 +991 @@
         if ($using_camtool) {
             next if $result->{fault};
-            updateCamRunCache($result);
-        }
+        } else {
+            $result->{path_base} = $result->{cam_path_base};
+        }
+        updateCamRunCache($result);
 
         $last_exp_id = $exp_id;
@@ -1008 +1066 @@
             cam_id    => $result->{cam_id},
             class_id  => $class_id,
-            component => $class_id
+            component => $class_id,
+            astrom    => $astrom
         };
         if ($stage eq "chip") {
@@ -1444 +1503 @@
 }
 
+sub selectComponentsByName {
+    my $ipprc = shift;
+    my $imagedb = shift;
+    my $req_type = shift;
+    my $stage  = shift;
+    my $component  = shift;
+    my $rowList = shift;
+    my $runList = shift;
+    my $verbose = shift;
+    my $results = [];
+
+    $component = "" if $component and lc($component) eq 'all';
+
+    # all runs match all rows so they can share a common row_index
+    # XXX: I'm not sure we ever actually get here with more than one
+    # row
+    my $numRows = scalar @$rowList;
+    my $row_index = [];
+    for (my $i=0; $i < $numRows; $i++) {
+        push @$row_index, $i;
+    }
+    
+    if (($req_type eq "byid") or ($req_type eq "byexp")) {
+        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
+        foreach my $run (@$runList) {
+            # now find the images for this run
+            #foreach my $c (keys %components) {
+            my $command;
+            my $stage_id;
+            if ($stage eq 'raw') {
+                $stage_id = $run->{exp_id};
+                $command = "$regtool -processedimfile -exp_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'chip') {
+                $stage_id = $run->{chip_id};
+                $command = "$chiptool -processedimfile -chip_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'warp') {
+                $stage_id = $run->{warp_id};
+                $command = "$warptool -warped -warp_id $stage_id";
+                $command .= " -skycell_id $component" if $component;
+            }
+            $command .= " -dbname $imagedb";
+            my $images = runToolAndParse($command, $verbose);
+            if (!defined $images) {
+                print "No  $component found for $stage ${stage}_id $stage_id\n";
+                next;
+            }
+            foreach my $image (@$images) {
+                $image->{row_index} = $row_index;
+                if (($stage eq "raw") or ($stage eq 'chip')) {
+                    $image->{cam_id} = $run->{cam_id};
+                    $image->{cam_path_base} = $run->{cam_path_base};
+                }
+                push @$results, $image;
+            }
+        }
+    } else {
+        &my_die ("selectComponentsByName not supported for REQ_TYPE: $req_type\n", $PS_EXIT_PROG_ERROR);
+    }
+    return $results;
+}
+
 sub filterRuns {
     my $stage      = shift;
@@ -1627 +1749 @@
 }
 
-sub get_release {
+sub get_release_info {
     my $row = shift;
 
     my $survey;
     my $release_name = $row->{IPP_RELEASE};
-    if (!isnull($release_name)) {
+    if (isnull($release_name)) {
+        $release_name = "";
         # no release check for survey
         $survey = $row->{SURVEY_NAME};
+    } else {
+        $survey = "";
     }