Changeset 35384 for trunk/PS-IPP-PStamp

Timestamp:

Apr 9, 2013, 2:08:27 PM (13 years ago)

Author:

bills

Message:

beginning rework of the postage stamp parser

Location:

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp

Files:

• Job.pm (modified) (25 diffs)
• RequestFile.pm (modified) (2 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -15 +15 @@
 our @EXPORT_OK = qw( 
                     locate_images
+                    new_locate_images
                     resolve_project
                     getCamRunByCamID
@@ -46 +47 @@
 my $last_project = "";
 
-my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool);
+my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool, $releasetool);
 my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump);
 
@@ -56 +57 @@
     my $rowList  = shift;   # required
     my $req_type = shift;   # required
-    my $img_type = shift;   # required
+    my $stage = shift;   # required
     my $id       = shift;   # required unless req_type eq bycoord or byskycell
     my $tess_id  = shift;
@@ -78 +79 @@
            ($req_type ne "byskycell");
 
-    $default_data_groups = load_data_groups($verbose);
+    if (!$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
 
     my $dateobs_begin;
@@ -104 +107 @@
         my $x = $row->{CENTER_X};
         my $y = $row->{CENTER_Y};
-        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
         return $results;
     }
 
-    if (($req_type eq "byid") and ($img_type eq "diff")) {
+    if (($req_type eq "byid") and ($stage eq "diff")) {
         # lookups of all of the information for diff images requires a two level lookup to
         # get the exposure information. Switching the req_type allows us to keep that code
         # in one place
         $req_type = "bydiff";
-        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose);
         return $results;
     }
@@ -120 +123 @@
         # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
         # the image from the diffRun
-        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose);
         if (!$results) {
             return undef;
         }
-        if ($img_type eq "diff") {
+        if ($stage eq "diff") {
             # lookup_diff has done all of the work
             return $results;
@@ -130 +133 @@
 
         my $image = $results->[0];
-        if ($img_type eq "raw") {
+        if ($stage eq "raw") {
             $req_type = "byid";
             $id = $image->{exp_id};
             return undef if !$id;
             # fall through and lookup byid
-        } elsif ($img_type eq "chip") {
+        } elsif ($stage eq "chip") {
             $req_type = "byid";
             $id = $image->{chip_id};
             return undef if !$id;
             # fall through and lookup byid
-        } elsif ($img_type eq "warp") {
+        } elsif ($stage eq "warp") {
             $req_type = "byid";
             $id = $image->{warp_id};
@@ -146 +149 @@
             return undef if !$id;
             # fall through and lookup by warp_id
-        } elsif ($img_type eq "stack") {
+        } elsif ($stage eq "stack") {
             $req_type = "byid";
             $id = $image->{stack_id};
@@ -153 +156 @@
         } else {
             # This is checked this elsewhere?
-            print STDERR "Error: $img_type is an unknown image type\n";
+            print STDERR "Error: $stage is an unknown image type\n";
             return undef;
         }
     } elsif ($req_type eq "byskycell") {
-        if (($img_type eq "raw") or ($img_type eq "chip")) {
+        if (($stage eq "raw") or ($stage eq "chip")) {
             print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
             return undef;
@@ -167 +170 @@
     }
 
-    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
         $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+
+    return $results;
+}
+
+sub new_locate_images {
+    my $ipprc    = shift; 
+    my $imagedb  = shift;
+    my $camera   = shift;
+    my $row      = shift;
+    my $verbose  = shift;
+
+    my $req_type  = $row->{REQ_TYPE};
+
+    # we die in response to bad data in request files
+    # The caller is responsible for updating the database
+    # pstampparse.pl error checks now so this shouldn't happen
+    &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) 
+        if ($req_type ne "byid") and
+           ($req_type ne "byexp") and 
+           ($req_type ne "bycoord") and 
+           ($req_type ne "bydiff") and 
+           ($req_type ne "byskycell");
+
+    if (!$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
+
+    my $stage     = $row->{IMG_TYPE};
+    my $id        = $row->{ID};
+    my $component = $row->{COMPONENT};
+    my $tess_id   = $row->{TESS_ID};
+
+    my $filter    = $row->{REQFILT};
+    my $mjd_min   = $row->{MJD_MIN};
+    my $mjd_max   = $row->{MJD_MAX};
+    my $data_group = $row->{DATA_GROUP};
+
+    my $rownum     = $row->{ROWNUM};
+    my $job_type   = $row->{JOB_TYPE};
+    my $option_mask= $row->{OPTION_MASK};
+
+    my $dateobs_begin;
+    my $dateobs_end;
+    if (!iszero($mjd_min)) {
+        $dateobs_begin = mjd_to_dateobs($mjd_min);
+    }
+    if (!iszero($mjd_max)) {
+        $dateobs_end = mjd_to_dateobs($mjd_max);
+    }
+    if (isnull($tess_id)) {
+        $tess_id = undef;
+    }
+    if (isnull($data_group)) {
+        $data_group = undef;
+    }
+    if (isnull($filter)) {
+        $filter = undef;
+    }
+
+
+
+    if ($req_type eq "bycoord") {
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, 0, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        return $results;
+    }
+
+    # for compatability with the lower level functions
+    my $rowList = [$row];
+
+    if (($req_type eq "byid") and ($stage eq "diff")) {
+        # lookups of all of the information for diff images requires a two level lookup to
+        # get the exposure information. Switching the req_type allows us to keep that code
+        # in one place
+        $req_type = "bydiff";
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose);
+        return $results;
+    }
+
+    if ($req_type eq "bydiff") {
+        # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
+        # the image from the diffRun
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose);
+        if (!$results) {
+            return undef;
+        }
+        if ($stage eq "diff") {
+            # lookup_diff has done all of the work
+            return $results;
+        } 
+
+        my $image = $results->[0];
+        if ($stage eq "raw") {
+            $req_type = "byid";
+            $id = $image->{exp_id};
+            return undef if !$id;
+            # fall through and lookup byid
+        } elsif ($stage eq "chip") {
+            $req_type = "byid";
+            $id = $image->{chip_id};
+            return undef if !$id;
+            # fall through and lookup byid
+        } elsif ($stage eq "warp") {
+            $req_type = "byid";
+            $id = $image->{warp_id};
+            $component = $image->{skycell_id};
+            return undef if !$id;
+            # fall through and lookup by warp_id
+        } elsif ($stage eq "stack") {
+            $req_type = "byid";
+            $id = $image->{stack_id};
+            return undef if !$id;
+            # fall though and lookup by stack_id
+        } else {
+            # This is checked this elsewhere?
+            print STDERR "Error: $stage is an unknown image type\n";
+            return undef;
+        }
+    } elsif ($req_type eq "byskycell") {
+        if (($stage eq "raw") or ($stage eq "chip")) {
+            print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
+            return undef;
+        }
+        if (!$tess_id or !$component) {
+            print STDERR "component and tess_id are required for REQ_TYPE byskycell\n";
+            return undef;
+        }
+    }
+
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
+        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
 
     return $results;
@@ -505 +640 @@
     my $byid     = shift;
     my $option_mask  = shift;
-    my $img_type = shift;
+    my $stage = shift;
     my $verbose = shift;
 
@@ -573 +708 @@
             # we have a stack - stack diff (well it might be stack - warp....)
             # but at any rate we only handle image type diff and stack
-            if (($img_type ne "diff") and ($img_type ne "stack")) {
-                print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n";
+            if (($stage ne "diff") and ($stage ne "stack")) {
+                print STDERR "lookup_diff: cannot lookup IMG_TYPE $stage bydiff from a stack stack diff run\n";
                 setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST);
                 # all images will be the same so we can stop
@@ -625 +760 @@
         }
 
-        if ($img_type eq "diff") {
+        if ($stage eq "diff") {
             my @imageList = ($image);
 
@@ -656 +791 @@
 sub lookup_bycoord {
     my $ipprc      = shift;
-    my $rowList    = shift;
+    my $row        = shift;
     my $imagedb    = shift;
-    my $img_type   = shift;
+    my $stage      = shift;
     my $tess_id    = shift;
     my $component  = shift;
@@ -673 +808 @@
     my $verbose    = shift;
 
+    my ($release_name, $survey) = get_release($row);
+
+    my $rowList = [$row];
+
     my $results = ();
-    if (($img_type eq "raw") or ($img_type eq "chip")) {
-
-        my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $img_type,
-            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
+    if (($stage eq "raw") or ($stage eq "chip")) {
+
+        my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $stage,
+            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose, $release_name, $survey, $row->{FWHM_MIN}, $row->{FWHM_MAX});
 
         if (!$chips or scalar @$chips == 0) {
@@ -684 +823 @@
             foreach my $chip (@$chips) {
                 next if $component and ($chip->{component} ne $component);
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id},
+                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
                     $tess_id, $chip->{component}, $need_magic, 
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
@@ -695 +834 @@
     } else {
         # this should have been checked elsewhere
-        die "unexpected image type $img_type" if ($img_type ne "warp") 
-                                              and ($img_type ne "stack") and ($img_type ne "diff");
+        die "unexpected image type $stage" if ($stage ne "warp") 
+                                              and ($stage ne "stack") and ($stage ne "diff");
 
         my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
@@ -704 +843 @@
         } else {
             foreach my $skycell (@$skycells) {
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
+                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
                     $skycell->{tess_id}, $skycell->{component}, $need_magic, 
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
@@ -722 +861 @@
 
 sub lookup_by_cam_id_and_coords {
-    my $ipprc      = shift;
-    my $imagedb   = shift;
-    my $img_type   = shift;
-    my $ra         = shift;
-    my $dec        = shift;
-    my $need_magic = shift;
+    my $ipprc       = shift;
+    my $imagedb     = shift;
+    my $stage       = shift;
+    my $ra          = shift;
+    my $dec         = shift;
+    my $need_magic  = shift;
     my $dateobs_begin  = shift;
     my $dateobs_end = shift;
@@ -733 +872 @@
     my $data_group = shift;
     my $verbose    = shift;
-
-    my $camruns;
-
-    {
-        my $search_radius = 1.6; # XXX: this should be camera specific
-
+    my $release_name = shift;
+    my $survey      = shift;
+    my $fwhm_min    = shift;
+    my $fwhm_max    = shift;
+
+    my $search_radius = 1.6; # XXX: this should be camera specific
+
+    my $command;
+    my $using_camtool = 0;
+    if ($release_name or $survey) {
+        my $command = "$releasetool -dbname $imagedb -listrelexp";
+        if ($survey) {
+            $command .= " -survey_name $survey";
+        }
+        if ($release_name) {
+            $command .= " -release_name $release_name";
+        } else {
+            $command .= " -priority_order";
+        }
+        if ($fwhm_min != 0) {
+            $command .= " -fwhm_min $fwhm_min";
+        }
+        if ($fwhm_max != 0) {
+            $command .= " -fwhm_max $fwhm_max";
+        }
+    } else {
+        $using_camtool = 1;
         my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
-           $command .= " -ra $ra -decl $dec -radius $search_radius";
-           $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
-           $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
-           $command .= " -filter $filter" if $filter;
            # NOTE: we are applying the data_group to the camera run.
            # If we're looking for chip stage images there is no guarentee that 
@@ -755 +911 @@
            }
            $command .= " -destreaked" if $need_magic;
-
-        # run the tool and parse the output
-        $camruns = runToolAndParse($command, $verbose);
-    }
-    if (!$camruns) {
+    }
+    $command .= " -ra $ra -decl $dec -radius $search_radius";
+    $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
+    $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
+    $command .= " -filter $filter" if $filter;
+
+    # run the tool and parse the output
+    my $results = runToolAndParse($command, $verbose);
+    if (!$results) {
         return undef;
     }
     my $components;
     my $last_exp_id = 0;
-    foreach my $camRun (@$camruns) {
-        my $cam_id = $camRun->{cam_id};
-        my $exp_id = $camRun->{exp_id};
-
+    foreach my $result (@$results) {
+        my $cam_id = $result->{cam_id};
+        my $exp_id = $result->{exp_id};
+
+        # go on if we already have a result for this exposure
         next if $exp_id eq $last_exp_id;
 
-        next if $camRun->{quality};
-        next if $camRun->{fault};
-
-        updateCamRunCache($camRun);
+        next if $result->{quality};
+
+        if ($using_camtool) {
+            next if $result->{fault};
+            updateCamRunCache($result);
+        }
 
         $last_exp_id = $exp_id;
 
         # XXX Use file rule
-        my $astrom = $camRun->{path_base} . ".smf";
+        my $astrom = $result->{path_base} . ".smf";
         my $astrom_resolved = $ipprc->file_resolve($astrom);
         next if !$astrom_resolved;
@@ -832 +995 @@
         }
 
-        print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n";
+        print "cam_id $cam_id exp_id $result->{exp_id} chip_id $result->{chip_id} $class_id\n";
 
         # build the hash to return
@@ -840 +1003 @@
         # problem
         my $comp = {
-            exp_id    => $camRun->{exp_id},
-            exp_name  => $camRun->{exp_name},
-            chip_id   => $camRun->{chip_id},
-            cam_id    => $camRun->{cam_id},
+            exp_id    => $result->{exp_id},
+            exp_name  => $result->{exp_name},
+            chip_id   => $result->{chip_id},
+            cam_id    => $result->{cam_id},
             class_id  => $class_id,
             component => $class_id
         };
-        if ($img_type eq "chip") {
-            $comp->{id} = $camRun->{chip_id};
-            $comp->{state} = $camRun->{chip_state};
-            $comp->{magicked} = $camRun->{chip_magicked};
-        } else {
-            $comp->{id} = $camRun->{exp_id};
+        if ($stage eq "chip") {
+            $comp->{id} = $result->{chip_id};
+            $comp->{state} = $result->{chip_state};
+            $comp->{magicked} = 0;
+        } else {
+            $comp->{id} = $result->{exp_id};
             $comp->{state} = 'full';
-            $comp->{magicked} = $camRun->{raw_magicked};
+            $comp->{magicked} = $0;
         }
         push @$components, $comp;
@@ -1407 +1570 @@
     $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" 
                                                                 and $missing_tools = 1);
+    $releasetool = can_run('releasetool') or (warn "Can't find releasetool" 
+                                                                and $missing_tools = 1);
     $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
     $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
@@ -1462 +1627 @@
 }
 
+sub get_release {
+    my $row = shift;
+
+    my $survey;
+    my $release_name = $row->{IPP_RELEASE};
+    if (!isnull($release_name)) {
+        # no release check for survey
+        $survey = $row->{SURVEY_NAME};
+    }
+
+    return ($release_name, $survey);
+}
+
 sub my_die
 {

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -52 +52 @@
                     $PSTAMP_NO_OVERLAP
                     $PSTAMP_NOT_AUTHORIZED
+                    $PSTAMP_UNKNOWN_PROJECT
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -101 +102 @@
 our $PSTAMP_NO_OVERLAP       = 28;
 our $PSTAMP_NOT_AUTHORIZED   = 29;
+our $PSTAMP_UNKNOWN_PROJECT  = 22;